Use Perl-based tflat, which is up to 90% faster than the awk-based one

(necause it was forking off "sort" subprocesses on every package).

2 files changed, 73 insertions, 161 deletions

diff --git a/mk/bulk/bsd.bulk-pkg.mk b/mk/bulk/bsd.bulk-pkg.mk
index 4cb48295957..54b4a595d7e 100644
--- a/mk/bulk/bsd.bulk-pkg.mk
+++ b/mk/bulk/bsd.bulk-pkg.mk
@@ -1,4 +1,4 @@
-#	$NetBSD: bsd.bulk-pkg.mk,v 1.86 2005/10/16 17:35:25 tv Exp $
+#	$NetBSD: bsd.bulk-pkg.mk,v 1.87 2005/10/26 16:54:04 tv Exp $
 
 #
 # Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
@@ -47,13 +47,6 @@ TEE?=	tee
 WC?=	wc
 TO_HTML?=	${SED} -e 's,&,\&amp;,g' -e 's,<,\&lt;,g' -e 's,>,\&gt;,g'
 
-# A sort(1) capable of very long lines is needed for full builds in "tflat".
-# Some platforms (namely, Interix) may not provide one, so override here.
-.if ${OPSYS} == "Interix"
-_SORT=			${LOCALBASE}/bin/${GNU_PROGRAM_PREFIX}sort
-.endif
-_SORT?=			${SORT}
-
 # This variable is set to 'yes' by the pkgsrc/mk/bulk/build script.  It enables
 # the use of several cache files (DEPENDSTREEFILE, DEPENDSFILE, SUPPORTSFILE,
 # and INDEXFILE) for speeding up the processing of dependencies.  If we're not
@@ -199,15 +192,17 @@ bulk-cache:
 		}} END{ \
 		for(pkg in pkgs) {if( pkg in listed ) {} else{ print pkg " " pkg;}} \
 		}' \
-		${BULK_DBFILE} | ${_SORT} -u > ${DEPENDSTREEFILE}
+		${BULK_DBFILE} | ${SORT} -u > ${DEPENDSTREEFILE}
 	@${ECHO_MSG} "BULK> Extracting package name <=> package directory cross reference file"
 	${AWK} '/^index/ {print $$2 " " $$3 " "}' ${BULK_DBFILE} > ${INDEXFILE}
 .endif
 	@${ECHO_MSG} "BULK> Sorting build order."
 	${TSORT} ${DEPENDSTREEFILE} > ${ORDERFILE}
 	@${ECHO_MSG} "BULK> Generating up and down dependency files."
-	${SETENV} SORT=${_SORT:Q} ${AWK} -f ${PKGSRCDIR}/mk/bulk/tflat up ${DEPENDSTREEFILE} > ${SUPPORTSFILE}
-	${SETENV} SORT=${_SORT:Q} ${AWK} -f ${PKGSRCDIR}/mk/bulk/tflat down ${DEPENDSTREEFILE} > ${DEPENDSFILE}
+	${LOCALBASE}/bin/perl ${PKGSRCDIR}/mk/bulk/tflat ${SUPPORTSFILE} ${DEPENDSFILE} < ${DEPENDSTREEFILE}
+
+# note explicit pathname on "perl" above, so that we do NOT auto-set
+# USE_TOOLS=perl (which may be undesirable for package building)
 
 # remove the bulk cache files
 clean-bulk-cache:
diff --git a/mk/bulk/tflat b/mk/bulk/tflat
index a9743946e74..63e8a046650 100755
--- a/mk/bulk/tflat
+++ b/mk/bulk/tflat
@@ -1,10 +1,10 @@
-# $NetBSD: tflat,v 1.14 2005/03/24 16:47:35 tv Exp $
+# $NetBSD: tflat,v 1.15 2005/10/26 16:54:04 tv Exp $
 #
-# Copyright (c) 2001 The NetBSD Foundation, Inc.
+# Copyright (c) 2001, 2005 The NetBSD Foundation, Inc.
 # All rights reserved.
 #
 # This code is derived from software contributed to The NetBSD Foundation
-# by Dan McMahill.
+# by Dan McMahill and Todd Vierling.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -35,170 +35,87 @@
 # POSSIBILITY OF SUCH DAMAGE.
 #
 
-BEGIN {
-	if (ARGC != 3){
-		printf("tflat:  wrong number of arguments\n");
-		usage();
-		exit(1);
-	}
+#
+# This perl version is very fast compared to other alternatives, which would
+# likely use things like forking "sort" subprocesses.
+#
+use strict;
 
-	if ( ARGV[1] == "up" ) {
-		up=1;
-	}
-	else{
-		if ( ARGV[1] == "down" ) { up=0; }
-		else{
-			printf("tflat: unknown option  \"%s\"\n",ARGV[1]);
-			usage();
-			exit(1);
-		}
-	}
-	
-	InFile = ARGV[2];
-
-	#
-	# Read in the entire depends tree
-	#
-	if (up){
-        	while(getline < InFile > 0){
-			if ($1 in topdepends)
-				topdepends[$1] = topdepends[$1] " " $2 " " ;
-			else{
-				topdepends[$1] = " " $2 " ";
-			}
-
-			# Note that it is possible for a package "foo/bar" to
-			# never appear in $1.  In fact if foo/bar is not
-			# depended upon by anything and it has depends then it
-			# will not show up in $1 however, we need to make sure
-			# we get a topdepends[foo/bar] entry so add it here if
-			# its not already there.
-			if (!($2 in topdepends))
-				topdepends[$2] = "";
-		}
-		depstr = " is depended on by: ";
-	}
-	else{
-        	while(getline < InFile > 0){
-			if ($2 in topdepends)
-				topdepends[$2] = topdepends[$2] " " $1 " " ;
-			else
-				topdepends[$2] = " " $1 " " ;
-		}
-		depstr = " depends on: ";
-	}
+sub usage () {
+	print STDERR <<EOF ;
+tflat -- flatten a depends tree.  tflat is used to show all
+         packages which depend upon a given package, and
+         all packages which are depend upon by a given package.
 
-	close(InFile);
-	#
-	# Now recurse the tree to give a flattened depends list for each pkg
-	#
+usage: $0 upfile downfile < depfile
 
+Input file format is in the form
+foo	bar
+foo	baz
+libbar	bar
 
-	for (toppkg in topdepends){
-		find_all_depends(toppkg);
-	}
+meaning "foo is depended upon by bar,
+         foo is depended upon by baz,
+         libbar is depended upon by bar"
 
-	for (x in alldepends){
-		print x depstr alldepends[x] | "${SORT-sort}";
-	}
+The typical use is:
+cd /usr/pkgsrc
+./mk/bulk/printdepends > .depends
+perl ./mk/bulk/tflat .supports .requires < .depends
 
-	printf("\n");
-        exit 0
+EOF
+	exit 1;
 }
 
-function find_all_depends(pkg,pkgreg,i,deps){
-	# If we find the package already has been fully depended
-	# then return the depends list.
-	if (pkg in alldepends){
-		return(alldepends[pkg]);	
-	}
+my $upfile = shift(@ARGV) || usage();
+my $downfile = shift(@ARGV) || usage();
+scalar(@ARGV) && usage();
 
-	# If we find the package listed in its own depends list, then
-	# return an empty list if we're going down the depends tree.
-	# When a package lists itself in the depends tree file, it simply
-	# is a place holder and means the package has no depends.  However
-	# other packages may still depend upon it, so we need to keep looking.
-	if ( (!up) && (topdepends[pkg]~reg2str(pkg)) ){
-		alldepends[pkg] = " ";
-		return(alldepends[pkg]);	
-	}
+open(UPF, ">$upfile") || die $!;
+open(DOWNF, ">$downfile") || die $!;
 
-	# Otherwise recursively gather depends that each of the depends
-	# has.
-	pkgreg=reg2str(pkg);
-	split(topdepends[pkg],deps);
-	i=1;
-	alldepends[pkg] = " ";
-	while ( i in deps ){
-		# Don't add ourselves to the list (a possibility when going up
-		# the tree).
-		if (" "deps[i]" "!~pkgreg){
-			alldepends[pkg] = alldepends[pkg] " " deps[i] " " find_all_depends(deps[i]);
-		}
-		i=i+1;
-	}
-	alldepends[pkg] = uniq(alldepends[pkg]);
-	return(alldepends[pkg]);	
-}
+# read in dependstree file
 
-#
-# Take a string which has special characters like '+' in it and
-# escape them.  Also put a space before and after since that's how
-# we'll distinguish things like gnome from gnome-libs
-#
-function reg2str(reg){
-	gsub(/\+/,"\\+",reg);
-	reg = " "reg" ";
-	return(reg);
+my %depended;
+my %depends;
+while (<>) {
+	chomp;
+	my ($dep, $pkg) = split;
+
+	push(@{$depended{$dep}}, $pkg);
+	push(@{$depends{$pkg}}, $dep);
 }
 
-#
-# Take the depends lists and uniq them.
-#
-function uniq(list,deps,i,ulist){
-	
-	# split out the depends
-	split(list,deps);
-
-	i=1;
-	ulist = " ";
-	while (i in deps){
-		if (ulist !~reg2str(deps[i])){
-			ulist = ulist deps[i]" ";
+# print out upfile and downfile
+
+sub getdeps ($$$@) {
+	my $curhash = shift;
+	my $alldeps = shift;
+	my $what = shift;
+
+	foreach my $dep (@{$curhash->{$what}}) {
+		if ($what eq $dep) {
+			next;
+		} elsif (grep { $_ eq $dep } @_) {
+			print STDERR "circular dependency in $dep\n";
+		} elsif (!grep { $_ eq $dep } @$alldeps) {
+			push(@$alldeps, $dep);
+			getdeps($curhash, $alldeps, $dep, $what, @_);
 		}
-		i++;
 	}
-	return(ulist);
 }
 
-#
-# show usage
-#
-function usage(){
-	printf("tflat -- flatten a depends tree.  tflat is used to show all\n");
-	printf("         packages which depend upon a given package or alternatively\n");
-	printf("         all packages which are depend upon by a given package.\n");
-	printf("\n");
-	printf("Usage:\ttflat up|down depfile\n");
-	printf("\n");
-	printf("Options:\tdown\tgo down the depends tree (ie \"foo depends on:\")\n");
-	printf("        \tup\tgo up the depends tree (ie \"foo is depended on by:\")\n");
-	printf("\n");
-	printf("Input file format is in the form\n");
-	printf("foo	bar\n");
-	printf("foo	baz\n");
-	printf("libbar	bar\n");
-	printf("\n");
-	printf("meaning \"foo is depended upon by bar,\n");
-	printf("         foo is depended upon by baz,\n");
-	printf("         libbar is depended upon by bar\"\n");
-	printf("\n");
-	printf("The typical use is:\n");
-	printf("cd /usr/pkgsrc\n");
-	printf("./mk/bulk/printdepends > .depends\n");
-	printf("./mk/bulk/tflat up .depends > .supports\n");
-	printf("./mk/bulk/tflat down .depends > .requires\n");
-	printf("\n");
+foreach my $pkg (sort(keys(%depended))) {
+	my @alldeps;
+	getdeps(\%depended, \@alldeps, $pkg);
+	print UPF "$pkg is depended on by:  ".join(' ', @alldeps)."\n";
+}
 
+foreach my $pkg (sort(keys(%depends))) {
+	my @alldeps;
+	getdeps(\%depends, \@alldeps, $pkg);
+	print DOWNF "$pkg depends on:  ".join(' ', @alldeps)."\n";
 }
 
+close(UPF);
+close(DOWNF);