add scripts for generating the README.html files in a much more efficient

way than the previous recursive make approach.  The 'mkreadme' script is
the top level script.  'mkreadme -h' or 'mkreadme --help' for complete
documentation on its use.  Generates README.html files more than two orders
of magnitude faster than the recursive make approach by only calling make(1)
once per pkg.

2 files changed, 1054 insertions, 0 deletions

diff --git a/mk/bulk/genreadme.awk b/mk/bulk/genreadme.awk
new file mode 100755
index 00000000000..90719f36bac
--- /dev/null
+++ b/mk/bulk/genreadme.awk
@@ -0,0 +1,609 @@
+ #!/usr/bin/awk -f
+# $NetBSD: genreadme.awk,v 1.1 2002/11/11 21:51:05 dmcmahill Exp $
+#
+# Copyright (c) 2002 The NetBSD Foundation, Inc.
+# All rights reserved.
+#
+# This code is derived from software contributed to The NetBSD Foundation
+# by Dan McMahill.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. All advertising materials mentioning features or use of this software
+#    must display the following acknowledgement:
+#        This product includes software developed by the NetBSD
+#        Foundation, Inc. and its contributors.
+# 4. Neither the name of The NetBSD Foundation nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+
+
+# Global variables
+#-----------------
+# The following associative arrays are used for storing the dependency
+# information and other information for the packages
+#
+# topdepends[]  : index=pkgdir (math/scilab)
+#                 List of explicitly listed depencencies by name.
+#                 I.e.  "xless-[0-9]* pvm-3.4.3"
+#
+# alldepends[]  : index=pkgdir (math/scilab)
+#                 Flattened dependency list by name.
+#                 
+BEGIN {
+     do_pkg_readme=1;
+# set to 1 to use "README-new.html" as the name
+     use_readme_new=0;
+     printf("Reading database file\n");
+     fflush("/dev/stdout");
+ }
+
+#conflicts /usr/pkgsrc/math/scilab 
+#depends /usr/pkgsrc/math/scilab xless-[0-9]*:../../x11/xless pvm-3.4.3:../../parallel/pvm3
+#
+
+/^(build_)?depends / {
+#
+# Read in the entire depends tree
+# These lines look like:
+#
+#depends /usr/pkgsrc/math/scilab xless-[0-9]*:../../x11/xless pvm-3.4.3:../../parallel/pvm3
+#build_depends /usr/pkgsrc/math/scilab libtool-base>=1.4.20010614nb9:../../devel/libtool-base
+#
+    deptype=$1;
+#    pkg=fulldir2pkgdir($2);
+    pkg = $2;
+    if(pkg in topdepends) {}
+    else {topdepends[pkg] = "";}
+    if(pkg in topbuilddepends) {}
+    else {topbuilddepends[pkg] = "";}
+
+    for(i=3; i<=NF; i++) {
+	split($i,a,":");
+	pkgpat=a[1];
+	pkgdir=a[2];
+	sub(/[\.\/]*/,"",pkgdir);
+	if(pkgdir !~ /\//) {
+	    pkgcat=pkg;
+	    gsub(/\/.*/,"",pkgcat);
+	    pkgdir=pkgcat "/" pkgdir;
+	    if(debug) printf("Corrected missing category directory to get \"%s\"\n",pkgdir);
+	}
+	if(debug){
+	    printf("package in directory %s %s on:\n",pkg,deptype);
+	    printf("\tpkgpat = %s\n",pkgpat);
+	    printf("\tpkgdir = %s\n",pkgdir);
+	}
+
+
+#
+# store the package directory in a associative array with the wildcard
+# pattern as the index since we'll need to be able to look this up later
+#
+	pat2dir[pkgpat]=pkgdir;
+
+	if(deptype == "depends") {
+	    topdepends[pkg] = topdepends[pkg] " " pkgpat " " ;
+	    if(debug) printf("Appending %s to topdepends[%s] (%s)\n",
+			     pkgpat,pkg,topdepends[pkg]);
+	}
+	else {
+	    if(debug) printf("Appending %s to topbuilddepends[%s] (%s)\n",
+			     pkgpat,pkg,topbuilddepends[pkg]);
+	    topbuilddepends[pkg] = topbuilddepends[pkg] " " pkgpat " " ;
+	}
+    }
+    
+    next;
+}
+
+/^comment /{
+    dir = $2;
+    gsub(/^comment[ \t]*/,"");
+    tmp=substr($0,length($1)+1);
+    gsub(/^[ \t]*/,"",tmp);
+    gsub(/&/,"\\\\\\&amp;",tmp);
+    comment[dir]=tmp;
+    next;
+}
+
+/^homepage /{
+    homepage[$2] = $3;
+    next;
+}
+
+
+/^htmlname / {
+#
+# read lines like:
+# htmlname /usr/pkgsrc/archivers/arc <a href=../../archivers/arc/README.html>arc-5.21e</A>
+#
+#    dir=fulldir2pkgdir($2);
+    dir = $2;
+    htmlname=$3;
+    for(i=4;i<=NF;i++){
+	htmlname=htmlname " " $i;
+    }
+    dir2htmlname[dir]=htmlname;
+    if(debug) printf("added dir2htmlname[%s]=%s\n",dir,htmlname);
+    next;
+}
+
+/^index / {
+#
+# read lines like:
+#index /usr/pkgsrc/math/scilab scilab-2.6nb3 
+# and store the directory name in a associative array where the index
+# is the package name and in a associative array that lets us lookup
+# name from directory.  We use fuldir2pkgdir to get "math/scilab"
+# and drop the /usr/pkgsrc part.
+#
+#	pkgname2dir[$3] = fulldir2pkgdir($2);
+#	pkgdir2name[fulldir2pkgdir($2)] = $3;
+    pkgname2dir[$3] = $2;
+    pkgdir2name[$2] = $3;
+    next;
+}
+
+/^license /{
+    license[$2] = $3;
+    next;
+}
+
+/^wildcard /{
+    wildcard[$2] = $3;
+}
+
+#
+# Now recurse the tree to give a flattened depends list for each pkg
+#
+
+END {
+    if(use_readme_new) {
+	readme_name="README-new.html"
+	    }
+    else {
+	readme_name="README.html";
+    }
+    readme=TMPDIR "/" readme_name;
+
+    if( dependsfile == "" ) dependsfile="/dev/stdout";
+    if( builddependsfile == "" ) builddependsfile="/dev/stdout";
+
+    printf("Flattening dependencies\n");
+    fflush("/dev/stdout");
+    printf("") > dependsfile;
+    for (toppkg in topdepends){
+	if(debug) printf("calling find_all_depends(%s)\n",toppkg);
+	find_all_depends(toppkg);
+	if(debug) printf("%s depends on: %s, topdepends on %s\n",
+			 toppkg,alldepends[toppkg],topdepends[toppkg]);
+	printf("%s depends on: %s\n",toppkg,alldepends[toppkg]) >> dependsfile;
+	flatdepends[toppkg] = alldepends[toppkg];
+    }
+    close(dependsfile);
+
+
+# clear out the flattened depends list and repeat for the build depends
+    delete alldepends;
+    printf("Flattening build dependencies\n");
+    fflush("/dev/stdout");
+    printf("") > builddependsfile;
+    for (toppkg in topbuilddepends){
+	find_all_depends(toppkg);
+	printf("%s build_depends on: %s\n",
+	       toppkg,alldepends[toppkg]) >> builddependsfile;
+    }
+    close(builddependsfile);
+
+
+    vfile=DISTDIR "/vulnerabilities";
+
+# extract date for vulnerabilities file
+    cmd="ls -l " vfile;
+    if((cmd | getline) > 0) {
+	vuldate=sprintf("at %s %s %s\n",$6,$7,$8);
+# read the vulnerabilities file
+	printf("Reading vulnerability file \"%s\"\n which was updated %s\n",
+	       vfile,vuldate);
+	fflush("/dev/stdout");
+	i=1;
+	while((getline < vfile) > 0) {
+	    if($0 !~ /^\#/) {
+		vulpkg[i]=$1;
+		vultype[i]=$2;
+		vulref[i]=$3;
+		i=i+1;
+	    }
+	}
+	printf("   Loaded %d vulnerabilities\n",i-1);
+	close(vfile);
+	have_vfile=1;
+    } else {
+	vuldate="<TR><TD><I>(no vulnerabilities list available)</I>";
+	printf("No vulnerability file found (%s).\n",vfile);
+	have_vfile=0;
+    }
+    close(cmd);
+    fflush("/dev/stdout");
+
+    printf("Generating README.html files\n");
+    fflush("/dev/stdout");
+    pkgcnt=0;
+    if(do_pkg_readme) {
+	templatefile=PKGSRCDIR "/templates/README.pkg";
+	for (toppkg in topdepends){
+	    pkgcnt++;
+	    pkgdir=PKGSRCDIR "/" toppkg;
+	    readmenew=pkgdir  "/" readme_name;
+
+	    if(debug)printf("Creating %s for %s\n",readme,readmenew);
+	    printf(".");
+	    if((pkgcnt % 100) == 0) {
+		printf("\n%d\n",pkgcnt);
+	    }
+	    fflush("/dev/stdout");
+	    printf("") > readme;
+	    htmldeps="";
+	    delete dpkgs;
+	    split(alldepends[toppkg],dpkgs);
+	    i=1;
+	    while(i in dpkgs){
+		if(debug) printf("\tdpkg=%s, pat2dir[%s] = %s\n",
+				 dpkgs[i],dpkgs[i],pat2dir[dpkgs[i]]);
+		nm=dpkgs[i];
+# we need a zillion escapes here because we need to end up with \\&lt; in 'htmldeps' so that when
+# we gsub htmldeps into the output file, we end up with &lt;
+		gsub(/</,"\\\\\\&lt;",nm);
+		gsub(/>/,"\\\\\\&gt;",nm);
+		htmldeps=htmldeps " <a href=\"../../" pat2dir[dpkgs[i]] "/"readme_name"\">" nm "</a>";
+		i=i+1;
+	    }
+	    if(debug) printf("htmldeps = \"%s\"\n",htmldeps);
+
+	    vul="";
+	    if(have_vfile) {
+		i=1;
+		pkgbase=gensub(/-[^-]*$/,"","G",pkgdir2name[toppkg]);
+		if(debug) printf("Checking for %s (%s) vulnerabilities\n",toppkg,pkgbase);
+		while(i in vulpkg) {
+		    nm=vulpkg[i];
+		    gsub(/</,"\\\\\\&lt;",nm);
+		    gsub(/>/,"\\\\\\&gt;",nm);
+		    if(vulpkg[i] ~ "^"pkgbase"[-<>=]+[0-9]") {
+			vul=sprintf("%s<STRONG><LI> %s has a %s exploit (see <a href=\"%s\">%s</a> for more details)</STRONG>\n", 
+				    vul,nm,vultype[i],vulref[i],vulref[i]); 
+		    }
+		    i=i+1;
+		}
+		if( vul == "" ){
+		    vul="<I>(no vulnerabilities known)</I>";
+		}
+	    }
+
+	    if(MULTIARCH == "no"){
+		cmd="ls -1 "PACKAGES "/" PKGREPOSITORYSUBDIR "/" wildcard[toppkg] PKG_SUFX " 2>/dev/null";
+		if(debug)printf("Checking for binary package with %s\n",cmd);
+		binpkgs="";
+		while((cmd | getline) > 0) {
+		    pkgfile=$0;
+		    gsub(/.*\//,"",pkgfile);
+		    pkgnm=pkgfile;
+		    gsub(/\.tgz$/,"",pkgnm);
+		    binpkgs=sprintf("%s\n<TR><TD>%s:<TD><a href=\"%s/%s\">%s</a><TD>(%s %s)",
+				    binpkgs,MACHINE_ARCH,PKG_URL,pkgfile,pkgnm,OPSYS,OS_VERSION);
+		}
+		close(cmd);}
+	    else {
+		cmd="ls -1 -d "PACKAGES"/[0-9].*/*/" PKGREPOSITORYSUBDIR "/" wildcard[toppkg] PKG_SUFX " 2>/dev/null";
+		oldfs=FS;
+		FS="[/]";
+		binpkgs="";
+		while((cmd | getline) > 0) {
+		    release = $(NF-3);
+		    arch = $(NF-2);
+		    pkg = $NF;
+		    pkgnm=pkg;
+		    gsub(PKG_SUFX "$","",pkgnm)
+			if(debug) printf("%s:%s:%s (%s)\n",release,arch,pkg,pkgnm);
+		    binpkgs=sprintf("%s\n<TR><TD>%s:<TD><a href=\"%s/%s/%s/%s/%s\">%s</a><TD>(%s-%s)",
+				    binpkgs,arch,PKG_URL,release,arch,PKGREPOSITORYSUBDIR,pkg,pkgnm,OPSYS,release);
+		}
+		FS=oldfs;
+		close(cmd);
+	    }
+
+# sort the binary pkgs (XXX would be nice to implement in memory in awk)
+	    sfile=TMPDIR "/sorted";
+	    spipe="sort > " sfile;
+	    printf("%s",binpkgs) | spipe;
+	    close(spipe);
+	    binpkgs="";
+	    while((getline < sfile) > 0) {
+		binpkgs=sprintf("%s\n%s",binpkgs,$0);
+	    }
+	    close(sfile);
+
+	    if(debug) printf("binary packages: \n%s\n\n",binpkgs);
+
+	    while((getline < templatefile) > 0){
+		gsub(/%%PORT%%/,toppkg);
+		gsub(/%%PKG%%/,dir2htmlname[toppkg]);
+		gsub(/%%COMMENT%%/,comment[toppkg]);
+		if(homepage[toppkg] == "") {
+		    gsub(/%%HOMEPAGE%%/,"");
+		} else {
+		    gsub(/%%HOMEPAGE%%/,"<p>This package has a home page at <a HREF=\"" homepage[toppkg] "\">" homepage[toppkg] "</a>.</p>");
+		}
+		if(license[toppkg] == "") {
+		    gsub(/%%LICENSE%%/,"");
+		} else {
+		    gsub(/%%LICENSE%%/,"<p>Please note that this package has a " license[toppkg] " license.</p>");
+		}
+		gsub(/%%VULNERABILITIES%%/,""vul"");
+		gsub(/%%VULDATE%%/,""vuldate"");
+		gsub(/%%BUILD_DEPENDS%%/,""htmldeps"");
+		gsub(/%%RUN_DEPENDS%%/,""flatdepends[toppkg]"");
+		gsub(/%%BIN_PKGS%%/,""binpkgs"");
+		gsub(/README.html/,readme_name);
+		print >> readme;
+	    }
+	    close(readme);
+	    close(templatefile);
+	    cmd="if [ -d " pkgdir " ]; then if ! cmp -s "readme" "readmenew" ; then mv -f " readme " " readmenew " ; fi ; else echo "pkgdir" does not exist ; exit 1 ; fi";
+	    if(debug) printf("Execute:  %s\n",cmd);
+	    rc=system(cmd);
+	    if(rc != 0) {
+		printf("**** WARNING ****\nCould not create %s (rc=%d)\n",readmenew,rc) > "/dev/stderr";
+		printf("**** ------- ****\n") > "/dev/stderr";
+	    }
+	}
+	printf("\n");
+    }
+    printf("\n");
+    printf("Generating category readmes\n");
+    templatefile=PKGSRCDIR "/templates/README.category";
+    print "" > readme;
+
+# string with URL's for all categories (used by the top README.html)
+    allcat="";
+# string with URL's for all pkgs (used by the top README-all.html)
+    tot_numpkg=0;
+    top_make=PKGSRCDIR"/Makefile";
+    while((getline < top_make) > 0){
+	if($0 ~ /^[ \t]*SUBDIR.*=[^\$]*$/) {
+	    category=$0;
+	    gsub(/^[ \t]*SUBDIR.*=[ \t]*/,"",category);
+	    catdir=PKGSRCDIR"/"category;
+	    readmenew=catdir"/"readme_name;
+	    printf("Category = %s\n",category);
+	    cat_make=catdir"/Makefile";
+	    pkgs="";
+	    numpkg=0;
+	    while((getline < cat_make) > 0){
+		if($0 ~ /^[ \t]*SUBDIR.*=[^\$]*$/) {
+		    pkg=$0;
+		    gsub(/^[ \t]*SUBDIR.*=[ \t]*/,"",pkg);
+		    dir=category"/"pkg;
+		    numpkg++;
+		    tot_numpkg++;
+		    if(debug) printf("\tAdding %s (%s : %s)\n",dir,pkgdir2name[dir],comment[dir]);
+		    pkgs=sprintf("%s<TR><TD VALIGN=TOP><a href=\"%s/%s\">%s</a>: %s<TD>\n",
+				 pkgs,pkg,readme_name,pkgdir2name[dir],comment[dir]);
+		    allpkg[tot_numpkg]=sprintf("<!-- %s (for sorting) --><TR VALIGN=TOP><TD><a href=\"%s/%s/%s\">%s</a>: <TD>(<a href=\"%s/%s\">%s</a>) <td>%s\n",
+					       pkgdir2name[dir],category,pkg,readme_name,pkgdir2name[dir],category,readme_name,category,comment[dir]);
+# we need slightly fewer escapes here since we're not gsub()-ing allpkg[] into the output files but
+# just printf()-ing it.
+		    gsub(/\\&/,"\\&",allpkg[tot_numpkg]);
+		} else if($0 ~ /^[ \t]*COMMENT/) {
+		    descr=$0;
+		    gsub(/^[ \t]*COMMENT.*=[ \t]*/,"",descr);
+		}
+	    }
+	    while((getline < templatefile) > 0){
+		gsub(/%%CATEGORY%%/,category);
+		gsub(/%%NUMITEMS%%/,numpkg);
+		gsub(/%%DESCR%%/,descr);
+		gsub(/%%SUBDIR%%/,pkgs);
+		gsub(/README.html/,readme_name);
+		print >> readme;
+	    }
+	    close(readme);
+	    close(templatefile);
+	    cmd="if [ ! -f "readmenew" ]; then cp "readme " " readmenew " ; fi ; if ! cmp -s "readme" "readmenew" ; then mv -f " readme " " readmenew " ; fi ";
+	    if(debug) printf("Execute:  %s\n",cmd);
+	    rc=system(cmd);
+	    if(rc != 0) {
+		printf("**** WARNING ****\nCould not create %s (rc=%d)\n",readmenew,rc) > "/dev/stderr";
+		printf("**** ------- ****\n") > "/dev/stderr";
+	    }
+	    
+	    gsub(/href=\"/,"href=\""category"/",pkgs);
+	    allcat=sprintf("%s<TR><TD VALIGN=TOP><a href=\"%s/%s\">%s</a>: %s<TD>\n",
+			   allcat,category,readme_name,category,descr);
+	    close(cat_make);
+	}
+    }
+    close(top_make);
+
+    printf("Generating toplevel readmes:\n");
+    templatefile=PKGSRCDIR "/templates/README.top";
+    readmenew=PKGSRCDIR "/"readme_name;
+    printf("\t%s\n",readmenew);
+    print "" > readme;
+    while((getline < templatefile) > 0){
+	gsub(/%%DESCR%%/,"");
+	gsub(/%%SUBDIR%%/,allcat);
+	gsub(/README.html/,readme_name);
+	print >> readme;
+    }
+    close(readme);
+    close(templatefile);
+    cmd="if [ ! -f "readmenew" ]; then cp "readme " " readmenew " ; fi ; if ! cmp -s "readme" "readmenew" ; then mv -f " readme " " readmenew " ; fi ";
+    if(debug) printf("Execute:  %s\n",cmd);
+    rc=system(cmd);
+    if(rc != 0) {
+	printf("**** WARNING ****\nCould not create %s (rc=%d)\n",readmenew,rc) > "/dev/stderr";
+	printf("**** ------- ****\n") > "/dev/stderr";
+    }
+
+    templatefile=PKGSRCDIR "/templates/README.all";
+    readmenew=PKGSRCDIR "/README-all.html";
+    printf("\t%s\n",readmenew);
+# sort the pkgs
+    sfile=TMPDIR"/unsorted";
+    spipe= "sort  " sfile;
+    i=1;
+    print "" >sfile;
+    while(i in allpkg) {
+	printf("%s",allpkg[i]) >> sfile;
+	i++;
+    }
+    close(sfile);
+
+    print "" > readme;
+    while((getline < templatefile) > 0){
+	line=$0;
+	if($0 ~ /%%PKGS%%/) {
+	    while((spipe | getline) > 0) {
+		print  >> readme;
+	    }
+	    close(spipe);
+	} else {
+	    gsub(/%%DESCR%%/,"",line);
+	    gsub(/%%NPKGS%%/,tot_numpkg,line);
+	    gsub(/README.html/,readme_name,line);
+	    print line >> readme;
+	}
+    }
+    close(readme);
+    close(templatefile);
+    cmd="if [ ! -f "readmenew" ]; then cp "readme " " readmenew " ; fi ; if ! cmp -s "readme" "readmenew" ; then mv -f " readme " " readmenew " ; fi ";
+    if(debug) printf("Execute:  %s\n",cmd);
+    rc=system(cmd);
+    if(rc != 0) {
+	printf("**** WARNING ****\nCould not create %s (rc=%d)\n",readmenew,rc) > "/dev/stderr";
+	printf("**** ------- ****\n") > "/dev/stderr";
+    }
+
+    close("/dev/stderr");
+    exit 0;
+} 
+
+function find_all_depends(pkg,pkgreg,i,deps,depdir){
+# pkg is the package directory, like math/scilab
+
+#    printf("find_all_depends(%s)\n",pkg);
+# if we find the package already has been fully depended
+# then return the depends list
+    if (pkg in alldepends){
+	if(debug) printf("\t%s is allready depended.  Returning %s\n",pkg,alldepends[pkg]);
+	return(alldepends[pkg]);
+    }
+
+# if this package has no top dependencies, enter an empty flat dependency
+# list for it.
+    if (topdepends[pkg] ~ "^[ \t]*$") {
+	alldepends[pkg] = " ";
+	if(debug) printf("\t%s has no depends(%s).  Returning %s\n",pkg,topdepends[pkg],alldepends[pkg]);
+	return(alldepends[pkg]);
+    }
+    
+# recursively gather depends that each of the depends has
+    pkgreg=reg2str(pkg);
+    split(topdepends[pkg],deps);
+    i=1;
+    alldepends[pkg] = " ";
+    while ( i in deps ){
+
+# figure out the directory name associated with the package hame
+# in (wild card/dewey) version form
+	depdir=pat2dir[deps[i]];
+	if(debug) printf("\tadding dependency #%d on \"%s\" (%s)\n",i,deps[i],depdir);
+# don't add ourselves to the list (shouldn't happen, but
+# we'd like to not get stuck in a loop if one exists)
+#	if (" "deps[i]" "!~pkgreg){
+
+# if we don't already have this dependency (deps[i]) listed, then add it.  However, we may
+# have already added it because another package we depend on may also have depended on
+# deps[i].
+	if (alldepends[pkg] !~reg2str(deps[i])){
+	    alldepends[pkg] = alldepends[pkg] " " deps[i] " " find_all_depends(depdir);
+	} 
+	else {
+	    if(debug) printf("\t%s is already listed in %s\n",deps[i],alldepends[pkg]);
+	}
+	
+#	}
+	i=i+1;
+    }
+    if(debug) printf("\tcalling uniq() on alldepends[%s] = %s\n",pkg,alldepends[pkg]);
+    alldepends[pkg] = uniq(alldepends[pkg]);
+    if(debug) printf("\tuniq() output alldepends[%s] = %s\n",pkg,alldepends[pkg]);
+    return(alldepends[pkg]);	
+}
+
+#
+# take a string which has special characters like '+' in it and
+# escape them.  Also put a space before and after since that's how
+# we'll distinguish things like gnome from gnome-libs
+#
+function reg2str(reg){
+    gsub(/\./,"\\\.",reg);
+    gsub(/\+/,"\\\+",reg);
+    gsub(/\*/,"\\\*",reg);
+    gsub(/\?/,"\\\?",reg);
+    gsub(/\[/,"\\\[",reg);
+    gsub(/\]/,"\\\]",reg);
+    reg = " "reg" ";
+    return(reg);
+}
+
+#
+# accepts a full path to a package directory, like "/usr/pkgsrc/math/scilab"
+# and returns just the last 2 directories, like "math/scilab"
+#
+function fulldir2pkgdir(d){
+    d=gensub(/^(.*)(\/)([^\/]*\/[^\/]*)$/,"\\3","g",d);
+    return(d);
+}
+
+#
+# take the depends lists and uniq them.
+#
+function uniq(list,deps,i,ulist){
+    
+# split out the depends
+    split(list,deps);
+    
+    i=1;
+    ulist = " ";
+    while (i in deps){
+#	printf("uniq():  Checking \"%s\"\n",ulist);
+#	printf("         for \"%s\"\n",reg2str(deps[i]));
+	if (ulist !~reg2str(deps[i])){
+	    ulist = ulist deps[i]" ";
+	}
+	i++;
+    }
+    return(ulist);
+}
diff --git a/mk/bulk/mkreadme b/mk/bulk/mkreadme
new file mode 100755
index 00000000000..2262bd083e1
--- /dev/null
+++ b/mk/bulk/mkreadme
@@ -0,0 +1,445 @@
+#!/bin/sh
+# $NetBSD: mkreadme,v 1.1 2002/11/11 21:51:04 dmcmahill Exp $
+#
+# Script for README.html generation
+#
+# Copyright (c) 2002 The NetBSD Foundation, Inc.
+# All rights reserved.
+#
+# This code is derived from software contributed to The NetBSD Foundation
+# by Dan McMahill.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. All advertising materials mentioning features or use of this software
+#    must display the following acknowledgement:
+#        This product includes software developed by the NetBSD
+#        Foundation, Inc. and its contributors.
+# 4. Neither the name of The NetBSD Foundation nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+
+
+TMPDIR=${TMPDIR:-/tmp/mkreadme}
+PKGSRCDIR=${PKGSRCDIR:-/usr/pkgsrc}
+BMAKE=${BMAKE:-make}
+AWK=${AWK:-/usr/bin/awk}
+
+usage(){
+    echo "$prog - Generates README.html files for a pkgsrc tree"
+    echo "Usage:      $prog [-c|--cdrom] [-C|--prune] [-d|--debug] [-f|--ftp] "
+    echo "                  [-m|--multi-arch] [-p|--pkgsrc directory] "
+    echo "                  [-P|--packages directory] [-r|--restart] "
+    echo "                  [-s|--single-arch]"
+    echo " "
+    echo "            $prog -h|--help"
+    echo " "
+    echo "            $prog -v|--version"
+    echo " "
+    echo "The options supported by $prog are: "
+    echo " "
+    echo "  -C|--prune          Prune unused README.html files which may exist in"
+    echo "                      pkg directories which have been removed from CVS."
+    echo " "
+    echo "  -c|--cdrom          Generates CD-ROM README.html files"
+    echo " "
+    echo "  -d|--debug          Enables (extremely verbose) debug output"
+    echo " "
+    echo "  -f|--ftp            Generates FTP README.html files"
+    echo " "
+    echo "  -h|--help           Displays this help message"
+    echo " "
+    echo "  -m|--multi-arch     Assumes a packages directory layout like:"
+    echo "                      OS_VERSION/MACHINE_ARCH/     for example:"
+    echo "                      1.{5,6}/{alpha,i386,sparc,vax}"
+    echo " "
+    echo "  -p|--pkgsrc dir     Specifies the pkgsrc directory.  Defaults to"
+    echo "                      The value of the PKGSRCDIR environment variable"
+    echo "                      if set or /usr/pkgsrc otherwise."
+    echo " "
+    echo "  -P|--packages dir   Specifies the packages directory."
+    echo " "
+    echo "  -r|--restart        Restart.  This option assumes that the database file"
+    echo "                      from a previous run still exists and that the script"
+    echo "                      should use that instead of recreating the database."
+    echo " "
+    echo "  -s|--single-arch    Assumes a single OS release and MACHINE_ARCH binary"
+    echo "                      packages directory structure."
+    echo " "
+    echo "  -v|--version        Displays the version of this script and exits."
+    echo " "
+    echo "Example:    $prog -p /pub/NetBSD/packages/pkgsrc -P /pub/NetBSD/packages -m -f"
+    echo " "
+}
+
+clean_and_exit(){
+    if [ "x$DEBUG" = "xno" -a "x$restart" = "xno" ]; then
+	rm -fr $TMPDIR
+    else
+	echo "Debugging output left in $TMP"
+    fi
+    exit 1
+}
+
+prog=$0
+
+
+######################################################################
+#
+#  Handle command line options
+#
+######################################################################
+
+ftp_readme=no
+restart=no
+prune=no
+
+while
+    test -n "$1"
+do
+    case "$1"
+    in
+
+    # We're generating README.html's for a CD-ROM 
+    -c|--cdrom)
+	ftp_readme=no
+	shift
+	;;
+
+    # Prune old README.html files from pkgs which no longer exist
+    -C|--prune)
+	prune=yes
+	shift
+	;;
+
+    # Turn on debugging
+    -d|--debug)
+	DEBUG=yes
+	shift
+	;;
+
+    # We're generating README.html's for a CD-ROM 
+    -f|--ftp)
+	ftp_readme=yes
+	shift
+	;;
+
+	
+    # Help
+    -h|--help)
+	usage
+	exit 0
+	;;
+
+    # assume a OS_RELEASE/MACHINE_ARCH/ layout for the binary pkgs
+    -m|--multi-arch)
+	multiarch=yes
+	shift
+	;;
+
+    # Specify pkgsrc directory
+    -p|--pkgsrc)
+	PKGSRCDIR=$2
+	shift 2
+	;;
+
+    # Specify PACKAGES directory
+    -P|--packages)
+	PKGDIR=$2
+	shift 2
+	;;
+
+    # Restart (ie, don't re-generate the database file)
+    -r|--restart)
+	restart=yes
+	shift
+	;;
+
+    # assume a single directory structure layout for the binary pkgs
+    -s|--single-arch)
+	multiarch=no
+	shift
+	;;
+
+    # Version
+    -v|--version) 
+	${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
+	exit 0
+        ;;
+
+    -*) echo "$prog:  ERROR:  $1 is not a valid option"
+	usage
+	clean_and_exit
+	;;
+
+    *)  
+	break
+	;;
+
+    esac
+done
+
+if [ "x$DEBUG" = "xyes" ]; then
+    set -v
+fi
+
+if [ ! -d ${PKGSRCDIR} ]; then
+    echo "ERROR:  package source directory ${PKGSRCDIR} does not exist"
+    echo ""
+    clean_and_exit
+fi
+
+if [ ! -d $TMPDIR ]; then
+	mkdir -p $TMPDIR
+fi
+
+DEPENDSTREEFILE=$TMPDIR/dependstree
+export DEPENDSTREEFILE
+DEPENDSFILE=$TMPDIR/depends
+export DEPENDSFILE
+SUPPORTSFILE=$TMPDIR/supports
+export SUPPORTSFILE
+INDEXFILE=$TMPDIR/index
+export SUPPORTSFILE
+ORDERFILE=$TMPDIR/order
+export ORDERFILE
+DATABASEFILE=$TMPDIR/database
+export DATABASEFILE
+BINPKGFILE=$TMPDIR/binpkglist
+
+
+######################################################################
+#
+#  Extract key pkgsrc configuration variables
+#
+######################################################################
+
+echo " "
+echo "Extracting configuration variables:"
+echo " "
+if [ -d ${PKGSRCDIR}/pkgtools/pkglint ]; then
+    cd ${PKGSRCDIR}/pkgtools/pkglint 
+
+    for v in DISTDIR PACKAGES PKGREPOSITORYSUBDIR PKG_SUFX MACHINE_ARCH PKG_URL OPSYS OS_VERSION FTP_PKG_URL_HOST FTP_PKG_URL_DIR CDROM_PKG_URL_HOST CDROM_PKG_URL_DIR
+    do
+	val=`make show-var VARNAME=${v}`
+	if [ $? != 0 ]; then
+	    echo "Error:  make show-var VARNAME=${v} in `pwd` "
+	    echo "Failed.  This is a fatal error"
+	    clean_and_exit
+	fi
+	echo "$v=$val"
+	eval ${v}=${val}
+    done
+else
+    echo "Error:   ${PKGSRCDIR}/pkgtools/pkglint does not seem to exist"
+    exit 1
+fi
+
+
+######################################################################
+#
+#  Decide on FTP vs CDROM README.html files
+#
+######################################################################
+
+if [ "$ftp_readme" = "yes" ]; then
+    PKG_URL=${FTP_PKG_URL_HOST}${FTP_PKG_URL_DIR}
+    echo "Will generate FTP readme files with PKG_URL=$PKG_URL"
+else
+    PKG_URL=${CDROM_PKG_URL_HOST}${CDROM_PKG_URL_DIR}
+    echo "Will generate CD-ROM readme files with PKG_URL=$PKG_URL"
+fi
+
+######################################################################
+#
+#  Check for command line switch for packages directory
+#
+######################################################################
+
+# we've been given the directory as a command line switch
+if [ ! -z "$PKGDIR" ]; then
+    PACKAGES=$PKGDIR
+    echo "PACKAGES specified on command line to be $PKGDIR"
+fi
+
+######################################################################
+#
+#  Decide in binary package directory layout
+#
+######################################################################
+
+
+# If PACKAGES is set to the default (../../pkgsrc/packages), the current
+# ${MACHINE_ARCH} and "release" (uname -r) will be used. Otherwise a directory
+# structure of ...pkgsrc/packages/`uname -r`/${MACHINE_ARCH} is assumed.
+#
+# This is the logic from bsd.pkg.mk, but I think I'd like to change it to
+# come from a command line switch
+
+if [ -z "$multiarch" -a -e ${PACKAGES} ]; then
+    cd ${PACKAGES}
+    case `pwd` in
+	*/pkgsrc/packages)
+	    multiarch=no
+	    ;;
+
+	*)
+	    multiarch=yes
+	    ;;
+    esac
+fi
+
+if [ "x$multiarch" = "xyes" ]; then
+    echo "Will generate multi-release, multi-arch readme files"
+else
+    echo "Will generate single-release, single-arch readme files"
+fi
+
+
+######################################################################
+#
+#  Extract Database for All Packages (longest step)
+#
+######################################################################
+
+if [ "x$restart" = "xno" ] ; then
+    echo " "
+    echo "Extracting data.  This could take a while"
+    echo " "
+    cd ${PKGSRCDIR} 
+    list=`grep '^[[:space:]]*'SUBDIR */Makefile | sed 's,/Makefile.*=[[:space:]]*,/,'`
+    for pkgdir in $list
+    do
+	    if [ ! -d $pkgdir ]; then
+		    echo "WARNING:  the package directory $pkgdir is listed in" > /dev/stderr
+		    echo $pkgdir | sed 's;/.*;/Makefile;g' > /dev/stderr
+		    echo "but the directory does not exist.  Please fix this!" > /dev/stderr
+	    else
+		    cd $pkgdir
+		    l=`${BMAKE} print-summary-data`
+		    if [ $? != 0 ]; then
+			    echo "WARNING (printdepends):  the package in $pkgdir had problem with" \
+				> /dev/stderr
+			    echo "    ${BMAKE} print-summary-data" > /dev/stderr
+			    echo "    database information for this package" > /dev/stderr
+			    echo "    will be dropped." > /dev/stderr
+			    ${BMAKE} print-summary-data  2>&1 > /dev/stderr
+		    else
+			    echo "$l" >> $DATABASEFILE
+		    fi
+	    fi
+	    cd $PKGSRCDIR
+    done
+else
+    echo " "
+    echo "Using existing database (are you sure you wanted the -r/--restart flag?)"
+    echo " "
+fi
+
+######################################################################
+#
+#  Prune README.html files which are no longer needed
+#
+######################################################################
+if [ "x$prune" = "xyes" ]; then
+    echo " "
+    echo "Pruning unused README.html files"
+    echo " "
+    cd ${PKGSRCDIR}
+    for d in `ls -d */*` ; do
+	if [ -d $d -a ! -f ${d}/Makefile -a -f ${d}/README.html ]; then
+	    echo "Pruning ${d}/README.html which is no longer used"
+	    rm -f ${d}/README.html
+	fi
+    done
+fi
+
+######################################################################
+#
+#  Generate the package and category README.html files
+#
+######################################################################
+
+echo " "
+echo "Generating package README.html files"
+echo " "
+if [ "x$DEBUG" = "xyes" ]; then
+    debug=1;
+else
+    debug=0;
+fi
+
+${AWK} -f ${PKGSRCDIR}/mk/bulk/genreadme.awk  \
+    builddependsfile=${TMPDIR}/pkgsrc.builddepends.debug \
+    debug=$debug \
+    dependsfile=${TMPDIR}/pkgsrc.depends.debug \
+    DISTDIR=$DISTDIR \
+    MACHINE_ARCH=$MACHINE_ARCH \
+    MULTIARCH=$multiarch \
+    OPSYS=$OPSYS \
+    OS_VERSION=$OS_VERSION \
+    PACKAGES=$PACKAGES \
+    PKG_SUFX=$PKG_SUFX \
+    PKG_URL=$PKG_URL \
+    PKGREPOSITORYSUBDIR=$PKGREPOSITORYSUBDIR \
+    PKGSRCDIR=$PKGSRCDIR \
+    TMPDIR=$TMPDIR \
+	${DATABASEFILE}
+
+if [ $? != 0 ]; then
+    echo "Error:  genreadme.awk failed to create README.html files"
+    clean_and_exit
+fi
+
+######################################################################
+#
+#  Generate the README-IPv6.html file
+#
+######################################################################
+
+echo " "
+echo "Generating the README-IPv6.html file"
+echo " "
+cd ${PKGSRCDIR}
+ipv6=${TMPDIR}/ipv6pkgs
+ipv6_entries=${TMPDIR}/ipv6_entries
+grep -l '^BUILD_DEFS.*=.*USE_INET6' */*/Makefile  | sed 's;Makefile;;g' > $ipv6
+fgrep -f $ipv6 README-all.html | sort -t/ +1 > $ipv6_entries
+sed \
+    -e "/%%TRS%%/r${ipv6_entries}" \
+    -e '/%%TRS%%/d' \
+    templates/README.ipv6 > README-IPv6.html
+if [ $? != 0 ]; then
+    echo "Error:  README-IPv6.html generation failed (on sed script)"
+    clean_and_exit
+fi
+
+echo " "
+echo "README.html generation finished"
+echo " "
+if [ "x$DEBUG" = "xno" -a "x$restart" = "xno" ]; then
+    rm -fr $TMPDIR
+else
+    echo "Debugging output left in $TMPDIR"
+fi
+