Rework the README.html generation code. Major changes are:

- completely redo the code which decides on the machine architecture,
  operating system, and operating system version for the binary packages.
  The old way just used to directory names to take a guess.  The new
  way creates a cache file containing meta-data for all the binary packages
  in each "All" directory.  This cache file is consulted when generating
  the lists of available binary packages.  The meta-data is obtained with
  pkg_info so it should always be correct even if you do something silly
  like mix OS_VERSION or MACHINE_ARCH packages up in the same directory.

  Among the benefits are:  works when PACKAGES is not $PKGSRC/packages,
  works with a more or less arbitrary subdirectory structure, works
  when there are subdirectories for multiple operating systems.

  This portion of the fix should address PR25390.

  The cache files are only updated when the contents of an "All" directory
  changes or if the cache file format changes.  There is some room for
  improving the updating of the cache files, but its not too bad the way
  it is.

- fix up some of the awk code so that generadme.awk works with Solaris
  nawk as well as NetBSD's nawk and gawk (for pre-2.0 systems).

- remove some "if ! foo" shell constructs to increase portability.

- be more consistent with what variables get passed to mkreadme from
  make and which ones are determined automatically.  Mostly this meant
  moving stuff into mkreadme to make it easier to run it standalone.

4 files changed, 551 insertions, 179 deletions

diff --git a/mk/bsd.pkg.mk b/mk/bsd.pkg.mk
index 50cb04be8ef..a11fdc8fd2e 100644
--- a/mk/bsd.pkg.mk
+++ b/mk/bsd.pkg.mk
@@ -1,4 +1,4 @@
-#	$NetBSD: bsd.pkg.mk,v 1.1667 2005/05/17 06:31:00 jlam Exp $
+#	$NetBSD: bsd.pkg.mk,v 1.1668 2005/05/17 21:46:59 dmcmahill Exp $
 #
 # This file is in the public domain.
 #
@@ -3994,16 +3994,18 @@ README.html: .PRECIOUS
 	${AWK} -f ../../mk/scripts/genreadme.awk \
 		builddependsfile=/dev/null \
 		dependsfile=/dev/null \
+		AWK=${AWK} \
+		CMP=${CMP} \
 		DISTDIR=${DISTDIR} \
-		MACHINE_ARCH=${MACHINE_ARCH} \
-		MULTIARCH=$$MULTIARCH \
-		OPSYS=${OPSYS} \
-		OS_VERSION=${OS_VERSION} \
+		GREP=${GREP} \
 		PACKAGES=${PACKAGES} \
+		PKG_INFO="${PKG_INFO}" \
 		PKG_SUFX=${PKG_SUFX} \
 		PKG_URL=${PKG_URL} \
-		PKGREPOSITORYSUBDIR=${PKGREPOSITORYSUBDIR} \
 		PKGSRCDIR=${.CURDIR:C|/[^/]*/[^/]*$||} \
+		SED=${SED} \
+		SETENV=${SETENV} \
+		SORT=${SORT} \
 		TMPDIR=${TMPDIR:U/tmp} \
 		SINGLEPKG=${PKGPATH} \
 		$@.tmp1
diff --git a/mk/scripts/binpkg-cache b/mk/scripts/binpkg-cache
new file mode 100755
index 00000000000..79ed68eb0db
--- /dev/null
+++ b/mk/scripts/binpkg-cache
@@ -0,0 +1,312 @@
+#!/bin/sh
+#
+# $NetBSD: binpkg-cache,v 1.1 2005/05/17 21:46:59 dmcmahill Exp $
+#
+# Script for generating a cache file with information about
+# all binary packages contained in a directory.
+#
+# Copyright (c) 2005 The NetBSD Foundation, Inc.
+# All rights reserved.
+#
+# This code is derived from software contributed to The NetBSD Foundation
+# by Dan McMahill.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. All advertising materials mentioning features or use of this software
+#    must display the following acknowledgement:
+#        This product includes software developed by the NetBSD
+#        Foundation, Inc. and its contributors.
+# 4. Neither the name of The NetBSD Foundation nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+
+
+TMPDIR=${TMPDIR:-/tmp}
+PACKAGES=${PACKAGES:-/usr/pkgsrc/packages/}
+AWK=${AWK:-awk}
+CMP=${CMP:-cmp}
+GREP=${GREP:-grep}
+PKG_INFO=${PKG_INFO:-pkg_info}
+PKG_SUFX=${PKG_SUFX:-.tgz}
+SED=${SED:-sed}
+SORT=${SORT:-sort}
+STAT=${STAT:-stat}
+
+cachefile=.pkgcache
+cacheversion=20050428
+
+prompt="----> "
+tab="      "
+
+tmpd=${TMPDIR}/pkg-cache.$$
+mkdir -m 0700 ${tmpd}
+if test $? -ne 0 ; then
+	echo "ERROR:  Could not create temporary directory ${tmpd}"
+	echo "Either you do not have write permission to ${tmpd} or"
+	echo "${tmpd} already exists"
+	exit 1
+fi
+
+prog=$0
+
+usage(){
+	echo "$prog - Generates cache files for each directory containing binary"
+	echo "        packages.  This cache file can then be used by the README.html"
+	echo "        generation code to avoid having to call pkg_info(1) over and over"
+	echo "        on the same binary package."
+	echo " "
+	echo "Usage:      $prog [-d|--debug] [-v|--verbose] [-p|--packages <dir>]"
+	echo " "
+	echo "            $prog -h|--help"
+	echo " "
+	echo "            $prog -V|--version"
+	echo " "
+	echo "The options supported by $prog are: "
+	echo " "
+	echo "  -d|--debug            Enables debugging output"
+	echo " "
+	echo "  -f|--database <file>  Writes the database into file specified by <file>"
+	echo " "
+	echo "  -h|--help             Displays this help message"
+	echo " "
+	echo "  -p|--packages <dir>   Specifies the top level directory to be searched"
+	echo "                        for binary packages."
+	echo " "
+	echo "  -v|--version          Displays the version of this script and exits."
+	echo " "
+	echo "Example:    $prog -v --packages /usr/pkgsrc/packages"
+	echo " "
+}
+
+clean_and_exit0(){
+	rm -fr ${tmpd}
+	exit 0
+}
+
+clean_and_exit1(){
+	rm -fr ${tmpd}
+	exit 1
+}
+
+all_cache_files=""
+
+process_binpkg_dir(){
+	all_cache_files="${all_cache_files} ${d}/${cachefile}"
+
+	need_update=no
+	if test -f ${d}/${cachefile} ; then
+		stale_entries=`find ${d} -type f -name \*${PKG_SUFX} -newer ${d}/${cachefile} -print`
+
+		# FIX_ME
+		#
+		# We also should find cache entries for files which no longer exist
+		# and nuke them.  Right now we simply declare the entire cache out
+		# of date.  Once we implement incremental updates to the cache,
+		# we need to remove the entries but not mark the entire cache as
+		# bad.
+		if test "X${DEBUG}" = "Xyes" ; then
+			echo "      Checking for cache entries with no corresponding pkg."
+		fi
+		# get the list of what pkgs belong in the cache
+		rm -f ${tmpd}/pkg_list ${tmpd}/cache_pkg_list
+		ls ${d}/*${PKG_SUFX} | ${SED} "s;^${PACKAGES}/*;;g" | ${SORT} > ${tmpd}/pkg_list
+
+		# and get the list of what is in the cache
+		${AWK} '/pkgcache_begin/ {gsub(/pkgcache_begin[ \t]*/, ""); print}' \
+			${d}/${cachefile} | ${SORT} > ${tmpd}/cache_pkg_list
+
+		if ${CMP} -s ${tmpd}/pkg_list ${tmpd}/cache_pkg_list ; then
+			if test "X${DEBUG}" = "Xyes" ; then
+				echo "      No extra cache entries in ${d}/${cachefile}"
+			fi
+		else
+			echo "      Entries found in ${d}/${cachefile} but no packages found"
+			need_update=yes
+		fi
+	else
+		stale_entries=""
+	fi
+
+	if test "X${force}" = "Xyes" -o "X${need_update}" = "Xyes" ; then
+		need_update=yes
+		echo "${tab}Forcing rebuild of cache ${d}/${cachefile}."
+	elif test ! -f ${d}/${cachefile} ; then
+		need_update=yes
+		echo "${tab}Missing cache file.  ${d}/${cachefile} will be generated."
+	elif test -n "${stale_entries}" ; then
+		need_update=yes
+		echo "${tab}Stale cache file.  ${d}/${cachefile} will be regenerated."
+	else
+		${GREP} "pkgcache_version ${cacheversion}" ${d}/${cachefile} >/dev/null 2>&1
+		if test $? -ne 0 ; then
+			need_update=yes
+			echo "${tab}Old version cache file.  ${d}/${cachefile} will be regenerated."
+		else
+			echo "${tab}Cache file ${d}/${cachefile} is up to date."
+		fi
+	fi
+
+	# FIX_ME
+	# We should use stale_entries in a way where we only update the 
+	# cache file entries corresponding to these if we're rebuilding
+	# due to stale entries.  That should save a good bit of time.
+	# 
+	if test "X${need_update}" = "Xyes" ; then
+		echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
+		for f in ${d}/*${PKG_SUFX} ; do
+			fn=`echo $f | ${SED} "s;^${PACKAGES}/*;;g"`
+			if test "X${DEBUG}" = "Xyes" ; then
+				echo "     Adding ${fn} (${f}) to the cache"
+			fi
+			echo " " >> ${tmpd}/${cachefile}
+			# stat(1) needs to be added to the bootstrap kit
+			# first if we want to use it here
+			#eval $(${STAT} -s ${f} 2>/dev/null)
+			echo "pkgcache_begin ${fn}" >> ${tmpd}/${cachefile}
+			#echo "pkgcache_mtime=${st_mtime}" >> ${tmpd}/${cachefile}
+			if test "X${DEBUG}" = "Xyes" ; then
+				echo "${PKG_INFO} -q -B ${f}"
+			fi
+			${PKG_INFO} -q -B ${f} >> ${tmpd}/${cachefile}
+			echo "pkgcache_end ${fn}" >> ${tmpd}/${cachefile}
+		done
+		mv ${tmpd}/${cachefile} ${d}/${cachefile}
+	fi
+
+
+}
+
+process_cache_files(){
+	echo "${prompt}Checking master cache file ${PACKAGES}/${cachefile}"
+	echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
+	if test -n "${all_cache_files}" ; then
+		for c in ${all_cache_files} ; do
+			echo "pkgcache_cachefile ${c}" >> ${tmpd}/${cachefile}
+		done
+	fi
+ 	if test ! -f ${PACKAGES}/${cachefile} ; then
+		echo "${tab}Creating master cache file ${PACKAGES}/${cachefile}"
+		cp ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
+	fi
+	if ${CMP} -s ${tmpd}/${cachefile} ${PACKAGES}/${cachefile} ; then
+		echo "${tab}Master cache file ${PACKAGES}/${cachefile} is up to date"
+	else
+		echo "${tab}Updating master cache file ${PACKAGES}/${cachefile}"
+		mv ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
+	fi
+}
+
+######################################################################
+#
+#  Handle command line options
+#
+######################################################################
+
+DEBUG=no
+verbose=no
+force=no
+
+while
+	test -n "$1"
+do
+	case "$1" in
+
+		# Turn on debugging
+		-d|--debug)
+			DEBUG=yes
+			verbose=yes
+			shift
+			;;
+
+		# Force a rebuilde of the cache
+		-F|--force)
+			force=yes
+			shift
+			;;
+		# Help
+		-h|--help)
+			usage
+			exit 0
+			;;
+
+		# Use the specified packages directory
+		-p|--packages)
+			PACKAGES=$2
+			shift 2
+			;;
+
+		# Version
+		-V|--version)
+			${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
+			exit 0
+	    		;;
+
+		# Turn on verbose output, but not as noisy as DEBUG
+		-v|--verbose)
+			verbose=yes
+			shift
+			;;
+
+		-*) echo "$prog:  ERROR:  $1 is not a valid option"
+			usage
+			clean_and_exit1
+			;;
+
+		*) 
+			break
+			;;
+
+	esac
+done
+
+# put a trailing / after ${PACKAGES} in case ${PACKAGES} is 
+# a link.
+for d in `find ${PACKAGES}/ -type d -print` ; do
+	if test "X${DEBUG}" = "Xyes" ; then
+		echo "${prompt}Processing directory ${d}"
+	fi
+	is_pkg_dir=no
+	for f in ${d}/*${PKG_SUFX} ; do
+		if test -f "${f}" -a ! -h "${f}" ; then
+			${PKG_INFO} ${f} >/dev/null 2>&1
+			if test $? -eq 0 ; then
+				is_pkg_dir=yes
+				break
+			fi
+		fi
+	done
+	if test "X${is_pkg_dir}" = "Xyes" ; then
+		echo "${prompt}Checking cache in ${d}"
+		process_binpkg_dir
+	else
+		if test "X${DEBUG}" = "Xyes" ; then
+			echo "${prompt}no binary packages in ${d}"
+		fi
+	fi
+		
+done
+
+process_cache_files
+
+clean_and_exit0
+
diff --git a/mk/scripts/genreadme.awk b/mk/scripts/genreadme.awk
index a653f592511..fdc9c507c6c 100755
--- a/mk/scripts/genreadme.awk
+++ b/mk/scripts/genreadme.awk
@@ -1,7 +1,7 @@
 #!/usr/bin/awk -f
-# $NetBSD: genreadme.awk,v 1.12 2004/07/06 22:49:18 wiz Exp $
+# $NetBSD: genreadme.awk,v 1.13 2005/05/17 21:46:59 dmcmahill Exp $
 #
-# Copyright (c) 2002, 2003 The NetBSD Foundation, Inc.
+# Copyright (c) 2002, 2003, 2005 The NetBSD Foundation, Inc.
 # All rights reserved.
 #
 # This code is derived from software contributed to The NetBSD Foundation
@@ -53,8 +53,13 @@ BEGIN {
 	do_pkg_readme=1;
 # set to 1 to use "README-new.html" as the name
 	use_readme_new=0;
+	if (use_readme_new) {
+		readme_name = "README-new.html";
+	}
+	else {
+		readme_name = "README.html";
+	}
 	printf("Reading database file\n");
-	fflush("/dev/stdout");
 }
 
 #conflicts /usr/pkgsrc/math/scilab
@@ -151,6 +156,10 @@ BEGIN {
 	for (i = 4; i <= NF; i++){
 		htmlname = htmlname " " $i;
 	}
+	# If we are using a name other than README.html, change it
+	# here.  This avoids having to process a huge line later which
+	# makes lesser awks puke.
+	gsub(/README.html/, readme_name, htmlname);
 	dir2htmlname[dir] = htmlname;
 	if (debug) printf("added dir2htmlname[%s]=%s\n", dir, htmlname);
 	next;
@@ -186,19 +195,27 @@ BEGIN {
 #
 
 END {
-	if (use_readme_new) {
-		readme_name = "README-new.html";
-	}
-	else {
-		readme_name = "README.html";
-	}
 	readme = TMPDIR "/" readme_name;
 
 	if ( dependsfile == "" ) dependsfile = "/dev/stdout";
 	if ( builddependsfile == "" ) builddependsfile = "/dev/stdout";
+
+	printf("Making sure binary package cache file is up to date...\n");
+	cmd = sprintf("%s AWK=%s CMP=%s GREP=%s PKG_INFO=\"%s\" PKG_SUFX=%s SED=%s SORT=%s %s/mk/scripts/binpkg-cache --packages %s",
+		SETENV, AWK, CMP, GREP, PKG_INFO, PKG_SUFX, SED, SORT, PKGSRCDIR, PACKAGES);
+	if (debug) printf("\nExecute:  %s\n",cmd);
+	rc = system(cmd);
+	if (rc != 0) {
+	  printf("\n**** WARNING ****\n") > "/dev/stderr";
+	  printf("Command: %s\nfailed.", cmd) > "/dev/stderr";
+	  printf("**** ------- ****\n") > "/dev/stderr";
+	  exit(1);
+	}
 	
+	printf("Loading binary package cache file...\n");
+	load_cache_file( PACKAGES "/.pkgcache" );
+
 	printf("Flattening dependencies\n");
-	fflush("/dev/stdout");
 	printf("") > dependsfile;
 	for (toppkg in topdepends){
 		if (debug) printf("calling find_all_depends(%s, run)\n", toppkg);
@@ -214,9 +231,10 @@ END {
 	
 
 # clear out the flattened depends list and repeat for the build depends
-	delete alldepends;
+	for( key in alldepends ) {
+		delete alldepends[key];
+	}
 	printf("Flattening build dependencies\n");
-	fflush("/dev/stdout");
 	printf("") > builddependsfile;
 	for (toppkg in topbuilddepends){
 		find_all_depends(toppkg, "build");
@@ -234,7 +252,6 @@ END {
 # read the vulnerabilities file
 		printf("Reading vulnerability file \"%s\"\n which was updated %s\n",
 		       vfile, vuldate);
-		fflush("/dev/stdout");
 		i = 1;
 		while((getline < vfile) > 0) {
 			if ($0 !~ /^\#/) {
@@ -253,16 +270,16 @@ END {
 		have_vfile = 0;
 	}
 	close(cmd);
-	fflush("/dev/stdout");
 
 	if (SINGLEPKG != "" ) {
 		printf("Only creating README for %s\n",SINGLEPKG);
-		delete topdepends;
+		for( key in topdepends ) {
+			delete topdepends[key];
+		}
 		topdepends[SINGLEPKG] = "yes";
 	}
 	
 	printf("Generating README.html files\n");
-	fflush("/dev/stdout");
 	pkgcnt = 0;
 	if (do_pkg_readme) {
 		templatefile = PKGSRCDIR "/templates/README.pkg";
@@ -278,12 +295,15 @@ END {
 			if ((pkgcnt % 100) == 0) {
 				printf("\n%d\n", pkgcnt);
 			}
-			fflush("/dev/stdout");
 			printf("") > readme;
 			htmldeps = "";
-			delete dpkgs;
+			for( key in dpkgs ) {
+				delete dpkgs[key];
+			}
 			split(alldepends[toppkg], dpkgs);
 			i = 1;
+			htmldeps_file = TMPDIR "/htmldep";
+			printf("") > htmldeps_file;
 			while(i in dpkgs){
 				if (debug) {
 				  printf("\tdpkg=%s, pat2dir[%s] = %s\n",
@@ -293,23 +313,30 @@ END {
 				}
 				nm=dpkgs[i];
 
-# we need a zillion escapes here because we need to end up
-# with \\&lt; in 'htmldeps' so that when
-# we gsub htmldeps into the output file, we end up with &lt;
-
-				gsub(/&/, "\\\\\\&amp;", nm);
-				gsub(/</, "\\\\\\&lt;", nm);
-				gsub(/>/, "\\\\\\&gt;", nm);
-				htmldeps=sprintf("%s<a href=\"../../%s/%s\">%s</a>\n",
+				gsub(/&/, "\\&amp;", nm);
+				gsub(/</, "\\&lt;", nm);
+				gsub(/>/, "\\&gt;", nm);
+#				htmldeps=sprintf("%s<a href=\"../../%s/%s\">%s</a>\n",
+#						 htmldeps,
+#						 pat2dir[dpkgs[i]],
+#						 readme_name, nm);
+# We use a temp file to hold the html dependencies because for
+# packages like gnome, this list can get very very large and 
+# become larger than what some awk implementations can deal
+# with.  The nawk shipped with solaris 9 is an example of 
+# such a limited awk.
+				printf("%s<a href=\"../../%s/%s\">%s</a>\n",
 						 htmldeps,
 						 pat2dir[dpkgs[i]],
-						 readme_name, nm);
+						 readme_name, nm) >> htmldeps_file;
 				i = i + 1;
 			}
-			if ( htmldeps == "" ) {
-				htmldeps = "<EM>none</EM>";
+			if ( i == 1 ) {
+			  printf("<EM>none</EM>") >> htmldeps_file;
 			}
-			if (debug) printf("htmldeps = \"%s\"\n", htmldeps);
+			close(htmldeps_file);
+			if (debug) printf("wrote = %d entries to \"%s\"\n",
+					  i-1, htmldeps_file);
 			
 			vul = "";
 			if (have_vfile) {
@@ -344,72 +371,28 @@ END {
 				}
 			}
 			
-			if (MULTIARCH == "no"){
-				cmd="ls -1 "PACKAGES "/" PKGREPOSITORYSUBDIR "/" wildcard[toppkg] PKG_SUFX " 2>/dev/null";
-				if (debug) {
-				  printf("Checking for binary package with %s\n",
-					 cmd);
-				}
-				binpkgs = "";
-				while((cmd | getline) > 0) {
-					pkgfile = $0;
-					gsub(/.*\//, "", pkgfile);
-					pkgnm = pkgfile;
-					gsub(/\.tgz$/, "", pkgnm);
-					binpkgs = sprintf("%s\n<TR><TD>%s:<TD><a href=\"%s/%s\">%s</a><TD>(%s %s)",
-							  binpkgs,
-							  MACHINE_ARCH,
-							  PKG_URL,
-							  pkgfile,
-							  pkgnm,
-							  OPSYS,
-							  OS_VERSION);
-				}
-				close(cmd);
-			}
-			else {
-				cmd = "ls -1 -d "PACKAGES"/[0-9].*/*/" 	PKGREPOSITORYSUBDIR "/" wildcard[toppkg] PKG_SUFX " 2>/dev/null";
-				oldfs = FS;
-				FS = "[/]";
-				binpkgs = "";
-				while((cmd | getline) > 0) {
-					release = $(NF-3);
-					arch = $(NF-2);
-					pkg = $NF;
-					pkgnm = pkg;
-					gsub(PKG_SUFX "$", "", pkgnm)
-						if (debug) {
-							printf("%s:%s:%s (%s)\n",
-							       release,
-							       arch,
-							       pkg,
-							       pkgnm);
-						}
-					binpkgs = sprintf("%s\n<TR><TD>%s:<TD><a href=\"%s/%s/%s/%s/%s\">%s</a><TD>(%s-%s)",
-						  binpkgs, arch,
-						  PKG_URL, release,
-						  arch,
-						  PKGREPOSITORYSUBDIR,
-						  pkg, pkgnm,
-						  OPSYS, release);
-				}
-				FS = oldfs;
-				close(cmd);
+
+			if (debug) {
+			  printf("Checking for binary package with lookup_cache( %s)\n",
+				 wc);
 			}
+# lookup_cache( wildcard ) will produce HTML for the packages which are found
+			binpkgs = lookup_cache( wildcard[toppkg] PKG_SUFX );
 			
+
 # sort the binary pkgs (XXX would be nice to implement in memory in awk)
 			sfile = TMPDIR "/sorted";
-			spipe = "sort > " sfile;
+			spipe = SORT " > " sfile;
 			printf("%s",binpkgs) | spipe;
 			close(spipe);
 			binpkgs = "";
 			while((getline < sfile) > 0) {
-				binpkgs = sprintf("%s\n%s", binpkgs, $0);
+			  binpkgs = sprintf("%s\n%s", binpkgs, $0);
 			}
 			close(sfile);
-			
-			if (debug) printf("binary packages: \n%s\n\n",
-					  binpkgs);
+
+                        if (debug) printf("binary packages: \n%s\n\n",
+                                          binpkgs);
 
 			if ( flatdepends[toppkg] ~ /^[ \t]*$/ ) {
 				rundeps = "<EM>none</EM>";
@@ -435,11 +418,21 @@ END {
 				}
 				gsub(/%%VULNERABILITIES%%/, ""vul"");
 				gsub(/%%VULDATE%%/, ""vuldate"");
-				gsub(/%%BUILD_DEPENDS%%/, ""htmldeps"");
+
 				gsub(/%%RUN_DEPENDS%%/, ""rundeps"");
 				gsub(/%%BIN_PKGS%%/, ""binpkgs"");
-				gsub(/README.html/, readme_name);
-				print >> readme;
+				
+				line = $0
+
+				if( line ~/%%BUILD_DEPENDS%%/ ) {
+				    gsub(/%%BUILD_DEPENDS%%/, "", line);
+				    while((getline < htmldeps_file) > 0) {
+				      print >> readme;
+				    }
+				    close( htmldeps_file );
+				}
+
+				print line >> readme;
 			}
 			close(readme);
 			close(templatefile);
@@ -484,6 +477,8 @@ END {
 			printf("Category = %s\n", category);
 			cat_make = catdir"/Makefile";
 			pkgs = "";
+			pkgs_file = TMPDIR "/pkgs_file";
+			printf("") > pkgs_file;
 			numpkg = 0;
 			print "" > readme;
 			while((getline < cat_make) > 0){
@@ -500,10 +495,19 @@ END {
 						 pkgdir2name[dir],
 						 comment[dir]);
 					}
-					pkgs =  sprintf("%s<TR><TD VALIGN=TOP><a href=\"%s/%s\">%s</a>: %s<TD>\n",
-							pkgs, pkg, readme_name,
+#					pkgs =  sprintf("%s<TR><TD VALIGN=TOP><a href=\"%s/%s\">%s</a>: %s<TD>\n",
+#							pkgs, pkg, readme_name,
+#							pkgdir2name[dir],
+#							comment[dir]);
+# We use a temp file to hold the list of all packages because
+# this list can get very very large and 
+# become larger than what some awk implementations can deal
+# with.  The nawk shipped with solaris 9 is an example of 
+# such a limited awk.
+					printf("<TR><TD VALIGN=TOP><a href=\"%s/%s\">%s</a>: %s<TD>\n",
+							pkg, readme_name,
 							pkgdir2name[dir],
-							comment[dir]);
+							comment[dir]) >> pkgs_file;
 					allpkg[tot_numpkg] =  sprintf("<!-- %s (for sorting) --><TR VALIGN=TOP><TD><a href=\"%s/%s/%s\">%s</a>: <TD>(<a href=\"%s/%s\">%s</a>) <td>%s\n",
 								      pkgdir2name[dir],
 								      category, pkg,
@@ -526,9 +530,19 @@ END {
 				gsub(/%%CATEGORY%%/, category);
 				gsub(/%%NUMITEMS%%/, numpkg);
 				gsub(/%%DESCR%%/, descr);
-				gsub(/%%SUBDIR%%/, pkgs);
-				gsub(/README.html/, readme_name);
-				print >> readme;
+
+				line = $0
+
+				if( $0 ~/%%SUBDIR%%/ ) {
+				    gsub(/%%SUBDIR%%/, "");
+				    while((getline < pkgs_file) > 0) {
+				      gsub(/README.html/, readme_name);
+				      print >> readme;
+				    }
+				    close( pkgs_file );
+				}
+
+				print line >> readme;
 			}
 			close(readme);
 			close(templatefile);
@@ -682,6 +696,18 @@ function reg2str(reg){
 }
 
 #
+# take a string which has a shell glob pattern and turn it into
+# an awk regular expression.
+#
+function glob2reg(reg){
+        gsub(/\./, "\\\.", reg);
+        gsub(/\+/, "\\\+", reg);
+        gsub(/\*/, ".*", reg);
+        gsub(/\?/, ".?", reg);
+        return(reg);
+}
+
+#
 # accepts a full path to a package directory, like "/usr/pkgsrc/math/scilab"
 # and returns just the last 2 directories, like "math/scilab"
 #
@@ -712,7 +738,7 @@ function uniq(list, deps, i, ulist){
 }
 
 function fatal_check_file(file, cmd){
-	cmd="test -f "file ;
+	cmd="test -f " file ;
 	if (debug) printf("Execute:  %s\n",cmd);
 	if (system(cmd) != 0) {
 		printf("**** FATAL ****\nRequired file %s does not exist\n",
@@ -747,7 +773,8 @@ function copy_readme(old, new, cmd, rc) {
 #	Compare the existing README.html file to the one we created.  If they are
 #	not the same, then copy over the one we created
 
-	cmd = " if ! cmp -s "new" "old" ; then mv -f " new " " old " ; fi";
+	cmd = sprintf("%s -s %s %s ; if test $? -ne 0 ; then mv -f %s %s ; fi",
+		CMP, new, old, new, old);
 	if (debug) printf("copy_readme()  execute:  %s\n",cmd);
 	rc = system(cmd);
 	if (rc != 0) {
@@ -768,3 +795,65 @@ function copy_readme(old, new, cmd, rc) {
 
 }
 
+
+function load_cache_file( file ) {
+
+  if( debug ) printf("load_cache_file(%s)\n", file);
+
+  # read the cache file
+  while( getline < file ) {
+
+    # if this line points to another cache file, then recursively
+    # load it
+    if( $0 ~ /^pkgcache_cachefile/ ) {
+      if( debug ) printf("\tFound pkgcache_cachefile line.\n");
+      load_cache_file( $2 );
+    } else if( $0 ~/^pkgcache_begin /) {
+      pkgfile = $2;
+      if( debug ) printf("\tStarting %s\n", pkgfile);
+      opsys = "unknown";
+      osver = "unknown";
+      march = "unknown";
+    } else if( $0 ~/^OPSYS=/ ) {
+      opsys = $1;
+      gsub(/OPSYS=/, "", opsys);
+    } else if( $0 ~/^OS_VERSION=/ ) {
+      osver = $1;
+      gsub(/OS_VERSION=/, "", osver);
+    } else if( $0 ~/^MACHINE_ARCH=/ ) {
+      march = $1;
+      gsub(/MACHINE_ARCH=/, "", march);
+    } else if( $0 ~/^pkgcache_end /) {
+      if( debug ) printf("\t%s, OPSYS=%s, OS_VERSION=%s, MACHINE_ARCH=%s\n",
+			 pkgfile, opsys, osver, march);
+      opsys_list[pkgfile] = opsys;
+      osver_list[pkgfile] = osver;
+      march_list[pkgfile] = march;
+      pkg_list[pkgfile] = pkgfile;
+      pkgnm_list[pkgfile] = pkgfile;
+      gsub(/.*\//, "", pkgnm_list[pkgfile]);
+    } else {
+      # skip this line
+    }
+  }
+
+  # close the cache file
+  close( file );
+}
+
+function lookup_cache( wc, r, binpkgs) {
+  if( debug ) printf("lookup_cache( %s )\n", wc);
+  r = ".*/" glob2reg( wc );
+  if( debug ) printf("lookup_cache():  Searching for \"%s\"\n", r );
+  
+  binpkgs = "";
+  for( key in pkg_list ) {
+    if( key ~ r ) {
+      binpkgs = sprintf("%s\n<TR><TD>%s:<TD><a href=\"%s/%s\">%s</a><TD>(%s %s)\n",
+			binpkgs, march_list[key], PKG_URL, key, pkgnm_list[key],
+			opsys_list[key], osver_list[key]);
+
+    }
+  }
+  return( binpkgs );
+}
diff --git a/mk/scripts/mkreadme b/mk/scripts/mkreadme
index 231cdb36ee1..8eea9444dc3 100755
--- a/mk/scripts/mkreadme
+++ b/mk/scripts/mkreadme
@@ -1,9 +1,9 @@
 #!/bin/sh
-# $NetBSD: mkreadme,v 1.4 2003/09/23 01:51:44 yyamano Exp $
+# $NetBSD: mkreadme,v 1.5 2005/05/17 21:46:59 dmcmahill Exp $
 #
 # Script for README.html generation
 #
-# Copyright (c) 2002 The NetBSD Foundation, Inc.
+# Copyright (c) 2002, 2005 The NetBSD Foundation, Inc.
 # All rights reserved.
 #
 # This code is derived from software contributed to The NetBSD Foundation
@@ -41,16 +41,27 @@
 
 TMPDIR=${TMPDIR:-/tmp/mkreadme}
 PKGSRCDIR=${PKGSRCDIR:-/usr/pkgsrc}
-BMAKE=${BMAKE:-make}
 AWK=${AWK:-/usr/bin/awk}
-EXPR=${EXPR:-expr}
+
+opsys=`uname -s`
+case "$opsys" in
+	*BSD)
+		makeprog=make
+		;;
+
+	*)
+		makeprog=bmake
+		;;
+esac
+
+BMAKE=${BMAKE:-${makeprog}}
 
 usage(){
     echo "$prog - Generates README.html files for a pkgsrc tree"
     echo "Usage:      $prog [-c|--cdrom] [-C|--prune] [-d|--debug] [-f|--ftp] "
-    echo "                  [-m|--multi-arch] [-p|--pkgsrc directory] "
+    echo "                  [-p|--pkgsrc directory] "
     echo "                  [-P|--packages directory] [-r|--restart] "
-    echo "                  [-s|--single-arch] [-S|--save-database]"
+    echo "                  [-S|--save-database]"
     echo " "
     echo "            $prog -h|--help"
     echo " "
@@ -69,10 +80,6 @@ usage(){
     echo " "
     echo "  -h|--help           Displays this help message"
     echo " "
-    echo "  -m|--multi-arch     Assumes a packages directory layout like:"
-    echo "                      OS_VERSION/MACHINE_ARCH/     for example:"
-    echo "                      1.{5,6}/{alpha,i386,sparc,vax}"
-    echo " "
     echo "  -p|--pkgsrc dir     Specifies the pkgsrc directory.  Defaults to"
     echo "                      The value of the PKGSRCDIR environment variable"
     echo "                      if set or /usr/pkgsrc otherwise."
@@ -83,9 +90,6 @@ usage(){
     echo "                      from a previous run still exists and that the script"
     echo "                      should use that instead of recreating the database."
     echo " "
-    echo "  -s|--single-arch    Assumes a single OS release and MACHINE_ARCH binary"
-    echo "                      packages directory structure."
-    echo " "
     echo "  -S|--save-database  Does not delete the database file after the run."
     echo "                      This is useful for debugging or re-running this script"
     echo "                      with the -r option."
@@ -114,6 +118,8 @@ prog=$0
 #
 ######################################################################
 
+cmdline=$*
+
 ftp_readme=no
 restart=no
 prune=no
@@ -157,12 +163,6 @@ do
 	exit 0
 	;;
 
-    # assume a OS_RELEASE/MACHINE_ARCH/ layout for the binary pkgs
-    -m|--multi-arch)
-	multiarch=yes
-	shift
-	;;
-
     # Specify pkgsrc directory
     -p|--pkgsrc)
 	PKGSRCDIR=$2
@@ -181,12 +181,6 @@ do
 	shift
 	;;
 
-    # assume a single directory structure layout for the binary pkgs
-    -s|--single-arch)
-	multiarch=no
-	shift
-	;;
-
     # Save the database files
     -S|--save-database)
 	save=yes
@@ -252,8 +246,9 @@ echo "Extracting configuration variables:"
 echo " "
 if [ -d ${PKGSRCDIR}/pkgtools/pkglint ]; then
     cd ${PKGSRCDIR}/pkgtools/pkglint
-
-    for v in DISTDIR PACKAGES PKGREPOSITORYSUBDIR PKG_SUFX MACHINE_ARCH PKG_URL OPSYS OS_VERSION FTP_PKG_URL_HOST FTP_PKG_URL_DIR CDROM_PKG_URL_HOST CDROM_PKG_URL_DIR
+    for v in AWK CDROM_PKG_URL_HOST CDROM_PKG_URL_DIR CMP DISTDIR ECHO EXPR \
+	FGREP FTP_PKG_URL_HOST FTP_PKG_URL_DIR GREP PACKAGES PKG_INFO PKG_SUFX \
+	SED SETENV SORT 
     do
 	val=`${BMAKE} show-var VARNAME=${v}`
 	if [ $? != 0 ]; then
@@ -261,15 +256,14 @@ if [ -d ${PKGSRCDIR}/pkgtools/pkglint ]; then
 	    echo "Failed.  This is a fatal error"
 	    clean_and_exit
 	fi
-	echo "$v=$val"
-	eval ${v}=${val}
+	echo "---->  ${v}=\"${val}\""
+	eval "${v}=\"${val}\""
     done
 else
     echo "Error:   ${PKGSRCDIR}/pkgtools/pkglint does not seem to exist"
     exit 1
 fi
 
-
 ######################################################################
 #
 #  Decide on FTP vs CDROM README.html files
@@ -296,39 +290,6 @@ if [ ! -z "$PKGDIR" ]; then
     echo "PACKAGES specified on command line to be $PKGDIR"
 fi
 
-######################################################################
-#
-#  Decide in binary package directory layout
-#
-######################################################################
-
-
-# If PACKAGES is set to the default (../../pkgsrc/packages), the current
-# ${MACHINE_ARCH} and "release" (uname -r) will be used. Otherwise a directory
-# structure of ...pkgsrc/packages/`uname -r`/${MACHINE_ARCH} is assumed.
-#
-# This is the logic from bsd.pkg.mk, but I think I'd like to change it to
-# come from a command line switch
-
-if [ -z "$multiarch" -a -e ${PACKAGES} ]; then
-    cd ${PACKAGES}
-    case `pwd` in
-	*/pkgsrc/packages)
-	    multiarch=no
-	    ;;
-
-	*)
-	    multiarch=yes
-	    ;;
-    esac
-fi
-
-if [ "x$multiarch" = "xyes" ]; then
-    echo "Will generate multi-release, multi-arch readme files"
-else
-    echo "Will generate single-release, single-arch readme files"
-fi
-
 
 ######################################################################
 #
@@ -339,6 +300,7 @@ fi
 if [ "x$restart" = "xno" ] ; then
     echo " "
     echo "Extracting data.  This could take a while"
+    echo "Started at: `date` "
     echo " "
     npkg=1
 
@@ -346,13 +308,13 @@ if [ "x$restart" = "xno" ] ; then
     rm -fr $DATABASEFILE
 
     cd ${PKGSRCDIR}
-    list=`grep '^[[:space:]]*'SUBDIR */Makefile | sed 's,/Makefile.*=[[:space:]]*,/,'`
+    list=`${GREP} '^[ \t]*'SUBDIR */Makefile | ${GREP} -v regress/ | ${SED} 's,/Makefile.*=[[:space:]]*,/,'`
     for pkgdir in $list
     do
 	    if [ ! -d $pkgdir ]; then
 		    echo " "
 		    echo "WARNING:  the package directory $pkgdir is listed in" > /dev/stderr
-		    echo $pkgdir | sed 's;/.*;/Makefile;g' > /dev/stderr
+		    echo $pkgdir | ${SED} 's;/.*;/Makefile;g' > /dev/stderr
 		    echo "but the directory does not exist.  Please fix this!" > /dev/stderr
 	    else
 		    cd $pkgdir
@@ -368,7 +330,7 @@ if [ "x$restart" = "xno" ] ; then
 			    echo "$l" >> $DATABASEFILE
 		    fi
 	    fi
-	    echo -n "."
+	    ${ECHO} -n "."
 	    if [ `${EXPR} $npkg % 100 = 0` -eq 1 ]; then
 		echo " "
 		echo "$npkg"
@@ -376,6 +338,8 @@ if [ "x$restart" = "xno" ] ; then
 	    npkg=`${EXPR} $npkg + 1`
 	    cd $PKGSRCDIR
     done
+    echo " "
+    echo "Finished extracting data for ${npkg} packages at: `date` "
 else
     echo " "
     echo "Using existing database (are you sure you wanted the -r/--restart flag?)"
@@ -408,16 +372,18 @@ ${AWK} -f ${PKGSRCDIR}/mk/scripts/genreadme.awk  \
     builddependsfile=${TMPDIR}/pkgsrc.builddepends.debug \
     debug=$debug \
     dependsfile=${TMPDIR}/pkgsrc.depends.debug \
+    AWK=$AWK \
+    CMP=$CMP \
     DISTDIR=$DISTDIR \
-    MACHINE_ARCH=$MACHINE_ARCH \
-    MULTIARCH=$multiarch \
-    OPSYS=$OPSYS \
-    OS_VERSION=$OS_VERSION \
+    GREP=$GREP \
     PACKAGES=$PACKAGES \
+    PKG_INFO="$PKG_INFO" \
     PKG_SUFX=$PKG_SUFX \
     PKG_URL=$PKG_URL \
-    PKGREPOSITORYSUBDIR=$PKGREPOSITORYSUBDIR \
     PKGSRCDIR=$PKGSRCDIR \
+    SED=$SED \
+    SETENV=$SETENV \
+    SORT=$SORT \
     TMPDIR=$TMPDIR \
 	${DATABASEFILE}
 
@@ -438,9 +404,9 @@ echo " "
 cd ${PKGSRCDIR}
 ipv6=${TMPDIR}/ipv6pkgs
 ipv6_entries=${TMPDIR}/ipv6_entries
-grep -l '^BUILD_DEFS.*=.*USE_INET6' */*/Makefile  | sed 's;Makefile;;g' > $ipv6
-fgrep -f $ipv6 README-all.html | sort -t/ +1 > $ipv6_entries
-sed \
+${GREP} -l '^BUILD_DEFS.*=.*USE_INET6' */*/Makefile  | ${SED} 's;Makefile;;g' > $ipv6
+${FGREP} -f $ipv6 README-all.html | sort -t/ +1 > $ipv6_entries
+${SED} \
     -e "/%%TRS%%/r${ipv6_entries}" \
     -e '/%%TRS%%/d' \
     templates/README.ipv6 > ${TMPDIR}/README-IPv6.html
@@ -448,10 +414,13 @@ if [ $? != 0 ]; then
     echo "Error:  README-IPv6.html generation failed (on sed script)"
     clean_and_exit
 fi
-if [ ! -f README-IPv6.html ] || ! cmp -s ${TMPDIR}/README-IPv6.html README-IPv6.html ; then
+
+if [ ! -f README-IPv6.html ]; then
 	mv -f ${TMPDIR}/README-IPv6.html README-IPv6.html
-else
+elif cmp -s ${TMPDIR}/README-IPv6.html README-IPv6.html ; then
 	echo "README-IPv6.html is unchanged (no changes were needed)"
+else
+	mv -f ${TMPDIR}/README-IPv6.html README-IPv6.html
 fi
 
 ######################################################################