-add support for a BULK_PREREQ make variable. BULK_PREREQ contains a list

 of packages which must stay installed during the build, but are not pulled
 in by the DEPENDS for each package.  Currently, BULK_PREREQ will always include
 pkgtools/digest.  At this time, the primary use will be to add
 BULK_PREREQ+=pkgtools/xpkgwedge
 in /etc/mk.conf to do an xpkgwedge'd bulk build.  It is up to the user to make
 sure that the list of packages in BULK_PREREQ is a flattened list (ie all the
 DEPENDS are listed too).  Again, at this time, xpkgwedge is really the only
 package that should be added to the list.

-add an ADMINSIG build.conf variable.  This is the signature at the end of the
 email report.  Maybe now I'll quit forwarding reports signed as "-Hubert".

-while here, eliminate grep|awk lines and `grep >/dev/null` replacing them
 with pure awk and grep -q.

5 files changed, 109 insertions, 32 deletions

diff --git a/mk/bulk/bsd.bulk-pkg.mk b/mk/bulk/bsd.bulk-pkg.mk
index 6e2470f3bfa..07a04fd94c0 100644
--- a/mk/bulk/bsd.bulk-pkg.mk
+++ b/mk/bulk/bsd.bulk-pkg.mk
@@ -1,4 +1,4 @@
-#	$NetBSD: bsd.bulk-pkg.mk,v 1.21 2001/02/17 15:48:51 dmcmahill Exp $
+#	$NetBSD: bsd.bulk-pkg.mk,v 1.22 2001/03/19 11:25:39 dmcmahill Exp $
 
 #
 # Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@netbsd.org>
@@ -101,6 +101,9 @@ INDEXFILE?=		${PKGSRCDIR}/.index
 ORDERFILE?=		${PKGSRCDIR}/.order
 .endif
 
+# a list of pkgs which we should _never_ delete during a build.  The primary use is for digest
+# and also for xpkgwedge.  Add pkgtools/xpkgwedge in /etc/mk.conf to do an xpkgwedged bulk build.
+BULK_PREREQ+=		pkgtools/digest
 
 # check if the $REF file is uptodate, i.e. is newer than any of
 # the pkg files; prints "1" if upto date, "0" if not.
@@ -192,7 +195,7 @@ bulk-package:
 		${ECHO_MSG} "BULK> Full rebuild  in progress..." ; \
 		${ECHO_MSG} "BULK> Cleaning package and its depends" ;\
 		if [ "${USE_BULK_CACHE}" = "yes" ]; then \
-			thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\
+			thisdir=`${AWK} '/ ${PKGNAME} / {print $$1}' ${INDEXFILE}`; \
 			for pkgdir in $$thisdir `${GREP} "^$$thisdir " ${DEPENDSFILE} | ${SED} -e 's;^.*:;;g'`; do \
 				${DO}       (cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} clean) ; \
 			done ;\
@@ -202,17 +205,24 @@ bulk-package:
 		fi; \
 		if [ "${PRECLEAN}" = "yes" -a "${USE_BULK_CACHE}" = "yes" ]; then \
 			${ECHO_MSG} "BULK> Removing installed packages which are not needed to build ${PKGNAME}" ; \
-			thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\
+			thisdir=`${AWK} '/ ${PKGNAME} / {print $$1}' ${INDEXFILE}`; \
 			for pkgname in `${PKG_INFO} | ${AWK} '{print $$1}'` ; \
 			do \
 				pkgdir=`${GREP} " $$pkgname " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\
-				if `${PKG_INFO} -qe $$pkgname` ; then \
-					if ! `${EGREP} "^$$thisdir.* $$pkgdir( |$$)" ${DEPENDSFILE} >/dev/null 2>&1` ; then \
-						${ECHO_MSG} ${PKG_DELETE} -r $$pkgname ; \
-						${DO}       ${PKG_DELETE} -r $$pkgname || true ; \
-						if `${PKG_INFO} -qe $$pkgname` ; then \
-							${DO}       ${PKG_DELETE} -f $$pkgname || true ; \
-						fi ;\
+				if ${PKG_INFO} -qe $$pkgname ; then \
+					if ! ${EGREP} -q "^$$thisdir.* $$pkgdir( |$$)" ${DEPENDSFILE} ; then \
+						case "${BULK_PREREQ}" in \
+							*$$pkgdir* ) \
+								${ECHO_MSG} "BULK> Keeping BULK_PREREQ: $$pkgname ($$pkgdir)" ;\
+								;; \
+							* ) \
+								${ECHO_MSG} ${PKG_DELETE} -r $$pkgname ; \
+								${DO}       ${PKG_DELETE} -r $$pkgname || true ; \
+								if `${PKG_INFO} -qe $$pkgname` ; then \
+									${DO}       ${PKG_DELETE} -f $$pkgname || true ; \
+								fi ;\
+								;; \
+						esac ; \
 					else \
 						${ECHO_MSG} "BULK> ${PKGNAME}  requires installed package $$pkgname ($$pkgdir) to build." ;\
 					fi ;\
@@ -220,9 +230,9 @@ bulk-package:
 			done ; \
 		fi ;\
 		if [ "${USE_BULK_CACHE}" = "yes" ]; then \
-			thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\
+			thisdir=`${AWK} '/ ${PKGNAME} / {print $$1}' ${INDEXFILE}` ;\
 			${ECHO_MSG} "BULK> Installing packages which are required to build ${PKGNAME}." ;\
-			for pkgdir in `${GREP} "^$$thisdir " ${DEPENDSFILE} | ${SED} -e 's;^.*:;;g'`; do \
+			for pkgdir in `${GREP} "^$$thisdir " ${DEPENDSFILE} | ${SED} -e 's;^.*:;;g'` ${BULK_PREREQ} ; do \
 				pkgname=`${GREP} "^$$pkgdir " ${INDEXFILE} | ${AWK} '{print $$2}'` ; \
 				pkgfile=${PACKAGES}/All/$${pkgname}.tgz ;\
 				if ! `${PKG_INFO} -qe $$pkgname` ; then \
@@ -248,7 +258,7 @@ bulk-package:
 			nbrokenby=0;\
 			if [ "${USE_BULK_CACHE}" = "yes" ]; then \
 				${ECHO_MSG} "BULK> Marking all packages which depend upon ${PKGNAME} as broken:"; \
-				thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\
+				thisdir=`${AWK} '/ ${PKGNAME} / {print $$1}' ${INDEXFILE}`; \
 				for pkgdir in `${GREP} "^$$thisdir " ${SUPPORTSFILE} | ${SED} -e 's;^.*:;;g'`; do \
 					pkgname=`${GREP} "^$$pkgdir " ${INDEXFILE} | ${AWK} '{print $$2}'` ;\
 					${ECHO_MSG} "BULK> marking package that requires ${PKGNAME} as broken:  $$pkgname ($$pkgdir)";\
@@ -266,7 +276,7 @@ bulk-package:
 					${ECHO_MSG} "BULK> $$pkgname ($$pkgdir) is broken because it depends upon ${PKGNAME} ($$thisdir) which is broken." \
 						>> ${PKGSRCDIR}/$$pkgdir/${BROKENFILE};\
 					nbrokenby=`expr $$nbrokenby + 1`;\
-					if ! `${GREP} " $$pkgdir/${BROKENFILE}" ${PKGSRCDIR}/${BROKENFILE} >/dev/null 2>&1` ; then \
+					if ! ${GREP} -q " $$pkgdir/${BROKENFILE}" ${PKGSRCDIR}/${BROKENFILE} ; then \
 						${ECHO} " $$pkgerr $$pkgdir/${BROKENFILE} 0 " >> ${PKGSRCDIR}/${BROKENFILE} ;\
 					fi ;\
 				done ;\
@@ -277,7 +287,7 @@ bulk-package:
 		fi ; \
 		${ECHO_MSG} "BULK> Cleaning packages and its depends" ;\
 		if [ "${USE_BULK_CACHE}" = "yes" ]; then \
-			thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\
+			thisdir=`${AWK} '/ ${PKGNAME} / {print $$1}' ${INDEXFILE}`; \
 			for pkgdir in $$thisdir `${GREP} "^$$thisdir " ${DEPENDSFILE} | ${SED} -e 's;^.*:;;g'`; do \
 				${DO}       (cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} clean) ; \
 			done ;\
diff --git a/mk/bulk/build b/mk/bulk/build
index b0e861c384c..de0161977ed 100644
--- a/mk/bulk/build
+++ b/mk/bulk/build
@@ -1,5 +1,5 @@
 #!/bin/sh
-# $NetBSD: build,v 1.9 2001/02/13 23:02:23 dmcmahill Exp $
+# $NetBSD: build,v 1.10 2001/03/19 11:25:39 dmcmahill Exp $
 #
 # Do builk build
 #
@@ -50,7 +50,23 @@ INDEXFILE=`make show-var VARNAME=INDEXFILE`
 ORDERFILE=`make show-var VARNAME=ORDERFILE`
 cd ${USR_PKGSRC}	
 
-# Figure out optimal build order, and build
+# get the list of packages which should always be installed during the build
+cd ${USR_PKGSRC}/pkgtools/pkglint
+BULK_PREREQ=`make show-var VARNAME=BULK_PREREQ`
+cd ${USR_PKGSRC}
+
+# install prerequisite packages.  Note:  we do this _before_ the depends tree
+# because some packages like xpkgwedge only become DEPENDS if its installed
+echo "Installing prerequisite packages specified with BULK_PREREQ..."
+for pkgdir in $BULK_PREREQ
+do
+	echo $pkgdir
+	# make sure its installed _and_ packaged
+	cd ${USR_PKGSRC}/$pkgdir && make bulk-install && make bulk-package && make clean
+done
+
+# Figure out optimal build order.
+cd ${USR_PKGSRC}
 if [ "$1" != "restart" ]; then
 	echo "Building dependency tree (this may take a while)..."
 	sh mk/bulk/printdepends > $DEPENDSTREEFILE
@@ -68,6 +84,7 @@ if [ "$1" != "restart" ]; then
 fi
 
 echo "Starting actual build using the order specified in $ORDERFILE..."
+cd ${USR_PKGSRC}
 nice -n 20 make \
 		USE_BULK_CACHE=yes \
 		SPECIFIC_PKGS=1 \
@@ -76,11 +93,23 @@ nice -n 20 make \
 
 echo "Build finished.  Removing all installed packages left over from build..."
 
-for pkg in `pkg_info`
+for pkgname in `pkg_info -e \*`
 do
-	if `pkg_info -qe $pkg`; then 
-		echo pkg_delete -r $pkg
-		pkg_delete -rR $pkg
+	if `pkg_info -qe $pkgname`; then 
+		pkgdir=`grep " $pkgname " $INDEXFILE | awk '{print $1}'`	
+		case "${BULK_PREREQ}" in 
+			*$pkgdir* ) 
+				echo "Keeping BULK_PREREQ: $pkgname ($pkgdir)" ;
+				;; 
+			* ) 
+				echo pkg_delete -r $pkgname
+				pkg_delete -r $pkgname
+				if `pkg_info -qe $pkgname` ; then
+					echo "$pkgname ($pkgdir) did not deinstall nicely.  Forcing the deinstall"
+					pkg_delete -f $pkgname || true 
+				fi 
+				;; 
+		esac
 	fi
 done
 
diff --git a/mk/bulk/build.conf-example b/mk/bulk/build.conf-example
index ca5eded0d88..6296efae3a4 100644
--- a/mk/bulk/build.conf-example
+++ b/mk/bulk/build.conf-example
@@ -1,5 +1,5 @@
 # build.conf
-# $NetBSD: build.conf-example,v 1.7 2000/12/30 14:53:28 dmcmahill Exp $
+# $NetBSD: build.conf-example,v 1.8 2001/03/19 11:25:39 dmcmahill Exp $
 #
 # config file in /bin/sh syntax for {,pre,post}-build
 #
@@ -14,6 +14,9 @@ FTPHOST="ftp://ftp.machi.ne"                    # host for broken.html
 # Where build will mail the report
 ADMIN="you@some.whe.re"
 
+# Who the report is signed by
+ADMINSIG="-Your Name"
+
 # The OS release this bulk build is happening on
 osrev=`uname -r`
 
diff --git a/mk/bulk/post-build b/mk/bulk/post-build
index 6de281d01b0..f36d8e5d5ec 100644
--- a/mk/bulk/post-build
+++ b/mk/bulk/post-build
@@ -1,5 +1,5 @@
 #!/usr/pkg/bin/perl
-# $NetBSD: post-build,v 1.11 2001/02/23 11:18:29 dmcmahill Exp $
+# $NetBSD: post-build,v 1.12 2001/03/19 11:25:39 dmcmahill Exp $
 #
 # Collect stuff after a pkg bulk build
 #
@@ -27,6 +27,7 @@ sub getconf
 }
 
 
+getconf("ADMINSIG");		# "-Your Name"
 getconf("FTPURL");		# "pub/NetBSD/pkgstat/`date +%Y%m%d.%H%M`"
 getconf("FTP");			# "/disk1/ftp/${FTPURL}"
 getconf("FTPHOST");		# ftp://ftp.machi.ne/
@@ -251,7 +252,7 @@ if ($verbose) {
 	print "for logs of builds broken or not resulting in a binary pkg.\n";
 	print "\n";
 	print "\n";
-	print " - Hubert\n";
+	print "$ADMINSIG\n";
 	print "\n";
 	print "[* This message was created automatically! *]\n";
 	print "\n";
diff --git a/mk/bulk/pre-build b/mk/bulk/pre-build
index b6b16533207..bc561362252 100644
--- a/mk/bulk/pre-build
+++ b/mk/bulk/pre-build
@@ -1,5 +1,5 @@
 #!/bin/sh
-# $NetBSD: pre-build,v 1.11 2001/03/14 15:04:03 hubertf Exp $
+# $NetBSD: pre-build,v 1.12 2001/03/19 11:25:39 dmcmahill Exp $
 #
 # Clean up system to be ready for bulk pkg build
 #
@@ -16,6 +16,11 @@ else
 fi
 
 PRUNEDISTFILES=${PRUNEDISTFILES:-"yes"}
+PRUNEPACKAGES=${PRUNEPACKAGES:-"no"}
+PRUNELINKS=${PRUNEPACKAGES:-"no"}
+PRUNEDISTFILES=`echo $PRUNEDISTFILES | tr "[:lower:]" "[:upper:]"`
+PRUNEPACKAGES=`echo $PRUNEPACKAGES | tr "[:lower:]" "[:upper:]"`
+PRUNELINKS=`echo $PRUNELINKS | tr "[:lower:]" "[:upper:]"`
 
 # extract the name of the files used for the build log and broken build log.
 # these have defaults set by bsd.bulk-pkg.mk and may be overridden in /etc/mk.conf
@@ -51,15 +56,42 @@ fi
 # Remove old/broken distfiles and binary packages
 #
 
-if [ $PRUNEDISTFILES = "yes" -o $PRUNEDISTFILES = "YES" ]; then
-	echo Removing old/broken distfiles and binary packages
+DISTDIR=`( cd $USR_PKGSRC/pkgtools/pkglint ; make show-var VARNAME=DISTDIR )`;
+PACKAGES=`( cd $USR_PKGSRC/pkgtools/pkglint ; make show-var VARNAME=PACKAGES )`;
+
+if [ $PRUNEDISTFILES = "YES" ]; then
+	echo "Removing old/broken distfiles"
 	( cd ${USR_PKGSRC}/pkgtools/pkglint ; make bulk-install )
-	lintpkgsrc -r -P${USR_PKGSRC}
+	lintpkgsrc -K $PACKAGES -P $USR_PKGSRC -M $DISTDIR -o -m -r
 	echo done.
 else
 	echo "Skipping distfile pruning."
 fi
 
+if [ $PRUNEPACKAGES = "YES" ]; then
+	echo "Removing old (out of date) binary packages"
+	( cd ${USR_PKGSRC}/pkgtools/pkglint ; make bulk-install )
+	lintpkgsrc -K $PACKAGES -P $USR_PKGSRC -M $DISTDIR -p -r
+	echo done.
+else
+	echo "Skipping packages pruning."
+fi
+
+if [ $PRUNELINKS = "YES" ]; then
+	echo "Checking for and removing orphaned packages links"
+	find $PACKAGES -type l -print |  \
+	while read f 
+	do
+		if [ ! -d $f -a ! -f $f ]; then
+			echo "Removing orphaned link:  \"$f\""
+			rm $f
+		fi
+	done
+	echo done.
+else
+	echo "Skipping pruning of packages links."
+fi
+
 #
 # Clean out everything and it's mother
 #
@@ -71,15 +103,17 @@ if [ -d /var/db/pkg ]; then
 	for pkg in *
 	do
 		if `pkg_info -qe $pkg`; then 
-			echo pkg_delete -rR $pkg
-			pkg_delete -rR $pkg
+			echo pkg_delete -r $pkg
+			pkg_delete -r $pkg
 		fi
 	done
+	# this should have removed everything.  Now force any broken pkgs
+	# to deinstall
 	for pkg in *
 	do
 		if `pkg_info -qe $pkg`; then 
-			echo pkg_delete -rR $pkg
-			pkg_delete -rR -f $pkg
+			echo pkg_delete -f $pkg
+			pkg_delete -f $pkg
 		fi
 	done