Pullup ticket 1770 - requested by rillig

improved upload script for bulk builds

Revisions pulled up:
- pkgsrc/mk/bulk/upload			1.31, 1.32, 1.33

   Module Name:		pkgsrc
   Committed By:	jschauma
   Date:		Sun Jul 30 14:31:10 UTC 2006

   Modified Files:
   	pkgsrc/mk/bulk: upload

   Log Message:
   - use ${BMAKE} instead of 'make' in one instance
   - if we can't build one of the required packages, bail out
---
   Module Name:		pkgsrc
   Committed By:	dmcmahill
   Date:		Tue Aug  1 00:53:24 UTC 2006

   Modified Files:
   	pkgsrc/mk/bulk: upload

   Log Message:
   - add a -h|--help flag

   - add a -n|--no-upload flag which does everything but actually executing
     the upload to help see what would happen

   - add a -d|--debug flag to preserve the temporary files to help with
     debugging

   - add a -V|--version flag
---
   Module Name:		pkgsrc
   Committed By:	rillig
   Date:		Tue Aug  1 13:16:42 UTC 2006

   Modified Files:
   	pkgsrc/mk/bulk: upload

   Log Message:
   Rewrote upload to use the newly introduced sort-packages program.

   While here, ...
   - Added stricter checking by using "set -eu".
   - The bulk build configuration file is properly included, and the
     MAKECONF definition that it may contain is properly exported.
   - All progress messages and error messages are prefixed by üpload>",
     so that it is obvious where the messages come from.
   - Since extracting the make(1) variables takes quite a long time, print
     an informational message before doing that.
   - Removed the use of the error-prone lintpkgsrc to detect whether a
     package is restricted or vulnerable.
   - If an error occurs, the upload program returns an exitcode of 1,
     which is common among Unix utilities.
   - Removed almost all pipe operators, since they tend to hide program
     failures.
   - All error messages are redirected to stderr instead of stdout.

1 files changed, 218 insertions, 90 deletions

diff --git a/mk/bulk/upload b/mk/bulk/upload
index dddfb1d6b90..9237da530d2 100644
--- a/mk/bulk/upload
+++ b/mk/bulk/upload
@@ -1,13 +1,136 @@
 #!/bin/sh
-# $NetBSD: upload,v 1.30 2006/06/22 11:09:18 joerg Exp $
+# $NetBSD: upload,v 1.30.2.1 2006/08/02 15:59:40 salo Exp $
 
 #
 # Upload non-restricted binary pkgs to ftp server
 #
 
+AWK=${AWK:-/usr/bin/awk}
+
+set -eu
+
+#
+# Find out where we are
+#
+scriptdir=`dirname "$0"`
+scriptdir=`cd "${scriptdir}" && pwd`
+
+usage()
+{
+cat << EOF
+
+$prog:  Uploads binary packages.
+
+Usage:  $prog [-n|--no-upload] [-d|--debug] [-v|--verbose]
+	$prog -h|--help
+        $prog -V|--version
+
+    -d|--debug          Preserves some intermediate files to help
+			debug any problems.
+
+    -h|--help           Displays this help message and exits.
+
+    -n|--no-upload      Does all of the work except for the actual upload.
+			This option may be used to generate the upload script
+			along with the list of packages to be excluded.
+
+    -i|--no-install	Do not install the required packages; assume instead
+			that they are already available.
+
+    -v|--verbose        Enables verbose output.
+
+    -V|--version        Displays the version of this script and exits.
+
+Example:
+
+    cd /usr/pkgsrc && $prog
+
+Environment Variables:
+    
+    AWK  --  If specified, sets the awk program to be used
+
+Files:
+
+    pkgsrc/mk/bulk/build.conf
+
+EOF
+}
+
+prog=$0
+debug=no
+do_upload=yes
+do_install=yes
+verbose=no
+while test $# -gt 0; do
+	case "$1" in
+	-d|--debug)
+		debug=yes
+		shift
+		;;
+
+	-h|--help)
+		usage
+		exit 0
+		;;
+
+	-i|--no-install)
+		do_install=no
+		shift
+		;;
+
+	-n|--no-upload)
+		do_upload=no
+		shift
+		;;
+
+	-v|--verbose)
+		verbose=yes
+		shift
+		;;
+
+	-V|--version)
+		${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
+		exit 0
+		;;
+
+	-*)
+		echo "$prog:  Unknown option:  $1" 1>&2
+		usage
+		exit 1
+		;;
+
+	*)
+		break
+		;;
+	esac
+done
+if test "$#" -gt 0; then
+	echo "$prog:  Unknown argument: $1" 1>&2
+	usage
+	exit 1
+fi
+
+install_required()
+{
+
+	[ "$do_install" = "yes" ] || return 0
+	pkg=$1
+	if [ "${verbose}" = "yes" ]; then
+		echo "upload> Installing ${pkg}"
+	fi
+	( cd "$pkg" && ${BMAKE} bulk-install) \
+	|| {
+		echo "upload> ERROR: Unable to install required package $pkg!" 1>&2
+		echo "        Bailing out -- you're on your own." 1>&2
+		exit 1
+	}
+}
+
 MD5="digest md5";
 SHA1="digest sha1";
 
+REQUIRED_PACKAGES="pkgtools/pkglint net/rsync security/audit-packages"
+
 opsys=`uname -s`
 case "$opsys" in
 NetBSD)	BMAKE=make;
@@ -30,17 +153,20 @@ BATCH=1
 DEPENDS_TARGET=bulk-install
 export BATCH DEPENDS_TARGET
 
-# Pull in RSYNC_DST, RSYNC_OPTS:
-if [ -f "$BULK_BUILD_CONF" ]; then
-    . $BULK_BUILD_CONF
-else
-    . `dirname $0`/build.conf
-fi
+#
+# Get the variables MAKECONF, RSYNC_DST, RSYNC_OPTS from the bulk build
+# configuration file.
+#
+: ${BULK_BUILD_CONF="${scriptdir}/build.conf"} #"
+. "${BULK_BUILD_CONF}"
+. "${scriptdir}/post-build-conf"
+check_config_vars
+export_config_vars
 
 cd $USR_PKGSRC
 
 if [ -z "$RSYNC_DST" ]; then
-	echo "You must set the variable RSYNC_DST, see build.conf-example."
+	echo "upload> ERROR: You must set the variable RSYNC_DST, see build.conf-example." 1>&2
 	exit 1
 fi
 
@@ -51,36 +177,40 @@ fi
 umask 022
 TMPDIR="${TMPDIR:-/tmp}"
 TMP="${TMPDIR}"/pkg_upload.$$
-(umask 077 && mkdir "${TMP}")
-if [ $? -ne 0 ]
-then
-        echo $0: cannot create temporary directory \""${TMP}"\" >&2
+(umask 077 && mkdir "${TMP}") \
+|| {
+        echo "upload> ERROR: cannot create temporary directory \"${TMP}\"." 1>&2
         exit 1
-fi
+}
+
+vulnerable_packages="$TMP/vulnerable_packages"
+restricted_packages="$TMP/restricted_packages"
+old_packages="$TMP/old_packages"
+good_packages="$TMP/regular_packages"
 
-exf="$TMP"/exclude
-vf="$TMP"/vulnerable
-upload="$TMP"/upload
 upload_general="$TMP"/upload_general
 upload_vulnerable="$TMP"/upload_vulnerable
 
 # May be different than $USR_PKGSRC:
+echo "upload> Running ${BMAKE} to get the pkgsrc variables"
 pkgsrcdir=`cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=_PKGSRCDIR`
 packages=`cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=PACKAGES`
 distdir=`cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=DISTDIR`
-gzip_cmd=`cd pkgtools/pkglint; make show-var VARNAME=GZIP_CMD USE_TOOLS=gzip`
+gzip_cmd=`cd pkgtools/pkglint; ${BMAKE} show-var VARNAME=GZIP_CMD USE_TOOLS=gzip`
+pkg_info=`cd pkgtools/pkglint && ${BMAKE} show-var VARNAME=PKG_INFO`
 
 # Pull in some pkgs needed
-( cd pkgtools/pkglint ;        ${BMAKE} bulk-install )
-( cd net/rsync ;               ${BMAKE} bulk-install )
-( cd security/audit-packages ; ${BMAKE} bulk-install )
+for pkg in ${REQUIRED_PACKAGES}; do
+	install_required $pkg
+done
 
-echo "Making sure vulnerability-list is up-to-date:"
+echo "upload> Making sure vulnerability-list is up-to-date:"
 if [ -z "$UPDATE_VULNERABILITY_LIST" -o "$UPDATE_VULNERABILITY_LIST" = "yes" ]
 then
 	env PKGVULNDIR=${distdir} download-vulnerability-list
+	echo "        done."
 else
-	echo '(skipped)'
+	echo "        (skipped)"
 fi
 
 case $LINTPKGSRC_CACHE in
@@ -92,29 +222,29 @@ yes|YES)
 	;;
 esac
 
-echo "Checking for restricted and out of date packages:"
+echo "upload> Checking for out of date packages:"
 # -p  =  report old versions of packages
-# -R  =  report restricted packages
-lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -pR  | sed 's@'$packages'/@@' > "$exf"
-
-echo "Checking for vulnerable packages:"
-lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -V  | sed 's@'$packages'/@@' > "$vf"
+lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -p > "${old_packages}.tmp"
+sed 's@'$packages'/@@' < "${old_packages}.tmp" > "$old_packages"
 
 RSFLAGS="-vap --progress $RSYNC_OPTS"
 
 failed=no
 cd $packages
 
+echo "upload> Checking for restricted and vulnerable packages"
+(cd All && env PKG_INFO="${pkg_info}" OUTDIR="${TMP}" PKGVULNDIR="${distdir}" sh "${pkgsrcdir}/mk/bulk/sort-packages")
+
 if [ "${MKSUMS}" = "yes" -o "${MKSUMS}" = "YES" ]; then
 
-	echo "Calculating checksum files..."
+	echo "upload> Calculating checksum files..."
 
 	SUMFILES="BSDSUM CKSUM MD5 SHA1 SYSVSUM"
 
 	rm -f ${SUMFILES}
 
-	if [ x"${SIGN_AS}" != x"" ]; then
-		( cd ${pkgsrcdir}/security/gnupg; ${BMAKE} bulk-install )
+	if [ "${SIGN_AS-}" != "" ]; then
+		install_required "security/gnupg"
 		for i in ${SUMFILES}; do
 			echo > $i
 			echo "This file is signed with ${SIGN_AS}'s PGP key." >> $i
@@ -122,94 +252,92 @@ if [ "${MKSUMS}" = "yes" -o "${MKSUMS}" = "YES" ]; then
 		done
 	fi
 
-	( cd ${pkgsrcdir}/pkgtools/digest; ${BMAKE} bulk-install )
+	install_required "pkgtools/digest"
 
 	[ -z "${BSDSUM}" ] && BSDSUM="echo"
 	[ -z "${CKSUM}" ] && CKSUM="echo"
 	[ -z "${SYSVSUM}" ] && SYSVSUM="echo"
 
-	for i in All/*; do
-		if grep $i $exf >/dev/null; then
-			:
-		else
-			${BSDSUM} $i >> BSDSUM
-			${CKSUM} $i >> CKSUM
-			${MD5} $i >> MD5
-			${SHA1} $i >> SHA1
-			${SYSVSUM} $i >> SYSVSUM
-		fi
+	for pkg in `cat "${good_packages}" "${vulnerable_packages}"`; do
+		pkg="All/$pkg"
+		${BSDSUM}	"$pkg" >> BSDSUM
+		${CKSUM}	"$pkg" >> CKSUM
+		${MD5}		"$pkg" >> MD5
+		${SHA1}		"$pkg" >> SHA1
+		${SYSVSUM}	"$pkg" >> SYSVSUM
 	done
 
 	[ "${BSDSUM}" = "echo" ] && rm BSDSUM
 	[ "${CKSUM}" = "echo" ] && rm CKSUM
 	[ "${SYSVSUM}" = "echo" ] && rm SYSVSUM
 	
-	if [ x"${SIGN_AS}" != x"" ]; then
+	if [ "${SIGN_AS-}" != "" ]; then
 		for i in ${SUMFILES}; do
 			if [ -s $i ]; then
-				echo "Signing $i"
+				echo "upload> Signing $i"
 				gpg --clearsign $i && rm $i
 			fi
 		done
 	else
-		echo "Checksum files not PGP-signed. Please do so manually!"
-		echo "(Run 'gpg --clearsign' on all of them)"
+		echo "upload> Checksum files not PGP-signed. Please do so manually!"
+		echo "        (Run 'gpg --clearsign' on all of them)"
 	fi
 fi
 
-if [ "${MKSUMMARY}" = "yes" -o "${MKSUMMARY}" = "YES" ]; then
-	echo "Creating summary file..."
+if [ "${MKSUMMARY-}" = "yes" -o "${MKSUMMARY-}" = "YES" ]; then
+	echo "upload> Creating summary file..."
 	(cd "${packages}/All" \
 		&& ls -t | grep '\.t[gb]z$' | while read n; do pkg_info -X "$n"; done) \
 		| ${gzip_cmd} > "${packages}"/All/pkg_summary.gz
 fi
 
-echo "#!/bin/sh" > "$upload"
-echo "packages=$packages" >> "$upload"
-echo "if cd $packages; then" >> "$upload"
-echo "  :" >> "$upload"
-echo "else" >> "$upload"
-echo "	echo \"could not cd to $packages\"" >> "$upload"
-echo "	exit 1" >> "$upload"
-echo "fi" >> "$upload"
-
-echo "Uploading non-vulnerable pkgs"
-cmd="rsync $RSFLAGS --exclude-from=\"$exf\" --exclude-from=\"$vf\" . \"$RSYNC_DST\""
-cp -f "$upload" "$upload_general"
-echo "$cmd" >> "$upload_general"
-chmod 755 "$upload_general"
-echo "$cmd"
-sh "$upload_general"
-if [ $? != 0 ]; then
-	echo "--------------------------------------------------"
-	echo " "
-	echo "WARNING: rsync failed.  To retry later, you can run"
-	echo "    $upload_general"
-	echo " "
-	echo "--------------------------------------------------"
-	failed=yes
+cat <<EOF > "$upload_general"
+#! /bin/sh
+set -e
+cd "$packages"
+rsync $RSFLAGS --include-from="${good_packages}" --exclude-from="${old_packages}" . "$RSYNC_DST/"
+EOF
+chmod +x "$upload_general"
+
+if [ "$do_upload" = "yes" ]; then
+	echo "upload> Uploading non-vulnerable packages"
+	sh "$upload_general" \
+	|| {
+		echo "upload> ERROR: rsync failed.  To retry later, you can run $upload_general" 1>&2
+		failed=yes
+	}
+else
+	echo "upload> Skipping upload of non-vulnerable packages."
+	echo "        Run \"$upload_general\" to upload them later."
 fi
 
-echo "Uploading vulnerable pkgs"
-sed -n "s@All/@@p" "$exf" > "$exf.new"
-sed -n "s@All/@@p" "$vf" > "$vf.new"
-cmd="rsync $RSFLAGS --exclude-from=\"$exf.new\" --include-from=\"$vf.new\" --exclude='*' All/ \"$RSYNC_DST/vulnerable/\""
-cp -f "$upload" "$upload_vulnerable"
-echo "$cmd" >> "$upload_vulnerable"
-chmod 755 "$upload_vulnerable"
-echo "$cmd"
-sh "$upload_vulnerable"
-if [ $? != 0 ]; then
-	echo "--------------------------------------------------"
-	echo " "
-	echo "WARNING: rsync failed.  To retry later, you can run"
-	echo "    $upload_vulnerable"
-	echo " "
-	echo "--------------------------------------------------"
-	failed=yes
+cat <<EOF > "$upload_vulnerable"
+#! /bin/sh
+set -e
+cd "$packages/All"
+rsync $RSFLAGS --files-from="${vulnerable_packages}" --exclude-from="${old_packages}" . "$RSYNC_DST/vulnerable/"
+EOF
+chmod +x "$upload_vulnerable"
+
+if [ "$do_upload" = "yes" ]; then
+	echo "upload> Uploading vulnerable packages"
+	sh "$upload_vulnerable" \
+	|| {
+		echo "upload> ERROR: rsync failed.  To retry later, you can run $upload_vulnerable" 1>&2
+		failed=yes
+	}
+else
+	echo "upload> Skipping upload of vulnerable packages."
+	echo "        Run \"$upload_vulnerable\" to upload them later."
 fi
 
 # clean up temp files
-if [ "$failed" = "no" ]; then
+if [ "$failed,$debug,$do_upload" = "no,no,yes" ]; then
 	rm -fr "$TMP"
+else
+	echo "upload> Preserving temporary directory ${TMP}"
+fi
+
+if [ "$failed" = "yes" ]; then
+	exit 1
 fi