- Added a new top level file that saves the REPORT_BASEDIR from the first

  call to the mk/bulk/build script. This is to allow restarting the bulk
  build without getting confused by the various timestamps that occur
  during a bulk build. It also fixes some problems where the text/plain
  report had ended up in a directory different from the other files.

- Changed the default filename for the text/plain report from
  "pkgsrc-results-${BUILDDATE}.txt" to simply "report.txt", as the
  ${BUILDDATE} variable didn't give a hint to when the bulk build had been
  started, which is often more important than when the bulk build stopped.

- Changed the way to specify the directory and the URL where the bulk build
  reports are made available. See

      http://mail-index.netbsd.org/tech-pkg/2005/12/05/0002.html

  for details. Specifying the directories using the FTPHOST and FTPURL
  variables will be possible for some months, after which it support for
  it will be dropped.

5 files changed, 124 insertions, 47 deletions

diff --git a/mk/bulk/bsd.bulk-pkg.mk b/mk/bulk/bsd.bulk-pkg.mk
index 277ebe48d4d..3ff3c2ec011 100644
--- a/mk/bulk/bsd.bulk-pkg.mk
+++ b/mk/bulk/bsd.bulk-pkg.mk
@@ -1,4 +1,4 @@
-#	$NetBSD: bsd.bulk-pkg.mk,v 1.111 2006/01/17 22:35:13 jdc Exp $
+#	$NetBSD: bsd.bulk-pkg.mk,v 1.112 2006/01/21 11:29:31 rillig Exp $
 
 #
 # Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
@@ -145,6 +145,10 @@ ORDERFILE?=	${BULKFILESDIR}/.order${BULK_ID}
 # eventually for looking for leftover files (files not properly deinstalled)
 STARTFILE?=	${BULKFILESDIR}/.start${BULK_ID}
 
+# This top level file saves the bulk build ID. As it most often has a time
+# stamp, it is only generated once and later retrieved from this file.
+BULK_BUILD_ID_FILE?=	${BULKFILESDIR}/.bulk_build_id
+
 # File created and used by lintpkgsrc(8) to cache package metadata for
 # pruning and bulk-upload exclusions.
 LINTPKGSRC_DB?=	${BULKFILESDIR}/.lintpkgsrc.db${BULK_ID}
diff --git a/mk/bulk/build b/mk/bulk/build
index 5929f7af43e..28b7e36bb82 100644
--- a/mk/bulk/build
+++ b/mk/bulk/build
@@ -1,5 +1,5 @@
 #!/bin/sh
-# $NetBSD: build,v 1.91 2005/12/05 10:19:14 rillig Exp $
+# $NetBSD: build,v 1.92 2006/01/21 11:29:32 rillig Exp $
 
 #
 # Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
@@ -48,8 +48,7 @@ The following options are supported:
 	one.
 
    -e | --no-email
-	Don't send email when the bulk build is finished, it will put
-	the results into a file (FTP/pkgsrc-results.txt).
+	Don't send email when the bulk build is finished.
 
    -h | --help
 	Displays this message.
@@ -279,6 +278,7 @@ load_pkgsrc_var INDEXFILE
 load_pkgsrc_var ORDERFILE
 load_pkgsrc_var STARTFILE
 load_pkgsrc_var SUPPORTSFILE
+load_pkgsrc_var BULK_BUILD_ID_FILE
 
 load_pkgsrc_var BUILDLOG
 load_pkgsrc_var BROKENFILE
@@ -307,6 +307,15 @@ if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then
 fi
 
 #
+# Save the bulk build ID in a file, as it most often contains a time
+# stamp.
+#
+case $restart in
+no)	echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \
+	|| die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";;
+esac
+
+#
 # Install prerequisite packages.
 #
 # Note: we do this _before_ the depends tree because some packages like
@@ -417,18 +426,22 @@ done
 # Generate the post-build report.
 #
 echo "build> Generating the bulk build report..."
-BUILDDATE=`date +%Y-%m-%d`
-mkdir -p "${FTP}"
+
+bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \
+|| die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}."
+
+report_dir="${REPORTS_DIR}/${bulk_build_id}"
+${MKDIR} "${report_dir}"
 ( cd "${pkgsrc_dir}" \
   && ${PERL5} mk/bulk/post-build \
-     > ${FTP}/pkgsrc-results-${BUILDDATE}.txt
+     > "${report_dir}/${REPORT_TXT_FILE}"
 ) || die "Could not write the results file."
 
 #
 # Notify the ADMIN of the finished build.
 #
 case $noemail in
-no)	cat "${FTP}/pkgsrc-results-${BUILDDATE}.txt" \
+no)	cat "${report_dir}/${REPORT_TXT_FILE}" \
 	| ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results $BUILDDATE" "$ADMIN"
 esac
 
diff --git a/mk/bulk/build.conf-example b/mk/bulk/build.conf-example
index bf288d45efc..ee06d32546c 100644
--- a/mk/bulk/build.conf-example
+++ b/mk/bulk/build.conf-example
@@ -1,4 +1,4 @@
-# $NetBSD: build.conf-example,v 1.32 2005/12/03 01:00:37 rillig Exp $
+# $NetBSD: build.conf-example,v 1.33 2006/01/21 11:29:32 rillig Exp $
 #
 # This is an example configuration file for pkgsrc bulk builds.
 # Actually it's a shell script that is sourced in by the pre-build,
@@ -61,11 +61,22 @@ ADMIN="you@some.whe.re"
 # Who the report is signed by
 ADMINSIG="-Your Name"
 
-# Some paths for output files and paths to log files
-FTPx=`date +%Y%m%d.%H%M`
-FTPURL="pub/NetBSD/pkgstat/${FTPx}"		# relative to ~ftp !
-FTP="/home/ftp/${FTPURL}"			# absolute base path
-FTPHOST="ftp://ftp.machi.ne"			# host for broken.html
+# The directory where the final reports are collected.
+#REPORTS_DIR="$HOME/bulk-logs"
+REPORTS_DIR="/home/ftp/pub/NetBSD/pkgstat"
+
+# The URL where the final reports will be available.
+REPORTS_URL="ftp://localhost/pub/NetBSD/pkgstat"
+
+# The basename of the directory of the current bulk build. This
+# directory will be created under ${REPORTS_DIR}. After uploading the
+# report, it will also be available under ${REPORTS_URL}.
+REPORT_BASEDIR=`date +%Y%m%d.%H%M`
+
+# The basenames of the the two report files, which will both be created
+# in ${REPORTS_DIR}/${REPORT_BASEDIR}.
+REPORT_HTML_FILE="report.html"
+REPORT_TXT_FILE="report.txt"
 
 #
 # Uploading binary packages
@@ -88,10 +99,3 @@ MKSUMS=yes
 # Destination for packages and rsync options 
 RSYNC_DST=ftp.NetBSD.org:/pub/NetBSD/packages/pkgsrc-200xQy/NetBSD-a.b.c/i386
 RSYNC_OPTS='-e ssh'
-
-###########################################################################
-### No changes should be needed below this line !!!
-###########################################################################
-
-# Name of the (generated) HTML file which lists all broken pkgs
-REPORT="$FTP/broken.html"
diff --git a/mk/bulk/post-build b/mk/bulk/post-build
index 5c23670bc3e..e6610f2da01 100644
--- a/mk/bulk/post-build
+++ b/mk/bulk/post-build
@@ -1,5 +1,5 @@
 #!/usr/pkg/bin/perl
-# $NetBSD: post-build,v 1.62 2005/12/03 13:39:04 rillig Exp $
+# $NetBSD: post-build,v 1.63 2006/01/21 11:29:32 rillig Exp $
 #
 # Collect stuff after a pkg bulk build
 #
@@ -41,6 +41,23 @@ sub my_system (@) {
 	return system(@_);
 }
 
+sub readfirstline($) {
+	my ($fname) = @_;
+	my ($contents);
+
+	open(F, "<", $fname)
+	or pb_die($fname, "Cannot be read: $!");
+
+	defined($contents = <F>)
+	or pb_die($fname, "Must not be empty.");
+	chomp($contents);
+
+	close(F)
+	or pb_die($fname, "Cannot be closed: $!");
+
+	return $contents;
+}
+
 #
 # Load configuration variables from the bulk.conf file, which is a shell
 # script.
@@ -59,7 +76,16 @@ sub get_build_conf_vars(@) {
 	my ($is_set, $value);
 
 	foreach my $varname (@varnames) {
-		open(CMD, ". '$BULK_BUILD_CONF'; echo \"\${${varname}+set}\"; echo \"\${${varname}-}\" |")
+		my $cmd = join("\n", (
+			#"set -eu",		# TODO: Should be enabled soon
+			". '${BULK_BUILD_CONF}'",
+			". mk/bulk/post-build-conf",
+			"check_config_vars",
+			"echo \"\${${varname}+set}\"",
+			"echo \"\${${varname}-}\""
+		));
+		
+		open(CMD, "${cmd} |")
 			or pb_die($BULK_BUILD_CONF, "Could not evaluate configuration file.");
 
 		chomp($is_set = <CMD>);
@@ -91,15 +117,17 @@ sub get_build_conf_vars(@) {
 
 get_build_conf_vars(
 	'ADMINSIG',		# "-Your Name"
-	'FTPURL',		# "pub/NetBSD/pkgstat/`date +%Y%m%d.%H%M`"
-	'FTP',			# "/disk1/ftp/${FTPURL}"
-	'FTPHOST',		# ftp://ftp.machi.ne/
-	'REPORT',		# "broken.html"
+	'REPORTS_URL',		# "ftp://ftp.example.com/pub/pkgsrc/misc/pkgstat"
+	'REPORTS_DIR',		# "$HOME/bulk-logs"
+# REPORT_BASEDIR often depends on a timestamp, which has been saved at
+# the beginning of the bulk build in the BULK_BUILD_ID_FILE. It will be
+# retrieved later.
+	'REPORT_HTML_FILE',	# "report.html"
+	'REPORT_TXT_FILE',	# "report.txt"
 	'USR_PKGSRC',		# "/usr/pkgsrc"
 	'osrev',		# `uname -r`
 );
 
-my $reportf = basename($vars{REPORT});
 
 my $os = `uname -s`;
 chomp $os;
@@ -151,6 +179,7 @@ get_mk_conf_vars(qw(
 	SUPPORTSFILE
 	X11BASE
 	PKG_TOOLS_BIN
+	BULK_BUILD_ID_FILE
 ));
 
 my $bulk_dbfile_base = basename($vars{BULK_DBFILE});
@@ -160,6 +189,17 @@ my $supportsfile_base = basename($vars{SUPPORTSFILE});
 my $indexfile_base = basename($vars{INDEXFILE});
 my $orderfile_base = basename($vars{ORDERFILE});
 
+my $reports_url = $vars{"REPORTS_URL"};
+my $reports_dir = $vars{"REPORTS_DIR"};
+my $report_basedir = readfirstline($vars{"BULK_BUILD_ID_FILE"});
+my $report_html_file = $vars{"REPORT_HTML_FILE"};
+my $report_txt_file = $vars{"REPORT_TXT_FILE"};
+
+my $report_url = "${reports_url}/${report_basedir}";
+my $report_dir = "${reports_dir}/${report_basedir}";
+my $report_html_path = "${report_dir}/${report_html_file}";
+my $report_txt_path = "${report_dir}/${report_txt_file}";
+
 my $startdate = (stat($vars{STARTFILE}))[9];
 my $enddate = '';
 if (!defined($startdate) || $startdate == 0) {
@@ -187,31 +227,27 @@ sub print_report_header() {
 	print("--------------------------------------------------------------\n");
 }
 
-my_system("mkdir", "-p", "--", $vars{FTP});
+my_system("mkdir", "-p", "--", $report_dir);
 
 # Copy over the output from the build process
 chdir($vars{"BULKFILESDIR"}) or pb_die($vars{"BULKFILESDIR"}, "Cannot change directory.");
-my_system("find . -name $vars{BROKENFILE} -print -o -name $vars{BROKENWRKLOG} -print | $vars{PAX} -r -w -X $vars{FTP}");
+my_system("find . -name $vars{BROKENFILE} -print -o -name $vars{BROKENWRKLOG} -print | $vars{PAX} -r -w -X ${report_dir}");
 
 # Copy over the cache files used during the build
 foreach my $f qw(BULK_DBFILE DEPENDSTREEFILE DEPENDSFILE SUPPORTSFILE INDEXFILE ORDERFILE) {
 	if (-f $vars{$f}) {
-		my_system("cp", "--", $vars{$f}, $vars{FTP});
+		my_system("cp", "--", $vars{$f}, $report_dir);
 	}
 }
 
-chdir($vars{FTP}) or pb_die($vars{"FTP"}, "Cannot change directory.");
+chdir($report_dir) or pb_die($report_dir, "Cannot change directory.");
 writeReport();
 
 #
 # Adjust "last" symlink
 #
-{
-	my ($base, $dir) = ($vars{FTP} =~ m|^(.*)/([^/]*)$|);
-
-	unlink("$base/last");
-	symlink($dir, "$base/last");
-}
+unlink("${reports_dir}/last");
+symlink($report_basedir, "${reports_dir}/last");
 
 #
 # Generate leftovers-$vars{MACHINE_ARCH}.html: files not deleted
@@ -219,7 +255,7 @@ writeReport();
 # and linked from leftovers-$vars{MACHINE_ARCH}.html
 #
 {
-	chdir($vars{FTP});
+	chdir($report_dir) or pb_die($report_dir, "Cannot change directory.");
 	my_system("mkdir", "-p", "leftovers-$vars{MACHINE_ARCH}");
 
 	# Find files since last build:
@@ -270,7 +306,7 @@ writeReport();
 EOOUT
 	foreach (@leftovers) {
 		chomp;
-		print OUT "<a href=\"$vars{FTPHOST}/$vars{FTPURL}/leftovers-$vars{MACHINE_ARCH}$_\">$_</a>\n";
+		print OUT "<a href=\"${report_url}/leftovers-$vars{MACHINE_ARCH}$_\">$_</a>\n";
 	}
 	print OUT <<EOOUT2;
 </pre>
@@ -369,11 +405,11 @@ EOF
 Packages not listed here resulted in a binary package. The build
 report, including logs of failed/not-packaged is available from:
 
-$vars{FTPHOST}/$vars{FTPURL}/$reportf
+${report_url}/${report_html_file}
 EOF
 	}
 
-	open(HTML, ">$vars{REPORT}") or die "Can't write $vars{REPORT}: $!\n";
+	open(HTML, ">", $report_html_path) or die "Can't write ${report_html_path}: $!";
 	print HTML <<EOHTML;
 <html>
 <head>
diff --git a/mk/bulk/post-build-conf b/mk/bulk/post-build-conf
index 6815fb40637..1fc24b80547 100644
--- a/mk/bulk/post-build-conf
+++ b/mk/bulk/post-build-conf
@@ -1,4 +1,4 @@
-# $NetBSD: post-build-conf,v 1.10 2006/01/07 22:27:56 rillig Exp $
+# $NetBSD: post-build-conf,v 1.11 2006/01/21 11:29:33 rillig Exp $
 #
 
 # This file is included after the build.conf file by the "build" and
@@ -39,7 +39,9 @@ show_config_vars() {
 	pbc_section "Building the packages" \
 		PKGLIST NICE_LEVEL
 	pbc_section "Generating the report" \
-		ADMIN ADMINSIG FTPHOST FTP
+		ADMIN ADMINSIG \
+		REPORTS_URL REPORTS_DIR \
+		REPORT_BASEDIR REPORT_HTML_FILE REPORT_TXT_FILE
 	pbc_section "Uploading binary packages" \
 		UPDATE_VULNERABILITY_LIST PRUNEPACKAGES MKSUMS SIGN_AS \
 		RSYNC_DST RSYNC_OPTS
@@ -51,7 +53,7 @@ export_config_vars() {
 	export PRUNEDISTFILES ftp_proxy http_proxy
 	export PKGLIST NICE_LEVEL ADMIN ADMINSIG
 	export UPDATE_VULNERABILITY_LIST PRUNEPACKAGES MKSUMS SIGN_AS
-	export RSYNC_DST RSYNC_OPTS FTPHOST FTP
+	export RSYNC_DST RSYNC_OPTS
 	case ${MAKECONF+set} in
 	"set")	export MAKECONF;;
 	esac
@@ -154,6 +156,20 @@ pbc_checkdefined() {
 
 # usage: check_config_vars
 check_config_vars() {
+
+	# Handle legacy variables FTP, FTPHOST, FTPURL, REPORT.
+	# Can be removed after 2006Q2.
+	case ${FTP+set},${REPORTS_DIR+set} in
+	set,set)
+		pbc_die "FTP and REPORTS_DIR must not be set both.";;
+	set,)
+		REPORTS_URL="${FTPHOST}"`dirname "${FTPURL}"`
+		REPORTS_DIR=`dirname "$FTP"`
+		REPORT_BASEDIR=`basename "${FTP}"`
+		REPORT_HTML_FILE=`basename "${REPORT}"`
+		REPORT_TXT_FILE="report.txt";;
+	esac
+
 	# section "System information"
 	pbc_checknonempty osrev
 	pbc_checknonempty arch
@@ -184,8 +200,12 @@ check_config_vars() {
 	# section "Generating the report"
 	# no checks for ADMIN
 	# no checks for ADMINSIG
-	# no checks for FTPHOST
-	# no checks for FTP
+	pbc_checkurl REPORTS_URL
+	pbc_checkpathname REPORTS_DIR
+	pbc_checkbasename REPORT_BASEDIR
+	pbc_checkbasename REPORT_HTML_FILE
+	pbc_checkbasename REPORT_TXT_FILE
+
 
 	# section "Uploading binary packages"
 	pbc_checkyesno UPDATE_VULNERABILITY_LIST