diff options
author | rillig <rillig> | 2006-01-21 11:29:31 +0000 |
---|---|---|
committer | rillig <rillig> | 2006-01-21 11:29:31 +0000 |
commit | 5f6d9fc52fea9eeb26466573b15015c169f7bb63 (patch) | |
tree | 5fa3342b20959eb19de695f81e860497ba90e28b /mk/bulk | |
parent | 30bba32ce9b2aecefdefc24f6b4c145c8333c577 (diff) | |
download | pkgsrc-5f6d9fc52fea9eeb26466573b15015c169f7bb63.tar.gz |
- Added a new top level file that saves the REPORT_BASEDIR from the first
call to the mk/bulk/build script. This is to allow restarting the bulk
build without getting confused by the various timestamps that occur
during a bulk build. It also fixes some problems where the text/plain
report had ended up in a directory different from the other files.
- Changed the default filename for the text/plain report from
"pkgsrc-results-${BUILDDATE}.txt" to simply "report.txt", as the
${BUILDDATE} variable didn't give a hint to when the bulk build had been
started, which is often more important than when the bulk build stopped.
- Changed the way to specify the directory and the URL where the bulk build
reports are made available. See
http://mail-index.netbsd.org/tech-pkg/2005/12/05/0002.html
for details. Specifying the directories using the FTPHOST and FTPURL
variables will be possible for some months, after which it support for
it will be dropped.
Diffstat (limited to 'mk/bulk')
-rw-r--r-- | mk/bulk/bsd.bulk-pkg.mk | 6 | ||||
-rw-r--r-- | mk/bulk/build | 27 | ||||
-rw-r--r-- | mk/bulk/build.conf-example | 30 | ||||
-rw-r--r-- | mk/bulk/post-build | 78 | ||||
-rw-r--r-- | mk/bulk/post-build-conf | 30 |
5 files changed, 124 insertions, 47 deletions
diff --git a/mk/bulk/bsd.bulk-pkg.mk b/mk/bulk/bsd.bulk-pkg.mk index 277ebe48d4d..3ff3c2ec011 100644 --- a/mk/bulk/bsd.bulk-pkg.mk +++ b/mk/bulk/bsd.bulk-pkg.mk @@ -1,4 +1,4 @@ -# $NetBSD: bsd.bulk-pkg.mk,v 1.111 2006/01/17 22:35:13 jdc Exp $ +# $NetBSD: bsd.bulk-pkg.mk,v 1.112 2006/01/21 11:29:31 rillig Exp $ # # Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org> @@ -145,6 +145,10 @@ ORDERFILE?= ${BULKFILESDIR}/.order${BULK_ID} # eventually for looking for leftover files (files not properly deinstalled) STARTFILE?= ${BULKFILESDIR}/.start${BULK_ID} +# This top level file saves the bulk build ID. As it most often has a time +# stamp, it is only generated once and later retrieved from this file. +BULK_BUILD_ID_FILE?= ${BULKFILESDIR}/.bulk_build_id + # File created and used by lintpkgsrc(8) to cache package metadata for # pruning and bulk-upload exclusions. LINTPKGSRC_DB?= ${BULKFILESDIR}/.lintpkgsrc.db${BULK_ID} diff --git a/mk/bulk/build b/mk/bulk/build index 5929f7af43e..28b7e36bb82 100644 --- a/mk/bulk/build +++ b/mk/bulk/build @@ -1,5 +1,5 @@ #!/bin/sh -# $NetBSD: build,v 1.91 2005/12/05 10:19:14 rillig Exp $ +# $NetBSD: build,v 1.92 2006/01/21 11:29:32 rillig Exp $ # # Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org> @@ -48,8 +48,7 @@ The following options are supported: one. -e | --no-email - Don't send email when the bulk build is finished, it will put - the results into a file (FTP/pkgsrc-results.txt). + Don't send email when the bulk build is finished. -h | --help Displays this message. @@ -279,6 +278,7 @@ load_pkgsrc_var INDEXFILE load_pkgsrc_var ORDERFILE load_pkgsrc_var STARTFILE load_pkgsrc_var SUPPORTSFILE +load_pkgsrc_var BULK_BUILD_ID_FILE load_pkgsrc_var BUILDLOG load_pkgsrc_var BROKENFILE @@ -307,6 +307,15 @@ if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then fi # +# Save the bulk build ID in a file, as it most often contains a time +# stamp. +# +case $restart in +no) echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \ + || die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";; +esac + +# # Install prerequisite packages. # # Note: we do this _before_ the depends tree because some packages like @@ -417,18 +426,22 @@ done # Generate the post-build report. # echo "build> Generating the bulk build report..." -BUILDDATE=`date +%Y-%m-%d` -mkdir -p "${FTP}" + +bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \ +|| die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}." + +report_dir="${REPORTS_DIR}/${bulk_build_id}" +${MKDIR} "${report_dir}" ( cd "${pkgsrc_dir}" \ && ${PERL5} mk/bulk/post-build \ - > ${FTP}/pkgsrc-results-${BUILDDATE}.txt + > "${report_dir}/${REPORT_TXT_FILE}" ) || die "Could not write the results file." # # Notify the ADMIN of the finished build. # case $noemail in -no) cat "${FTP}/pkgsrc-results-${BUILDDATE}.txt" \ +no) cat "${report_dir}/${REPORT_TXT_FILE}" \ | ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results $BUILDDATE" "$ADMIN" esac diff --git a/mk/bulk/build.conf-example b/mk/bulk/build.conf-example index bf288d45efc..ee06d32546c 100644 --- a/mk/bulk/build.conf-example +++ b/mk/bulk/build.conf-example @@ -1,4 +1,4 @@ -# $NetBSD: build.conf-example,v 1.32 2005/12/03 01:00:37 rillig Exp $ +# $NetBSD: build.conf-example,v 1.33 2006/01/21 11:29:32 rillig Exp $ # # This is an example configuration file for pkgsrc bulk builds. # Actually it's a shell script that is sourced in by the pre-build, @@ -61,11 +61,22 @@ ADMIN="you@some.whe.re" # Who the report is signed by ADMINSIG="-Your Name" -# Some paths for output files and paths to log files -FTPx=`date +%Y%m%d.%H%M` -FTPURL="pub/NetBSD/pkgstat/${FTPx}" # relative to ~ftp ! -FTP="/home/ftp/${FTPURL}" # absolute base path -FTPHOST="ftp://ftp.machi.ne" # host for broken.html +# The directory where the final reports are collected. +#REPORTS_DIR="$HOME/bulk-logs" +REPORTS_DIR="/home/ftp/pub/NetBSD/pkgstat" + +# The URL where the final reports will be available. +REPORTS_URL="ftp://localhost/pub/NetBSD/pkgstat" + +# The basename of the directory of the current bulk build. This +# directory will be created under ${REPORTS_DIR}. After uploading the +# report, it will also be available under ${REPORTS_URL}. +REPORT_BASEDIR=`date +%Y%m%d.%H%M` + +# The basenames of the the two report files, which will both be created +# in ${REPORTS_DIR}/${REPORT_BASEDIR}. +REPORT_HTML_FILE="report.html" +REPORT_TXT_FILE="report.txt" # # Uploading binary packages @@ -88,10 +99,3 @@ MKSUMS=yes # Destination for packages and rsync options RSYNC_DST=ftp.NetBSD.org:/pub/NetBSD/packages/pkgsrc-200xQy/NetBSD-a.b.c/i386 RSYNC_OPTS='-e ssh' - -########################################################################### -### No changes should be needed below this line !!! -########################################################################### - -# Name of the (generated) HTML file which lists all broken pkgs -REPORT="$FTP/broken.html" diff --git a/mk/bulk/post-build b/mk/bulk/post-build index 5c23670bc3e..e6610f2da01 100644 --- a/mk/bulk/post-build +++ b/mk/bulk/post-build @@ -1,5 +1,5 @@ #!/usr/pkg/bin/perl -# $NetBSD: post-build,v 1.62 2005/12/03 13:39:04 rillig Exp $ +# $NetBSD: post-build,v 1.63 2006/01/21 11:29:32 rillig Exp $ # # Collect stuff after a pkg bulk build # @@ -41,6 +41,23 @@ sub my_system (@) { return system(@_); } +sub readfirstline($) { + my ($fname) = @_; + my ($contents); + + open(F, "<", $fname) + or pb_die($fname, "Cannot be read: $!"); + + defined($contents = <F>) + or pb_die($fname, "Must not be empty."); + chomp($contents); + + close(F) + or pb_die($fname, "Cannot be closed: $!"); + + return $contents; +} + # # Load configuration variables from the bulk.conf file, which is a shell # script. @@ -59,7 +76,16 @@ sub get_build_conf_vars(@) { my ($is_set, $value); foreach my $varname (@varnames) { - open(CMD, ". '$BULK_BUILD_CONF'; echo \"\${${varname}+set}\"; echo \"\${${varname}-}\" |") + my $cmd = join("\n", ( + #"set -eu", # TODO: Should be enabled soon + ". '${BULK_BUILD_CONF}'", + ". mk/bulk/post-build-conf", + "check_config_vars", + "echo \"\${${varname}+set}\"", + "echo \"\${${varname}-}\"" + )); + + open(CMD, "${cmd} |") or pb_die($BULK_BUILD_CONF, "Could not evaluate configuration file."); chomp($is_set = <CMD>); @@ -91,15 +117,17 @@ sub get_build_conf_vars(@) { get_build_conf_vars( 'ADMINSIG', # "-Your Name" - 'FTPURL', # "pub/NetBSD/pkgstat/`date +%Y%m%d.%H%M`" - 'FTP', # "/disk1/ftp/${FTPURL}" - 'FTPHOST', # ftp://ftp.machi.ne/ - 'REPORT', # "broken.html" + 'REPORTS_URL', # "ftp://ftp.example.com/pub/pkgsrc/misc/pkgstat" + 'REPORTS_DIR', # "$HOME/bulk-logs" +# REPORT_BASEDIR often depends on a timestamp, which has been saved at +# the beginning of the bulk build in the BULK_BUILD_ID_FILE. It will be +# retrieved later. + 'REPORT_HTML_FILE', # "report.html" + 'REPORT_TXT_FILE', # "report.txt" 'USR_PKGSRC', # "/usr/pkgsrc" 'osrev', # `uname -r` ); -my $reportf = basename($vars{REPORT}); my $os = `uname -s`; chomp $os; @@ -151,6 +179,7 @@ get_mk_conf_vars(qw( SUPPORTSFILE X11BASE PKG_TOOLS_BIN + BULK_BUILD_ID_FILE )); my $bulk_dbfile_base = basename($vars{BULK_DBFILE}); @@ -160,6 +189,17 @@ my $supportsfile_base = basename($vars{SUPPORTSFILE}); my $indexfile_base = basename($vars{INDEXFILE}); my $orderfile_base = basename($vars{ORDERFILE}); +my $reports_url = $vars{"REPORTS_URL"}; +my $reports_dir = $vars{"REPORTS_DIR"}; +my $report_basedir = readfirstline($vars{"BULK_BUILD_ID_FILE"}); +my $report_html_file = $vars{"REPORT_HTML_FILE"}; +my $report_txt_file = $vars{"REPORT_TXT_FILE"}; + +my $report_url = "${reports_url}/${report_basedir}"; +my $report_dir = "${reports_dir}/${report_basedir}"; +my $report_html_path = "${report_dir}/${report_html_file}"; +my $report_txt_path = "${report_dir}/${report_txt_file}"; + my $startdate = (stat($vars{STARTFILE}))[9]; my $enddate = ''; if (!defined($startdate) || $startdate == 0) { @@ -187,31 +227,27 @@ sub print_report_header() { print("--------------------------------------------------------------\n"); } -my_system("mkdir", "-p", "--", $vars{FTP}); +my_system("mkdir", "-p", "--", $report_dir); # Copy over the output from the build process chdir($vars{"BULKFILESDIR"}) or pb_die($vars{"BULKFILESDIR"}, "Cannot change directory."); -my_system("find . -name $vars{BROKENFILE} -print -o -name $vars{BROKENWRKLOG} -print | $vars{PAX} -r -w -X $vars{FTP}"); +my_system("find . -name $vars{BROKENFILE} -print -o -name $vars{BROKENWRKLOG} -print | $vars{PAX} -r -w -X ${report_dir}"); # Copy over the cache files used during the build foreach my $f qw(BULK_DBFILE DEPENDSTREEFILE DEPENDSFILE SUPPORTSFILE INDEXFILE ORDERFILE) { if (-f $vars{$f}) { - my_system("cp", "--", $vars{$f}, $vars{FTP}); + my_system("cp", "--", $vars{$f}, $report_dir); } } -chdir($vars{FTP}) or pb_die($vars{"FTP"}, "Cannot change directory."); +chdir($report_dir) or pb_die($report_dir, "Cannot change directory."); writeReport(); # # Adjust "last" symlink # -{ - my ($base, $dir) = ($vars{FTP} =~ m|^(.*)/([^/]*)$|); - - unlink("$base/last"); - symlink($dir, "$base/last"); -} +unlink("${reports_dir}/last"); +symlink($report_basedir, "${reports_dir}/last"); # # Generate leftovers-$vars{MACHINE_ARCH}.html: files not deleted @@ -219,7 +255,7 @@ writeReport(); # and linked from leftovers-$vars{MACHINE_ARCH}.html # { - chdir($vars{FTP}); + chdir($report_dir) or pb_die($report_dir, "Cannot change directory."); my_system("mkdir", "-p", "leftovers-$vars{MACHINE_ARCH}"); # Find files since last build: @@ -270,7 +306,7 @@ writeReport(); EOOUT foreach (@leftovers) { chomp; - print OUT "<a href=\"$vars{FTPHOST}/$vars{FTPURL}/leftovers-$vars{MACHINE_ARCH}$_\">$_</a>\n"; + print OUT "<a href=\"${report_url}/leftovers-$vars{MACHINE_ARCH}$_\">$_</a>\n"; } print OUT <<EOOUT2; </pre> @@ -369,11 +405,11 @@ EOF Packages not listed here resulted in a binary package. The build report, including logs of failed/not-packaged is available from: -$vars{FTPHOST}/$vars{FTPURL}/$reportf +${report_url}/${report_html_file} EOF } - open(HTML, ">$vars{REPORT}") or die "Can't write $vars{REPORT}: $!\n"; + open(HTML, ">", $report_html_path) or die "Can't write ${report_html_path}: $!"; print HTML <<EOHTML; <html> <head> diff --git a/mk/bulk/post-build-conf b/mk/bulk/post-build-conf index 6815fb40637..1fc24b80547 100644 --- a/mk/bulk/post-build-conf +++ b/mk/bulk/post-build-conf @@ -1,4 +1,4 @@ -# $NetBSD: post-build-conf,v 1.10 2006/01/07 22:27:56 rillig Exp $ +# $NetBSD: post-build-conf,v 1.11 2006/01/21 11:29:33 rillig Exp $ # # This file is included after the build.conf file by the "build" and @@ -39,7 +39,9 @@ show_config_vars() { pbc_section "Building the packages" \ PKGLIST NICE_LEVEL pbc_section "Generating the report" \ - ADMIN ADMINSIG FTPHOST FTP + ADMIN ADMINSIG \ + REPORTS_URL REPORTS_DIR \ + REPORT_BASEDIR REPORT_HTML_FILE REPORT_TXT_FILE pbc_section "Uploading binary packages" \ UPDATE_VULNERABILITY_LIST PRUNEPACKAGES MKSUMS SIGN_AS \ RSYNC_DST RSYNC_OPTS @@ -51,7 +53,7 @@ export_config_vars() { export PRUNEDISTFILES ftp_proxy http_proxy export PKGLIST NICE_LEVEL ADMIN ADMINSIG export UPDATE_VULNERABILITY_LIST PRUNEPACKAGES MKSUMS SIGN_AS - export RSYNC_DST RSYNC_OPTS FTPHOST FTP + export RSYNC_DST RSYNC_OPTS case ${MAKECONF+set} in "set") export MAKECONF;; esac @@ -154,6 +156,20 @@ pbc_checkdefined() { # usage: check_config_vars check_config_vars() { + + # Handle legacy variables FTP, FTPHOST, FTPURL, REPORT. + # Can be removed after 2006Q2. + case ${FTP+set},${REPORTS_DIR+set} in + set,set) + pbc_die "FTP and REPORTS_DIR must not be set both.";; + set,) + REPORTS_URL="${FTPHOST}"`dirname "${FTPURL}"` + REPORTS_DIR=`dirname "$FTP"` + REPORT_BASEDIR=`basename "${FTP}"` + REPORT_HTML_FILE=`basename "${REPORT}"` + REPORT_TXT_FILE="report.txt";; + esac + # section "System information" pbc_checknonempty osrev pbc_checknonempty arch @@ -184,8 +200,12 @@ check_config_vars() { # section "Generating the report" # no checks for ADMIN # no checks for ADMINSIG - # no checks for FTPHOST - # no checks for FTP + pbc_checkurl REPORTS_URL + pbc_checkpathname REPORTS_DIR + pbc_checkbasename REPORT_BASEDIR + pbc_checkbasename REPORT_HTML_FILE + pbc_checkbasename REPORT_TXT_FILE + # section "Uploading binary packages" pbc_checkyesno UPDATE_VULNERABILITY_LIST |