summaryrefslogtreecommitdiff
path: root/mk
diff options
context:
space:
mode:
authorrillig <rillig@pkgsrc.org>2006-01-21 11:29:31 +0000
committerrillig <rillig@pkgsrc.org>2006-01-21 11:29:31 +0000
commit1278e33918bbbb4ffb083ba697210758e13510ca (patch)
tree5fa3342b20959eb19de695f81e860497ba90e28b /mk
parent8358f4d35eee8bbad0d364f5dbc4f80920576015 (diff)
downloadpkgsrc-1278e33918bbbb4ffb083ba697210758e13510ca.tar.gz
- Added a new top level file that saves the REPORT_BASEDIR from the first
call to the mk/bulk/build script. This is to allow restarting the bulk build without getting confused by the various timestamps that occur during a bulk build. It also fixes some problems where the text/plain report had ended up in a directory different from the other files. - Changed the default filename for the text/plain report from "pkgsrc-results-${BUILDDATE}.txt" to simply "report.txt", as the ${BUILDDATE} variable didn't give a hint to when the bulk build had been started, which is often more important than when the bulk build stopped. - Changed the way to specify the directory and the URL where the bulk build reports are made available. See http://mail-index.netbsd.org/tech-pkg/2005/12/05/0002.html for details. Specifying the directories using the FTPHOST and FTPURL variables will be possible for some months, after which it support for it will be dropped.
Diffstat (limited to 'mk')
-rw-r--r--mk/bulk/bsd.bulk-pkg.mk6
-rw-r--r--mk/bulk/build27
-rw-r--r--mk/bulk/build.conf-example30
-rw-r--r--mk/bulk/post-build78
-rw-r--r--mk/bulk/post-build-conf30
5 files changed, 124 insertions, 47 deletions
diff --git a/mk/bulk/bsd.bulk-pkg.mk b/mk/bulk/bsd.bulk-pkg.mk
index 277ebe48d4d..3ff3c2ec011 100644
--- a/mk/bulk/bsd.bulk-pkg.mk
+++ b/mk/bulk/bsd.bulk-pkg.mk
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.bulk-pkg.mk,v 1.111 2006/01/17 22:35:13 jdc Exp $
+# $NetBSD: bsd.bulk-pkg.mk,v 1.112 2006/01/21 11:29:31 rillig Exp $
#
# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
@@ -145,6 +145,10 @@ ORDERFILE?= ${BULKFILESDIR}/.order${BULK_ID}
# eventually for looking for leftover files (files not properly deinstalled)
STARTFILE?= ${BULKFILESDIR}/.start${BULK_ID}
+# This top level file saves the bulk build ID. As it most often has a time
+# stamp, it is only generated once and later retrieved from this file.
+BULK_BUILD_ID_FILE?= ${BULKFILESDIR}/.bulk_build_id
+
# File created and used by lintpkgsrc(8) to cache package metadata for
# pruning and bulk-upload exclusions.
LINTPKGSRC_DB?= ${BULKFILESDIR}/.lintpkgsrc.db${BULK_ID}
diff --git a/mk/bulk/build b/mk/bulk/build
index 5929f7af43e..28b7e36bb82 100644
--- a/mk/bulk/build
+++ b/mk/bulk/build
@@ -1,5 +1,5 @@
#!/bin/sh
-# $NetBSD: build,v 1.91 2005/12/05 10:19:14 rillig Exp $
+# $NetBSD: build,v 1.92 2006/01/21 11:29:32 rillig Exp $
#
# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
@@ -48,8 +48,7 @@ The following options are supported:
one.
-e | --no-email
- Don't send email when the bulk build is finished, it will put
- the results into a file (FTP/pkgsrc-results.txt).
+ Don't send email when the bulk build is finished.
-h | --help
Displays this message.
@@ -279,6 +278,7 @@ load_pkgsrc_var INDEXFILE
load_pkgsrc_var ORDERFILE
load_pkgsrc_var STARTFILE
load_pkgsrc_var SUPPORTSFILE
+load_pkgsrc_var BULK_BUILD_ID_FILE
load_pkgsrc_var BUILDLOG
load_pkgsrc_var BROKENFILE
@@ -307,6 +307,15 @@ if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then
fi
#
+# Save the bulk build ID in a file, as it most often contains a time
+# stamp.
+#
+case $restart in
+no) echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \
+ || die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";;
+esac
+
+#
# Install prerequisite packages.
#
# Note: we do this _before_ the depends tree because some packages like
@@ -417,18 +426,22 @@ done
# Generate the post-build report.
#
echo "build> Generating the bulk build report..."
-BUILDDATE=`date +%Y-%m-%d`
-mkdir -p "${FTP}"
+
+bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \
+|| die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}."
+
+report_dir="${REPORTS_DIR}/${bulk_build_id}"
+${MKDIR} "${report_dir}"
( cd "${pkgsrc_dir}" \
&& ${PERL5} mk/bulk/post-build \
- > ${FTP}/pkgsrc-results-${BUILDDATE}.txt
+ > "${report_dir}/${REPORT_TXT_FILE}"
) || die "Could not write the results file."
#
# Notify the ADMIN of the finished build.
#
case $noemail in
-no) cat "${FTP}/pkgsrc-results-${BUILDDATE}.txt" \
+no) cat "${report_dir}/${REPORT_TXT_FILE}" \
| ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results $BUILDDATE" "$ADMIN"
esac
diff --git a/mk/bulk/build.conf-example b/mk/bulk/build.conf-example
index bf288d45efc..ee06d32546c 100644
--- a/mk/bulk/build.conf-example
+++ b/mk/bulk/build.conf-example
@@ -1,4 +1,4 @@
-# $NetBSD: build.conf-example,v 1.32 2005/12/03 01:00:37 rillig Exp $
+# $NetBSD: build.conf-example,v 1.33 2006/01/21 11:29:32 rillig Exp $
#
# This is an example configuration file for pkgsrc bulk builds.
# Actually it's a shell script that is sourced in by the pre-build,
@@ -61,11 +61,22 @@ ADMIN="you@some.whe.re"
# Who the report is signed by
ADMINSIG="-Your Name"
-# Some paths for output files and paths to log files
-FTPx=`date +%Y%m%d.%H%M`
-FTPURL="pub/NetBSD/pkgstat/${FTPx}" # relative to ~ftp !
-FTP="/home/ftp/${FTPURL}" # absolute base path
-FTPHOST="ftp://ftp.machi.ne" # host for broken.html
+# The directory where the final reports are collected.
+#REPORTS_DIR="$HOME/bulk-logs"
+REPORTS_DIR="/home/ftp/pub/NetBSD/pkgstat"
+
+# The URL where the final reports will be available.
+REPORTS_URL="ftp://localhost/pub/NetBSD/pkgstat"
+
+# The basename of the directory of the current bulk build. This
+# directory will be created under ${REPORTS_DIR}. After uploading the
+# report, it will also be available under ${REPORTS_URL}.
+REPORT_BASEDIR=`date +%Y%m%d.%H%M`
+
+# The basenames of the the two report files, which will both be created
+# in ${REPORTS_DIR}/${REPORT_BASEDIR}.
+REPORT_HTML_FILE="report.html"
+REPORT_TXT_FILE="report.txt"
#
# Uploading binary packages
@@ -88,10 +99,3 @@ MKSUMS=yes
# Destination for packages and rsync options
RSYNC_DST=ftp.NetBSD.org:/pub/NetBSD/packages/pkgsrc-200xQy/NetBSD-a.b.c/i386
RSYNC_OPTS='-e ssh'
-
-###########################################################################
-### No changes should be needed below this line !!!
-###########################################################################
-
-# Name of the (generated) HTML file which lists all broken pkgs
-REPORT="$FTP/broken.html"
diff --git a/mk/bulk/post-build b/mk/bulk/post-build
index 5c23670bc3e..e6610f2da01 100644
--- a/mk/bulk/post-build
+++ b/mk/bulk/post-build
@@ -1,5 +1,5 @@
#!/usr/pkg/bin/perl
-# $NetBSD: post-build,v 1.62 2005/12/03 13:39:04 rillig Exp $
+# $NetBSD: post-build,v 1.63 2006/01/21 11:29:32 rillig Exp $
#
# Collect stuff after a pkg bulk build
#
@@ -41,6 +41,23 @@ sub my_system (@) {
return system(@_);
}
+sub readfirstline($) {
+ my ($fname) = @_;
+ my ($contents);
+
+ open(F, "<", $fname)
+ or pb_die($fname, "Cannot be read: $!");
+
+ defined($contents = <F>)
+ or pb_die($fname, "Must not be empty.");
+ chomp($contents);
+
+ close(F)
+ or pb_die($fname, "Cannot be closed: $!");
+
+ return $contents;
+}
+
#
# Load configuration variables from the bulk.conf file, which is a shell
# script.
@@ -59,7 +76,16 @@ sub get_build_conf_vars(@) {
my ($is_set, $value);
foreach my $varname (@varnames) {
- open(CMD, ". '$BULK_BUILD_CONF'; echo \"\${${varname}+set}\"; echo \"\${${varname}-}\" |")
+ my $cmd = join("\n", (
+ #"set -eu", # TODO: Should be enabled soon
+ ". '${BULK_BUILD_CONF}'",
+ ". mk/bulk/post-build-conf",
+ "check_config_vars",
+ "echo \"\${${varname}+set}\"",
+ "echo \"\${${varname}-}\""
+ ));
+
+ open(CMD, "${cmd} |")
or pb_die($BULK_BUILD_CONF, "Could not evaluate configuration file.");
chomp($is_set = <CMD>);
@@ -91,15 +117,17 @@ sub get_build_conf_vars(@) {
get_build_conf_vars(
'ADMINSIG', # "-Your Name"
- 'FTPURL', # "pub/NetBSD/pkgstat/`date +%Y%m%d.%H%M`"
- 'FTP', # "/disk1/ftp/${FTPURL}"
- 'FTPHOST', # ftp://ftp.machi.ne/
- 'REPORT', # "broken.html"
+ 'REPORTS_URL', # "ftp://ftp.example.com/pub/pkgsrc/misc/pkgstat"
+ 'REPORTS_DIR', # "$HOME/bulk-logs"
+# REPORT_BASEDIR often depends on a timestamp, which has been saved at
+# the beginning of the bulk build in the BULK_BUILD_ID_FILE. It will be
+# retrieved later.
+ 'REPORT_HTML_FILE', # "report.html"
+ 'REPORT_TXT_FILE', # "report.txt"
'USR_PKGSRC', # "/usr/pkgsrc"
'osrev', # `uname -r`
);
-my $reportf = basename($vars{REPORT});
my $os = `uname -s`;
chomp $os;
@@ -151,6 +179,7 @@ get_mk_conf_vars(qw(
SUPPORTSFILE
X11BASE
PKG_TOOLS_BIN
+ BULK_BUILD_ID_FILE
));
my $bulk_dbfile_base = basename($vars{BULK_DBFILE});
@@ -160,6 +189,17 @@ my $supportsfile_base = basename($vars{SUPPORTSFILE});
my $indexfile_base = basename($vars{INDEXFILE});
my $orderfile_base = basename($vars{ORDERFILE});
+my $reports_url = $vars{"REPORTS_URL"};
+my $reports_dir = $vars{"REPORTS_DIR"};
+my $report_basedir = readfirstline($vars{"BULK_BUILD_ID_FILE"});
+my $report_html_file = $vars{"REPORT_HTML_FILE"};
+my $report_txt_file = $vars{"REPORT_TXT_FILE"};
+
+my $report_url = "${reports_url}/${report_basedir}";
+my $report_dir = "${reports_dir}/${report_basedir}";
+my $report_html_path = "${report_dir}/${report_html_file}";
+my $report_txt_path = "${report_dir}/${report_txt_file}";
+
my $startdate = (stat($vars{STARTFILE}))[9];
my $enddate = '';
if (!defined($startdate) || $startdate == 0) {
@@ -187,31 +227,27 @@ sub print_report_header() {
print("--------------------------------------------------------------\n");
}
-my_system("mkdir", "-p", "--", $vars{FTP});
+my_system("mkdir", "-p", "--", $report_dir);
# Copy over the output from the build process
chdir($vars{"BULKFILESDIR"}) or pb_die($vars{"BULKFILESDIR"}, "Cannot change directory.");
-my_system("find . -name $vars{BROKENFILE} -print -o -name $vars{BROKENWRKLOG} -print | $vars{PAX} -r -w -X $vars{FTP}");
+my_system("find . -name $vars{BROKENFILE} -print -o -name $vars{BROKENWRKLOG} -print | $vars{PAX} -r -w -X ${report_dir}");
# Copy over the cache files used during the build
foreach my $f qw(BULK_DBFILE DEPENDSTREEFILE DEPENDSFILE SUPPORTSFILE INDEXFILE ORDERFILE) {
if (-f $vars{$f}) {
- my_system("cp", "--", $vars{$f}, $vars{FTP});
+ my_system("cp", "--", $vars{$f}, $report_dir);
}
}
-chdir($vars{FTP}) or pb_die($vars{"FTP"}, "Cannot change directory.");
+chdir($report_dir) or pb_die($report_dir, "Cannot change directory.");
writeReport();
#
# Adjust "last" symlink
#
-{
- my ($base, $dir) = ($vars{FTP} =~ m|^(.*)/([^/]*)$|);
-
- unlink("$base/last");
- symlink($dir, "$base/last");
-}
+unlink("${reports_dir}/last");
+symlink($report_basedir, "${reports_dir}/last");
#
# Generate leftovers-$vars{MACHINE_ARCH}.html: files not deleted
@@ -219,7 +255,7 @@ writeReport();
# and linked from leftovers-$vars{MACHINE_ARCH}.html
#
{
- chdir($vars{FTP});
+ chdir($report_dir) or pb_die($report_dir, "Cannot change directory.");
my_system("mkdir", "-p", "leftovers-$vars{MACHINE_ARCH}");
# Find files since last build:
@@ -270,7 +306,7 @@ writeReport();
EOOUT
foreach (@leftovers) {
chomp;
- print OUT "<a href=\"$vars{FTPHOST}/$vars{FTPURL}/leftovers-$vars{MACHINE_ARCH}$_\">$_</a>\n";
+ print OUT "<a href=\"${report_url}/leftovers-$vars{MACHINE_ARCH}$_\">$_</a>\n";
}
print OUT <<EOOUT2;
</pre>
@@ -369,11 +405,11 @@ EOF
Packages not listed here resulted in a binary package. The build
report, including logs of failed/not-packaged is available from:
-$vars{FTPHOST}/$vars{FTPURL}/$reportf
+${report_url}/${report_html_file}
EOF
}
- open(HTML, ">$vars{REPORT}") or die "Can't write $vars{REPORT}: $!\n";
+ open(HTML, ">", $report_html_path) or die "Can't write ${report_html_path}: $!";
print HTML <<EOHTML;
<html>
<head>
diff --git a/mk/bulk/post-build-conf b/mk/bulk/post-build-conf
index 6815fb40637..1fc24b80547 100644
--- a/mk/bulk/post-build-conf
+++ b/mk/bulk/post-build-conf
@@ -1,4 +1,4 @@
-# $NetBSD: post-build-conf,v 1.10 2006/01/07 22:27:56 rillig Exp $
+# $NetBSD: post-build-conf,v 1.11 2006/01/21 11:29:33 rillig Exp $
#
# This file is included after the build.conf file by the "build" and
@@ -39,7 +39,9 @@ show_config_vars() {
pbc_section "Building the packages" \
PKGLIST NICE_LEVEL
pbc_section "Generating the report" \
- ADMIN ADMINSIG FTPHOST FTP
+ ADMIN ADMINSIG \
+ REPORTS_URL REPORTS_DIR \
+ REPORT_BASEDIR REPORT_HTML_FILE REPORT_TXT_FILE
pbc_section "Uploading binary packages" \
UPDATE_VULNERABILITY_LIST PRUNEPACKAGES MKSUMS SIGN_AS \
RSYNC_DST RSYNC_OPTS
@@ -51,7 +53,7 @@ export_config_vars() {
export PRUNEDISTFILES ftp_proxy http_proxy
export PKGLIST NICE_LEVEL ADMIN ADMINSIG
export UPDATE_VULNERABILITY_LIST PRUNEPACKAGES MKSUMS SIGN_AS
- export RSYNC_DST RSYNC_OPTS FTPHOST FTP
+ export RSYNC_DST RSYNC_OPTS
case ${MAKECONF+set} in
"set") export MAKECONF;;
esac
@@ -154,6 +156,20 @@ pbc_checkdefined() {
# usage: check_config_vars
check_config_vars() {
+
+ # Handle legacy variables FTP, FTPHOST, FTPURL, REPORT.
+ # Can be removed after 2006Q2.
+ case ${FTP+set},${REPORTS_DIR+set} in
+ set,set)
+ pbc_die "FTP and REPORTS_DIR must not be set both.";;
+ set,)
+ REPORTS_URL="${FTPHOST}"`dirname "${FTPURL}"`
+ REPORTS_DIR=`dirname "$FTP"`
+ REPORT_BASEDIR=`basename "${FTP}"`
+ REPORT_HTML_FILE=`basename "${REPORT}"`
+ REPORT_TXT_FILE="report.txt";;
+ esac
+
# section "System information"
pbc_checknonempty osrev
pbc_checknonempty arch
@@ -184,8 +200,12 @@ check_config_vars() {
# section "Generating the report"
# no checks for ADMIN
# no checks for ADMINSIG
- # no checks for FTPHOST
- # no checks for FTP
+ pbc_checkurl REPORTS_URL
+ pbc_checkpathname REPORTS_DIR
+ pbc_checkbasename REPORT_BASEDIR
+ pbc_checkbasename REPORT_HTML_FILE
+ pbc_checkbasename REPORT_TXT_FILE
+
# section "Uploading binary packages"
pbc_checkyesno UPDATE_VULNERABILITY_LIST