From 11ff5b93802116af306f193bf22b381eec69ce69 Mon Sep 17 00:00:00 2001 From: jschauma Date: Mon, 27 Nov 2006 00:05:22 +0000 Subject: - modularize functionality into a number of distinct functions - add the option "-p|--post-build" to allow you to only run the post-build portion of the bulk-build --- mk/bulk/build | 658 +++++++++++++++++++++++++++++++--------------------------- 1 file changed, 357 insertions(+), 301 deletions(-) (limited to 'mk') diff --git a/mk/bulk/build b/mk/bulk/build index 25311e5d34f..8361cd9e979 100644 --- a/mk/bulk/build +++ b/mk/bulk/build @@ -1,5 +1,5 @@ #!/bin/sh -# $NetBSD: build,v 1.95 2006/11/19 03:06:26 jschauma Exp $ +# $NetBSD: build,v 1.96 2006/11/27 00:05:22 jschauma Exp $ # # Copyright (c) 1999, 2000 Hubert Feyrer @@ -30,11 +30,27 @@ # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # +## +## Globals +## +scriptdir=`dirname "$0"` +scriptdir=`cd "${scriptdir}" && pwd` + # -# Do bulk build +# Default values for command line options. # +resume=no +mirror_only=no +target=bulk-package +makeargs="" +noemail=no +post_only=no -usage() { +## +## Functions +## + +usage () { cat <&2 for i in "$@"; do echo " $i" 1>&2 @@ -93,152 +113,7 @@ post_filter_cmd() { ${SED} "s;^;`date '+%Y/%m/%d %H:%M:%S'` ${percent} ${pkgdir} @ ${MACHINE_ARCH}> ;g" } - -# -# Find out where we are -# -scriptdir=`dirname "$0"` -scriptdir=`cd "${scriptdir}" && pwd` - -# -# Default values for command line options. -# -resume=no -mirror_only=no -target=bulk-package -makeargs="" -noemail=no - -# -# Parse the command line. -# -while test $# -gt 0; do - case $1 in - -c|--config) - shift - BULK_BUILD_CONF=$1; shift - ;; - -e|--no-email) - noemail=yes - shift - ;; - -h|--help) - usage - exit 0 - ;; - -m|--mirror_only) - mirror_only=yes - target=mirror-distfiles - shift - ;; - -r|--resume|--restart|restart) - resume=yes - shift - ;; - -s|--specific-pkgs) - makeargs="$makeargs SPECIFIC_PKGS=1" - shift - ;; - *) - echo "unknown option: $1" 1>&2 - usage 1>&2 - exit 1 - ;; - esac -done - -# -# Choose an appropriate value for BMAKE depending on the operating system. -# -opsys=`uname -s` -case "$opsys" in -NetBSD) BMAKE=make ;; -*) BMAKE=bmake ;; -esac -export BMAKE - -# -# Set resource limits as high as possible -# -ulimit -S -s `ulimit -H -s` -ulimit -S -d `ulimit -H -d` - -# -# Find the configuration file. -# -BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}" -case $BULK_BUILD_CONF in -/*) ;; -*) BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}" -esac - -# -# Load the variables from the configuration file. -# -{ test -f "${BULK_BUILD_CONF}" \ - && . "${BULK_BUILD_CONF}" \ - && . "${scriptdir}/post-build-conf" \ - && check_config_vars \ - && export_config_vars -} || die "Cannot load config file ${BULK_BUILD_CONF}, aborting." - -# -# Set the paths to commonly used directories. -# -pkgsrc_dir="${USR_PKGSRC}" -pkglint_dir="${USR_PKGSRC}/pkgtools/pkglint" - -# -# Set up variables specific for the bulk build. -# -BATCH="1" -DEPENDS_TARGET="bulk-install" -export BATCH DEPENDS_TARGET - -# -# Unset some environment variables that could disturbe the build. -# -unset CDPATH || true # ensure cd does not print new cwd to stdout, which - # confuses the printindex script. -unset DISPLAY || true # allow sane failure for gimp, xlispstat - -# -# It starts ... -# -echo "build> Bulk build started: `date`" -echo "" -show_config_vars - -# -# Check that the package tools are up to date. -# -echo "build> Checking if the pkgtools are up-to-date" -( cd "${pkglint_dir}" \ - && ${BMAKE} fetch >/dev/null 2>&1 -) || { - echo "build> Updating pkgtools" - ( cd "${pkgsrc_dir}/pkgtools/pkg_install" \ - && ${BMAKE} clean \ - && ${BMAKE} install \ - && ${BMAKE} clean - ) || die "Could not update the package tools." -} - -# -# Run the pre-build script if necessary. -# -case $resume in -yes) echo "build> Resuming -- skipping pre-build script";; -*) # make veryveryclean :) - ( cd "${pkgsrc_dir}" \ - && /bin/sh mk/bulk/pre-build - ) || die "Error during bulk-build preparations, aborting.";; -esac - -# -# Load pkgsrc variables that affect the build process. -# - +# find and set a pkgsrc variable # usage: load_pkgsrc_var load_pkgsrc_var() { case $1 in @@ -260,94 +135,194 @@ load_pkgsrc_var() { printf "%-15s = %s\\n" "${lpv_varname}" "${lpv_value}" } -echo "+----------------------------------------+" -echo "| Some variables used in the bulk build: |" -echo "+----------------------------------------+" -load_pkgsrc_var OPSYS -load_pkgsrc_var OS_VERSION -load_pkgsrc_var MACHINE_ARCH -load_pkgsrc_var -z BULK_PREREQ - -load_pkgsrc_var BULKFILESDIR -load_pkgsrc_var BULK_DBFILE -load_pkgsrc_var DEPENDSFILE -load_pkgsrc_var DEPENDSTREEFILE -load_pkgsrc_var INDEXFILE -load_pkgsrc_var ORDERFILE -load_pkgsrc_var STARTFILE -load_pkgsrc_var SUPPORTSFILE -load_pkgsrc_var BULK_BUILD_ID_FILE - -load_pkgsrc_var BUILDLOG -load_pkgsrc_var BROKENFILE -load_pkgsrc_var BROKENWRKLOG - -load_pkgsrc_var AWK USE_TOOLS=awk -load_pkgsrc_var GREP USE_TOOLS=grep -load_pkgsrc_var MAIL_CMD USE_TOOLS=mail -load_pkgsrc_var MKDIR USE_TOOLS=mkdir -load_pkgsrc_var PERL5 USE_TOOLS=perl -load_pkgsrc_var PKG_DELETE -load_pkgsrc_var PKG_INFO -load_pkgsrc_var SED USE_TOOLS=sed -echo "------------------------------------------" +# perform post-processing of the bulk-build results +do_post_build () { + echo "build> Post processing bulk build results..." -# -# Get the location of commonly used files -# -main_buildlog="${BULKFILESDIR}/${BUILDLOG}" + [ -z "${BULK_BUILD_ID_FILE}" ] && load_pkgsrc_var BULK_BUILD_ID_FILE + [ -z "${MKDIR}" ] && load_pkgsrc_var MKDIR USE_TOOLS=mkdir + [ -z "${PERL5}" ] && load_pkgsrc_var PERL5 USE_TOOLS=perl -# -# Create the directory for the log files if necessary -# -if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then - ${MKDIR} "${BULKFILESDIR}" -fi + # Re-install BULK_PREREQ as we may need functionality (e.g. SMTP) provided by + # them for post-build to run. + echo "build> Re-installing prerequisite packages specified with BULK_PREREQ..." + for pkgdir in $BULK_PREREQ lang/perl5; do + echo "build> Installing prerequisite package $pkgdir" + ( cd "${pkgsrc_dir}/${pkgdir}" \ + && ${BMAKE} bulk-install + ) || die "Failed to install prerequisite packages." + done -# -# Save the bulk build ID in a file, as it most often contains a time -# stamp. -# -case $resume in -no) echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \ - || die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";; -esac + # + # Generate the post-build report. + # + echo "build> Generating the bulk build report..." + + bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \ + || die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}." + + report_dir="${REPORTS_DIR}/${bulk_build_id}" + ${MKDIR} "${report_dir}" + ( cd "${pkgsrc_dir}" \ + && ${PERL5} mk/bulk/post-build \ + > "${report_dir}/${REPORT_TXT_FILE}" + ) || die "Could not write the results file." +} + +# Notify the ADMIN of the finished build. +do_email () { + case $noemail in + no) cat "${report_dir}/${REPORT_TXT_FILE}" \ + | ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results ${bulk_build_id}" "$ADMIN" + esac +} + +# output final note that we're done +do_done () { + echo "" + echo "build> Bulk build ended: `date`" +} + +# set all commonly used variables, prepare files etc. +do_common_setup () { + + # + # Choose an appropriate value for BMAKE depending on the operating + # system. + # + opsys=`uname -s` + case "$opsys" in + NetBSD) BMAKE=make ;; + *) BMAKE=bmake ;; + esac + export BMAKE + + # + # Set resource limits as high as possible + # + ulimit -S -s `ulimit -H -s` + ulimit -S -d `ulimit -H -d` + + # + # Find the configuration file. + # + BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}" + case $BULK_BUILD_CONF in + /*) ;; + *) BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}" + esac + + # + # Load the variables from the configuration file. + # + { test -f "${BULK_BUILD_CONF}" \ + && . "${BULK_BUILD_CONF}" \ + && . "${scriptdir}/post-build-conf" \ + && check_config_vars \ + && export_config_vars + } || die "Cannot load config file ${BULK_BUILD_CONF}, aborting." + + # + # Set the paths to commonly used directories. + # + pkgsrc_dir="${USR_PKGSRC}" + pkglint_dir="${USR_PKGSRC}/pkgtools/pkglint" + + # + # Set up variables specific for the bulk build. + # + BATCH="1" + DEPENDS_TARGET="bulk-install" + export BATCH DEPENDS_TARGET + + # + # Unset some environment variables that could disturbe the build. + # + unset CDPATH || true # ensure cd does not print new cwd to stdout, which + # confuses the printindex script. + unset DISPLAY || true # allow sane failure for gimp, xlispstat +} + +# Check that the package tools are up to date. +check_tools () { + echo "build> Checking if the pkgtools are up-to-date" + ( cd "${pkglint_dir}" \ + && ${BMAKE} fetch >/dev/null 2>&1 + ) || { + echo "build> Updating pkgtools" + ( cd "${pkgsrc_dir}/pkgtools/pkg_install" \ + && ${BMAKE} clean \ + && ${BMAKE} install \ + && ${BMAKE} clean + ) || die "Could not update the package tools." + } +} + +# Run the pre-build script if necessary. +run_pre_build () { + case $resume in + yes) echo "build> Resuming -- skipping pre-build script";; + *) # make veryveryclean :) + ( cd "${pkgsrc_dir}" \ + && /bin/sh mk/bulk/pre-build + ) || die "Error during bulk-build preparations, aborting.";; + esac +} + +# Load pkgsrc variables that affect the build process. +load_vars () { + echo "+----------------------------------------+" + echo "| Some variables used in the bulk build: |" + echo "+----------------------------------------+" + load_pkgsrc_var OPSYS + load_pkgsrc_var OS_VERSION + load_pkgsrc_var MACHINE_ARCH + load_pkgsrc_var -z BULK_PREREQ + + load_pkgsrc_var BULKFILESDIR + load_pkgsrc_var BULK_DBFILE + load_pkgsrc_var DEPENDSFILE + load_pkgsrc_var DEPENDSTREEFILE + load_pkgsrc_var INDEXFILE + load_pkgsrc_var ORDERFILE + load_pkgsrc_var STARTFILE + load_pkgsrc_var SUPPORTSFILE + load_pkgsrc_var BULK_BUILD_ID_FILE + + load_pkgsrc_var BUILDLOG + load_pkgsrc_var BROKENFILE + load_pkgsrc_var BROKENWRKLOG + + load_pkgsrc_var AWK USE_TOOLS=awk + load_pkgsrc_var GREP USE_TOOLS=grep + load_pkgsrc_var MAIL_CMD USE_TOOLS=mail + load_pkgsrc_var MKDIR USE_TOOLS=mkdir + load_pkgsrc_var PERL5 USE_TOOLS=perl + load_pkgsrc_var PKG_DELETE + load_pkgsrc_var PKG_INFO + load_pkgsrc_var SED USE_TOOLS=sed + echo "------------------------------------------" + + # Get the location of commonly used files + main_buildlog="${BULKFILESDIR}/${BUILDLOG}" +} -# # Install prerequisite packages. # # Note: we do this _before_ the depends tree because some packages like # xpkgwedge only become DEPENDS if they are installed. -# -echo "build> Installing prerequisite packages specified with BULK_PREREQ..." -for pkgdir in $BULK_PREREQ; do - echo "build> Installing prerequisite package $pkgdir" - ( cd "${pkgsrc_dir}/${pkgdir}" \ - && ${BMAKE} bulk-install - ) || die "Could not install prerequisite packages." -done - -# -# Create the bulk cache files. -# -if [ "x$resume" != "xyes" ]; then - ( cd "${pkgsrc_dir}" \ - && env PKGLIST="${PKGLIST-}" ${BMAKE} bulk-cache $makeargs - ) || die "Could not create the bulk build cache." -else - if [ ! -f "${ORDERFILE}" ]; then - die "The ${ORDERFILE} does not exist." \ - "(You cannot resume a bulk build that has not yet started.)" - fi -fi +install_prereqs () { + echo "build> Installing prerequisite packages specified with BULK_PREREQ..." + for pkgdir in $BULK_PREREQ; do + echo "build> Installing prerequisite package $pkgdir" + ( cd "${pkgsrc_dir}/${pkgdir}" \ + && ${BMAKE} bulk-install + ) || die "Could not install prerequisite packages." + done +} -# # Everything is prepared. We can start building the real packages now. # -cd "${pkgsrc_dir}" || die "The pkgsrc directory does not exist." - -echo "build> Starting actual build using the order specified in $ORDERFILE..." - # Loop over every package in the correct order. Before building # each one, check to see if we've already processed this package # before. This could happen if the build got interrupted and we @@ -357,94 +332,175 @@ echo "build> Starting actual build using the order specified in $ORDERFILE..." # it to the top level buildlog # (usually '.make' or '.make.${MACHINE}'). As a side benefit, this # can make a progress-meter very simple to add! +do_real_bulk_build () { + + cd "${pkgsrc_dir}" || die "The pkgsrc directory does not exist." + + echo "build> Starting actual build using the order specified in $ORDERFILE..." + + # make sure we have something to grep in in the build loop + touch "${main_buildlog}" || die "Cannot write to ${main_buildlog}." + + tot=`${AWK} 'END { print NR }' "${ORDERFILE}"` + for pkgdir in `cat "${ORDERFILE}"` + do + if ${GREP} -q "^${pkgdir}\$" "${main_buildlog}"; then + : "skip this package" + else + percent=`${AWK} -v tot="${tot}" ' + END { + printf("%d/%d=%4.1f%%", NR, tot, NR*100/tot); + }' "${main_buildlog}"` + ( cd "${pkgsrc_dir}/${pkgdir}" \ + && ${NICE_LEVEL} ${BMAKE} USE_BULK_CACHE=yes "${target}" \ + $makeargs > "${main_buildlog}" + fi + done -# make sure we have something to grep in in the build loop -touch "${main_buildlog}" || die "Cannot write to ${main_buildlog}." + echo "build> Build finished." +} -tot=`${AWK} 'END { print NR }' "${ORDERFILE}"` -for pkgdir in `cat "${ORDERFILE}"` -do - if ${GREP} -q "^${pkgdir}\$" "${main_buildlog}"; then - : "skip this package" - else - percent=`${AWK} -v tot="${tot}" ' - END { - printf("%d/%d=%4.1f%%", NR, tot, NR*100/tot); - }' "${main_buildlog}"` - ( cd "${pkgsrc_dir}/${pkgdir}" \ - && ${NICE_LEVEL} ${BMAKE} USE_BULK_CACHE=yes "${target}" \ - $makeargs > "${main_buildlog}" +# clean up installed packages left over +do_bulk_cleanup () { + + echo "build> Removing all installed packages left over from build..." + for pkgname in `${PKG_INFO} -e \*` + do + if ${PKG_INFO} -qe "${pkgname}"; then + pkgdir=`${AWK} '$2 == "'"$pkgname"'" { print $1; }' "$INDEXFILE"` + case "${BULK_PREREQ}" in + *"${pkgdir}"* ) + echo "build> Keeping BULK_PREREQ: $pkgname ($pkgdir)" ; + ;; + * ) + echo "build> ${PKG_DELETE} -r ${pkgname}" + ${PKG_DELETE} -r "${pkgname}" + if ${PKG_INFO} -qe "${pkgname}"; then + echo "build> $pkgname ($pkgdir) did not deinstall nicely. Forcing the deinstall" + ${PKG_DELETE} -f "${pkgname}" || true + fi + ;; + esac + fi + done +} + +# start the full bulk-build +do_bulk_build () { + echo "build> Bulk build started: `date`" + echo "" + + # this function from post-build-conf + show_config_vars + + check_tools + run_pre_build + load_vars + + # + # Create the directory for the log files if necessary + # + if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then + ${MKDIR} "${BULKFILESDIR}" fi -done -echo "build> Build finished. Removing all installed packages left over from build..." -for pkgname in `${PKG_INFO} -e \*` -do - if ${PKG_INFO} -qe "${pkgname}"; then - pkgdir=`${AWK} '$2 == "'"$pkgname"'" { print $1; }' "$INDEXFILE"` - case "${BULK_PREREQ}" in - *"${pkgdir}"* ) - echo "build> Keeping BULK_PREREQ: $pkgname ($pkgdir)" ; - ;; - * ) - echo "build> ${PKG_DELETE} -r ${pkgname}" - ${PKG_DELETE} -r "${pkgname}" - if ${PKG_INFO} -qe "${pkgname}"; then - echo "build> $pkgname ($pkgdir) did not deinstall nicely. Forcing the deinstall" - ${PKG_DELETE} -f "${pkgname}" || true - fi - ;; - esac + # + # Save the bulk build ID in a file, as it most often contains a time + # stamp. + # + case $resume in + no) echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \ + || die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";; + esac + + install_prereqs + + # + # Create the bulk cache files. + # + if [ "x$resume" != "xyes" ]; then + ( cd "${pkgsrc_dir}" \ + && env PKGLIST="${PKGLIST-}" ${BMAKE} bulk-cache $makeargs + ) || die "Could not create the bulk build cache." + else + if [ ! -f "${ORDERFILE}" ]; then + die "The ${ORDERFILE} does not exist." \ + "(You cannot resume a bulk build that has not yet started.)" + fi fi + + do_real_bulk_build + do_bulk_cleanup +} + + +## +## main +## + +# +# Parse the command line. +# +while test $# -gt 0; do + case $1 in + -c|--config) + shift + BULK_BUILD_CONF=$1; shift + ;; + -e|--no-email) + noemail=yes + shift + ;; + -h|--help) + usage + exit 0 + ;; + -m|--mirror_only) + mirror_only=yes + target=mirror-distfiles + shift + ;; + -p|--post-build) + post_only=yes + shift + ;; + -r|--resume|--restart|restart) + resume=yes + shift + ;; + -s|--specific-pkgs) + makeargs="$makeargs SPECIFIC_PKGS=1" + shift + ;; + *) + echo "unknown option: $1" 1>&2 + usage 1>&2 + exit 1 + ;; + esac done +do_common_setup + +if [ "x$post_only" = "xyes" ]; then + do_post_build + exit 0 +fi + +do_bulk_build + # for now, just quit if we were only mirroring distfiles. At somepoint we # should teach the post-build script to generate a nice report about how many # distfiles were downloaded, how many had bad checksums, failed master sites, # network speed, etc. - if [ "x$mirror_only" = "xyes" ]; then echo "build> Bulk mirror of distfiles completed: `date`" exit 0 fi -echo "build> Post processing bulk build results..." +do_post_build +do_email +do_done -# Re-install BULK_PREREQ as we may need functionality (e.g. SMTP) provided by -# them for post-build to run. -echo "build> Re-installing prerequisite packages specified with BULK_PREREQ..." -for pkgdir in $BULK_PREREQ lang/perl5; do - echo "build> Installing prerequisite package $pkgdir" - ( cd "${pkgsrc_dir}/${pkgdir}" \ - && ${BMAKE} bulk-install - ) || die "Failed to install prerequisite packages." -done - -# -# Generate the post-build report. -# -echo "build> Generating the bulk build report..." - -bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \ -|| die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}." - -report_dir="${REPORTS_DIR}/${bulk_build_id}" -${MKDIR} "${report_dir}" -( cd "${pkgsrc_dir}" \ - && ${PERL5} mk/bulk/post-build \ - > "${report_dir}/${REPORT_TXT_FILE}" -) || die "Could not write the results file." - -# -# Notify the ADMIN of the finished build. -# -case $noemail in -no) cat "${report_dir}/${REPORT_TXT_FILE}" \ - | ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results ${bulk_build_id}" "$ADMIN" -esac - -# Done! -echo "" -echo "build> Bulk build ended: `date`" -- cgit v1.2.3