diff options
| -rw-r--r-- | Makefile | 5 | ||||
| -rw-r--r-- | mk/bsd.pkg.mk | 15 | ||||
| -rw-r--r-- | mk/bulk/TODO | 3 | ||||
| -rw-r--r-- | mk/bulk/bsd.bulk-pkg.mk | 583 | ||||
| -rw-r--r-- | mk/bulk/build | 505 | ||||
| -rw-r--r-- | mk/bulk/build.conf-example | 106 | ||||
| -rw-r--r-- | mk/bulk/do-sandbox-build | 14 | ||||
| -rw-r--r-- | mk/bulk/do-sandbox-upload | 14 | ||||
| -rw-r--r-- | mk/bulk/environment.txt | 105 | ||||
| -rw-r--r-- | mk/bulk/parallel.txt | 207 | ||||
| -rw-r--r-- | mk/bulk/post-build | 722 | ||||
| -rw-r--r-- | mk/bulk/post-build-conf | 203 | ||||
| -rw-r--r-- | mk/bulk/pre-build | 287 | ||||
| -rw-r--r-- | mk/bulk/printdepends | 164 | ||||
| -rw-r--r-- | mk/bulk/printindex | 202 | ||||
| -rw-r--r-- | mk/bulk/printindex.1 | 104 | ||||
| -rw-r--r-- | mk/bulk/sort-packages | 83 | ||||
| -rwxr-xr-x | mk/bulk/tflat | 121 | ||||
| -rw-r--r-- | mk/bulk/upload | 356 | ||||
| -rw-r--r-- | mk/misc/toplevel.mk | 9 |
20 files changed, 3 insertions, 3805 deletions
@@ -1,4 +1,4 @@ -# $NetBSD: Makefile,v 1.85 2009/05/16 07:21:05 rillig Exp $ +# $NetBSD: Makefile,v 1.86 2014/06/02 10:24:05 wiz Exp $ # # This is the top-level Makefile of pkgsrc. It contains a list of the # categories of packages, as well as some targets that operate on the @@ -32,9 +32,6 @@ USE_TOOLS+= [ awk cat cmp echo env expr false fgrep grep mv rm sed \ # additional tools used by bsd.pkg.subdir.mk USE_TOOLS+= basename touch -# additional tools used by bsd.bulk-pkg.mk -USE_TOOLS+= egrep find ls sh tee true tsort - PKGSRCTOP= yes .include "mk/bsd.prefs.mk" diff --git a/mk/bsd.pkg.mk b/mk/bsd.pkg.mk index d070044e96f..ee15aba97a6 100644 --- a/mk/bsd.pkg.mk +++ b/mk/bsd.pkg.mk @@ -1,4 +1,4 @@ -# $NetBSD: bsd.pkg.mk,v 1.1999 2014/05/06 14:48:06 jperkin Exp $ +# $NetBSD: bsd.pkg.mk,v 1.2000 2014/06/02 10:24:05 wiz Exp $ # # This file is in the public domain. # @@ -386,11 +386,6 @@ USE_TOOLS+= \ # bsd.wrapper.mk USE_TOOLS+= expr -# bsd.bulk-pkg.mk uses certain tools -.if defined(BATCH) -USE_TOOLS+= tee tsort -.endif - # scripts/shlib-type .if ${_OPSYS_SHLIB_TYPE} == "ELF/a.out" USE_TOOLS+= file @@ -747,14 +742,6 @@ tags: .include "subst.mk" -# -# For bulk build targets (bulk-install, bulk-package), the -# BATCH variable must be set in /etc/mk.conf: -# -.if defined(BATCH) -. include "bulk/bsd.bulk-pkg.mk" -.endif - # README generation code. .include "bsd.pkg.readme.mk" diff --git a/mk/bulk/TODO b/mk/bulk/TODO deleted file mode 100644 index d3add521435..00000000000 --- a/mk/bulk/TODO +++ /dev/null @@ -1,3 +0,0 @@ - * integrate pkgtools/libkver - * Generate list of packages that were "Successfully packaged" - * allow parallel builds (see parallel.txt) diff --git a/mk/bulk/bsd.bulk-pkg.mk b/mk/bulk/bsd.bulk-pkg.mk deleted file mode 100644 index baa3d280b01..00000000000 --- a/mk/bulk/bsd.bulk-pkg.mk +++ /dev/null @@ -1,583 +0,0 @@ -# $NetBSD: bsd.bulk-pkg.mk,v 1.152 2011/09/08 20:17:15 abs Exp $ - -# -# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org> -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. All advertising materials mentioning features or use of this software -# must display the following acknowledgement: -# This product includes software developed by Hubert Feyrer for -# the NetBSD Project. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -## -## For this to work, put -## DEPENDS_TARGET= bulk-install -## BATCH= yes -## into your /etc/mk.conf, then cd /usr/pkgsrc; make bulk-package -## -## bulk-install will only compile pkgs if necessary, and install binary -## pkgs if possible/available. -## bulk-package will build binary packages, and remove them (together with -## any depending packages) immediately after that, leaving only the -## binary package back. Broken packages are marked (``.broken''). -## - -LS?= ls -TEE?= tee -WC?= wc -TO_HTML?= ${SED} -e 's,&,\&,g' -e 's,<,\<,g' -e 's,>,\>,g' - -# This variable is set to 'yes' by the pkgsrc/mk/bulk/build script. It enables -# the use of several cache files (DEPENDSTREEFILE, DEPENDSFILE, SUPPORTSFILE, -# and INDEXFILE) for speeding up the processing of dependencies. If we're not -# actively in the middle of a bulk build, however, we don't know that these files -# exist or are up to date and they take quite a while to rebuild. So unless -# they're known to exist and be up to date, don't use them. -USE_BULK_CACHE?= no - -# If set to 'yes', avoids inflating the 'breaks' column of the final report -# by not counting dependent packages that are broken all by themselves. -# If set to 'no', may speed up building on some platforms where fork() is slow. -# (Only takes effect if USE_BULK_CACHE=yes.) -USE_BULK_BROKEN_CHECK?= yes - -# This variable may be set to 'no' to avoid automatic rebuilding of dependent -# packages based solely on timestamps of the package's pkgsrc files and/or -# its dependency binary packages. This will cause rebuilding only when the -# full version number of the package changes (e.g., via PKGREVISION bump). -USE_BULK_TIMESTAMPS?= yes - -# Shall we remove any packages which are installed, but not required -# to build this package -PRECLEAN?= yes - -# Sometimes it's useful to not only keep the build logs from broken -# packages, but also the ones from packages that worked. -# This feature is still experimental. -KEEP_BUILDLOGS?= no - -# If OBJHOSTNAME is set, use first component of hostname in cache and log files -# If OBJMACHINE is set, use ${MACHINE_ARCH} in the cache and log files -.if defined(OBJHOSTNAME) -. if !defined(_HOSTNAME) -_HOSTNAME!= ${UNAME} -n -. endif -BULK_ID?= .${_HOSTNAME:C|\..*||} -.elif defined(OBJMACHINE) -BULK_ID?= .${MACHINE_ARCH} -.else -BULK_ID?= -.endif - -.if ${_USE_DESTDIR} != "no" -_PACKAGE_TARGET= package package-install -.else -_PACKAGE_TARGET= package -.endif -# -# Package-specific files -# - -# If the package has been built successfully, and if KEEP_BUILDLOGS is -# set to "yes", the build log is saved in this file. -BUILD_SUCCEEDED_FILE?= .build_succeeded${BULK_ID} - -# This file exists to mark a package as broken -BROKENFILE?= .broken${BULK_ID}.html - -# When this file exists for a package, it has been marked as not to be -# available on this platform. -NOT_AVAILABLE_FILE?= .bulk-not_available - -# This file is the work log for a broken package -BROKENWRKLOG?= .broken${BULK_ID}.work.txt - -# This file is human-created to force a package to show up as broken -# (it is never cleaned by the bulk build, and contains the broken reason) -FORCEBROKENFILE?= .forcebroken - -# This file is where the log of the build goes -BUILDLOG?= .make${BULK_ID} - -# -# Top level databases and log files -# - -# This is the directory in which all temporary files and log files from the -# bulk build are kept. -# It defaults to ${PKGSRCDIR}, but may be better suited to another directory -# if pkgsrc is on a remote (e.g., nfs) filesystem. -BULKFILESDIR?= ${PKGSRCDIR} - -# This is a top level file which lists the entire pkgsrc depends tree in the -# format: -# foo/bar devel/libfoo -# meaning 'foo/bar' is requied to build 'devel/libfoo' -# this is in the format needed by tsort(1) -DEPENDSTREEFILE?= ${BULKFILESDIR}/.dependstree${BULK_ID} - -# This is a top level file which lists the entire pkgsrc depends tree in the -# format: -# foo/bar depends on: devel/libfoo devel/libbar devel/baz ..... -# ie, to build foo/bar we need devel/libfoo devel/libbar devel/baz ... installed -DEPENDSFILE?= ${BULKFILESDIR}/.depends${BULK_ID} - -# This is a top level file which lists the entire pkgsrc depends tree in the -# format: -# devel/libfoo is depended upon by: foo/bar graphics/gtkfoo ... -# ie, to build foo/bar we need devel/libfoo to be installed. -# to build graphics/gtkfoo we need devel/libfoo to be installed -SUPPORTSFILE?= ${BULKFILESDIR}/.supports${BULK_ID} - -# This is a top level file which cross-references each package name and pkg -# directory in the format: -# devel/libfoo libfoo-1.3 -INDEXFILE?= ${BULKFILESDIR}/.index${BULK_ID} - -# File containing a list of all the packages in the correct order for a bulk -# build. The correct order is one where packages that are required by others -# are built before the packages which require them. -ORDERFILE?= ${BULKFILESDIR}/.order${BULK_ID} - -# File which is used as a timestamp for when the build started. This is used -# eventually for looking for leftover files (files not properly deinstalled) -STARTFILE?= ${BULKFILESDIR}/.start${BULK_ID} - -# This top level file saves the bulk build ID. As it most often has a time -# stamp, it is only generated once and later retrieved from this file. -BULK_BUILD_ID_FILE?= ${BULKFILESDIR}/.bulk_build_id${BULK_ID} - -# File created and used by lintpkgsrc(8) to cache package metadata for -# pruning and bulk-upload exclusions. -LINTPKGSRC_DB?= ${BULKFILESDIR}/.lintpkgsrc.db${BULK_ID} - -# File which is used as a database for bulk builds in which SPECIFIC_PKGS is -# defined. This database is used to hold all the dependency and index -# information for the specific packages as well as their dependencies. In a -# SPECIFIC_PKGS bulk build, this file is created and then used to create the -# INDEXFILE and DEPENDSTREEFILE. -BULK_DBFILE?= ${BULKFILESDIR}/.bulk_db${BULK_ID} - -# A list of pkgs which we should _never_ delete during a build. The primary -# use is for digest and also for xpkgwedge. Add pkgtools/xpkgwedge in -# /etc/mk.conf to do an xpkgwedged bulk build. -BULK_PREREQ+= pkgtools/digest - -# Commands for printing informational messages from the bulk build. -# BULK_MSG_CONT is used for continuing a message started with BULK_MSG -BULK_MSG?= ${ECHO_MSG} "BULK>" -BULK_MSG_CONT?= ${ECHO_MSG} " " - -# -# Private variables -# - -# If this is "yes", the working directory will not be cleaned after building -# the package. It can also be set to "pack-if-failed" to save a .tar.gz -# file of the working directory in BULKFILESDIR instead of keeping the files, -# but only if the package failed to build. -_PRESERVE_WRKDIR?= no - -# create an escaped version of PKGPATH. We do this because -# in several places we want to be able to use something like -# foo/bar.baz++ in a regular expression but have it be interpreted -# literally. So, turn it into foo\/bar\.baz\+\+ -_ESCPKGPATH= ${PKGPATH:C@\/@\\/@g:C@\+@\\+@g:C@\.@\\.@g:Q} - -# The directory where package-specific log files are saved. -_BULK_PKGLOGDIR= ${BULKFILESDIR}/${PKGPATH} - -# Package-specific log files. -_BROKENFILE= ${_BULK_PKGLOGDIR}/${BROKENFILE} -_BROKENWRKLOG= ${_BULK_PKGLOGDIR}/${BROKENWRKLOG} -_BUILDLOG= ${_BULK_PKGLOGDIR}/${BUILDLOG} -_FORCEBROKENFILE= ${_BULK_PKGLOGDIR}/${FORCEBROKENFILE} -_BUILD_SUCCEEDED_FILE= ${_BULK_PKGLOGDIR}/${BUILD_SUCCEEDED_FILE} - -# Only create directories if ${PKGSRCDIR} != ${BULKFILESDIR} -.if ${PKGSRCDIR} != ${BULKFILESDIR} -_BULK_MKDIR= ${MKDIR} -.else -_BULK_MKDIR= ${DO_NADA} -.endif - -# -# Sanity checks -# - -# Allow FORCEBROKENFILE files to be created in either PKGSRCDIR or BULKFILESDIR. -.if exists(${FORCEBROKENFILE}) -PKG_FAIL_REASON+= "${PKGNAME} is marked as broken by the bulk build administrator: `cat ${FORCEBROKENFILE:Q:Q}`" -.elif exists(${_FORCEBROKENFILE}) -PKG_FAIL_REASON+= "${PKGNAME} is marked as broken by the bulk build administrator: `cat ${_FORCEBROKENFILE:Q:Q}`" -.endif - -# -# Make targets -# - -# build the cache files used as part of a full bulk build -# Note: we have to install the BULK_PREREQ packages _before_ -# extracting the depends tree because some packages like -# xpkgwedge only become DEPENDS if it is installed -.PHONY: bulk-cache -bulk-cache: - @${BULK_MSG} "Installing BULK_PREREQ packages" -.for __prereq in ${BULK_PREREQ} lang/perl5 - cd ${PKGSRCDIR}/${__prereq} && ${RECURSIVE_MAKE} ${MAKEFLAGS} bulk-install -.endfor - ${RM} -f ${BULK_DBFILE} - ${TOUCH} ${BULK_DBFILE} -.if !defined(SPECIFIC_PKGS) - @${ECHO} "This file is unused for a full pkgsrc bulk build" >> ${BULK_DBFILE} - @${ECHO} "It is only used for a SPECIFIC_PKGS bulk build" >> ${BULK_DBFILE} - @${BULK_MSG} "Building complete pkgsrc dependency tree (this may take a while)." - cd ${PKGSRCDIR} && ${PKGSRC_SETENV} BMAKE=${MAKE:Q} ${SH} mk/bulk/printdepends ${BROKENFILE:Q} ${BULKFILESDIR:Q} > ${DEPENDSTREEFILE:Q} - @${BULK_MSG} "Generating package name <=> package directory cross reference file" - @${BULK_MSG_CONT} "(this may take a while)." - cd ${PKGSRCDIR} && ${PKGSRC_SETENV} BMAKE=${MAKE:Q} ${SH} mk/bulk/printindex ${NOT_AVAILABLE_FILE} ${BULKFILESDIR:Q} > ${INDEXFILE:Q} -.else - @${BULK_MSG} "Extracting database for SPECIFIC_PKGS subset of pkgsrc" - @${BULK_MSG_CONT} "along with their dependencies" -. for __tmp__ in ${SUBDIR} ${BULK_PREREQ} lang/perl5 pkgtools/lintpkgsrc - @if [ ! -d ${PKGSRCDIR}/${__tmp__} ]; then \ - ${BULK_MSG} "WARN: Skipping nonexisting directory ${__tmp__}"; \ - ${ECHO} " 0 ${__tmp__} 0" >> ${BULKFILESDIR:Q}/${BROKENFILE:Q}; \ - else \ - cd ${PKGSRCDIR}/${__tmp__} && ${PKGSRC_SETENV} BMAKE=${MAKE:Q} AWK=${AWK:Q} EXPR=${EXPR:Q} ${SH} ../../mk/scripts/mkdatabase -a -f ${BULK_DBFILE:Q}; \ - fi -. endfor - @${BULK_MSG} "Extracting dependency tree file" - ${AWK} '/^(build_)?depends/ {pkgs[$$2] = 1; cat=$$2; sub(/\/.*/, "", cat); \ - for(i=3; i<=NF; i=i+1){ \ - listed[$$2] = 1; \ - sub(/[^:]*:\.\.\/\.\.\//, "", $$i); \ - sub(/[^:]*:\.\./, cat , $$i); \ - print $$i " " $$2; \ - }} END{ \ - for(pkg in pkgs) {if( pkg in listed ) {} else{ print pkg " " pkg;}} \ - }' \ - ${BULK_DBFILE} | ${SORT} -u > ${DEPENDSTREEFILE} - @${BULK_MSG} "Extracting package name <=> package directory cross reference file" - ${AWK} '/^index/ {print $$2 " " $$3 " "}' ${BULK_DBFILE} > ${INDEXFILE} -.endif - @${BULK_MSG} "Sorting build order." - cd ${BULKFILESDIR} && ${TSORT} ${DEPENDSTREEFILE} > ${ORDERFILE} - @${BULK_MSG} "Generating up and down dependency files." - ${LOCALBASE}/bin/perl ${PKGSRCDIR}/mk/bulk/tflat ${SUPPORTSFILE} ${DEPENDSFILE} < ${DEPENDSTREEFILE} - -# note explicit pathname on "perl" above, so that we do NOT auto-set -# USE_TOOLS=perl (which may be undesirable for package building) - -# remove the bulk cache files -.PHONY: clean-bulk-cache -clean-bulk-cache: - ${RM} -f ${BULK_DBFILE} \ - ${DEPENDSTREEFILE} \ - ${DEPENDSFILE} \ - ${SUPPORTSFILE} \ - ${INDEXFILE} \ - ${ORDERFILE} - -# check if the $REF file is up to date, i.e. is newer than any of -# the pkg files; prints "1" if up to date, "0" if not. -.PHONY: bulk-check-uptodate -bulk-check-uptodate: - @uptodate=1; \ - if [ -f "${REF}" ]; then \ - if [ "${REF:T}" != "${PKGFILE:T}" ]; then \ - ${BULK_MSG} 1>&2 "${REF} is out of date (new version ${PKGNAME}); rebuilding..."; \ - uptodate=0; \ - elif [ "${USE_BULK_TIMESTAMPS}" = "yes" ]; then \ - ${SHCOMMENT} "Check files of this package"; \ - newfiles="`${FIND} . \( \( -name CVS -o -name ${WRKDIR_BASENAME} \) -prune \) -o \( -type f -newer "${REF}" ! \( -name README.html -o -name DESCR -o -name COMMENT -o -name '.*' \) \) -print || ${TRUE}`"; \ - nnewfiles="`echo $$newfiles | ${WC} -w`"; \ - if [ "$$nnewfiles" -gt 0 ]; then \ - ${BULK_MSG} 1>&2 "Package ${PKGNAME} ($$newfiles) modified since last 'make package' re-packaging..."; \ - uptodate=0; \ - else \ - ${BULK_MSG} 1>&2 "${REF} is up to date."; \ - fi; \ - else \ - ${BULK_MSG} 1>&2 "${REF} is up to date."; \ - fi; \ - else \ - ${BULK_MSG} 1>&2 "Package ${PKGNAME} not built yet, packaging..."; \ - uptodate=0; \ - fi; \ - if [ "$$uptodate" = "1" ]; then \ - ${SHCOMMENT} "Check required binary packages"; \ - (${DEPENDS:C/:.*$//:@d@${ECHO} ${d:Q};@} ${TRUE}) | \ - (while read dep; do \ - ${SHCOMMENT} "check against the binary pkg that pkg_add would pick, too:"; \ - ${SHCOMMENT} "(Only one should be returned here, really...)"; \ - pkg=`${PKG_ADMIN} lsbest "${PACKAGES}/All/$$dep"`; \ - if [ -z "$$pkg" ]; then \ - ${BULK_MSG} 1>&2 "Required binary package $$dep does not exist, rebuilding..."; \ - exit 1; \ - elif [ "${USE_BULK_TIMESTAMPS}" = "yes" ] && [ -n "`${FIND} $$pkg -prune -newer ${REF} -print`" ]; then \ - ${BULK_MSG} 1>&2 "Required binary package $$dep (`basename $$pkg`) is newer, rebuilding..."; \ - exit 1; \ - else \ - ${BULK_MSG} 1>&2 "Required binary package $$dep (`basename $$pkg`) is usable."; \ - fi; \ - done) || uptodate=0; \ - fi; \ - ${ECHO} $$uptodate - -# rebuild binpkg if any of the pkg files is newer than the binary archive -# set DO to ":" to not actually do anything (debugging, ...) -.PHONY: bulk-package -bulk-package: - @${_BULK_MKDIR} ${_BULK_PKGLOGDIR:Q} - @if [ -f ${_BROKENFILE:Q} ]; then \ - ${BULK_MSG} "Package ${PKGNAME} is already marked as broken."; \ - exit 1; \ - fi - ${RUN} \ - { ${ECHO} "###"; \ - ${ECHO} "### `${DATE}`"; \ - ${ECHO} "### pkgsrc build log for ${PKGNAME}"; \ - ${ECHO} "###"; \ - ${ECHO} ""; \ - } >> ${_BUILDLOG} - ${RUN} set +e; \ - uptodate=`${RECURSIVE_MAKE} ${MAKEFLAGS} bulk-check-uptodate REF=${PKGFILE}` ; \ - if [ $$uptodate = 1 ]; then \ - { ${BULK_MSG} "Nothing to be done for ${PKGNAME}." ; \ - } 2>&1 | ${TEE} -a ${_BUILDLOG:Q}; \ - else \ - ( if ${PKG_INFO} -qe ${PKGWILDCARD:Q}; then \ - oldpkg=`${PKG_INFO} -e ${PKGWILDCARD}`; \ - ${BULK_MSG} "Removing outdated (installed) package $$oldpkg first." ; \ - ${DO} ${PKG_DELETE} -r ${PKGWILDCARD:Q} ;\ - fi ; \ - if [ -f ${PKGFILE} ]; then \ - ${BULK_MSG} "Removing old binary package ${PKGFILE}." ; \ - ${DO} ${RM} -f ${PKGFILE} ; \ - for cat in ${CATEGORIES}; do \ - ${DO} ${RM} -f ${PACKAGES}/$$cat/${PKGNAME}${PKG_SUFX}; \ - done ;\ - fi; \ - ${BULK_MSG} "Bulk building ${PKGNAME}" ; \ - ${DO} ${RECURSIVE_MAKE} ${MAKEFLAGS} clean;\ - if [ "${PRECLEAN}" = "yes" ]; then \ - ${BULK_MSG} "Currently installed packages:"; \ - ${PKG_INFO} -e "*" | ${SED} -e "s,^,* ,"; \ - ${PHASE_MSG} "Removing installed packages which are not needed to build ${PKGNAME}" ; \ - if [ "${USE_BULK_CACHE}" = "yes" ]; then \ - for pkgname in `${PKG_INFO} -e \\*`; do \ - : "The package may have been deinstalled in between."; \ - ${PKG_INFO} -qe "$$pkgname" || continue; \ - pkgdir=`${PKG_INFO} -Q PKGPATH "$$pkgname"`; \ - [ "$$pkgdir" ] || { ${FAIL_MSG} "Empty PKGPATH for $$pkgname"; continue; }; \ - required=`${AWK} '$$1 == "${PKGPATH}" { for (i = 4; i <= NF; i++) { if ($$i == "'"$$pkgdir"'") { print $$i; } } }' ${DEPENDSFILE}`; \ - if true; then \ - if [ "$$required" ]; then \ - ${STEP_MSG} "Keeping dependency $$pkgname." ;\ - else \ - case "${BULK_PREREQ}" in \ - *$$pkgdir* ) \ - ${STEP_MSG} "Keeping BULK_PREREQ $$pkgname ($$pkgdir)" ;\ - ;; \ - * ) \ - ${STEP_MSG} "Deinstalling $$pkgname"; \ - ${DO} ${PKG_DELETE} -r $$pkgname || ${TRUE}; \ - ;; \ - esac ; \ - fi ;\ - fi ;\ - done; \ - else \ - ${STEP_MSG} "Deinstalling all packages"; \ - ${DO} ${PKG_DELETE} -r "*" || ${TRUE}; \ - fi ;\ - fi ;\ - if [ "${USE_BULK_CACHE}" = "yes" ]; then \ - ${SHCOMMENT} "Install required depends via binarypkgs XXX" ; \ - ${BULK_MSG} "Installing dependencies for ${PKGNAME}." ;\ - for pkgdir in `${SED} -n -e "/^${_ESCPKGPATH} / s;^[^:]*:;;p" ${DEPENDSFILE}` ${BULK_PREREQ} ; do \ - pkgname=`${AWK} '$$1 == "'"$$pkgdir"'" { print $$2; }' ${INDEXFILE}`; \ - if [ -z "$$pkgname" ]; then ${BULK_MSG} "WARNING: could not find package name for directory $$pkgdir"; continue ; fi ;\ - if [ -f "${BULKFILESDIR}/$$pkgdir/${NOT_AVAILABLE_FILE}" ]; then \ - ${ECHO} "The dependency $$pkgname ($$pkgdir) is not available." >> ${BULKFILESDIR}/${PKGPATH}/${NOT_AVAILABLE_FILE}; \ - exit 1; \ - fi; \ - pkgfile=${PACKAGES}/All/$${pkgname}${PKG_SUFX} ;\ - if ${PKG_INFO} -qe $$pkgname ; then \ - ${BULK_MSG} "Dependency $$pkgname is already installed" ; \ - else \ - if [ -f $$pkgfile ]; then \ - ${BULK_MSG} "Installing $$pkgfile"; \ - ${DO} ${PKG_ADD} ${PKG_ARGS_ADD} $$pkgfile || ${WARNING_MSG} "could not add $$pkgfile." ; \ - else \ - ${BULK_MSG} "warning: $$pkgfile does not exist. It will be rebuilt." ;\ - fi ;\ - fi ;\ - done ;\ - fi ;\ - ${ECHO_MSG} ${MAKE} package '(${PKGNAME})' 2>&1 ; \ - ${DO} ${RECURSIVE_MAKE} ${MAKEFLAGS} ${_PACKAGE_TARGET};\ - ${ECHO} ""; \ - ${ECHO} "===> Warnings from the wrapper log (sorted):"; \ - ${GREP} "^WARNING" ${WRKLOG} | ${SORT} -u | ${TO_HTML}; \ - ${ECHO} ""; \ - ) 2>&1 | ${TEE} -a ${_BUILDLOG:Q} ; \ - if [ -f ${PKGFILE} ]; then \ - case ${KEEP_BUILDLOGS} in \ - yes) ${MV} ${_BUILDLOG:Q} ${_BUILD_SUCCEEDED_FILE:Q}; \ - ;; \ - no) ${RM} ${_BUILDLOG:Q}; \ - ;; \ - esac; \ - else \ - { ${ECHO} "<html>"; \ - ${ECHO} "<head><title>pkgsrc build log for ${PKGNAME}</title></head>"; \ - ${ECHO} ""; \ - ${ECHO} "<body>"; \ - ${ECHO} "<pre>"; \ - ${TO_HTML} < ${_BUILDLOG:Q}; \ - ${MAKE} debug 2>&1 | ${TO_HTML}; \ - ${ECHO} "</pre>"; \ - } > ${_BROKENFILE:Q}; \ - ${RM} -f ${_BUILDLOG:Q}; \ - if [ -f "${WRKLOG}" ]; then \ - ${CP} ${WRKLOG:Q} ${_BROKENWRKLOG:Q}; \ - fi; \ - ( \ - if [ -f ${_BROKENWRKLOG:Q} ]; then \ - ${ECHO} "<p>"; \ - ${ECHO} "Please view the <a href=\"../../${PKGPATH}/${BROKENWRKLOG}\">work log for ${PKGNAME}</a>"; \ - ${ECHO} "</p>"; \ - fi >> ${_BROKENFILE:Q}; \ - ${ECHO} "<pre>" >> ${_BROKENFILE:Q}; \ - ${BULK_MSG} "${PKGNAME} was marked as broken:" ; \ - ${LS} -la ${_BROKENFILE:Q} ; \ - ${ECHO_MSG} ${MAKE} deinstall ; \ - ${DO} ${RECURSIVE_MAKE} ${MAKEFLAGS} deinstall ; \ - ${ECHO} "</pre>" >> ${_BROKENFILE:Q}; \ - nbrokenby=0;\ - if [ "${USE_BULK_CACHE}" = "yes" ]; then \ - ${BULK_MSG} "Marking all packages which depend upon ${PKGNAME} as broken:"; \ - tmp=`${SED} -n -e "/^${_ESCPKGPATH} / s;^[^:]*:[ ]*;;p" ${SUPPORTSFILE}` ; \ - if test -n "$$tmp" ; then \ - ${ECHO} "<ul>"; \ - for pkgdir in $$tmp ; do \ - pkg_brokendir=${BULKFILESDIR:Q}/"$$pkgdir"; \ - pkg_brokenfile="$$pkg_brokendir"/${BROKENFILE:Q}; \ - pkgname=`${AWK} '$$1 == "'"$$pkgdir"'" { print $$2; }' ${INDEXFILE}`; \ - case $$pkgname in \ - "") pkgname="unknown"; \ - ${BULK_MSG} "WARNING: unknown pkgname in $$pkgdir.";; \ - esac; \ - ${ECHO} "<li>$$pkgname ($$pkgdir)</li>";\ - pkgerr='-1'; pkgignore=''; pkgskip=''; \ - if [ "${USE_BULK_BROKEN_CHECK}" = 'yes' ]; then \ - pkgignore=`(cd ${PKGSRCDIR}/$$pkgdir && ${RECURSIVE_MAKE} ${MAKEFLAGS} show-var VARNAME=PKG_FAIL_REASON)`; \ - pkgskip=`(cd ${PKGSRCDIR}/$$pkgdir && ${RECURSIVE_MAKE} ${MAKEFLAGS} show-var VARNAME=PKG_SKIP_REASON)`; \ - fi; \ - if [ ! -z "$${pkgignore}$${pkgskip}" -a ! -f "$${pkg_brokenfile}" ]; then \ - { ${BULK_MSG} "$$pkgname ($$pkgdir) may not be packaged because:"; \ - ${BULK_MSG} "$$pkgignore"; \ - ${BULK_MSG} "$$pkgskip"; \ - } >> "$${pkg_brokenfile}"; \ - if [ "${USE_BULK_BROKEN_CHECK}" != 'yes' ] || [ -z "`(cd ${PKGSRCDIR}/$$pkgdir && ${RECURSIVE_MAKE} ${MAKEFLAGS} show-var VARNAME=BROKEN)`" ]; then \ - pkgerr="0"; \ - else \ - pkgerr="1"; \ - fi; \ - fi; \ - ${_BULK_MKDIR} "$${pkg_brokendir}"; \ - { ${ECHO} "$$pkgname ($$pkgdir) is broken because it depends upon ${PKGNAME} (${PKGPATH}) which is broken."; \ - ${ECHO} "Please view the <a href=\"../../${PKGPATH}/${BROKENFILE}\">build log for ${PKGNAME}</a>.<br />"; \ - } >> "$${pkg_brokenfile}"; \ - nbrokenby=`expr $$nbrokenby + 1`;\ - if ${GREP} -q " $$pkgdir/${BROKENFILE}" ${BULKFILESDIR:Q}/${BROKENFILE:Q} ; then :; \ - else \ - ${ECHO} " $$pkgerr $$pkgdir/${BROKENFILE} 0 " >> ${BULKFILESDIR:Q}/${BROKENFILE:Q} ;\ - fi ;\ - done ;\ - ${ECHO} "</ul>"; \ - fi ;\ - fi ;\ - nerrors=`${GREP} -c '^\*\*\* Error code' ${_BROKENFILE:Q} || true`; \ - ${ECHO} " $$nerrors ${PKGPATH}/${BROKENFILE} $$nbrokenby " >> ${BULKFILESDIR:Q}/${BROKENFILE:Q} \ - ) 2>&1 | ${TEE} -a ${_BROKENFILE:Q}; \ - fi ; \ - if [ ${_PRESERVE_WRKDIR} = "pack-if-failed" ] && [ ! -f ${PKGFILE} ]; then \ - ${BULK_MSG} "Making an archive from the working directory."; \ - (cd ${WRKDIR} \ - && ${PAX} -wz -f ${_BULK_PKGLOGDIR}/wrkdir.tar.gz . \ - ) || ${TRUE}; \ - fi; \ - case ${_PRESERVE_WRKDIR} in \ - yes|YES) ;; \ - *) ${DO} ${RECURSIVE_MAKE} ${MAKEFLAGS} clean;; \ - esac; \ - fi - @if [ ! -f ${PKGFILE} ]; then \ - ${BULK_MSG} "Build for ${PKGNAME} was not successful, aborting." | ${TEE} -a ${_BROKENFILE:Q} ; \ - ${ECHO} "<pre>" >> ${_BROKENFILE:Q}; \ - { ${ECHO} "</pre>"; \ - ${ECHO} "</body>"; \ - ${ECHO} "</html>"; \ - } >> ${_BROKENFILE:Q}; \ - exitcode=1; \ - else \ - case ${KEEP_BUILDLOGS} in \ - yes) ${TEST} ! -f ${_BUILDLOG:Q} \ - || ${MV} ${_BUILDLOG:Q} ${_BUILD_SUCCEEDED_FILE:Q}; \ - ;; \ - no) ${RM} -f ${_BUILDLOG:Q}; \ - ;; \ - esac; \ - exitcode=0; \ - fi; \ - exit $$exitcode -.if ${BULKFILESDIR} != ${PKGSRCDIR} - ${RUN} \ - ${RMDIR} ${_BULK_PKGLOGDIR:Q} 2>/dev/null 1>&2 || ${TRUE} -.endif - -# Install pkg - if possible from binary pkg (i.e. available & up-to-date) -# else revert to the old recompiling. -# Don't rely upon pkg_add to snarf in depending pkgs as they may have -# been modified and need rebuilding. -.PHONY: bulk-install -bulk-install: - ${RUN} if ${PKG_INFO} -qe ${PKGNAME} ; then exit 0; fi; \ - if [ `${RECURSIVE_MAKE} ${MAKEFLAGS} bulk-check-uptodate REF=${PKGFILE}` = 1 ]; then \ - if ${PKG_INFO} -qe ${PKGNAME} ; then :; \ - else \ - ${DO} ${RECURSIVE_MAKE} ${MAKEFLAGS} install-depends ; \ - ${BULK_MSG} ${PKG_ADD} ${PKG_ARGS_ADD} ${PKGFILE} ; \ - ${DO} ${PKG_ADD} ${PKG_ARGS_ADD} ${PKGFILE} ; \ - fi ; \ - else \ - ${ECHO_MSG} ${MAKE} bulk-package PRECLEAN=no; \ - ${DO} ${RECURSIVE_MAKE} ${MAKEFLAGS} bulk-package PRECLEAN=no; \ - fi - -bulk-info: .PHONY - @${ECHO} pkgname ${PKGPATH} ${PKGNAME} - @:; ${DEPENDS:@d@${ECHO} ${PKGPATH} ${d:Q}@; } diff --git a/mk/bulk/build b/mk/bulk/build deleted file mode 100644 index f9a2f4430ee..00000000000 --- a/mk/bulk/build +++ /dev/null @@ -1,505 +0,0 @@ -#!/bin/sh -# $NetBSD: build,v 1.106 2009/07/09 12:46:49 he Exp $ - -# -# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org> -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. All advertising materials mentioning features or use of this software -# must display the following acknowledgement: -# This product includes software developed by Hubert Feyrer for -# the NetBSD Project. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -## -## Globals -## -scriptdir=`dirname "$0"` -scriptdir=`cd "${scriptdir}" && pwd` - -# -# Default values for command line options. -# -resume=no -mirror_only=no -target=bulk-package -makeargs="" -noemail=no -post_only=no -prepare_only=no - -## -## Functions -## - -usage () { - cat <<EOF -usage: $0 [options] - $0 -h | --help - -Runs a bulk pkgsrc build. - -The following options are supported: - - -c | --config <file> - Load the following configuration file instead of the default - one. - - -e | --no-email - Don't send email when the bulk build is finished. - - -h | --help - Displays this message. - - -m | --mirror_only - Downloads all distfiles needed for the build but does not run - the build. IMPORTANT: Note that this will still run all the - pre-build stuff which involves removing all of your installed - packages. - - The only difference between this option and a regular bulk build - is that the packages are not actually built. - - -p | --post-build - Run the post-build processing and generate the report only. - - --prepare - Only generate the package database that will be used when bulk - building the packages. - - -r | --resume - Resume a previously interrupted bulk build. - - The --resume option may be combined with the --mirror_only - option. - - -s | --specific-pkgs - Sets SPECIFIC_PKGS=1 when building packages. This option is - used for building a subset of pkgsrc. - -EOF -} - -# print out error message and exit 1 -die () { - echo "$0: error:" 1>&2 - for i in "$@"; do - echo " $i" 1>&2 - done - exit 1 -} - -# This function can be overridden in the build.conf file to change the -# output format of the bulk build. It is used in a pipe, so if you want -# the original output, just define post_filter_cmd() { cat; }. -# -# For more sophisticated output, you may use all the variables that this -# example function uses. -post_filter_cmd() { - ${SED} "s;^;`date '+%Y/%m/%d %H:%M:%S'` ${percent} ${pkgdir} @ ${MACHINE_ARCH}> ;g" -} - -# perform post-processing of the bulk-build results -do_post_build () { - echo "build> Post processing bulk build results..." - - # Re-install BULK_PREREQ as we may need functionality (e.g. SMTP) provided by - # them for post-build to run. - echo "build> Re-installing prerequisite packages specified with BULK_PREREQ..." - for pkgdir in $BULK_PREREQ lang/perl5; do - echo "build> Installing prerequisite package $pkgdir" - ( cd "${pkgsrc_dir}/${pkgdir}" \ - && ${BMAKE} bulk-install - ) || die "Failed to install prerequisite packages." - done - - # - # Generate the post-build report. - # - echo "build> Generating the bulk build report..." - - bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \ - || die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}." - - report_dir="${REPORTS_DIR}/${bulk_build_id}" - ${MKDIR} "${report_dir}" - ( cd "${pkgsrc_dir}" \ - && ${PERL5} mk/bulk/post-build \ - > "${report_dir}/${REPORT_TXT_FILE}" - ) || die "Could not write the results file." -} - -# Notify the ADMIN of the finished build. -do_email () { - case $noemail in - no) cat "${report_dir}/${REPORT_TXT_FILE}" \ - | ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results ${bulk_build_id}" "$ADMIN" - esac -} - -# output final note that we're done -do_done () { - echo "" - echo "build> Bulk build ended: `date`" -} - -# set all commonly used variables, prepare files etc. -do_common_setup () { - - # - # Choose an appropriate value for BMAKE depending on the operating - # system. - # - opsys=`uname -s` - case "$opsys" in - NetBSD) BMAKE=make ;; - *) BMAKE=bmake ;; - esac - export BMAKE - - # - # Set resource limits as high as possible - # - ulimit -S -s `ulimit -H -s` # XXX: why do we need unlimited stack? - ulimit -S -d `ulimit -H -d` # some processes grow rather large - ulimit -S -t `ulimit -H -t` # pkgsrc bulk builds need _much_ time - - # - # Find the configuration file. - # - BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}" - case $BULK_BUILD_CONF in - /*) ;; - *) BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}" - esac - - # - # Load the variables from the configuration file. - # - { test -f "${BULK_BUILD_CONF}" \ - && . "${BULK_BUILD_CONF}" \ - && . "${scriptdir}/post-build-conf" \ - && check_config_vars \ - && export_config_vars - } || die "Cannot load config file ${BULK_BUILD_CONF}, aborting." - - # - # Set the paths to commonly used directories. - # - pkgsrc_dir="${USR_PKGSRC}" - lintpkgsrc_dir="${USR_PKGSRC}/pkgtools/lintpkgsrc" - - # - # Set up variables specific for the bulk build. - # - BATCH="1" - DEPENDS_TARGET="bulk-install" - export BATCH DEPENDS_TARGET - - # - # Unset some environment variables that could disturbe the build. - # - unset CDPATH || true # ensure cd does not print new cwd to stdout, which - # confuses the printindex script. - unset DISPLAY || true # allow sane failure for gimp, xlispstat -} - -# Check that the package tools are up to date. -check_tools () { - echo "build> Checking if the pkgtools are up-to-date" - ( cd "${lintpkgsrc_dir}" \ - && ${BMAKE} fetch >/dev/null 2>&1 - ) || { - echo "build> Updating pkgtools" - ( cd "${pkgsrc_dir}/pkgtools/pkg_install" \ - && ${BMAKE} clean \ - && ${BMAKE} update - ) || die "Could not update the package tools." - } -} - -# Run the pre-build script if necessary. -run_pre_build () { - case $resume in - yes) echo "build> Resuming -- skipping pre-build script";; - *) # make veryveryclean :) - ( cd "${pkgsrc_dir}" \ - && /bin/sh mk/bulk/pre-build - ) || die "Error during bulk-build preparations, aborting.";; - esac -} - -# Load pkgsrc variables that affect the build process. -load_vars () { - echo "+----------------------------------------+" - echo "| Some variables used in the bulk build: |" - echo "+----------------------------------------+" - - vars=" OPSYS OS_VERSION MACHINE_ARCH - BULK_PREREQ - BULKFILESDIR - BULK_DBFILE DEPENDSFILE INDEXFILE ORDERFILE STARTFILE - SUPPORTSFILE BULK_BUILD_ID_FILE BUILDLOG BROKENFILE - BROKENWRKLOG - PREFIX - AWK GREP MAIL_CMD MKDIR PAX PERL5 SED - PKG_DELETE PKG_INFO PKGBASE" - - values=`cd "$lintpkgsrc_dir" && $BMAKE show-vars VARNAMES="$vars" USE_TOOLS="awk grep mail mkdir pax perl sed"` - - for v in $vars; do - eval "read $v" || die "Could not read value for $v" - eval "value=\$$v" - if [ "$v" != "BULK_PREREQ" ] && [ ! "$value" ]; then - die "$v must not be empty." - fi - printf "%-15s = %s\\n" "$v" "$value" - done <<EOF -$values -EOF - echo "------------------------------------------" - [ "$PKGBASE" = "lintpkgsrc" ] \ - || die "Error reading the variables." \ - "Try running $0 with another shell." - - # Get the location of commonly used files - main_buildlog="${BULKFILESDIR}/${BUILDLOG}" -} - -# Install prerequisite packages. -# -# Note: we do this _before_ the depends tree because some packages like -# xpkgwedge only become DEPENDS if they are installed. -install_prereqs () { - echo "build> Installing prerequisite packages specified with BULK_PREREQ..." - for pkgdir in $BULK_PREREQ; do - echo "build> Installing prerequisite package $pkgdir" - ( cd "${pkgsrc_dir}/${pkgdir}" \ - && ${BMAKE} bulk-install - ) || die "Could not install prerequisite packages." - done -} - -# Everything is prepared. We can start building the real packages now. -# -# Loop over every package in the correct order. Before building -# each one, check to see if we've already processed this package -# before. This could happen if the build got interrupted and we -# started it again with the '-resume' option. This prevents us -# from having to do a potentially very large number of make's to -# get back to where we let off. After we build each package, add -# it to the top level buildlog -# (usually '.make' or '.make.${MACHINE}'). As a side benefit, this -# can make a progress-meter very simple to add! -do_real_bulk_build () { - - cd "${pkgsrc_dir}" || die "The pkgsrc directory does not exist." - - echo "build> Starting actual build using the order specified in $ORDERFILE..." - - # make sure we have something to grep in in the build loop - touch "${main_buildlog}" || die "Cannot write to ${main_buildlog}." - - tot=`${AWK} 'END { print NR }' "${ORDERFILE}"` - for pkgdir in `cat "${ORDERFILE}"` - do - if ${GREP} -q "^${pkgdir}\$" "${main_buildlog}"; then - : "skip this package" - else - percent=`${AWK} -v tot="${tot}" ' - END { - printf("%d/%d=%4.1f%%", NR, tot, NR*100/tot); - }' "${main_buildlog}"` - ( cd "${pkgsrc_dir}/${pkgdir}" \ - && ${NICE_LEVEL} ${BMAKE} USE_BULK_CACHE=yes "${target}" \ - $makeargs </dev/null | post_filter_cmd - ) || true - echo "$pkgdir" >> "${main_buildlog}" - check_pkg_dirs - fi - done - - echo "build> Build finished." -} - -check_pkg_dirs () { - for d in bin etc include info lib libexec man sbin share; do - if [ -f $PREFIX/$d ]; then - echo "Removing file, should be dir: $PREFIX/$d" >&2 - rm -f $PREFIX/$d - fi - done -} - -# clean up installed packages left over -do_bulk_cleanup () { - - echo "build> Removing all installed packages left over from build..." - for pkgname in `${PKG_INFO} -e \*` - do - if ${PKG_INFO} -qe "${pkgname}"; then - pkgdir=`${AWK} '$2 == "'"$pkgname"'" { print $1; }' "$INDEXFILE"` - case "${BULK_PREREQ}" in - *"${pkgdir}"* ) - echo "build> Keeping BULK_PREREQ: $pkgname ($pkgdir)" ; - ;; - * ) - echo "build> ${PKG_DELETE} -r ${pkgname}" - ${PKG_DELETE} -r "${pkgname}" - if ${PKG_INFO} -qe "${pkgname}"; then - echo "build> $pkgname ($pkgdir) did not deinstall nicely. Forcing the deinstall" - ${PKG_DELETE} -f "${pkgname}" || true - fi - ;; - esac - fi - done -} - -# start the full bulk-build -do_bulk_build () { - echo "build> Bulk build started: `date`" - echo "" - - # this function from post-build-conf - show_config_vars - - check_tools - run_pre_build - load_vars - - # - # Create the directory for the log files if necessary - # - if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then - ${MKDIR} "${BULKFILESDIR}" - fi - - # - # Save the bulk build ID in a file, as it most often contains a time - # stamp. - # - case $resume in - no) echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \ - || die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";; - esac - - install_prereqs - - # - # Create the bulk cache files. - # - if [ "x$resume" != "xyes" ]; then - ( cd "${pkgsrc_dir}" \ - && env PKGLIST="${PKGLIST-}" ${BMAKE} bulk-cache $makeargs - ) || die "Could not create the bulk build cache." - else - if [ ! -f "${ORDERFILE}" ]; then - die "The ${ORDERFILE} does not exist." \ - "(You cannot resume a bulk build that has not yet started.)" - fi - fi - - # XXX: This looks like a hack, and indeed, the functions in this file - # should be reorganized to better reflect the phases of the bulk build. - if [ $prepare_only = yes ]; then - exit 0 - fi - - do_real_bulk_build - do_bulk_cleanup -} - - -## -## main -## - -# -# Parse the command line. -# -while test $# -gt 0; do - case $1 in - -c|--config) - shift - BULK_BUILD_CONF=$1; shift - ;; - -e|--no-email) - noemail=yes - shift - ;; - -h|--help) - usage - exit 0 - ;; - -m|--mirror_only) - mirror_only=yes - target=mirror-distfiles - shift - ;; - -p|--post-build) - post_only=yes - shift - ;; - --prepare) - prepare_only=yes - shift - ;; - -r|--resume|--restart|restart) - resume=yes - shift - ;; - -s|--specific-pkgs) - makeargs="$makeargs SPECIFIC_PKGS=1" - shift - ;; - *) - echo "unknown option: $1" 1>&2 - usage 1>&2 - exit 1 - ;; - esac -done - -do_common_setup - -if [ "x$post_only" = "xyes" ]; then - load_vars - do_post_build - exit 0 -fi - -do_bulk_build - -# for now, just quit if we were only mirroring distfiles. At somepoint we -# should teach the post-build script to generate a nice report about how many -# distfiles were downloaded, how many had bad checksums, failed master sites, -# network speed, etc. -if [ "x$mirror_only" = "xyes" ]; then - echo "build> Bulk mirror of distfiles completed: `date`" - exit 0 -fi - -do_post_build -do_email -do_done - diff --git a/mk/bulk/build.conf-example b/mk/bulk/build.conf-example deleted file mode 100644 index 1581ccf2a1e..00000000000 --- a/mk/bulk/build.conf-example +++ /dev/null @@ -1,106 +0,0 @@ -# $NetBSD: build.conf-example,v 1.39 2011/04/22 09:03:52 spz Exp $ -# -# This is an example configuration file for pkgsrc bulk builds. -# Actually it's a shell script that is sourced in by the pre-build, -# build and post-build programs. -# - -# -# System information -# - -# The OS release this bulk build is happening on. -osrev=`uname -r` - -# Where our pkgsrc is located. -USR_PKGSRC="/usr/pkgsrc" - -# An absolute pathname to your mk.conf file. -# The file must be outside ${PREFIX}, or it will be overwritten. -#MAKECONF="$HOME/etc/mk.conf" - - -# -# Getting distfiles -# - -# Prune distfiles to remove those which are out of date. -# Can be set to 'yes' or 'no'. -PRUNEDISTFILES=no - -# Proxies (optional): -#ftp_proxy=http://proxy.machi.ne:3128/ -#http_proxy=http://proxy.machi.ne:3128/ - - -# -# Building the packages -# - -# List of package directories to build. -# Defaults to all packages if PKGLIST is empty or undefined. -#PKGLIST="www/mozilla meta-pkgs/kde3" - -# Nice level for builds -NICE_LEVEL="nice -n 20" - -# Whether to cache the lintpkgsrc database for PRUNE* or uploading. -# Can be set to 'yes' or 'no'. -LINTPKGSRC_CACHE=no - -# -# Generating the report -# - -# Where build will mail the report. -ADMIN="you@some.whe.re" - -# Who the report is signed by. -ADMINSIG="-Your Name" - -# The directory where the final reports are collected. -#REPORTS_DIR="$HOME/bulk-logs" -REPORTS_DIR="/home/ftp/pub/NetBSD/pkgstat" - -# The URL where the final reports will be available. -REPORTS_URL="ftp://localhost/pub/NetBSD/pkgstat" - -# The basename of the directory of the current bulk build. This -# directory will be created under ${REPORTS_DIR}. After uploading the -# report, it will also be available under ${REPORTS_URL}. -REPORT_BASEDIR=`date +%Y%m%d.%H%M` - -# The basenames of the the two report files, which will both be created -# in ${REPORTS_DIR}/${REPORT_BASEDIR}. -REPORT_HTML_FILE="report.html" -REPORT_TXT_FILE="report.txt" - -# -# Uploading binary packages -# - -# Optional flags, however these variables must be left defined (even if empty) to -# satisfy 'set -eu' in bulk scripts. -AUDIT_PACKAGES_FLAGS= -DOWNLOAD_VULNERABILITY_LIST_FLAGS= - -# Update vulnerability database before uploading. -# Can be set to 'yes' or 'no'. -UPDATE_VULNERABILITY_LIST=yes - -# Prune binary packages to remove those which are out of date. -# Can be set to 'yes' or 'no'. -PRUNEPACKAGES=yes - -# Whether or not to create checksum files. -MKSUMS=yes - -# Whether or not to create the summary file. -MKSUMMARY=no - -# If you want to PGP sign the checksum files, set and correct. -#SIGN_AS=username@NetBSD.org - -# Destination for packages and rsync options. -RSYNC_DST=ftp.NetBSD.org:/pub/pkgsrc/packages/NetBSD/SomeArch/${osrev}-201xQy/ -RSYNC_OPTS='-e ssh' diff --git a/mk/bulk/do-sandbox-build b/mk/bulk/do-sandbox-build deleted file mode 100644 index 31c2df1e2f7..00000000000 --- a/mk/bulk/do-sandbox-build +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -# $NetBSD: do-sandbox-build,v 1.12 2005/07/13 22:07:46 rillig Exp $ - -# -# Script to start a sandbox build -# -# See pkgsrc/doc/pkgsrc.txt for documentation! -# - -sh="/bin/sh" -build="mk/bulk/build" - -chroot /usr/sandbox \ - $sh -c "cd /usr/pkgsrc && exec $sh $build \"\$@\"" $build "$@" diff --git a/mk/bulk/do-sandbox-upload b/mk/bulk/do-sandbox-upload deleted file mode 100644 index e2b0df344f3..00000000000 --- a/mk/bulk/do-sandbox-upload +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -# $NetBSD: do-sandbox-upload,v 1.5 2005/07/13 22:07:46 rillig Exp $ - -# -# Script to start a sandbox build -# -# See pkgsrc/doc/pkgsrc.txt for documentation! -# - -sh="/bin/sh" -upload="mk/bulk/upload" - -chroot /usr/sandbox \ - $sh -c "cd /usr/pkgsrc && exec $sh $upload \"\$@\"" $upload "$@" diff --git a/mk/bulk/environment.txt b/mk/bulk/environment.txt deleted file mode 100644 index de67e877ed8..00000000000 --- a/mk/bulk/environment.txt +++ /dev/null @@ -1,105 +0,0 @@ -# $NetBSD: environment.txt,v 1.5 2006/12/15 13:15:06 martti Exp $ -# - -===> Environment variables used by the bulk build scripts - -This file lists all(?) the environment and pkgsrc variables that -directly influence the bulk build process, grouped by the script in -which they are used. - -===> mk/bulk/build - - => Settable by the user - - BULK_BUILD_CONF: Pathname - The configuration file for the bulk build. - http_proxy: URL - The HTTP proxy used for fetching distfiles. - ftp_proxy: URL - The FTP proxy used for fetching distfiles. - USR_PKGSRC: Pathname - The location of the pkgsrc root directory. - - => User-defined in mk.conf - - BULK_DBFILE - DEPENDSTREEFILE - DEPENDSFILE - SUPPORTSFILE - INDEXFILE - ORDERFILE - BROKENFILE - BROKENWRKLOG - FORCEBROKENFILE - BUILDLOG - STARTFILE - - BULK_PREREQ - - => Set by the script - - BATCH=1 - Used by pkgsrc. - BMAKE={make,bmake} - The make(1) program used for building the packages. - DEPENDS_TARGET=bulk-install - Used by pkgsrc. - -### to be continued in line 191 - -===> mk/bulk/pre-build - - => User-definable - - BULK_BUILD_CONF: Pathname - PRUNEDISTFILES: Boolean := no - PRUNEPACKAGES: Boolean := yes - PRUNELINKS: Boolean := yes - UPDATE_VULNERABILITY_LIST: Boolean := yes - USR_PKGSRC: Pathname - - => User-defined in mk.conf - - BROKENFILE - BROKENWRKLOG - BUILDLOG - LINTPKGSRC_DB - STARTFILE - LOCALBASE - X11BASE - DISTDIR - PKG_DBDIR - DEPOTBASE - PKG_TOOLS_BIN - PACKAGES - - => Set by the script - - BROKENF: Filename - Filename where the build log is saved when the build fails. - BRKWRKLOG: Filename - Filename where the work log is saved when the build fails. - BLDLOG: Filename - Filename where ? is saved when the build fails. - LINTPKGSRC_DB: Filename - Filename of the lintpkgsrc database. - STARTFILE: Filename - Cookie file to determine whether the build has started. - LOCALBASE: Pathname - Like in pkgsrc. - X11BASE: Pathname - Like in pkgsrc. - DISTDIR: Pathname - Like in pkgsrc. - PKG_DBDIR: Pathname - Like in pkgsrc. - DEPOTBASE: Pathname - Like in pkgsrc. - PKG_TOOLS_BIN: Pathname - Like in pkgsrc. - PACKAGES: Pathname - Like in pkgsrc. - - => Files used by the script - - mk/bulk/pre-build.local: Shellscript diff --git a/mk/bulk/parallel.txt b/mk/bulk/parallel.txt deleted file mode 100644 index 301d1c7966b..00000000000 --- a/mk/bulk/parallel.txt +++ /dev/null @@ -1,207 +0,0 @@ -# $Id: parallel.txt,v 1.7 2006/12/15 12:46:24 martti Exp $ -# - -These are my (<dmcmahill>) thoughts on how one would want a parallel -bulk build to work. - - -==================================================================== -Single Machine Build Process -==================================================================== - -The current (as of 2003-03-16) bulk build system works in the -following manner: - -1) All installed packages are removed. - -2) Packages listed in the BULK_PREREQ variable are installed. This - must be done before step 3 as some packages (like xpkgwedge) can - affect the dependencies of other packages when installed. - -3) Each package directory is visited and its explicitly listed - dependencies are extracted and put in a 'dependstree' file. The - mk/bulk/tflat script is used to generate flattened dependencies - for all packages from this dependstree file in both the up and - down directions. The result is a file 'dependsfile' which has one - line per package that lists all build dependencies. Additionally, - a 'supportsfile' is created which has one line for each package - and lists all packages which depend upon the listed pacakge. - Finally, tsort(1) is applied to the 'dependstree' file to - determine the correct build order for the bulk build. The build - order is stored in a 'buildorder' file. This is all achieved via - the 'bulk-cache' top level target. By extracting dependencies in - this fashion, we avoid highly redundant recursive make calls. For - example, we no longer need to use a recursive make to find the - dependencies for libtool literally thousands and thousands of - times throughout the build. - -4) During the build, the 'buildorder' file is consulted to figure out - which package should be built next. Then to build the package, - the following steps are taken: - - a) Check for the existance of a '.broken' file in the package - directory. If this file exists, then the package is already - broken for some reason so move on to the next package. - - b) Remove all packages which are not needed to build the current - package. This dependency list is obtained from the 'dependsfile' - created in step 3 and the BULK_PREREQ variable. - - c) Install via pkg_add all packages which are needed to build the - current package. We are able to do this because we have been - building our packages in a bottom up order so all dependencies - should have been built. - - d) Build and package the package. - - e) If the package build fails, then we copy over the build log to - a .broken file and in addition, we consult the 'supportsfile' and - mark all packages which depend upon this one as broken by adding a - line to their .broken files (creating them if needed). By going - ahead and marking these packages as broken, we avoid wasting time - on them later. - - f) Append the package directory name to the top level pkgsrc - '.make' file to indicate that we have processed this package. - -5) Run the mk/bulk/post-build script to collect the summary and - generate html pages and the email we've all seen. - -==================================================================== -Single Machine Build Comments -==================================================================== - -There are several features of this approach that are worth mentioning -explicitly. - -1) Packages are built in the correct order. We don't want to rebuild - the gnome meta-pkg and then rebuild gnome-libs for example. - -2) Restarting the build is a cheap operation. Remember that this - build can take weeks or more. In fact the 1.6 build took nearly 6 - weeks on a sparc 20! If for some reason, the build needs to be - interrupted, it can be easily restarted because in step 4f we keep - track of what has been built in a file. The lines in the build - script which control this are: - - for pkgdir in `cat $ORDERFILE` ; do - if ! grep -q "^${pkgdir}\$" $BUILDLOG ; then - (cd $pkgdir && \ - nice -n 20 ${BMAKE} USE_BULK_CACHE=yes bulk-package) - fi - done - - In addition to storing the progress to disk, the bulk cache files - (the 'dependstreefile', 'dependsfile', 'supportsfile', and - 'orderfile') are stored on disk so they do not need to be - recreated if a build is stopped and then restarted. - -3) By leaving packages installed and only deleting the ones which are - not needed before each build, we reduce the amount of installing - and deinstalling needed during the build. For example, it is - quite common to build several packages in a row which all need GNU - make or perl. - -4) Using the 'supportsfile' to mark all packages which depend upon a - package which has just failed to build can greatly reduce the time - wasted on trying to build packages which known broken dependencies. - -==================================================================== -Parallel Build Thoughts -==================================================================== - -To exploit multiple machines in an attempt to reduce the build time, -many of the same ideas used in the single machine build can still be -used. My view of how a parallel build should work is detailed here. - -master == master machine. This machine is in charge of directing - the build and may or may not actively participate in it. - In addition, this machine might not be of the same - architecture or operating system as the slaves (unless it - is to be used as a slave as well). - -slave#x == slave machine #x. All slave machines are of the same - MACHINE_ARCH and have the same operating system and access - the same pkgsrc tree via NFS and access the same binary - packages directory. - - If the master machine is also to be used as a build - machine, then it is also considered a slave. - -Prior to starting the build, the master directs one of the slaves to -extract the dependency information per steps 1-3 in the single machine -case. - -The actually build should progress as follows: - -1) For each slave which needs a job, the master assigns a package to - build based on the rule that only packages that have had all their - dependencies built will be sent to slaves for compilation. - -2) When a slave finishes, the master either notes that the binary - package is now available for use as a depends _or_ notes failure - and marks all packages which depend upon it as broken as in step - 4e of the single machine build. - - -Each slave builds a package in the same way as it would in a single -machine build (steps 4a-d). - -==================================================================== -Important Parallel Build Considerations -==================================================================== - - -1) Security. Packages are installed as root prior to packaging. - -2) All state kept by the master should be stored to disk to - facilitate restarting a build. Remember this could take weeks so - we don't want to have to start over. - -3) The master needs to be able to monitor all slaves for signs of - life. I.e., if a slave machine is simply shut off, the master - should detect that it's no longer there and re-assign that slaves - current job. - -3a) The master must be able to distinguish between a slave failing to - compile a package due to the package failing vs a - network/power/disk/etc. failure. The former causes the package to - be marked as broken, the latter causes the slave to be marked as - broken. - -4) Ability to add and remove slaves from the cluster during a build. - Again, a build may take a long time so we want to add/remove - slaves while the build is in progress. - -==================================================================== -Additional Thoughts -==================================================================== - -This is mostly related to using slaves which are not on a local -network. - -- maybe a hook could be put in place which rsync's the binary package - tree between the binary package repository machine and the slave - machine before and after each package is built? - -- security - -- support for Kerberos? - -==================================================================== -Implementation Thoughts -==================================================================== - -- Can this all be written around using ssh to send out tasks? How do - we monitor slaves for signs of life? How do we indicate 'build - failed/build succeeded/slave failed' conditions? - -- Maybe we could have a file listing slaves and the master consults - this each time it needs a slave. That would make adding/removing - slaves easy. There would need to be another file to keep track of - which slaves are busy (and with what). - -- Do we want to use something like pvm instead? There is a - p5-Parallel-Pvm package and perl nicely deals with parsing some of - these files and sorting dependencies although I hate to add any - extra dependencies to the build system. diff --git a/mk/bulk/post-build b/mk/bulk/post-build deleted file mode 100644 index 7007f670ac4..00000000000 --- a/mk/bulk/post-build +++ /dev/null @@ -1,722 +0,0 @@ -#!/usr/pkg/bin/perl -# $NetBSD: post-build,v 1.71 2008/06/13 21:52:16 sketch Exp $ -# -# Collect stuff after a pkg bulk build -# -# (c) 2000 Hubert Feyrer, All Rights Reserved. -# - -use File::Basename; -use POSIX qw(strftime); -use strict; -use warnings; - -# -# Global variables -# - -my %vars; -my $verbose = 1; # set to 2 to see more command execution detail - -# -# Helper functions -# - -sub pb_die($$) { - my ($fname, $msg) = @_; - my ($text, $sep); - - $text = "[post-build] error: "; - $sep = ""; - if (defined($fname)) { - $text .= "${sep}${fname}"; - $sep = ": "; - } - $text .= "${sep}${msg}"; - die "${text}\n"; -} - -sub my_system (@) { - print STDERR '> '.join(' ', @_)."\n" if ($verbose >= 2); - return system(@_); -} - -sub readfirstline($) { - my ($fname) = @_; - my ($contents); - - open(F, "<", $fname) - or pb_die($fname, "Cannot be read: $!"); - - defined($contents = <F>) - or pb_die($fname, "Must not be empty."); - chomp($contents); - - close(F) - or pb_die($fname, "Cannot be closed: $!"); - - return $contents; -} - -# -# Load configuration variables from the bulk.conf file, which is a shell -# script. -# - -my $BULK_BUILD_CONF = $ENV{BULK_BUILD_CONF} || (dirname($0).'/build.conf'); -$BULK_BUILD_CONF = "./$BULK_BUILD_CONF" if ($BULK_BUILD_CONF !~ m:^/:); - -if (!-f $BULK_BUILD_CONF) { - pb_die($BULK_BUILD_CONF, "Does not exist."); -} - -# Dig given variable out of config file, and set it -sub get_build_conf_vars(@) { - my (@varnames) = @_; - my ($is_set, $value); - - foreach my $varname (@varnames) { - my $cmd = join("\n", ( - #"set -eu", # TODO: Should be enabled soon - ". '${BULK_BUILD_CONF}'", - ". mk/bulk/post-build-conf", - "check_config_vars", - "echo \"\${${varname}+set}\"", - "echo \"\${${varname}-}\"" - )); - - open(CMD, "${cmd} |") - or pb_die($BULK_BUILD_CONF, "Could not evaluate configuration file."); - - chomp($is_set = <CMD>); - { local $/ = undef; $value = <CMD>; } - chomp($value); # This must be outside the above {...} block - - close(CMD) or pb_die($BULK_BUILD_CONF, "Could not evaluate configuration file (close)."); - - # - # Sanity checks - # - - if ($is_set ne "set") { - pb_die($BULK_BUILD_CONF, "${varname} must be set."); - } - if ($value =~ qr"^\s+$") { - pb_die($BULK_BUILD_CONF, "${varname} must be non-empty."); - } - if ($value =~ qr"\n") { - pb_die($BULK_BUILD_CONF, "${varname} must not contain newlines."); - } - - $vars{$varname} = $value; - if ($verbose >= 2) { - print STDERR "> $varname=$vars{$varname}\n"; - } - } -} - -get_build_conf_vars( - 'ADMINSIG', # "-Your Name" - 'REPORTS_URL', # "ftp://ftp.example.com/pub/pkgsrc/misc/pkgstat" - 'REPORTS_DIR', # "$HOME/bulk-logs" -# REPORT_BASEDIR often depends on a timestamp, which has been saved at -# the beginning of the bulk build in the BULK_BUILD_ID_FILE. It will be -# retrieved later. - 'REPORT_HTML_FILE', # "report.html" - 'REPORT_TXT_FILE', # "report.txt" - 'USR_PKGSRC', # "/usr/pkgsrc" - 'osrev', # `uname -r` -); - - -my $os = `uname -s`; -chomp $os; - -my $BMAKE = $ENV{BMAKE} || pb_die(undef, "The BMAKE environment variable must be set."); - -sub get_mk_conf_vars (@) { - my ($rest); - - open(I, "set -e; . '$BULK_BUILD_CONF'; . '$vars{USR_PKGSRC}/mk/bulk/post-build-conf'; export_config_vars; cd $vars{USR_PKGSRC}/pkgtools/lintpkgsrc && $BMAKE show-vars BATCH=1 VARNAMES='".join(' ', @_)."' USE_TOOLS=\"pax\" |") - or pb_die(undef, "Cannot get mk.conf variables."); - - foreach my $var (@_) { - chomp($vars{$var} = <I>); - - if ($vars{$var} eq "") { - pb_die(undef, "\${$var} must be defined in your mk.conf"); - } - - print STDERR "> $var=$vars{$var}\n" if ($verbose >= 2); - } - - { local $/ = undef; $rest = <I>; } - if (defined($rest) && $rest ne "") { - pb_die(undef, "Output of $BMAKE show-vars too long:\n${rest}"); - } - - close(I) or die pb_die(undef, "Cannot get mk.conf variables (close)."); -} - -# Extract the names of the files used for the build log and broken build logs. -# These have defaults set by bsd.bulk-pkg.mk and may be overridden in -# /etc/mk.conf -get_mk_conf_vars(qw( - BROKENFILE - BROKENWRKLOG - BULKFILESDIR - BULK_DBFILE - DEPENDSFILE - DEPENDSTREEFILE - FIND - INDEXFILE - LOCALBASE - MACHINE_ARCH - NOT_AVAILABLE_FILE - ORDERFILE - PAX - PKG_DBDIR - PKGSRC_COMPILER - STARTFILE - SUPPORTSFILE - X11BASE - PKG_TOOLS_BIN - BULK_BUILD_ID_FILE -)); - -my $bulk_dbfile_base = basename($vars{BULK_DBFILE}); -my $dependstreefile_base = basename($vars{DEPENDSTREEFILE}); -my $dependsfile_base = basename($vars{DEPENDSFILE}); -my $supportsfile_base = basename($vars{SUPPORTSFILE}); -my $indexfile_base = basename($vars{INDEXFILE}); -my $orderfile_base = basename($vars{ORDERFILE}); - -my $reports_url = $vars{"REPORTS_URL"}; -my $reports_dir = $vars{"REPORTS_DIR"}; -my $report_basedir = readfirstline($vars{"BULK_BUILD_ID_FILE"}); -my $report_html_file = $vars{"REPORT_HTML_FILE"}; -my $report_txt_file = $vars{"REPORT_TXT_FILE"}; - -my $report_url = "${reports_url}/${report_basedir}"; -my $report_dir = "${reports_dir}/${report_basedir}"; -my $report_html_path = "${report_dir}/${report_html_file}"; -my $report_txt_path = "${report_dir}/${report_txt_file}"; - -my $startdate = (stat($vars{STARTFILE}))[9]; -my $enddate = ''; -if (!defined($startdate) || $startdate == 0) { - $startdate = "unknown"; -} else { - local $ENV{TZ} = "UTC"; - $startdate = strftime("%c %Z", gmtime($startdate)); - $enddate = strftime("%c %Z", gmtime(time())); -} - -sub print_summary_line($$) { - my ($name, $value) = @_; - - printf(" %-30s %s\n", $name, $value); -} - -sub print_report_line($$$) { - my ($pkgpath, $breaks, $maintainer) = @_; - - printf("%-26s %-7s %s\n", $pkgpath, $breaks, $maintainer); -} - -sub print_report_header() { - print_report_line("Package", "Breaks", "Maintainer"); - print("--------------------------------------------------------------\n"); -} - -my_system("mkdir", "-p", "--", $report_dir); - -# Copy over the output from the build process -chdir($vars{"BULKFILESDIR"}) or pb_die($vars{"BULKFILESDIR"}, "Cannot change directory."); -my_system("find . -name $vars{BROKENFILE} -print -o -name $vars{BROKENWRKLOG} -print | $vars{PAX} -r -w -X ${report_dir}"); - -# Copy over the cache files used during the build -foreach my $f qw(BULK_DBFILE DEPENDSTREEFILE DEPENDSFILE SUPPORTSFILE INDEXFILE ORDERFILE) { - if (-f $vars{$f}) { - my_system("cp", "--", $vars{$f}, $report_dir); - } -} - -chdir($report_dir) or pb_die($report_dir, "Cannot change directory."); -writeReport(); - -# -# Adjust "last" symlink -# -unlink("${reports_dir}/last"); -symlink($report_basedir, "${reports_dir}/last"); - -# -# Generate leftovers-$vars{MACHINE_ARCH}.html: files not deleted -# Leftover files are copied to leftovers-$vars{MACHINE_ARCH} dir, -# and linked from leftovers-$vars{MACHINE_ARCH}.html -# -{ - chdir($report_dir) or pb_die($report_dir, "Cannot change directory."); - my_system("mkdir", "-p", "leftovers-$vars{MACHINE_ARCH}"); - - # Find files since last build: - my $leftovers_txt = "leftovers-$vars{MACHINE_ARCH}.txt"; - my $leftovers_html = "leftovers-$vars{MACHINE_ARCH}.html"; - - my_system("$vars{FIND} $vars{LOCALBASE}/ -newer $vars{STARTFILE} -type f -print >>$leftovers_txt"); - my_system("$vars{FIND} $vars{X11BASE}/ -newer $vars{STARTFILE} -type f -print >>$leftovers_txt"); - - # Strip perl-files: - my $perlfiles; - { - local $/; - undef $/; - $perlfiles = `$vars{PKG_TOOLS_BIN}/pkg_info -qL perl*`; - } - - my $perlfiles_pattern = $perlfiles; - $perlfiles_pattern =~ s/\n/|/g; - $perlfiles_pattern =~ s/|$//; - - open (LEFT, $leftovers_txt) or die "can't read $leftovers_txt: $!"; - my @left = <LEFT>; - close (LEFT); - my @leftovers = grep(!/^(?:${perlfiles_pattern})$/, @left); - - if (index($vars{PKG_DBDIR}, $vars{LOCALBASE}) == 0) { - # If PKG_DBDIR is inside LOCALBASE, exclude it from the leftovers. - @leftovers = grep { index($_, $vars{PKG_DBDIR}) != 0 } @leftovers; - } - - open (LEFT, ">$leftovers_txt") or die "can't write $leftovers_txt: $!"; - print LEFT @leftovers; - close (LEFT); - - if (scalar(@leftovers)) { - # Store leftovers, for easier identification: - my_system("$vars{PAX} -r -w -X leftovers-$vars{MACHINE_ARCH} < $leftovers_txt"); - } - - # Add links to leftover list: - open (OUT, "> $leftovers_html") - or die "can't write $leftovers_html"; - print OUT <<EOOUT; -<html> -<body> -<pre> -EOOUT - foreach (@leftovers) { - chomp; - print OUT "<a href=\"${report_url}/leftovers-$vars{MACHINE_ARCH}$_\">$_</a>\n"; - } - print OUT <<EOOUT2; -</pre> -</body> -</html> -EOOUT2 - close(OUT); -} - -# print the result of a single broken package -sub pkgResult ($$) { - my ($pinfo, $state) = @_; - my $pkg = $pinfo->{pkg}; - my $nbrokenby = $pinfo->{nbrokenby}; - my $nerrors = $pinfo->{nerrors}; - my $pkgdirmissing = 0; - my $DIR; - - if (not opendir($DIR, "$vars{USR_PKGSRC}/$pkg")) { - $pkgdirmissing=1; - } - else { - closedir($DIR); - } - - my @idents = ""; - if (not $pkgdirmissing) { - @idents = `$vars{FIND} $vars{USR_PKGSRC}/$pkg -type f -print | xargs grep \\\$NetBSD`; - } - my $datetime = ""; - my $file = ""; - my $ver = ""; - foreach my $ident (@idents) { - $ident =~ /\$[N]etBSD: ([^ ]*),v [^ ]* ([^ ]*) ([^ ]*) [^ ]* Exp \$/; - if (defined($2) && defined($3) && ("$2 $3" gt $datetime)) { - $datetime = "$2 $3"; - $file = $1; - $ver = $1; - } - } - - my $maintainer; - if (not $pkgdirmissing) { - $maintainer = `grep ^MAINTAINER $vars{USR_PKGSRC}/$pkg/Makefile`; - } else { - $maintainer = "directory_does_not_exist"; - } - $maintainer =~ s/MAINTAINER=[ \t]*//; - if (! $maintainer) { - $maintainer = `cd $vars{USR_PKGSRC}/$pkg ; $BMAKE show-var VARNAME=MAINTAINER`; - } - $maintainer =~ s/</</g; - $maintainer =~ s/>/>/g; - chomp($maintainer); - - (my $state_style = $state) =~ s/ //g; - - my $nbrokenby_html = '<td> </td>'; - $nbrokenby_html = - '<td align="right" class="pkg-'.$state_style.'">'.$nbrokenby.'</td>' - if $nbrokenby > 0; - - if ($pinfo->{nerrors} != 0 && $verbose && ($state eq "broken" || $state eq "topten")) { - print_report_line($pkg, $nbrokenby > 0 ? $nbrokenby : "", $maintainer); - } - - return <<EOHTML; -<tr> - <td><a class="pkg-$state_style" href="$pinfo->{bf}" title="build log for $pkg">$pkg</a></td> - $nbrokenby_html - <td>$file</td> - <td>$maintainer</td> -</tr> - -EOHTML -} - -# write the build report -sub writeReport { - my $broken = getBroken(); - my $nbroken = scalar(@{$broken->{"broken"}}); - my $nbrokendep = scalar(@{$broken->{"broken depends"}}); - my $nunpackaged = scalar(@{$broken->{"not packaged"}}); - my $nnot_available = scalar(@{$broken->{"not available"}}); - my $nbrokentot = $nbroken + $nbrokendep; - my $ntotal = $nunpackaged + $nbroken + $nbrokendep; - - # determine the number of packages attempted, and then successful - open(ORDER, $vars{ORDERFILE}) || die "can't open $vars{ORDERFILE}: $!"; - my @order = <ORDER>; - close(ORDER); - my $nattempted = scalar(@order); - my $nsuccessful = $nattempted - $ntotal; - - if ($verbose) { - print <<EOF; -pkgsrc bulk build results -$os $vars{osrev}/$vars{MACHINE_ARCH} -Compiler: $vars{PKGSRC_COMPILER} - -Summary: - -EOF - print_summary_line("Build started:", $startdate); - print_summary_line("Build ended:", $enddate); - print("\n"); - print_summary_line("Successfully packaged:", $nsuccessful); - print_summary_line("Packages really broken:", $nbroken); - print_summary_line("Pkgs broken due to them:", $nbrokendep); - print_summary_line("Total broken:", $nbrokentot); - print_summary_line("Not packaged:", $nunpackaged); - print_summary_line("Not available:", $nnot_available); - print_summary_line("Total:", $ntotal); - print <<EOF; - -Packages not listed here resulted in a binary package. The build -report, including logs of failed/not-packaged is available from: - -${report_url}/${report_html_file} -EOF - } - - open(HTML, ">", $report_html_path) or die "Can't write ${report_html_path}: $!"; - print HTML <<EOHTML; -<html> -<head> -<title>$os $vars{osrev}/$vars{MACHINE_ARCH} bulk package build</title> -<style type="text/css"> -<!-- - -body { - Font-Family: Tahoma, Verdana, sans-serif; - Line-Height: 1.3em; - Text-Decoration: None; - Color: black; - Background-Color: white; - Border-Width: 0; -} - -table { - Border-Width: 0; -} - -table td { - Font-Family: Tahoma, Verdana, sans-serif; - line-height: 1em; -} - -a:link { - Color: #3535c5; -} - -a:visited { - Color: #700080; -} - -a:hover { - Color: #6565e5; - Text-Decoration: underline; -} - -tr { - Vertical-Align: top; -} - -td { - Vertical-Align: top; -} - -h1 { - Font-Size: 3.5ex; - Line-Height: 1em; - Color: #000066; -} - -h2 { - Font-Size: 2.5ex; - Line-Height: 1em; - Color: #660000; -} - -h3 { - Font-Size: 2ex; - Color: #660066; -} - -h4 { - Font-Size: 1.8ex; - Color: #006600; -} - -tt.filename { - Line-Height: 1.3em; - Color: #AA0000; -} - -.pkgname { - Font-Family: Arial, Helvetica, Courier, fixed; - Font-Style: Italic; - Text-Decoration: none; - Line-Height: 1.3em; -} - -.pkg-broken { - Color: red; -} - -.pkg-brokendepends { - Color: orange; -} - -.pkg-notpackaged { - Color: blue; -} ---> -</style> -</head> - -<body bgcolor="white" text="black" link="#3535c5" vlink="#700080" alink="#3535c5"> - -<a name="top"></a> -<h1>pkgsrc bulk build results</h1> -<h2>$os $vars{osrev}/$vars{MACHINE_ARCH}</h2> - -<h3>Summary</h3> - -<table> -<tr> - <td>Build started: <td align="right">$startdate</td> -</tr> -<tr> - <td>Build ended: <td align="right">$enddate</td> -</tr> -<tr> - <td> </td> <td> </td> -</tr> -<tr> - <td>Successfully packaged:</td> <td align="right">$nsuccessful</td> -</tr> -<tr class="pkg-broken"> - <td>Packages really broken:</td> <td align="right">$nbroken</td> -</tr> -<tr class="pkg-brokendepends"> - <td>Packages broken due to them:</td> <td align="right">$nbrokendep</td> -</tr> -<tr> - <td>Total broken:</td> <td align="right">$nbrokentot</td> -</tr> -<tr class="pkg-notpackaged"> - <td>Not packaged:</td> <td align="right">$nunpackaged</td> -</tr> -<tr class="pkg-notavailable"> - <td>Not available:</td> <td align="right">$nnot_available</td> -</tr> -<tr> - <td>Total:</td> <td align="right">$ntotal</td> -</tr> -</table> - -<p> - Packages not listed here resulted in a <a - href="../../packages/" title="binary packages for $os $vars{osrev}/$vars{MACHINE_ARCH}">binary - package</a>. Results of failed packages are available below. -</p> - -<p> - Files leftover from the build (because of broken PLISTs, etc.) can be - found in <a href="leftovers-$vars{MACHINE_ARCH}.html" title="leftover files">this - list</a>. -</p> - -<p> - Jump to:<br/> - <ul> - <li><a href="#topten">Top Ten Offenders</a></li> - <li><a href="#broken">Broken packages</a></li> - <li><a href="#broken depends">Broken dependencies</a></li> - <li><a href="#not packaged">Not packaged</a></li> - <li><a href="#not available">Not available</a></li> - </ul> -</p> - -EOHTML - - my %state_head = ( - "topten" => "Top Ten Offenders", - "broken" => "Broken packages", - "broken depends" => "Broken dependencies", - "not packaged" => "Not packaged", - "not available" => "Not available", - ); - - foreach my $state ("topten", "broken", "broken depends", "not packaged", "not available") { - next unless scalar(@{$broken->{$state}}); - - if ($verbose && ($state eq "topten" || $state eq "broken")) { - print "\n\n$state_head{$state}\n\n"; - print_report_header(); - } - - - print HTML <<EOHTML; - -<a name="$state"></a> -<h2>$state_head{$state}</h2> -<table width="100%"> -<tr align="left"> - <th width="30%">Package</th> - <th>Breaks</th> - <th>File touched last</th> - <th>Maintainer</th> -</tr> - -EOHTML - foreach my $pinfo (@{$broken->{$state}}) { - print HTML pkgResult($pinfo, $state); - } - - print HTML <<EOHTML; -</table> -<hr> -<a href="#top">Up to top</a><br/> -<hr> -EOHTML - } - - print HTML <<EOHTML; -<hr> -<p> -The following cache files were used during the build: -</p> -<ul> -<li>The <a href="$bulk_dbfile_base">SPECIFIC_PKGS bulk database file</a>.</li> -<li>The <a href="$dependstreefile_base">depends tree file</a>.</li> -<li>The <a href="$dependsfile_base">depends file</a>.</li> -<li>The <a href="$supportsfile_base">supports file</a>.</li> -<li>The <a href="$indexfile_base">index file</a>.</li> -<li>The <a href="$orderfile_base">build order file</a>.</li> -</ul> -<hr> - -<p> -<ul> -<!-- <li>See the list of <a href="../index.html">all log files</a>. --> -<li>Visit the <a href="http://www.NetBSD.org">NetBSD web site</a>. -<li>Learn more about - <a href="http://www.NetBSD.org/docs/software/packages.html"> - The NetBSD Packages Collection</a>. -</ul> -</p> -</body> -</html> -EOHTML - close(HTML); - - if ($verbose) { - print "\n\n$vars{ADMINSIG}\n\n"; - print "[* This message was created by the Packages Collection bulk build software *]\n"; - } -} - -# get and sort the broken packages -sub getBroken { - my $res = { - 'broken' => [], - 'broken depends' => [], - 'not packaged' => [], - 'topten' => [], - "not available" => [], - }; - - open (BF, $vars{BROKENFILE}) || return $res; - my @in = <BF>; - close (BF); - - foreach (@in) { - chomp; - my ($nerrors, $bf, $nbrokenby) = split; - my $pkg = $bf; - $pkg =~ s,/$vars{BROKENFILE},,; - my $tmp = { - bf => $bf, - pkg => $pkg, - nbrokenby => $nbrokenby, - nerrors => $nerrors, - }; - - if (-f "$vars{BULKFILESDIR}/$pkg/$vars{NOT_AVAILABLE_FILE}") { - push(@{$res->{"not available"}}, $tmp); - } elsif ($nerrors > 0) { - push(@{$res->{"broken"}}, $tmp); - } elsif ($nerrors == -1) { - push(@{$res->{"broken depends"}}, $tmp); - } else { - push(@{$res->{"not packaged"}}, $tmp); - } - } - - # sort pkgs in each state - foreach my $state ("broken", "broken depends", "not packaged", "not available") { - $res->{$state} = [ sort { $a->{pkg} cmp $b->{pkg} } @{$res->{$state}} ]; - } - - $res->{"topten"} = [ sort { $b->{nbrokenby} <=> $a->{nbrokenby} } @{$res->{"broken"}} ]; - - for (my $count = $#{$res->{"topten"}}; $count >= 10; $count--) { - pop(@{$res->{"topten"}}); - } - - return $res; -} diff --git a/mk/bulk/post-build-conf b/mk/bulk/post-build-conf deleted file mode 100644 index 05578bfea6a..00000000000 --- a/mk/bulk/post-build-conf +++ /dev/null @@ -1,203 +0,0 @@ -# $NetBSD: post-build-conf,v 1.14 2007/03/06 10:57:11 rillig Exp $ -# - -# This file is included after the build.conf file by the "build" and -# "pre-build" scripts. It provides functions for printing, checking and -# exporting the configuration variables. - -# Note: All functions whose names start with "pbc_" are considered private -# to this file. The "pbc" prefix is an abbreviation for "post-build-conf". - -# usage: pbc_showvar varname -pbc_showvar() { - eval "pbc_isset=\${$1+set}" - case $pbc_isset in - "set") eval "fnv_val=\${$1-}" - printf " %-25s = %s\\n" "$1" "${fnv_val}" - ;; - *) printf " %-25s (undefined)\\n" "$1" - ;; - esac -} - -# usage: pbc_section title varname... -pbc_section() { - printf "%s\\n" "$1" - shift - for pbc_var in "$@"; do - pbc_showvar "${pbc_var}" - done - printf "\\n" -} - -# usage: show_config_vars -show_config_vars() { - pbc_section "System information" \ - osrev BULK_BUILD_CONF USR_PKGSRC MAKECONF - pbc_section "Getting distfiles" \ - PRUNEDISTFILES ftp_proxy http_proxy - pbc_section "Building the packages" \ - PKGLIST NICE_LEVEL - pbc_section "Generating the report" \ - ADMIN ADMINSIG \ - REPORTS_URL REPORTS_DIR \ - REPORT_BASEDIR REPORT_HTML_FILE REPORT_TXT_FILE - pbc_section "Uploading binary packages" \ - UPDATE_VULNERABILITY_LIST PRUNEPACKAGES MKSUMS SIGN_AS \ - RSYNC_DST RSYNC_OPTS -} - -# usage: export_config_vars -export_config_vars() { - export osrev BULK_BUILD_CONF USR_PKGSRC - export PRUNEDISTFILES ftp_proxy http_proxy - export PKGLIST NICE_LEVEL ADMIN ADMINSIG - export UPDATE_VULNERABILITY_LIST PRUNEPACKAGES MKSUMS SIGN_AS - export RSYNC_DST RSYNC_OPTS - case ${MAKECONF+set} in - "set") export MAKECONF;; - esac -} - -# usage: pbc_die error-message -pbc_die() { - exec 1>&2 - printf "error: %s\\n" "$@" - printf " Please check your bulk build configuration file:\\n" - case ${BULK_BUILD_CONF+set} in - "set") printf " %s\\n" "${BULK_BUILD_CONF}" - esac - exit 1 -} - -# usage: pbc_checkyesno varname -pbc_checkyesno() { - eval "pbc_val=\${$1-}" - case $pbc_val in - [Yy][Ee][Ss]|[Nn][Oo]) ;; - *) pbc_die "$1 must be set to one of YES, yes, NO, no.";; - esac -} - -# usage: pbc_checkpathname varname -pbc_checkpathname() { - eval "pbc_val=\${$1-}" - - case $pbc_val in - /*) ;; - *) pbc_die "$1 must be an absolute pathname.";; - esac -} - -# usage: pbc_checkbasename varname -pbc_checkbasename() { - eval "pbc_val=\${$1-}" - - case $pbc_val in - */*) pbc_die "$1 must not contain slashes.";; - esac -} - -# usage: pbc_checkexistingfile varname -pbc_checkexistingfile() { - - pbc_checkpathname "$1" - - eval "pbc_val=\${$1-}" - - test -f "${pbc_val}" \ - || pbc_die "$1 must be the name of an existing file." -} - -# usage: pbc_checkexistingdir varname -pbc_checkexistingdir() { - - pbc_checkpathname "$1" - - eval "pbc_val=\${$1-}" - - test -d "${pbc_val}" \ - || pbc_die "$1 must be the name of an existing directory." -} - -# usage: pbc_checkurl varname -pbc_checkurl() { - eval "pbc_val=\${$1-}" - - case $pbc_val in - file:///*|ftp://*|http://*) - ;; - *) pbc_die "$1 is not a valid URL.";; - esac -} - -# usage: pbc_checknonempty varname -pbc_checknonempty() { - eval "pbc_isset=\${$1+set}" - eval "pbc_val=\${$1-}" - - case $pbc_isset in - "set") case $pbc_val in - "") pbc_die "$1 must be non-empty.";; - esac;; - *) pbc_die "$1 must be defined and non-empty.";; - esac -} - -# usage: pbc_checkdefined varname -pbc_checkdefined() { - eval "pbc_val=\${$1+set}" - - case $pbc_val in - "set") ;; - *) pbc_die "$1 must be defined.";; - esac -} - -# usage: check_config_vars -check_config_vars() { - - case ${FTP+set},${REPORTS_DIR+set} in - set,set) - pbc_die "FTP and REPORTS_DIR must not be set both.";; - esac - - # section "System information" - pbc_checknonempty osrev - pbc_checkexistingfile BULK_BUILD_CONF - pbc_checkexistingdir USR_PKGSRC - case ${MAKECONF+set} in - "set") pbc_checkexistingfile MAKECONF;; - esac - - # section "Getting distfiles" - case ${PRUNEDISTFILES+set} in - "set") pbc_checkyesno PRUNEDISTFILES - esac - # no checks for ftp_proxy - # no checks for http_proxy - - # section "Building the packages" - # no checks for PKGLIST - # no checks for NICE_LEVEL - - # section "Generating the report" - # no checks for ADMIN - # no checks for ADMINSIG - pbc_checkurl REPORTS_URL - pbc_checkpathname REPORTS_DIR - pbc_checkbasename REPORT_BASEDIR - pbc_checkbasename REPORT_HTML_FILE - pbc_checkbasename REPORT_TXT_FILE - - - # section "Uploading binary packages" - pbc_checkyesno UPDATE_VULNERABILITY_LIST - case ${PRUNEPACKAGES+set} in - "set") pbc_checkyesno PRUNEPACKAGES - esac - pbc_checkyesno MKSUMS - # no checks for SIGN_AS - # no checks for RSYNC_DST - # no checks for RSYNC_OPTS -} diff --git a/mk/bulk/pre-build b/mk/bulk/pre-build deleted file mode 100644 index b71f470a654..00000000000 --- a/mk/bulk/pre-build +++ /dev/null @@ -1,287 +0,0 @@ -#!/bin/sh -# $NetBSD: pre-build,v 1.76 2010/05/11 18:36:15 mbalmer Exp $ -# -# Clean up system to be ready for bulk pkg build -# -# (c) 2000 Hubert Feyrer, All Rights Reserved. -# - -#set -v # Debug - -# Pull in USR_PKGSRC: -if [ -f "$BULK_BUILD_CONF" ]; then - . "$BULK_BUILD_CONF" -else - . `dirname $0`/build.conf -fi - -PRUNEDISTFILES=${PRUNEDISTFILES:-"no"} -PRUNEPACKAGES=${PRUNEPACKAGES:-"yes"} -PRUNELINKS=${PRUNEPACKAGES:-"yes"} -PKG_ADMIN="pkg_admin" -PKG_INFO="pkg_info" - -some_package_dir=${USR_PKGSRC}/pkgtools/lintpkgsrc - -# Extract the name of the files used for the build log and broken build log. -# These have defaults set by bsd.bulk-pkg.mk and may be overridden in -# /etc/mk.conf -BULKFILESDIR=`( cd ${some_package_dir} ; ${BMAKE} show-var VARNAME=BULKFILESDIR )`; -if [ "$BULKFILESDIR" = "" ]; then - echo "pre-build> Had problems determining the directory of .broken* files" - exit 1 -fi -BROKENF=`( cd ${some_package_dir} ; ${BMAKE} show-var VARNAME=BROKENFILE )`; -if [ "$BROKENF" = "" ]; then - echo "pre-build> Had problems determining the name of the .broken files" - exit 1 -fi -NOTAVAILF=`( cd ${some_package_dir} ; ${BMAKE} show-var VARNAME=NOT_AVAILABLE_FILE )`; -if [ "$NOTAVAILF" = "" ]; then - echo "pre-build> Had problems determining the name of the .bulk-not_available files" - exit 1 -fi -BRKWRKLOG=`( cd ${some_package_dir} ; ${BMAKE} show-var VARNAME=BROKENWRKLOG )`; -if [ "$BRKWRKLOG" = "" ]; then - echo "pre-build> Had problems determining the name of the .broken.work files" - exit 1 -fi -BLDLOG=`( cd ${some_package_dir} ; ${BMAKE} show-var VARNAME=BUILDLOG )`; -if [ "$BLDLOG" = "" ]; then - echo "pre-build> Had problems determining the name of the .make files" - exit 1 -fi -LINTPKGSRC_DB=`( cd ${some_package_dir} ; ${BMAKE} show-var VARNAME=LINTPKGSRC_DB )`; -if [ "$LINTPKGSRC_DB" = "" ]; then - echo "pre-build> Had problems determining the name of the .lintpkgsrc.db file" - exit 1 -fi -STARTFILE=`( cd ${some_package_dir} ; ${BMAKE} show-var VARNAME=STARTFILE )`; -if [ "$STARTFILE" = "" ]; then - echo "pre-build> Had problems determining the name of the .start file" - exit 1 -fi - -LOCALBASE=`( cd ${some_package_dir} ; ${BMAKE} show-var VARNAME=LOCALBASE )`; -X11BASE=`( cd ${some_package_dir} ; ${BMAKE} show-var VARNAME=X11BASE )`; -DISTDIR=`( cd ${some_package_dir} ; ${BMAKE} show-var VARNAME=DISTDIR )`; - -# Create ${BULKFILESDIR} if necessary -echo "pre-build> Creating ${BULKFILESDIR} if necessary" -mkdir -p "${BULKFILESDIR}" - -# Make sure the pkg-vulnerabilities file is up to date -_INFO_VER=`${PKG_INFO} -V`; -echo "pre-build> Making sure vulnerability-list is up to date:" -if [ -z "$UPDATE_VULNERABILITY_LIST" -o "$UPDATE_VULNERABILITY_LIST" = yes ]; then - _PKGVULNDIR=`audit-packages ${AUDIT_PACKAGES_FLAGS} -Q PKGVULNDIR` - download-vulnerability-list ${DOWNLOAD_VULNERABILITY_LIST_FLAGS} - if [ "x${_PKGVULNDIR}" != "x${DISTDIR}" ]; then - cp ${_PKGVULNDIR}/pkg-vulnerabilities ${DISTDIR} - fi - echo 'pre-build> done.' -else - echo 'pre-build> (skipped)' -fi - -# On non-NetBSD platforms we need to keep the bootstrap-files! -if [ x"$BMAKE" = x"bmake" ]; then - ( cd "$LOCALBASE" && tar cf /tmp/$$.tar \ - sbin/audit-packages sbin/download-vulnerability-list \ - sbin/pkg_add sbin/pkg_admin sbin/pkg_create \ - sbin/pkg_delete sbin/pkg_info \ - share/mk/sys.mk share/mk/bsd.dep.mk share/mk/bsd.depall.mk \ - share/mk/bsd.files.mk share/mk/bsd.hostprog.mk \ - share/mk/bsd.doc.mk share/mk/bsd.inc.mk share/mk/bsd.info.mk \ - share/mk/bsd.kernobj.mk share/mk/bsd.kinc.mk \ - share/mk/bsd.kmod.mk share/mk/bsd.lib.mk \ - share/mk/bsd.links.mk share/mk/bsd.man.mk \ - share/mk/bsd.nls.mk share/mk/bsd.obj.mk \ - share/mk/bsd.prog.mk share/mk/bsd.subdir.mk \ - share/mk/bsd.sys.mk share/mk/bsd.own.mk share/mk/bsd.README \ - bin/bmake bin/nawk bin/nbsed bin/pdksh bin/install-sh \ - man/cat1/nbsed.0 man/cat1/pkg_add.0 man/cat1/pkg_admin.0 \ - man/cat1/pkg_create.0 man/cat1/pkg_delete.0 \ - man/cat1/pkg_info.0 man/cat5/pkg_install.conf.0 \ - man/cat5/pkg_summary.0 man/cat7/pkgsrc.0 man/man1/bmake.1 \ - man/man1/nawk.1 man/man1/nbsed.1 man/man1/pkg_add.1 \ - man/man1/pkg_admin.1 man/man1/pkg_create.1 \ - man/man1/pkg_delete.1 man/man1/pkg_info.1 man/man1/pdksh.1 \ - man/man5/pkg_install.conf.5 man/man5/pkg_summary.5 \ - man/man7/pkgsrc.7 etc/mk.conf ) -fi - -# -# Clean out everything and its mother -# -PKG_DBDIR=`(cd "${some_package_dir}" && ${BMAKE} show-var VARNAME=PKG_DBDIR)` -DEPOTBASE=`(cd "${some_package_dir}" && ${BMAKE} show-var VARNAME=DEPOTBASE)` -PKG_TOOLS_BIN=`(cd "${some_package_dir}" && ${BMAKE} show-var VARNAME=PKG_TOOLS_BIN)` - -for dbdir in "${PKG_DBDIR}" "${DEPOTBASE}"; do - echo "pre-build> Removing all installed packages in ${dbdir}" - - # The Solaris /bin/sh requires the following weird-looking code. - if (cd "$dbdir" 1>/dev/null 2>&1); then - cd "$dbdir" - - for pkg in * - do - if ${PKG_TOOLS_BIN}/pkg_info -K "$dbdir" -qe "$pkg"; then - echo "pre-build> ${PKG_TOOLS_BIN}/pkg_delete -r ${pkg}" - ${PKG_TOOLS_BIN}/pkg_delete -K "$dbdir" -r "$pkg" - fi - done - # - # this should have removed everything. - # Now force any broken pkgs to deinstall. - # - for pkg in * - do - if ${PKG_TOOLS_BIN}/pkg_info -K "$dbdir" -qe "$pkg"; then - echo "pre-build> ${PKG_TOOLS_BIN}/pkg_delete -f ${pkg}" - ${PKG_TOOLS_BIN}/pkg_delete -K "$dbdir" -f "$pkg" - fi - done - - # We've tried our best to get rid of the pkgs, now do it - # the hard way. If it wasn't for stuff in $X11BASE, I - # wouldn't have hard feelings about this! - rm -fr * - fi -done - -cd "$LOCALBASE" && rm -fr * && rm -fr .??*; -if [ x"$BMAKE" = x"bmake" ]; then - ( cd "$LOCALBASE" && tar xf /tmp/$$.tar ) -fi - -# Stuff known to be broken -rm -fr "$X11BASE/share/gimp" -rm -fr "$X11BASE/share/gnome" -rm -fr "$X11BASE/share/kde" -rm -fr "$X11BASE/share/netscape" -rm -fr /var/tmp/inst* -rm -fr "$X11BASE/lib/libgimp"* # gimp doesn't build with old libs around -rm -fr /nonexistent # broken useradd on pop3d -rm -fr /home/majordom # mail/majordomo pkg -rm -fr /home/nut # sysutils/ups-net -rm -fr /var/wwwoffle # www/wwwoffle -rm -fr /tmp/config-guess-* -rm -fr /tmp/mod* - - -# Clean up state files -cd "${USR_PKGSRC}" -echo "pre-build> Cleaning up leftover state files from previous runs" -rm -f "${BULKFILESDIR}"/*/*/$BROKENF "${BULKFILESDIR}"/*/*/$BRKWRKLOG "${BULKFILESDIR}"/*/*/$BLDLOG -rm -f "${BULKFILESDIR}"/*/*/$NOTAVAILF -rm -f "${BULKFILESDIR}/$BROKENF" "${BULKFILESDIR}/$BRKWRKLOG" "${BULKFILESDIR}/$BLDLOG" "$STARTFILE" - - -# -# Remove old/broken distfiles and binary packages -# -PACKAGES=`( cd "${some_package_dir}" && ${BMAKE} show-var VARNAME=PACKAGES )` - -case "$LINTPKGSRC_CACHE$PRUNEDISTFILES$PRUNEPACKAGES" in -*yes*|*YES*) - # install lintpkgsrc to support any of these options - ( cd "${USR_PKGSRC}/pkgtools/lintpkgsrc" && ${BMAKE} PRECLEAN=yes bulk-install ) - ;; -esac - -case "$LINTPKGSRC_CACHE" in -yes|YES) - echo "pre-build> Creating lintpkgsrc cache database." - lintpkgsrc -K "$PACKAGES" -P "$USR_PKGSRC" -M "$DISTDIR" -E "$LINTPKGSRC_DB" - lintpkgsrc_cache="-I $LINTPKGSRC_DB" - echo "pre-build> done." - ;; -*) - lintpkgsrc_cache='' - ;; -esac - -case "$PRUNEDISTFILES" in -yes|YES) - echo "pre-build> Removing old/broken distfiles" - lintpkgsrc $lintpkgsrc_cache -K "$PACKAGES" -P "$USR_PKGSRC" -M "$DISTDIR" -o -m -r - echo "pre-build> done." - ;; -*) - echo "pre-build> Skipping distfile pruning." - ;; -esac - -case "$PRUNEPACKAGES" in -yes|YES) - echo "pre-build> Removing old (out of date) binary packages" - lintpkgsrc $lintpkgsrc_cache -K "$PACKAGES" -P "$USR_PKGSRC" -M "$DISTDIR" -p -r - echo "pre-build> done." - ;; -*) - echo "pre-build> Skipping packages pruning." - ;; -esac - -case "$PRUNELINKS" in -yes|YES) - echo "pre-build> Checking for and removing orphaned packages links" - find "$PACKAGES" -type l -print | \ - while read f - do - if [ ! -d "$f" -a ! -f "$f" ]; then - echo "pre-build> Removing orphaned link: \"$f\"" - rm "$f" - fi - done - echo "pre-build> done." - ;; -*) - echo "pre-build> Skipping pruning of packages links." - ;; -esac - -# Clean out the packages added above so that the real bulk build -# is started in a consistent state. -for dbdir in "${PKG_DBDIR}" "${DEPOTBASE}"; do - echo "pre-build> Removing all installed packages in ${dbdir}" - if [ -d "$dbdir" ]; then - cd "$dbdir" - for pkg in * - do - if ${PKG_TOOLS_BIN}/pkg_info -K "$dbdir" -qe "$pkg"; then - echo "pre-build> ${PKG_TOOLS_BIN}/pkg_delete -r ${pkg}" - ${PKG_TOOLS_BIN}/pkg_delete -K "$dbdir" -r "$pkg" - fi - done - fi -done - -# on non-NetBSD, we don't want these to build as they overwrite -# bootstrap-pkgsrc generated files and thus would break the following builds. - -if [ x"$BMAKE" = x"bmake" ]; then - for pkg in devel/bmake devel/mk-files lang/nawk \ - pkgtools/bootstrap-mk-files pkgtools/bootstrap-extras \ - pkgtools/pkg_install \ - shells/pdksh sysutils/install-sh textproc/nbsed; do - mkdir -p "${BULKFILESDIR}/${pkg}" - echo "Don't blow away bootstrap-pkgsrc stuff!" \ - > "${BULKFILESDIR}/${pkg}/${BROKENF}" - done -fi - -cd "${USR_PKGSRC}" - -if [ -f mk/bulk/pre-build.local ]; then - export BROKENF - . mk/bulk/pre-build.local -fi - -if [ ! -d "${PKG_DBDIR}" ]; then - mkdir -p "${PKG_DBDIR}" -fi - -echo "This file marks the start time of the bulk build." > "$STARTFILE" diff --git a/mk/bulk/printdepends b/mk/bulk/printdepends deleted file mode 100644 index 0eda6dcdc8f..00000000000 --- a/mk/bulk/printdepends +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/sh -# $NetBSD: printdepends,v 1.25 2008/01/04 15:49:08 rillig Exp $ -# - -# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org> -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. All advertising materials mentioning features or use of this software -# must display the following acknowledgement: -# This product includes software developed by Hubert Feyrer for -# the NetBSD Project. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -# usage: printdepends [brokenfile [bulkfilesdir]] -# -# Generates a list of package dependencies suitable for tsort(1) on -# stdout. Each line of the output has two fields: the package directory -# of the dependency and the package directory of the depending package -# (both in the form category/package). -# -# If <brokenfile> is given, package-specific errors are logged to -# <bulkfilesdir>/${pkgdir}/<brokenfile>. Otherwise no additional logging -# takes place. -# -# The default for <bulkfilesdir> is the pkgsrc directory itself. If -# <bulkfilesdir> differs from the pkgsrc directory, all directories -# that are needed for the log files are created automatically. -# -# Note: printdepends must be called from a pkgsrc root directory. - -set -e - -# Pull in PKGLIST -if [ -f "$BULK_BUILD_CONF" ]; then - . $BULK_BUILD_CONF -else - . `dirname $0`/build.conf -fi - -# -# Global variables -# - -# The brokenfile_flag variable tells whether we want package-specific -# log files at all. If it is set to "yes", the mkdirs_flag -# variable tells whether the directories of the package-specific log -# files are created if necessary. -brokenfile_flag="yes" -mkdirs_flag="no" -pkgsrcdir="${PWD}" -bulkfilesdir="${pkgsrcdir}" -brokenfile="" - -# -# Command line parsing -# - -case $# in -0) brokenfile_flag="no" - ;; -1) brokenfile="$1" - ;; -2) brokenfile="$1" - bulkfilesdir="$2" - if [ "${bulkfilesdir}" != "${pkgsrcdir}" ]; then - mkdirs_flag="yes" - fi - ;; -*) echo "usage: $0 [brokenfile [bulkfilesdir]]" 1>&2 - exit 1 - ;; -esac - -# -# Sanity checks -# - -case ${BMAKE-""} in -"") echo "$0: error: BMAKE must be set and non-empty." 1>&2 - exit 1;; -esac -case ${bulkfilesdir} in -/*) ;; -*) echo "$0: error: The <bulkfilesdir> argument must be absolute." 1>&2 - exit 1;; -esac -case ${brokenfile} in -*/*) echo "$0: error: The <brokenfile> argument must not contain a slash." 1>&2 - exit 1;; -esac - -# -# Get additional system information -# - -cd "${pkgsrcdir}/pkgtools/lintpkgsrc" -GREP=`${BMAKE} show-var VARNAME=GREP USE_TOOLS=grep` -MKDIR=`${BMAKE} show-var VARNAME=MKDIR USE_TOOLS=mkdir` -SED=`${BMAKE} show-var VARNAME=SED USE_TOOLS=sed` -cd "${pkgsrcdir}" - -case $mkdirs_flag in -yes) mkbulkdir="${MKDIR}";; -*) mkbulkdir=":";; -esac - -case ${PKGLIST} in -"") # List of all packages, from pkgsrc/*/Makefile - list=`${GREP} '^SUBDIR+=' */Makefile | ${GREP} -v '^regress/' | ${SED} -e 's,/Makefile:SUBDIR+=[[:space:]]*,/,' -e 's,#.*,,'` - ;; -*) list="${PKGLIST}" - ;; -esac - -for pkgdir in $list; do - if cd "${pkgsrcdir}/${pkgdir}"; then - if deps=`${BMAKE} show-depends-dirs`; then - case ${deps} in - "") # Make the package depend on itself. - # Otherwise it would not show up in the - # dependency graph. - echo "${pkgdir} ${pkgdir}";; - *) for dep in $deps; do - echo "${dep} ${pkgdir}" - done;; - esac - else - echo "$0: error: could not extract dependencies for ${pkgdir} -- skipping." 1>&2 - - [ $brokenfile_flag = yes ] || continue - - ${mkbulkdir} "${bulkfilesdir}/${pkgdir}" - { echo "[printdepends] command failed: ${BMAKE} show-depends-dirs" - ( ${BMAKE} show-depends-dirs - ) || true - } >> "${bulkfilesdir}/${pkgdir}/${brokenfile}" 2>&1 - fi - else - ${mkbulkdir} "${bulkfilesdir}" - { echo "[printdepends] command failed: cd ${pkgsrcdir}/${pkgdir}" - ( cd "${pkgsrcdir}/${pkgdir}" - ) || true - } 1>&2 - fi -done diff --git a/mk/bulk/printindex b/mk/bulk/printindex deleted file mode 100644 index 621856db0d8..00000000000 --- a/mk/bulk/printindex +++ /dev/null @@ -1,202 +0,0 @@ -#!/bin/sh -# $NetBSD: printindex,v 1.32 2008/04/29 05:46:09 martin Exp $ -# - -# Copyright (c) 2001 The NetBSD Foundation, Inc. -# All rights reserved. -# -# This code is derived from software contributed to The NetBSD Foundation -# by -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS -# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# - -set -e - -# -# Global variables -# - -# The brokenfile_flag variable tells whether we want package-specific -# log files at all. If it is set to "yes", the mkdirs_flag -# variable tells whether the directories of the package-specific log -# files are created if necessary. -brokenfile_flag="yes" -mkdirs_flag="no" -pkgsrcdir="${PWD}" -bulkfilesdir="${pkgsrcdir}" -brokenfile="" - -# -# Command line parsing -# - -case $# in -0) brokenfile_flag="no" - ;; -1) brokenfile="$1" - ;; -2) brokenfile="$1" - bulkfilesdir="$2" - if [ "${bulkfilesdir}" != "${pkgsrcdir}" ]; then - mkdirs_flag="yes" - fi - ;; -*) echo "usage: $0 [brokenfile [bulkfilesdir]]" 1>&2 - exit 1 - ;; -esac - -# -# Sanity checks -# - -case ${BMAKE-""} in -"") echo "$0: error: BMAKE must be set and non-empty." 1>&2 - exit 1;; -esac -case ${bulkfilesdir} in -/*) ;; -*) echo "$0: error: The <bulkfilesdir> argument must be absolute." 1>&2 - exit 1;; -esac -case ${brokenfile} in -*/*) echo "$0: error: The <brokenfile> argument must not contain a slash." 1>&2 - exit 1;; -esac - -# -# Get additional system information -# - -cd "${pkgsrcdir}/pkgtools/lintpkgsrc" -BULK_PREREQ=`${BMAKE} show-var VARNAME=BULK_PREREQ` -GREP=`${BMAKE} show-var VARNAME=GREP USE_TOOLS=grep` -MKDIR=`${BMAKE} show-var VARNAME=MKDIR USE_TOOLS=mkdir` -SED=`${BMAKE} show-var VARNAME=SED USE_TOOLS=sed` -cd "${pkgsrcdir}" - -case $mkdirs_flag in -yes) mkbulkdir="${MKDIR}";; -*) mkbulkdir=":";; -esac - -case ${PKGLIST-""} in -"") # List of all packages, from pkgsrc/*/Makefile - list=`${GREP} '^[[:space:]]*'SUBDIR */Makefile | ${GREP} -v regress/ | ${SED} 's,/Makefile.*=[[:space:]]*,/,'` - ;; -*) list="${PKGLIST}" - ;; -esac - -# -# Sanity checks, part 2 -# - -allowed=" - +./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_" -case $BULK_PREREQ in -*[!-"${allowed}"]*) - inval=`echo "${BULK_PREREQ}" | tr -d " +\\\\-${allowed}"` - echo "$0: error: BULK_PREREQ contains invalid characters (${inval})." 1>&2 - exit 1 -esac -case $list in -*[!-"${allowed}"]*) - inval=`echo "${list}" | tr -d " +\\\\-${allowed}"` - echo "$0: error: The package list contains invalid characters (${inval})." 1>&2 - exit 1 -esac - -newline=" -" - -# -# Print the table of PKGPATH and PKGNAME. -# -# The check for duplicates is necessary to prevents double entries in -# the table. The use of the :detect_duplicates: delimiter prevents the -# inner "case" from occuring too often, as the $done_pkgs string grows -# to about 100k during one run of the program. This saves about -# 40 seconds on a 1 GHz Athlon. -# -done_pkgs="" -detect_duplicates=no -for pkgdir in $list :detect_duplicates: $BULK_PREREQ; do - - case $pkgdir in :detect_duplicates:) - detect_duplicates=yes - continue;; - esac - case $detect_duplicates in yes) - case $done_pkgs in *="${pkgdir}"=*) - continue;; - esac;; - esac - - if cd "${pkgsrcdir}/${pkgdir}"; then - - # The NetBSD /bin/sh creates lots of zombies for the backticks - # line below, and it doesn't clean them up until a real - # command is executed. (See PR 35227) - # - cat </dev/null - - if output=`${BMAKE} show-var VARNAME=PKGNAME can-be-built-here`; then - reasons="" - { - read pkgname - read can_be_built_here - while read reason; do - reasons="$reasons$reason$newline" - done - } <<EOF -$output -EOF - echo "${pkgdir} ${pkgname}" - done_pkgs="${done_pkgs} =${pkgdir}=" - - if [ "$can_be_built_here" != "yes" ]; then - ${mkbulkdir} "${bulkfilesdir}/${pkgdir}" - echo "$reasons" > "${bulkfilesdir}/${pkgdir}/${brokenfile}" - fi - else - echo "$0: error: could not extract PKGNAME for ${pkgdir} -- skipping." 1>&2 - - case $brokenfile_flag in - no) continue;; - esac - - ${mkbulkdir} "${bulkfilesdir}/${pkgdir}" - { echo "[printindex] command failed: ${BMAKE} show-var VARNAME=PKGNAME" - ( ${BMAKE} show-var VARNAME=PKGNAME - ) || true - } >> "${bulkfilesdir}/${pkgdir}/${brokenfile}" 2>&1 - fi - else - ${mkbulkdir} "${bulkfilesdir}" - { echo "[printindex] command failed: cd ${pkgsrcdir}/${pkgdir}" - ( cd "${pkgsrcdir}/${pkgdir}" - ) || true - } 1>&2 - fi -done diff --git a/mk/bulk/printindex.1 b/mk/bulk/printindex.1 deleted file mode 100644 index 1a9d4bca1ab..00000000000 --- a/mk/bulk/printindex.1 +++ /dev/null @@ -1,104 +0,0 @@ -.\" $NetBSD: printindex.1,v 1.8 2006/12/23 08:11:30 wiz Exp $ -.\" -.\" Copyright (c) 2005 The NetBSD Foundation, Inc. -.\" All rights reserved. -.\" -.\" This code is derived from software contributed to The NetBSD Foundation -.\" by Roland Illig <rillig@NetBSD.org>. -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted provided that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" 3. All advertising materials mentioning features or use of this software -.\" must display the following acknowledgement: -.\" This product includes software developed by the NetBSD -.\" Foundation, Inc. and its contributors. -.\" 4. Neither the name of The NetBSD Foundation nor the names of its -.\" contributors may be used to endorse or promote products derived -.\" from this software without specific prior written permission. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS -.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS -.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.\" POSSIBILITY OF SUCH DAMAGE. -.\" -.Dd November 26, 2006 -.Dt PRINTINDEX 1 -.Os -.Sh NAME -.Nm printindex -.Nd generate a PKGPATH to PKGNAME mapping database file -.Sh SYNOPSIS -.Nm -.Op Ar brokenfile Op Ar bulkfilesdir -.Sh DESCRIPTION -Generates a mapping from pkgsrc directories to the current package -version and prints it on stdout. -It also checks whether the package can be built on this system at all -and marks those that cannot be built as broken. -.Pp -Each line of the output has two fields: -the package directory (in the form -.Ql category/package ) -and the package version (in the form -.Ql package-1.456nb3 ) . -.Pp -If -.Ar brokenfile -is given, package-specific errors are logged to -.Ar bulkfilesdir Ns / Ns -.Va ${pkgdir} Ns / Ns -.Ar brokenfile . -Otherwise no additional logging takes place. -.Pp -The default for -.Ar bulkfilesdir -is the pkgsrc directory itself. -If -.Ar bulkfilesdir -differs from the pkgsrc directory, all directories that are needed for -the log files are created automatically. -.Sh LIMITATIONS -.Nm -must be called from a pkgsrc root directory. -.Sh EXIT STATUS -Non-zero if writing to a log file fails, zero otherwise. -.Sh ENVIRONMENT -.Bl -tag -width XXXXXX -.It Ev BMAKE Pq mandatory -The -.Xr make 1 -program used to build packages. -.It Ev PKGLIST Pq optional -The list of packages from which to build an index. -.El -.\" .Sh EXAMPLES -.Sh DIAGNOSTICS -Diagnostics have one of the two forms -.Ql printindex: error: ... -or -.Ql [printindex] ... . -The latter may span multiple lines. -.Sh SEE ALSO -.Xr printdepends 1 -.\" .Sh STANDARDS -.Sh HISTORY -.Nm -appeared in 2001 as part of the bulk build framework. -.Sh AUTHORS -Many (see the CVS ChangeLog). -.\" .Sh CAVEATS -.\" .Sh BUGS -.\" .Sh SECURITY CONSIDERATIONS diff --git a/mk/bulk/sort-packages b/mk/bulk/sort-packages deleted file mode 100644 index 0b22474455b..00000000000 --- a/mk/bulk/sort-packages +++ /dev/null @@ -1,83 +0,0 @@ -#! /bin/sh -# $NetBSD: sort-packages,v 1.16 2011/08/04 14:39:33 reinoud Exp $ - -# This program scans all binary packages in the current directory and -# creates two lists of files in OUTDIR: -# -# restricted_packages -# contains all packages that must not be published on the FTP -# server, for whatever reason -# -# regular_packages -# contains all the other ("good") packages. -# - -set -eu - -: ${OUTDIR="/tmp"} -: ${PKG_SUFX=".tgz"} -: ${PKG_ADMIN="pkg_admin"} -: ${PKG_INFO="pkg_info"} - -regular_packages="${OUTDIR}/regular_packages" -restricted_packages="${OUTDIR}/restricted_packages" -newline=" -" - -: > "${regular_packages}" -: > "${restricted_packages}" - -for pkg in *${PKG_SUFX}; do - build_info=`${PKG_INFO} -B "${pkg}"` - - # Note: this code needs to be that complicated because licensing - # issues are critical to pkgsrc, and we really don't want - # anything unexpected to happen here. The worst case would be - # that some file is sorted wrongly because some change in the - # output of pkg_info which had not been foreseen. Therefore it - # is better to check as strictly as possible to make those - # changes immediately visible. - - no_bin_on_ftp="unknown" - case "${newline}${build_info}${newline}" in - *"${newline}NO_BIN_ON_FTP=${newline}"*) - no_bin_on_ftp="no" - ;; - *"${newline}NO_BIN_ON_FTP="*) - no_bin_on_ftp="yes" - ;; - esac - - restricted="unknown" - case "${newline}${build_info}${newline}" in - *"${newline}RESTRICTED=${newline}"*) - restricted="no" - ;; - *"${newline}RESTRICTED="*) - restricted="yes" - ;; - esac - - category="unknown" - if [ "${restricted}" = "no" ] && [ "${no_bin_on_ftp}" = "no" ]; then - category="regular" - else - if [ "${restricted}" = "yes" ] || [ "${no_bin_on_ftp}" = "yes" ]; then - category="restricted" - fi - fi - - : echo "upload> ${pkg} is ${category}." - - case "${category}" in - "regular") - echo "${pkg}" >> "${regular_packages}" - ;; - "restricted") - echo "${pkg}" >> "${restricted_packages}" - ;; - *) - echo "sort-packages> WARNING: Could not sort ${pkg} into a category." 1>&2 - ;; - esac -done diff --git a/mk/bulk/tflat b/mk/bulk/tflat deleted file mode 100755 index 63e8a046650..00000000000 --- a/mk/bulk/tflat +++ /dev/null @@ -1,121 +0,0 @@ -# $NetBSD: tflat,v 1.15 2005/10/26 16:54:04 tv Exp $ -# -# Copyright (c) 2001, 2005 The NetBSD Foundation, Inc. -# All rights reserved. -# -# This code is derived from software contributed to The NetBSD Foundation -# by Dan McMahill and Todd Vierling. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. All advertising materials mentioning features or use of this software -# must display the following acknowledgement: -# This product includes software developed by the NetBSD -# Foundation, Inc. and its contributors. -# 4. Neither the name of The NetBSD Foundation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS -# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# - -# -# This perl version is very fast compared to other alternatives, which would -# likely use things like forking "sort" subprocesses. -# -use strict; - -sub usage () { - print STDERR <<EOF ; -tflat -- flatten a depends tree. tflat is used to show all - packages which depend upon a given package, and - all packages which are depend upon by a given package. - -usage: $0 upfile downfile < depfile - -Input file format is in the form -foo bar -foo baz -libbar bar - -meaning "foo is depended upon by bar, - foo is depended upon by baz, - libbar is depended upon by bar" - -The typical use is: -cd /usr/pkgsrc -./mk/bulk/printdepends > .depends -perl ./mk/bulk/tflat .supports .requires < .depends - -EOF - exit 1; -} - -my $upfile = shift(@ARGV) || usage(); -my $downfile = shift(@ARGV) || usage(); -scalar(@ARGV) && usage(); - -open(UPF, ">$upfile") || die $!; -open(DOWNF, ">$downfile") || die $!; - -# read in dependstree file - -my %depended; -my %depends; -while (<>) { - chomp; - my ($dep, $pkg) = split; - - push(@{$depended{$dep}}, $pkg); - push(@{$depends{$pkg}}, $dep); -} - -# print out upfile and downfile - -sub getdeps ($$$@) { - my $curhash = shift; - my $alldeps = shift; - my $what = shift; - - foreach my $dep (@{$curhash->{$what}}) { - if ($what eq $dep) { - next; - } elsif (grep { $_ eq $dep } @_) { - print STDERR "circular dependency in $dep\n"; - } elsif (!grep { $_ eq $dep } @$alldeps) { - push(@$alldeps, $dep); - getdeps($curhash, $alldeps, $dep, $what, @_); - } - } -} - -foreach my $pkg (sort(keys(%depended))) { - my @alldeps; - getdeps(\%depended, \@alldeps, $pkg); - print UPF "$pkg is depended on by: ".join(' ', @alldeps)."\n"; -} - -foreach my $pkg (sort(keys(%depends))) { - my @alldeps; - getdeps(\%depends, \@alldeps, $pkg); - print DOWNF "$pkg depends on: ".join(' ', @alldeps)."\n"; -} - -close(UPF); -close(DOWNF); diff --git a/mk/bulk/upload b/mk/bulk/upload deleted file mode 100644 index 81ab685a9ea..00000000000 --- a/mk/bulk/upload +++ /dev/null @@ -1,356 +0,0 @@ -#!/bin/sh -# $NetBSD: upload,v 1.48 2010/04/10 21:44:44 wiz Exp $ - -# -# Upload non-restricted binary pkgs to ftp server -# - -AWK=${AWK:-/usr/bin/awk} -PKG_ADMIN="pkg_admin" -PKG_INFO="pkg_info" - -set -eu - -# -# Find out where we are -# -scriptdir=`dirname "$0"` -scriptdir=`cd "${scriptdir}" && pwd` - -usage() -{ -cat << EOF - -$prog: Uploads binary packages. - -Usage: $prog [-c|--config <file>] [-n|--no-upload] [-d|--debug] [-v|--verbose] - $prog -h|--help - $prog -V|--version - - -c | --config <file> - Load the following configuration file instead of - the default one. - - -d|--debug Preserves some intermediate files to help - debug any problems. - - -h|--help Displays this help message and exits. - - -n|--no-upload Does all of the work except for the actual upload. - This option may be used to generate the upload script - along with the list of packages to be excluded. - - -i|--no-install Do not install the required packages; assume instead - that they are already available. - - -v|--verbose Enables verbose output. - - -V|--version Displays the version of this script and exits. - -Example: - - cd /usr/pkgsrc && $prog - -Environment Variables: - - AWK -- If specified, sets the awk program to be used - -Files: - - pkgsrc/mk/bulk/build.conf - -EOF -} - -# print out error message and exit 1 -die () { - echo "$0: error:" 1>&2 - for i in "$@"; do - echo " $i" 1>&2 - done - exit 1 -} - -prog=$0 -debug=no -do_upload=yes -do_install=yes -verbose=no -while test $# -gt 0; do - case "$1" in - -c|--config) - shift - BULK_BUILD_CONF=$1; shift - ;; - - -d|--debug) - debug=yes - shift - ;; - - -h|--help) - usage - exit 0 - ;; - - -i|--no-install) - do_install=no - shift - ;; - - -n|--no-upload) - do_upload=no - shift - ;; - - -v|--verbose) - verbose=yes - shift - ;; - - -V|--version) - ${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s: Version %s, %s\n",$3,$4,$5); exit 0;}' $prog - exit 0 - ;; - - -*) - echo "$prog: Unknown option: $1" 1>&2 - usage - exit 1 - ;; - - *) - break - ;; - esac -done -if test "$#" -gt 0; then - echo "$prog: Unknown argument: $1" 1>&2 - usage - exit 1 -fi - -install_required() -{ - - [ "$do_install" = "yes" ] || return 0 - pkg=$1 - if [ "${verbose}" = "yes" ]; then - echo "upload> Installing ${pkg}" - fi - (cd "$USR_PKGSRC" && cd "$pkg" && ${BMAKE} bulk-install) \ - || { - echo "upload> ERROR: Unable to install required package $pkg!" 1>&2 - echo " Bailing out -- you're on your own." 1>&2 - exit 1 - } -} - -MD5="digest md5"; -SHA1="digest sha1"; - -_INFO_VER=`${PKG_INFO} -V`; -REQUIRED_PACKAGES="pkgtools/lintpkgsrc net/rsync" - -opsys=`uname -s` -case "$opsys" in -NetBSD) BMAKE=make; - BSDSUM="sum -o 1"; - CKSUM="cksum"; - SYSVSUM="sum -o 2"; - ;; -IRIX*) BMAKE=bmake; - BSDSUM="sum -r"; - CKSUM="cksum"; - SYSVSUM="sum"; - ;; -*) BMAKE=bmake; - BSDSUM=""; - CKSUM=""; - SYSVSUM=""; - ;; -esac - -export BMAKE - -# Bring in variables for bulk-install -BATCH=1 -DEPENDS_TARGET=bulk-install -export BATCH DEPENDS_TARGET - -# -# Find the configuration file. -# -BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}" -case $BULK_BUILD_CONF in - /*) ;; - *) BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}" -esac - -# -# Get the variables MAKECONF, RSYNC_DST, RSYNC_OPTS from the bulk build -# configuration file. -# -{ test -f "${BULK_BUILD_CONF}" \ - && . "${BULK_BUILD_CONF}" \ - && . "${scriptdir}/post-build-conf" \ - && check_config_vars \ - && export_config_vars -} || die "Cannot load config file ${BULK_BUILD_CONF}, aborting." - -cd $USR_PKGSRC - -if [ -z "$RSYNC_DST" ]; then - echo "upload> ERROR: You must set the variable RSYNC_DST, see build.conf-example." 1>&2 - exit 1 -fi - -# -# Some temp files -# - -umask 022 -TMPDIR="${TMPDIR:-/tmp}" -TMP="${TMPDIR}"/pkg_upload.$$ -(umask 077 && mkdir "${TMP}") \ -|| { - echo "upload> ERROR: cannot create temporary directory \"${TMP}\"." 1>&2 - exit 1 -} - -restricted_packages="$TMP/restricted_packages" -old_packages="$TMP/old_packages" -good_packages="$TMP/regular_packages" -all_good_packages="$TMP/all_regular_packages" - -upload_general="$TMP"/upload_general - -# May be different than $USR_PKGSRC: -echo "upload> Running ${BMAKE} to get the pkgsrc variables" -pkgsrcdir=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=_PKGSRCDIR` -packages=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=PACKAGES` -distdir=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=DISTDIR` -gzip_cmd=`cd pkgtools/lintpkgsrc; ${BMAKE} show-var VARNAME=GZIP_CMD USE_TOOLS=gzip` -pkg_info=`cd pkgtools/lintpkgsrc && ${BMAKE} show-var VARNAME=PKG_INFO` -shell=`cd pkgtools/lintpkgsrc && ${BMAKE} show-var VARNAME=TOOLS_PLATFORM.sh` - -# Pull in some pkgs needed -for pkg in ${REQUIRED_PACKAGES}; do - install_required $pkg -done - -case $LINTPKGSRC_CACHE in -yes|YES) - lintpkgsrc_cache="-I `cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=LINTPKGSRC_DB`" - ;; -*) - lintpkgsrc_cache='' - ;; -esac - -echo "upload> Checking for out of date packages:" -# -p = report old versions of packages -lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -p > "${old_packages}.tmp" -sed 's@'$packages'/@@' < "${old_packages}.tmp" > "$old_packages" - -RSFLAGS="-vap --progress $RSYNC_OPTS" - -failed=no -cd $packages - -echo "upload> Checking for restricted packages" -(cd All && env PKG_INFO="${pkg_info}" OUTDIR="${TMP}" \ - ${shell} "${pkgsrcdir}/mk/bulk/sort-packages") - -# Add the name of the package file, including all its symlinks to the -# list of files to be uploaded. -while read package; do - ls -1 */"$package" -done < "$good_packages" > "$all_good_packages" - -if [ "${MKSUMS}" = "yes" -o "${MKSUMS}" = "YES" ]; then - - echo "upload> Calculating checksum files..." - - SUMFILES="BSDSUM CKSUM MD5 SHA1 SYSVSUM" - - rm -f ${SUMFILES} - - if [ "${SIGN_AS-}" != "" ]; then - install_required "security/gnupg" - for i in ${SUMFILES}; do - echo > $i - echo "This file is signed with ${SIGN_AS}'s PGP key." >> $i - echo >> $i - done - fi - - install_required "pkgtools/digest" - - [ -z "${BSDSUM}" ] && BSDSUM="echo" - [ -z "${CKSUM}" ] && CKSUM="echo" - [ -z "${SYSVSUM}" ] && SYSVSUM="echo" - - for pkg in `cat "${good_packages}"`; do - pkg="All/$pkg" - ${BSDSUM} "$pkg" >> BSDSUM - ${CKSUM} "$pkg" >> CKSUM - ${MD5} "$pkg" >> MD5 - ${SHA1} "$pkg" >> SHA1 - ${SYSVSUM} "$pkg" >> SYSVSUM - done - - [ "${BSDSUM}" = "echo" ] && rm BSDSUM - [ "${CKSUM}" = "echo" ] && rm CKSUM - [ "${SYSVSUM}" = "echo" ] && rm SYSVSUM - - if [ "${SIGN_AS-}" != "" ]; then - for i in ${SUMFILES}; do - if [ -s $i ]; then - echo "upload> Signing $i" - gpg --clearsign $i && rm $i - fi - done - else - echo "upload> Checksum files not PGP-signed. Please do so manually!" - echo " (Run 'gpg --clearsign' on all of them)" - fi -fi - -if [ "${MKSUMMARY-}" = "yes" -o "${MKSUMMARY-}" = "YES" ]; then - echo "upload> Creating summary file..." - (cd "${packages}/All" \ - && ls -t | grep '\.t[gb]z$' | while read n; do pkg_info -X "$n"; done) \ - | ${gzip_cmd} > "${packages}"/All/pkg_summary.gz -fi - -cat <<EOF > "$upload_general" -#! /bin/sh -set -e -cd "$packages" -rsync $RSFLAGS --files-from="${all_good_packages}" --exclude-from="${old_packages}" . "$RSYNC_DST/" -EOF -chmod +x "$upload_general" - -if [ "$do_upload" = "yes" ]; then - echo "upload> Uploading packages" - ${shell} "$upload_general" \ - || { - echo "upload> ERROR: rsync failed. To retry later, you can run $upload_general" 1>&2 - failed=yes - } -else - echo "upload> Skipping upload of packages." - echo " Run \"$upload_general\" to upload them later." -fi - -# clean up temp files -if [ "$failed,$debug,$do_upload" = "no,no,yes" ]; then - rm -fr "$TMP" -else - echo "upload> Preserving temporary directory ${TMP}" -fi - -if [ "$failed" = "yes" ]; then - exit 1 -fi diff --git a/mk/misc/toplevel.mk b/mk/misc/toplevel.mk index 95e48bd984c..e2486be7ab0 100644 --- a/mk/misc/toplevel.mk +++ b/mk/misc/toplevel.mk @@ -1,4 +1,4 @@ -# $NetBSD: toplevel.mk,v 1.5 2011/09/08 20:17:16 abs Exp $ +# $NetBSD: toplevel.mk,v 1.6 2014/06/02 10:24:05 wiz Exp $ # # This file contains the make targets that can be used from the # top-level Makefile. They are in this separate file to keep the @@ -29,13 +29,6 @@ README.html: .PRECIOUS .include "../bsd.pkg.subdir.mk" -# the bulk-cache and clean-bulk-cache targets are a global-pkgsrc -# thing and thus it makes sense to run it from the top level pkgsrc -# directory. -.if make(bulk-cache) || make(clean-bulk-cache) -.include "../bulk/bsd.bulk-pkg.mk" -.endif - .PHONY: ${.CURDIR}/PKGDB ${.CURDIR}/PKGDB: @${RM} -f ${.CURDIR}/PKGDB |