diff options
Diffstat (limited to 'mk/bulk')
-rw-r--r-- | mk/bulk/TODO | 2 | ||||
-rw-r--r-- | mk/bulk/bsd.bulk-pkg.mk | 487 | ||||
-rw-r--r-- | mk/bulk/build | 365 | ||||
-rw-r--r-- | mk/bulk/build.conf-example | 80 | ||||
-rw-r--r-- | mk/bulk/do-sandbox-build | 11 | ||||
-rw-r--r-- | mk/bulk/do-sandbox-upload | 11 | ||||
-rwxr-xr-x | mk/bulk/mksandbox | 358 | ||||
-rw-r--r-- | mk/bulk/parallel.txt | 207 | ||||
-rw-r--r-- | mk/bulk/post-build | 591 | ||||
-rw-r--r-- | mk/bulk/pre-build | 286 | ||||
-rw-r--r-- | mk/bulk/printdepends | 112 | ||||
-rw-r--r-- | mk/bulk/printindex | 121 | ||||
-rwxr-xr-x | mk/bulk/tflat | 204 | ||||
-rw-r--r-- | mk/bulk/upload | 204 |
14 files changed, 0 insertions, 3039 deletions
diff --git a/mk/bulk/TODO b/mk/bulk/TODO deleted file mode 100644 index db2875b26f6..00000000000 --- a/mk/bulk/TODO +++ /dev/null @@ -1,2 +0,0 @@ - * allow using non-/etc mk.conf - * integrate libkver diff --git a/mk/bulk/bsd.bulk-pkg.mk b/mk/bulk/bsd.bulk-pkg.mk deleted file mode 100644 index 7c5bea35fc2..00000000000 --- a/mk/bulk/bsd.bulk-pkg.mk +++ /dev/null @@ -1,487 +0,0 @@ -# $NetBSD: bsd.bulk-pkg.mk,v 1.65.2.6 2005/05/17 18:29:44 tv Exp $ - -# -# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org> -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. All advertising materials mentioning features or use of this software -# must display the following acknowledgement: -# This product includes software developed by Hubert Feyrer for -# the NetBSD Project. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -## -## For this to work, put -## DEPENDS_TARGET= bulk-install -## BATCH= yes -## into your /etc/mk.conf, then cd /usr/pkgsrc; make bulk-package -## -## bulk-install will only compile pkgs if necessary, and install binary -## pkgs if possible/available. -## bulk-package will build binary packages, and remove them (together with -## any depending packages) immediately after that, leaving only the -## binary package back. Broken packages are marked (``.broken''). -## - -LS?= ls -WC?= wc -TO_HTML?= ${SED} -e 's,&,\&,g' -e 's,<,\<,g' -e 's,>,\>,g' - -# A sort(1) capable of very long lines is needed for full builds in "tflat". -# Some platforms (namely, Interix) may not provide one, so override here. -.if ${OPSYS} == "Interix" -_SORT= ${LOCALBASE}/bin/${GNU_PROGRAM_PREFIX}sort -.endif -_SORT?= ${SORT} - -# This variable is set to 'yes' by the pkgsrc/mk/bulk/build script. It enables -# the use of several cache files (DEPENDSTREEFILE, DEPENDSFILE, SUPPORTSFILE, -# and INDEXFILE) for speeding up the processing of dependencies. If we're not -# actively in the middle of a bulk build, however, we don't know that these files -# exist or are up to date and they take quite a while to rebuild. So unless -# they're known to exist and be up to date, don't use them. -USE_BULK_CACHE?= no - -# This variable may be set to 'no' to avoid automatic rebuilding of dependent -# packages based solely on timestamps of the package's pkgsrc files and/or -# its dependency binary packages. This will cause rebuilding only when the -# full version number of the package changes (e.g., via PKGREVISION bump). -USE_BULK_TIMESTAMPS?= yes - -# Shall we remove any packages which are installed, but not required -# to build this package -PRECLEAN?= yes - -# If OBJHOSTNAME is set, use first component of hostname in cache and log files -# If OBJMACHINE is set, use ${MACHINE_ARCH} in the cache and log files -.if defined(OBJHOSTNAME) -. if !defined(_HOSTNAME) -_HOSTNAME!= ${UNAME} -n -. endif -BULK_ID?= .${_HOSTNAME:C|\..*||} -.elif defined(OBJMACHINE) -BULK_ID?= .${MACHINE_ARCH} -.else -BULK_ID?= -.endif - -# This file exists to mark a package as broken -BROKENFILE?= .broken${BULK_ID}.html - -# This file is the work log for a broken package -BROKENWRKLOG?= .broken${BULK_ID}.work.html - -# This file is where the log of the build goes -BUILDLOG?= .make${BULK_ID} - -# This is the directory in which bulk build global files should be kept. -# It defaults to ${PKGSRCDIR}, but may be better suited to another directory -# if pkgsrc is on a remote (e.g., nfs) filesystem. -BULKFILESDIR?= ${PKGSRCDIR} - -# This is a top level file which lists the entire pkgsrc depends tree in the -# format: -# foo/bar devel/libfoo -# meaning 'foo/bar' is requied to build 'devel/libfoo' -# this is in the format needed by tsort(1) -DEPENDSTREEFILE?= ${BULKFILESDIR}/.dependstree${BULK_ID} - -# This is a top level file which lists the entire pkgsrc depends tree in the -# format: -# foo/bar depends on: devel/libfoo devel/libbar devel/baz ..... -# ie, to build foo/bar we need devel/libfoo devel/libbar devel/baz ... installed -DEPENDSFILE?= ${BULKFILESDIR}/.depends${BULK_ID} - -# This is a top level file which lists the entire pkgsrc depends tree in the -# format: -# devel/libfoo is depended upon by: foo/bar graphics/gtkfoo ... -# ie, to build foo/bar we need devel/libfoo to be installed. -# to build graphics/gtkfoo we need devel/libfoo to be installed -SUPPORTSFILE?= ${BULKFILESDIR}/.supports${BULK_ID} - -# This is a top level file which cross-references each package name and pkg -# directory in the format: -# devel/libfoo libfoo-1.3 -INDEXFILE?= ${BULKFILESDIR}/.index${BULK_ID} - -# File containing a list of all the packages in the correct order for a bulk -# build. The correct order is one where packages that are required by others -# are built before the packages which require them. -ORDERFILE?= ${BULKFILESDIR}/.order${BULK_ID} - -# File which is used as a timestamp for when the build started. This is used -# eventually for looking for leftover files (files not properly deinstalled) -STARTFILE?= ${BULKFILESDIR}/.start${BULK_ID} - -# File created and used by lintpkgsrc(8) to cache package metadata for -# pruning and bulk-upload exclusions. -LINTPKGSRC_DB?= ${BULKFILESDIR}/.lintpkgsrc.db${BULK_ID} - -# File which is used as a database for bulk builds in which SPECIFIC_PKGS is -# defined. This database is used to hold all the dependency and index -# information for the specific packages as well as their dependencies. In a -# SPECIFIC_PKGS bulk build, this file is created and then used to create the -# INDEXFILE and DEPENDSTREEFILE. -BULK_DBFILE?= ${BULKFILESDIR}/.bulk_db${BULK_ID} - -# A list of pkgs which we should _never_ delete during a build. The primary -# use is for digest and also for xpkgwedge. Add pkgtools/xpkgwedge in -# /etc/mk.conf to do an xpkgwedged bulk build. -BULK_PREREQ+= pkgtools/digest - -# by default, clean up any broken packages -_PRESERVE_WRKDIR?= no - -# create an escaped version of PKGPATH. We do this because -# in several places we want to be able to use something like -# foo/bar.baz++ in a regular expression but have it be interpreted -# literally. So, turn it into foo\/bar\.baz\+\+ -_ESCPKGPATH= ${PKGPATH:C@\/@\\/@g:C@\+@\\+@g:C@\.@\\.@g:Q} - -# build the cache files used as part of a full bulk build -# Note: we have to install the BULK_PREREQ packages _before_ -# extracting the depends tree because some packages like -# xpkgwedge only become DEPENDS if it is installed -bulk-cache: - @${ECHO_MSG} "BULK> Installing BULK_PREREQ packages" -.for __prereq in ${BULK_PREREQ} - cd ${PKGSRCDIR}/${__prereq} && ${MAKE} bulk-install -.endfor - ${RM} -f ${BULK_DBFILE} - ${TOUCH} ${BULK_DBFILE} -.if !defined(SPECIFIC_PKGS) - @${ECHO} "This file is unused for a full pkgsrc bulk build" >> ${BULK_DBFILE} - @${ECHO} "It is only used for a SPECIFIC_PKGS bulk build" >> ${BULK_DBFILE} - @${ECHO_MSG} "BULK> Building complete pkgsrc dependency tree (this may take a while)." - cd ${PKGSRCDIR} && ${SETENV} BMAKE=${MAKE} ${SH} mk/bulk/printdepends ${BROKENFILE} > ${DEPENDSTREEFILE} - @${ECHO_MSG} "BULK> Generating package name <=> package directory cross reference file" - @${ECHO_MSG} " (this may take a while)." - cd ${PKGSRCDIR} && ${SETENV} BMAKE=${MAKE} ${SH} mk/bulk/printindex ${BROKENFILE} > ${INDEXFILE} -.else - @${ECHO_MSG} "BULK> Extracting database for SPECIFIC_PKGS subset of pkgsrc" - @${ECHO_MSG} " along with their dependencies" -.for __tmp__ in ${SUBDIR} ${BULK_PREREQ} ${PERL5_PKGSRCDIR:C|.*/([^/]*/[^/]*)$|\1|} pkgtools/pkglint - cd ${PKGSRCDIR}/${__tmp__} && ${SETENV} BMAKE=${MAKE} AWK=${AWK} EXPR=${EXPR} ${SH} ../../mk/scripts/mkdatabase -a -f ${BULK_DBFILE} -.endfor - @${ECHO_MSG} "BULK> Extracting dependency tree file" - ${AWK} '/^(build_)?depends/ {pkgs[$$2] = 1; cat=$$2; sub(/\/.*/, "", cat); \ - for(i=3; i<=NF; i=i+1){ \ - listed[$$2] = 1; \ - sub(/[^:]*:\.\.\/\.\.\//, "", $$i); \ - sub(/[^:]*:\.\./, cat , $$i); \ - print $$i " " $$2; \ - }} END{ \ - for(pkg in pkgs) {if( pkg in listed ) {} else{ print pkg " " pkg;}} \ - }' \ - ${BULK_DBFILE} | ${_SORT} -u > ${DEPENDSTREEFILE} - @${ECHO_MSG} "BULK> Extracting package name <=> package directory cross reference file" - ${AWK} '/^index/ {print $$2 " " $$3 " "}' ${BULK_DBFILE} > ${INDEXFILE} -.endif - @${ECHO_MSG} "BULK> Sorting build order." - ${TSORT} ${DEPENDSTREEFILE} > ${ORDERFILE} - @${ECHO_MSG} "BULK> Generating up and down dependency files." - ${SETENV} SORT=${_SORT:Q} ${AWK} -f ${PKGSRCDIR}/mk/bulk/tflat up ${DEPENDSTREEFILE} > ${SUPPORTSFILE} - ${SETENV} SORT=${_SORT:Q} ${AWK} -f ${PKGSRCDIR}/mk/bulk/tflat down ${DEPENDSTREEFILE} > ${DEPENDSFILE} - -# remove the bulk cache files -clean-bulk-cache: - ${RM} -f ${BULK_DBFILE} \ - ${DEPENDSTREEFILE} \ - ${DEPENDSFILE} \ - ${SUPPORTSFILE} \ - ${INDEXFILE} \ - ${ORDERFILE} - -# check if the $REF file is uptodate, i.e. is newer than any of -# the pkg files; prints "1" if upto date, "0" if not. -bulk-check-uptodate: - @uptodate=1; \ - if [ -f "${REF}" ]; then \ - if [ "${REF:T}" != "${PKGFILE:T}" ]; then \ - ${ECHO_MSG} >&2 "BULK> ${REF} is out of date (new version ${PKGNAME}); rebuilding..."; \ - uptodate=0; \ - elif [ "${USE_BULK_TIMESTAMPS}" = "yes" ]; then \ - ${SHCOMMENT} "Check files of this package"; \ - newfiles="`${FIND} . -type f -newer "${REF}" -print | ${EGREP} -v -e ./work -e COMMENT -e DESCR -e README.html -e CVS -e '^\./\.' || ${TRUE}`"; \ - nnewfiles="`${FIND} . -type f -newer "${REF}" -print | ${EGREP} -v -e ./work -e COMMENT -e DESCR -e README.html -e CVS -e '^\./\.' | ${WC} -l`"; \ - if [ "$$nnewfiles" -gt 0 ]; then \ - ${ECHO_MSG} >&2 "BULK> Package ${PKGNAME} ($$newfiles) modified since last 'make package' re-packaging..."; \ - uptodate=0; \ - else \ - ${ECHO_MSG} >&2 "BULK> ${REF} is up to date."; \ - fi; \ - else \ - ${ECHO_MSG} >&2 "BULK> ${REF} is up to date."; \ - fi; \ - else \ - ${ECHO_MSG} >&2 "BULK> Package ${PKGNAME} not built yet, packaging..."; \ - uptodate=0; \ - fi; \ - if [ "$$uptodate" = "1" ]; then \ - ${SHCOMMENT} "Check required binary packages"; \ - (${DEPENDS:C/:.*$//:@d@${ECHO} ${d:Q};@} ${TRUE}) | \ - (while read dep; do \ - ${SHCOMMENT} "check against the binary pkg that pkg_add would pick, too:"; \ - ${SHCOMMENT} "(Only one should be returned here, really...)"; \ - pkg=`${PKG_ADMIN} lsbest "${PACKAGES}/All/$$dep"`; \ - if [ -z "$$pkg" ]; then \ - ${ECHO_MSG} >&2 "BULK> Required binary package $$dep does not exist, rebuilding..."; \ - exit 1; \ - elif [ "${USE_BULK_TIMESTAMPS}" = "yes" ] && [ -n "`${FIND} $$pkg -prune -newer ${REF} -print`" ]; then \ - ${ECHO_MSG} >&2 "BULK> Required binary package $$dep (`basename $$pkg`) is newer, rebuilding..."; \ - exit 1; \ - else \ - ${ECHO_MSG} >&2 "BULK> Required binary package $$dep (`basename $$pkg`) is usable."; \ - fi; \ - done) || uptodate=0; \ - fi; \ - ${ECHO_MSG} $$uptodate - -# rebuild binpkg if any of the pkg files is newer than the binary archive -# set DO to ":" to not actually do anything (debugging, ...) -bulk-package: - @if [ -f ${BROKENFILE} ]; then \ - ${ECHO_MSG} "BULK> *** Package ${PKGNAME} seems broken and needs attention:" ; \ - ${LS} -la ${BROKENFILE}; \ - exit 1; \ - fi - @( \ - if [ "${PRECLEAN}" = "yes" ]; then \ - ${ECHO_MSG} '<pre>' ; \ - ${ECHO_MSG} '' ; \ - ${ECHO_MSG} '###' ; \ - ${ECHO_MSG} '###' `date`: ; \ - ${ECHO_MSG} '### ${MAKE} ${.TARGET} for ${PKGNAME}' ; \ - ${ECHO_MSG} '### Current pkg count: ' `${LS} -l ${PKG_DBDIR} | ${GREP} '^d' | ${WC} -l` installed packages: `${LS} ${PKG_DBDIR} | ${GREP} -v pkgdb.byfile.db`; \ - ${ECHO_MSG} '###' ; \ - fi \ - ) 2>&1 | ${TEE} -a ${BUILDLOG} - @uptodate=`${MAKE} ${MAKEFLAGS} bulk-check-uptodate REF=${PKGFILE}` ; \ - if ${PKG_INFO} -qe "${PKGNAME:C/-[^-]*$/-[0-9]*/}" ; then \ - installed=1; \ - else \ - installed=0; \ - fi ; \ - if [ $$uptodate = 1 ]; then \ - ( if [ $$installed = 1 ]; then \ - ${ECHO_MSG} "BULK> Package ${PKGNAME} is upto-date, and still installed" ; \ - else \ - ${ECHO_MSG} "BULK> Nothing to be done." ; \ - fi \ - ) 2>&1 | ${TEE} -a ${BUILDLOG}; \ - else \ - ( if [ $$installed = 1 ]; then \ - ${ECHO_MSG} "BULK> Removing outdated (installed) package ${PKGNAME} first." ; \ - ${ECHO_MSG} ${MAKE} deinstall ; \ - ${DO} ${MAKE} deinstall ; \ - if ${PKG_INFO} -qe ${PKGWILDCARD} ; then \ - ${ECHO_MSG} ${PKG_DELETE} -r ${PKGWILDCARD} ;\ - ${DO} ${PKG_DELETE} -r ${PKGWILDCARD} ;\ - fi ;\ - fi ; \ - if [ -f ${PKGFILE} ]; then \ - ${ECHO_MSG} "BULK> Removing old binary package..." ; \ - ${ECHO_MSG} ${RM} -f ${PKGFILE} ; \ - ${DO} ${RM} -f ${PKGFILE} ; \ - for cat in ${CATEGORIES} ;\ - do \ - ${ECHO_MSG} ${RM} -f ${PACKAGES}/$$cat/${PKGNAME}${PKG_SUFX}; \ - ${DO} ${RM} -f ${PACKAGES}/$$cat/${PKGNAME}${PKG_SUFX}; \ - done ;\ - fi; \ - ${ECHO_MSG} "BULK> Full rebuild in progress..." ; \ - ${ECHO_MSG} "BULK> Cleaning package ${PKGNAME} and pre-requisite packages" ;\ - if [ "${USE_BULK_CACHE}" = "yes" ]; then \ - for pkgdir in ${PKGPATH} `${SED} -n -e "/^${_ESCPKGPATH} / s;^[^:]*:;;p" ${DEPENDSFILE}`; do \ - ${DO} (cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} clean) ; \ - done ;\ - else \ - ${ECHO_MSG} ${MAKE} clean CLEANDEPENDS=YES;\ - ${DO} ${MAKE} clean CLEANDEPENDS=YES;\ - fi; \ - if [ "${PRECLEAN}" = "yes" ]; then \ - ${ECHO_MSG} "BULK> Removing installed packages which are not needed to build ${PKGNAME}" ; \ - for pkgname in `${PKG_INFO} -e \*` ; \ - do \ - if [ "${USE_BULK_CACHE}" = "yes" ]; then \ - pkgdir=`${GREP} " $$pkgname " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\ - if [ -z "$$pkgdir" ]; then \ - pkgdir=unknown ; \ - fi; \ - if ${PKG_INFO} -qe $$pkgname ; then \ - ${SHCOMMENT} "Remove only unneeded pkgs" ; \ - pkgdir2=`${ECHO} "$$pkgdir" | ${AWK} '{gsub(/\//,"\\\\/"); gsub(/\+/,"\\\\+"); gsub(/ /,"\\\\ "); gsub(/\./,"\\\\."); print}'` ; \ - tmp=`${SED} -n -e "/^${_ESCPKGPATH} .* $$pkgdir2 / s;.*;yes;p" ${DEPENDSFILE}` ; \ - if test "X$$tmp" = "Xyes" ; then \ - ${ECHO_MSG} "BULK> ${PKGNAME} requires installed package $$pkgname ($$pkgdir) to build." ;\ - else \ - case "${BULK_PREREQ}" in \ - *$$pkgdir* ) \ - ${ECHO_MSG} "BULK> Keeping BULK_PREREQ: $$pkgname ($$pkgdir)" ;\ - ;; \ - * ) \ - ${ECHO_MSG} ${PKG_DELETE} -r $$pkgname ; \ - ${DO} ${PKG_DELETE} -r $$pkgname || true ; \ - if ${PKG_INFO} -qe $$pkgname ; then \ - ${DO} ${PKG_DELETE} -f $$pkgname || true ; \ - fi ;\ - ;; \ - esac ; \ - fi ;\ - fi ;\ - else \ - ${SHCOMMENT} "Remove all pkgs" ; \ - ${ECHO_MSG} ${PKG_DELETE} -r $$pkgname ; \ - ${DO} ${PKG_DELETE} -r $$pkgname || true ; \ - if ${PKG_INFO} -qe $$pkgname ; then \ - ${DO} ${PKG_DELETE} -f $$pkgname || true ; \ - fi ;\ - fi ;\ - done ; \ - fi ;\ - if [ "${USE_BULK_CACHE}" = "yes" ]; then \ - ${SHCOMMENT} "Install required depends via binarypkgs XXX" ; \ - ${ECHO_MSG} "BULK> Installing packages which are required to build ${PKGNAME}." ;\ - for pkgdir in `${SED} -n -e "/^${_ESCPKGPATH} / s;^[^:]*:;;p" ${DEPENDSFILE}` ${BULK_PREREQ} ; do \ - pkgname=`${GREP} "^$$pkgdir " ${INDEXFILE} | ${AWK} '{print $$2}'` ; \ - if [ -z "$$pkgname" ]; then continue ; fi ;\ - pkgfile=${PACKAGES}/All/$${pkgname}${PKG_SUFX} ;\ - if ${PKG_INFO} -qe $$pkgname ; then \ - ${ECHO_MSG} "BULK> Required package $$pkgname ($$pkgdir) is already installed" ; \ - else \ - if [ -f $$pkgfile ]; then \ - ${ECHO_MSG} "BULK> ${PKG_ADD} ${PKG_ARGS_ADD} $$pkgfile"; \ - ${DO} ${PKG_ADD} ${PKG_ARGS_ADD} $$pkgfile || ${ECHO_MSG} "warning: could not add $$pkgfile." ; \ - else \ - ${ECHO_MSG} "BULK> warning: $$pkgfile does not exist. It will be rebuilt." ;\ - fi ;\ - fi ;\ - done ;\ - fi ;\ - if [ -f ${INTERACTIVE_COOKIE} ]; then \ - ${ECHO_MSG} "BULK> Removing old marker for INTERACTIVE_STAGE..." ; \ - ${ECHO_MSG} ${RM} -f ${INTERACTIVE_COOKIE} ; \ - ${DO} ${RM} -f ${INTERACTIVE_COOKIE} ; \ - fi ;\ - ${ECHO_MSG} ${MAKE} package '(${PKGNAME})' 2>&1 ; \ - ${DO} ( ${MAKE} package 2>&1 ); \ - ) 2>&1 | ${TEE} -a ${BUILDLOG} ; \ - if [ -f ${PKGFILE} ]; then \ - ${RM} ${BUILDLOG} ; \ - else \ - ${MV} ${BUILDLOG} ${BROKENFILE} ;\ - if [ -f "${WRKLOG}" ]; then \ - (${ECHO_MSG} "<pre>"; \ - ${ECHO_MSG} ""; \ - ${TO_HTML} ${WRKLOG}; \ - ${ECHO_MSG} "</pre>"; \ - ) >> ${BROKENWRKLOG}; \ - fi; \ - ( \ - if [ -f "${BROKENWRKLOG}" ]; then \ - ${ECHO_MSG} "</pre>"; \ - ${ECHO_MSG} "<p>"; \ - ${ECHO_MSG} "Please view the <a href=\"../../${PKGPATH}/${BROKENWRKLOG}\">work log for ${PKGNAME}</a>"; \ - ${ECHO_MSG} "</p>"; \ - ${ECHO_MSG} "<pre>"; \ - ${ECHO_MSG} ""; \ - fi ; \ - ${ECHO_MSG} "BULK> ${PKGNAME} was marked as broken:" ; \ - ${LS} -la ${BROKENFILE} ; \ - ${ECHO_MSG} ${MAKE} deinstall ; \ - ${DO} ${MAKE} deinstall ; \ - nbrokenby=0;\ - if [ "${USE_BULK_CACHE}" = "yes" ]; then \ - ${ECHO_MSG} "BULK> Marking all packages which depend upon ${PKGNAME} as broken:"; \ - tmp=`${SED} -n -e "/^${_ESCPKGPATH} / s;^[^:]*:[ ]*;;p" ${SUPPORTSFILE}` ; \ - if test -n "$$tmp" ; then \ - for pkgdir in $$tmp ; do \ - pkgname=`${GREP} "^$$pkgdir " ${INDEXFILE} | ${AWK} '{print $$2}'` ;\ - if [ -z "$$pkgname" ]; then pkgname=unknown ; fi ; \ - ${ECHO_MSG} "BULK> marking package that requires ${PKGNAME} as broken: $$pkgname ($$pkgdir)";\ - pkgerr="-1"; \ - pkgignore=`(cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} show-var VARNAME=PKG_FAIL_REASON)`; \ - pkgskip=`(cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} show-var VARNAME=PKG_SKIP_REASON)`; \ - if [ ! -z "$${pkgignore}$${pkgskip}" -a ! -f ${PKGSRCDIR}/$$pkgdir/${BROKENFILE} ]; then \ - ${ECHO_MSG} "BULK> $$pkgname ($$pkgdir) may not be packaged because:" >> ${PKGSRCDIR}/$$pkgdir/${BROKENFILE};\ - ${ECHO_MSG} "BULK> $$pkgignore" >> ${PKGSRCDIR}/$$pkgdir/${BROKENFILE};\ - ${ECHO_MSG} "BULK> $$pkgskip" >> ${PKGSRCDIR}/$$pkgdir/${BROKENFILE};\ - if [ -z "`(cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} show-var VARNAME=BROKEN)`" ]; then \ - pkgerr="0"; \ - else \ - pkgerr="1"; \ - fi; \ - fi; \ - ${ECHO_MSG} "BULK> $$pkgname ($$pkgdir) is broken because it depends upon ${PKGNAME} (${PKGPATH}) which is broken." \ - >> ${PKGSRCDIR}/$$pkgdir/${BROKENFILE};\ - ${ECHO_MSG} "Please view the <a href=\"../../${PKGPATH}/${BROKENFILE}\">build log for ${PKGNAME}</a>.<br />" \ - >> ${PKGSRCDIR}/$$pkgdir/${BROKENFILE};\ - nbrokenby=`expr $$nbrokenby + 1`;\ - if ${GREP} -q " $$pkgdir/${BROKENFILE}" ${PKGSRCDIR}/${BROKENFILE} ; then :; else \ - ${ECHO} " $$pkgerr $$pkgdir/${BROKENFILE} 0 " >> ${PKGSRCDIR}/${BROKENFILE} ;\ - fi ;\ - done ;\ - fi ;\ - fi ;\ - nerrors=`${GREP} -c '^\*\*\* Error code' ${BROKENFILE} || true`; \ - if [ -f ${INTERACTIVE_COOKIE} ]; then \ - nerrors="0"; \ - fi; \ - ${ECHO_MSG} " $$nerrors ${PKGPATH}/${BROKENFILE} $$nbrokenby " >> ${PKGSRCDIR}/${BROKENFILE} \ - ) 2>&1 | ${TEE} -a ${BROKENFILE}; \ - fi ; \ - case ${_PRESERVE_WRKDIR} in \ - yes|YES) ;; \ - *) ${ECHO_MSG} "BULK> Cleaning package ${PKGNAME} and pre-requisite packages"; \ - if [ "${USE_BULK_CACHE}" = "yes" ]; then \ - for pkgdir in ${PKGPATH} `${SED} -n -e "/^${_ESCPKGPATH} / s;^[^:]*:;;p" ${DEPENDSFILE}`; do \ - ${DO} (cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} clean) ; \ - done; \ - else \ - ${ECHO_MSG} ${MAKE} clean CLEANDEPENDS=YES;\ - ${DO} ${MAKE} clean CLEANDEPENDS=YES; \ - fi ;; \ - esac; \ - fi - @if [ ! -f ${PKGFILE} ]; then \ - ${ECHO_MSG} "BULK> Build for ${PKGNAME} was not successful, aborting." | ${TEE} -a ${BROKENFILE} ; \ - false; \ - else \ - ${RM} -f ${BUILDLOG} ;\ - fi - -# Install pkg - if possible from binary pkg (i.e. available & upto date) -# else revert to the old recompiling. -# Don't rely upon pkg_add to snarf in depending pkgs as they may have -# been modified and need rebuilding. -bulk-install: - @if [ `${MAKE} bulk-check-uptodate REF=${PKGFILE}` = 1 ]; then \ - if ${PKG_INFO} -qe ${PKGNAME} ; then :; else \ - ${DO} ${MAKE} install-depends ; \ - ${ECHO_MSG} "BULK>" ${PKG_ADD} ${PKG_ARGS_ADD} ${PKGFILE} ; \ - ${DO} ${PKG_ADD} ${PKG_ARGS_ADD} ${PKGFILE} ; \ - fi ; \ - else \ - ${ECHO_MSG} ${MAKE} bulk-package PRECLEAN=no; \ - ${DO} ${MAKE} bulk-package PRECLEAN=no; \ - fi - diff --git a/mk/bulk/build b/mk/bulk/build deleted file mode 100644 index fe376f6ee6e..00000000000 --- a/mk/bulk/build +++ /dev/null @@ -1,365 +0,0 @@ -#!/bin/sh -# $NetBSD: build,v 1.45.2.4 2005/05/17 18:29:45 tv Exp $ - -# -# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org> -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. All advertising materials mentioning features or use of this software -# must display the following acknowledgement: -# This product includes software developed by Hubert Feyrer for -# the NetBSD Project. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -# -# Do bulk build -# - -usage() { - echo "Usage:" - echo " $0 [-m | --mirror_only] [-r | --restart | restart]" - echo " [-s | --specific-pkgs]" - echo " $0 -h|--help" - echo " " - echo "Runs a bulk pkgsrc build." - echo " " - echo "The following options are supported:" - echo " " - echo " -h|--help Displays this message." - echo " " - echo " -m|--mirror_only Downloads all distfiles need for the" - echo " build but does not run the build." - echo " IMPORTANT: Note that this will still" - echo " run all the pre-build stuff which involves" - echo " removing all of your installed packages." - echo " " - echo " The only difference between this option" - echo " and a regular bulk build is that the packages" - echo " are not actually built." - echo " " - echo " -r|--restart|restart Restart a previously interrupted bulk build." - echo " last form of this option is for backwards" - echo " compatibility and may be removed in future" - echo " versions of this script." - echo " " - echo " The --restart option may be combined with the" - echo " --mirror_only option." - echo " " - echo " -s|--specific-pkgs Sets SPECIFIC_PKGS=1 when building packages." - echo " This option is used for building a subset of" - echo " pkgsrc." - echo " " -} - -restart=no -mirror_only=no -target=bulk-package -makeargs="" - -while [ ${#} -ge 1 ] ; do - case $1 in - - --help|-h ) - usage - exit 0 - ;; - - --mirror_only|-m ) - mirror_only=yes - target=mirror-distfiles - shift - ;; - - --restart|-r|restart ) - restart=yes - shift - ;; - - --specific-pkgs|-s ) - makeargs="$makeargs SPECIFIC_PKGS=1" - shift - ;; - - -* ) - echo "unknown option: $1" - usage - exit 1 - ;; - - * ) - break - ;; - - esac -done - -if [ $# -ne 0 ]; then - usage - exit 1 -fi - -opsys=`uname -s` -case "$opsys" in -NetBSD) BMAKE=make ;; -*) BMAKE=bmake ;; -esac - -export BMAKE - -# Set resource limits as high as possible -ulimit -S -s `ulimit -H -s` -ulimit -S -d `ulimit -H -d` - - -echo Bulk build started: `date` -echo "" - -# Pull in ADMIN etc.: -if [ -f "$BULK_BUILD_CONF" ]; then - . $BULK_BUILD_CONF -else - conf=`dirname $0`/build.conf - if [ -f "$conf" ]; then - . $conf - else - echo "$0: Cannot find config file $conf, aborting." - exit 1 - fi -fi - -# set up variables specifically for the bulk build -BATCH=1 -DEPENDS_TARGET=bulk-install -export BATCH DEPENDS_TARGET - -if [ "$http_proxy" != "" ]; then - echo "Using HTTP proxy $http_proxy" - export http_proxy -fi -if [ "$ftp_proxy" != "" ]; then - echo "Using FTP proxy $ftp_proxy" - export ftp_proxy -fi -echo "" - -unset CDPATH # ensure cd does not print new cwd to stdout, which - # confuses the printindex script. -unset DISPLAY # allow sane failure for gimp, xlispstat - -# Check that the pkg_tools are up to date -(cd ${USR_PKGSRC}/pkgtools/pkglint && \ - ${BMAKE} fetch >/dev/null 2>&1 ) || \ - ( echo "Updating pkgtools" && \ - cd ${USR_PKGSRC}/pkgtools/pkg_install && ${BMAKE} clean && \ - ${BMAKE} install && ${BMAKE} clean ) - -cd ${USR_PKGSRC} - - -if [ "x$restart" = "xyes" ]; then - echo Restarting - skipping pre-build script -else - sh mk/bulk/pre-build # make veryveryclean :) - if [ $? != 0 ] - then - echo "Error during bulk-build preparations, aborting." - exit 1 - fi -fi - -fail=no -if [ -d pkgtools/pkglint ]; then - cd pkgtools/pkglint - PERL5_PKGSRCDIR=`${BMAKE} show-var VARNAME=PERL5_PKGSRCDIR` || fail=yes - BULK_DBFILE=`${BMAKE} show-var VARNAME=BULK_DBFILE` || fail=yes - DEPENDSTREEFILE=`${BMAKE} show-var VARNAME=DEPENDSTREEFILE` || fail=yes - DEPENDSFILE=`${BMAKE} show-var VARNAME=DEPENDSFILE` || fail=yes - SUPPORTSFILE=`${BMAKE} show-var VARNAME=SUPPORTSFILE` || fail=yes - INDEXFILE=`${BMAKE} show-var VARNAME=INDEXFILE` || fail=yes - ORDERFILE=`${BMAKE} show-var VARNAME=ORDERFILE` || fail=yes - BROKENFILE=`${BMAKE} show-var VARNAME=BROKENFILE` || fail=yes - BROKENWRKLOG=`${BMAKE} show-var VARNAME=BROKENWRKLOG ` || fail=yes - BUILDLOG=`${BMAKE} show-var VARNAME=BUILDLOG` || fail=yes - STARTFILE=`${BMAKE} show-var VARNAME=STARTFILE` || fail=yes - AWK=`${BMAKE} show-var VARNAME=AWK USE_TOOLS=awk` || fail=yes - GREP=`${BMAKE} show-var VARNAME=GREP USE_TOOLS=grep` || fail=yes - SED=`${BMAKE} show-var VARNAME=SED USE_TOOLS=sed` || fail=yes - MAIL_CMD=`${BMAKE} show-var VARNAME=MAIL_CMD USE_TOOLS=mail` || fail=yes - MACHINE_ARCH=`${BMAKE} show-var VARNAME=MACHINE_ARCH` || fail=yes - OPSYS=`${BMAKE} show-var VARNAME=OPSYS` || fail=yes - OS_VERSION=`${BMAKE} show-var VARNAME=OS_VERSION` || fail=yes -else - echo "The pkgtools/pkglint directory does not exist. Please update" - echo "your pkgsrc tree in ${USR_PKGSRC}" - exit 1 -fi - -echo "----------------------------------" -echo "| Build Temporary Files: |" -echo "----------------------------------" -echo "BULK_DBFILE = $BULK_DBFILE" -echo "DEPENDSTREEFILE = $DEPENDSTREEFILE" -echo "DEPENDSFILE = $DEPENDSFILE" -echo "SUPPORTSFILE = $SUPPORTSFILE" -echo "INDEXFILE = $INDEXFILE" -echo "ORDERFILE = $ORDERFILE" -echo "BROKENFILE = $BROKENFILE" -echo "BROKENWRKLOG = $BROKENWRKLOG" -echo "BUILDLOG = $BUILDLOG" -echo "STARTFILE = $STARTFILE" -echo "----------------------------------" - -# make sure we have values for these very important -# variables -if [ $fail = "yes" -o \ - -z "$PERL5_PKGSRCDIR" -o \ - -z "$BULK_DBFILE" -o \ - -z "$DEPENDSTREEFILE" -o \ - -z "$DEPENDSFILE" -o \ - -z "$SUPPORTSFILE" -o \ - -z "$INDEXFILE" -o \ - -z "$ORDERFILE" -o \ - -z "$BROKENFILE" -o \ - -z "$BROKENWRKLOG" -o \ - -z "$BUILDLOG" -o \ - -z "$STARTFILE" -o \ - -z "$AWK" -o \ - -z "$GREP" -o \ - -z "$SED" -o \ - -z "$MAIL_CMD" -o \ - -z "$MACHINE_ARCH" -o \ - -z "$OPSYS" -o \ - -z "$OS_VERSION" \ - ]; then - echo "ERROR: build failed to extract certain key variables." - echo " please examine the above list and correct the" - echo " problem." - exit 1 -fi - -cd ${USR_PKGSRC} - -# get the list of packages which should always be installed during the build -cd ${USR_PKGSRC}/pkgtools/pkglint -BULK_PREREQ=`${BMAKE} show-var VARNAME=BULK_PREREQ` -cd ${USR_PKGSRC} - -# install prerequisite packages. Note: we do this _before_ the depends tree -# because some packages like xpkgwedge only become DEPENDS if its installed -echo "Installing prerequisite packages specified with BULK_PREREQ..." -for pkgdir in $BULK_PREREQ -do - echo $pkgdir - # make sure its installed _and_ packaged - cd ${USR_PKGSRC}/$pkgdir && ${BMAKE} bulk-install -done - -# Create the bulk cache files -cd ${USR_PKGSRC} -if [ "x$restart" != "xyes" ]; then - cd ${USR_PKGSRC} && ${BMAKE} bulk-cache $makeargs - if [ $? != 0 ]; then - echo "$0: Cache creation failed. Aborting build." - exit 1 - fi -fi - -echo "Starting actual build using the order specified in $ORDERFILE..." -cd ${USR_PKGSRC} - -# make sure we have something to grep in in the build loop -touch $BUILDLOG - -# set the nice level for bulk builds -#case "${NICE_LEVEL}" in -#"") NICE_LEVEL="nice -n 20" ;; -#esac - -# Loop over every package in the correct order. Before building -# each one, check to see if we've already processed this package -# before. This could happen if the build got interrupted and we -# started it again with the 'restart' option. This prevents us -# from having to do a potentially very large number of make's to -# get back to where we let off. After we build each package, add -# it to the top level buildlog -# (usually '.make' or '.make.${MACHINE}'). As a side benefit, this -# can make a progress-meter very simple to add! - -tot=`wc -l $ORDERFILE | ${AWK} '{print $1}'` -for pkgdir in `cat $ORDERFILE` -do - ${GREP} -q "^${pkgdir}\$" $BUILDLOG - if [ $? -ne 0 ]; then - built=`wc -l $BUILDLOG | ${AWK} '{print $1}'` - percent=`echo $built $tot | ${AWK} '{printf("%4.1f%%",$1*100/$2);}'` - (cd $pkgdir && ${NICE_LEVEL} ${BMAKE} USE_BULK_CACHE=yes $target \ - $makeargs </dev/null | \ - ${SED} "s;^;`date '+%Y/%m/%d %H:%M:%S'` $built/${tot}=$percent $pkgdir @ ${MACHINE_ARCH}> ;g") - echo "$pkgdir" >> $BUILDLOG - fi -done - -echo "Build finished. Removing all installed packages left over from build..." - -for pkgname in `pkg_info -e \*` -do - pkg_info -qe $pkgname - if [ $? -eq 0 ]; then - pkgdir=`${GREP} " $pkgname " $INDEXFILE | ${AWK} '{print $1}'` - case "${BULK_PREREQ}" in - *$pkgdir* ) - echo "Keeping BULK_PREREQ: $pkgname ($pkgdir)" ; - ;; - * ) - echo pkg_delete -r $pkgname - pkg_delete -r $pkgname - pkg_info -qe $pkgname - if [ $? -eq 0 ]; then - echo "$pkgname ($pkgdir) did not deinstall nicely. Forcing the deinstall" - pkg_delete -f $pkgname || true - fi - ;; - esac - fi -done - -# for now, just quit if we were only mirroring distfiles. At somepoint we -# should teach the post-build script to generate a nice report about how many -# distfiles were downloaded, how many had bad checksums, failed master sites, -# network speed, etc. - -if [ "x$mirror_only" = "xyes" ]; then - echo "Bulk mirror of distfiles completed: `date`" - exit 0 -fi - -echo "Post processing bulk build results..." - -#rm $DEPENDSTREEFILE $DEPENDSFILE $SUPPORTSFILE $INDEXFILE $ORDERFILE - -# Perl was wiped, reinstall it! -( cd pkgtools/pkglint && cd ${PERL5_PKGSRCDIR} && ${BMAKE} bulk-install ) -perl mk/bulk/post-build | ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results `date +%Y-%m-%d`" $ADMIN - -# Done! -echo "" -echo Bulk build ended: `date` diff --git a/mk/bulk/build.conf-example b/mk/bulk/build.conf-example deleted file mode 100644 index 54abda7295d..00000000000 --- a/mk/bulk/build.conf-example +++ /dev/null @@ -1,80 +0,0 @@ -# build.conf -# $NetBSD: build.conf-example,v 1.22.4.3 2005/05/17 18:29:45 tv Exp $ -# -# config file in /bin/sh syntax for {,pre,post}-build -# - -# List of package directories to build. -# Defaults to all packages if PKGLIST is empty or undefined. -#PKGLIST="www/mozilla meta-pkgs/kde3" - -# Some paths for output files and paths to log files -FTPx=`date +%Y%m%d.%H%M` -FTPURL="pub/NetBSD/pkgstat/${FTPx}" # relative to ~ftp ! -FTP="/home/ftp/${FTPURL}" # absolute base path -FTPHOST="ftp://ftp.machi.ne" # host for broken.html - -# Where build will mail the report -ADMIN="you@some.whe.re" - -# Who the report is signed by -ADMINSIG="-Your Name" - -# The OS release this bulk build is happening on -osrev=`uname -r` - -# The machine architecture this is happening on -arch=`uname -m` - -# Where our pkgsrc is located -USR_PKGSRC="/usr/pkgsrc" - -# As which user to run "cvs updates". Leave empty for no update. -CVS_USER="yourlogin" - -# Flags to pass to 'cvs update', e.g. to get a certain branch -#CVS_FLAGS="-rnetbsd-2005Q1" -#CVS_FLAGS="-A" # pkgsrc-current - -# prune distfiles to remove those which are out of date -# can be set to 'yes' or 'no'. -PRUNEDISTFILES=no - -# prune binary packages to remove those which are out of date -# can be set to 'yes' or 'no'. -PRUNEPACKAGES=yes - -# whether to cache the lintpkgsrc database for PRUNE* or uploading -# can be set to 'yes' or 'no'. -LINTPKGSRC_CACHE=no - -# Proxies (optional): -#ftp_proxy=http://proxy.machi.ne:3128/ -#http_proxy=http://proxy.machi.ne:3128/ - -# Uploading binary packages: - -# Update vulnerability database before uploading. -# can be set to 'yes' or 'no'. -UPDATE_VULNERABILITY_LIST=yes - -# Destination for packages and rsync options -RSYNC_DST=$CVS_USER@ftp.NetBSD.org:/pub/NetBSD/packages/pkgsrc-200xQy/NetBSD-a.b.c/i386 -RSYNC_OPTS='-e ssh' - -# Nice level for builds -NICE_LEVEL="nice -n 20" - -# Whether or not to create checksum files -MKSUMS=yes - -# If you want to PGP sign the checksum files, set and correct -#SIGN_AS=username@NetBSD.org - -########################################################################### -### No changes should be needed below this line !!! -########################################################################### - -# Name of the (generated) HTML file which lists all broken pkgs -REPORT="$FTP/broken.html" - diff --git a/mk/bulk/do-sandbox-build b/mk/bulk/do-sandbox-build deleted file mode 100644 index 58d2ab2d6a2..00000000000 --- a/mk/bulk/do-sandbox-build +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# $NetBSD: do-sandbox-build,v 1.7.8.3 2005/05/17 18:29:45 tv Exp $ - -# -# Script to start a sandbox build -# -# See pkgsrc/doc/pkgsrc.txt for documentation! -# - -sh=`which sh` -chroot /usr/sandbox $sh -c "cd /usr/pkgsrc && $sh mk/bulk/build $@" diff --git a/mk/bulk/do-sandbox-upload b/mk/bulk/do-sandbox-upload deleted file mode 100644 index 1aac9f0a455..00000000000 --- a/mk/bulk/do-sandbox-upload +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# $NetBSD: do-sandbox-upload,v 1.2.2.3 2005/05/17 18:29:45 tv Exp $ - -# -# Script to start a sandbox build -# -# See pkgsrc/doc/pkgsrc.txt for documentation! -# - -sh=`which sh` -chroot /usr/sandbox $sh -c "cd /usr/pkgsrc/ ; $sh mk/bulk/upload $@" diff --git a/mk/bulk/mksandbox b/mk/bulk/mksandbox deleted file mode 100755 index 2ad6696bcf0..00000000000 --- a/mk/bulk/mksandbox +++ /dev/null @@ -1,358 +0,0 @@ -#! /bin/sh - -# $NetBSD: mksandbox,v 1.38.2.2 2005/05/17 18:29:45 tv Exp $ -# -# -# Copyright (c) 2002 Alistair G. Crooks. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. All advertising materials mentioning features or use of this software -# must display the following acknowledgement: -# This product includes software developed by Alistair G. Crooks -# for the NetBSD project. -# 4. The name of the author may not be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE -# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -# Usage: mksandbox [--pkgsrc=dir] [--src=srcdir] [--xsrc=xsrcdir] [--verbose] sandbox-dir -# -# A small shell script to set up a sandbox (usually for a pkgsrc bulk -# build), using null mounts. - -pkgsrc=/usr/pkgsrc -src=/usr/src -xsrc=/usr/xsrc - -kernel="" -sandboxMountDirs="/bin /sbin /lib /libexec /usr/X11R6 /usr/bin /usr/games /usr/include /usr/lib /usr/libdata /usr/libexec /usr/lkm /usr/share /usr/sbin /var/mail" -sandboxEmptyDirs="/var/run /var/log /var/spool/mqueue" -sandboxEmptyFiles="/var/run/utmp /var/run/utmpx /var/log/wtmp /var/log/wtmpx /var/log/lastlog /var/log/lastlogx" - -usage() -{ - echo "Usage: mksandbox [--pkgsrc=dir] [--src=srcdir] [--xsrc=xsrcdir] [--verbose] sandbox-dir" - exit 1 -} - -err() -{ - echo "error: $1" - exit 1 -} - -# by default, don't require src and xsrc to be available -need_src=no -need_xsrc=no - -opsys=`uname -s` -case "$opsys" in -Darwin) - bmakeprog=bmake - chmodprog=/bin/chmod - chownprog=/usr/sbin/chown - cpprog=/bin/cp - gtarprog=/usr/bin/gnutar - idprog=/usr/bin/id - mkdirprog="/bin/mkdir -p" - mountflags="-t null" - mountprog=/sbin/mount - paxprog=/bin/pax - sedprog=/usr/bin/sed - ;; -Linux) - bmakeprog=bmake - chmodprog=/bin/chmod - chownprog=/bin/chown - cpprog=/bin/cp - gtarprog=/bin/tar - idprog=/usr/bin/id - mkdirprog="/bin/mkdir -p" - mountflags="--bind" - mountprog=/sbin/mount - paxprog="" - sedprog=/bin/sed - sandboxMountDirs="$sandboxMountDirs /proc" - ;; -NetBSD) - bmakeprog=make - chmodprog=/bin/chmod - chownprog=/usr/sbin/chown - cpprog=/bin/cp - gtarprog=/usr/bin/tar - idprog=/usr/bin/id - kernel=/netbsd - mkdirprog="/bin/mkdir -p" - mountflags="-t null" - mountprog=/sbin/mount - paxprog=/bin/pax - sedprog=/usr/bin/sed - need_src=yes - need_xsrc=yes - ;; -SunOS) - bmakeprog=bmake - chmodprog=/usr/bin/chmod - chownprog=/usr/bin/chown - cpprog=/usr/bin/cp - gtarprog="" - idprog=/usr/xpg4/bin/id - mkdirprog="/usr/bin/mkdir -p" - mountflags="-F lofs" - mountprog=/sbin/mount - paxprog=/bin/pax - sedprog=/usr/xpg4/bin/sed - sandboxMountDirs="/bin /sbin /kernel /lib /proc /opt/SUNWspro /usr/X11R6 /usr/5bin /usr/bin /usr/ccs /usr/dt /usr/games /usr/include /usr/lib /usr/openwin /usr/share /usr/sbin /usr/ucb /usr/ucblib /usr/xpg4 /var/mail /var/sadm" - sandboxEmptyDirs="$sandboxEmptyDirs /usr/tmp /var/tmp" - ;; -*) - echo "Unknown Operating System ($opsys) - good luck" - bmakeprog=bmake - chmodprog=chmod - chownprog=chown - cpprog=cp - gtarprog="tar" - idprog="id" - mkdirprog="mkdir -p" - mountflags="-t null" - mountprog=mount - paxprog=pax - sedprog=sed - ;; -esac - -while [ $# -gt 0 ]; do - case "$1" in - --pkgsrc=*) pkgsrc=`echo $1 | $sedprog -e 's|^--pkgsrc=||'` ;; - --src=*) src=`echo $1 | $sedprog -e 's|^--src=||'` ;; - --xsrc=*) xsrc=`echo $1 | $sedprog -e 's|^--xsrc=||'` ;; - --verbose) set -x ;; - -*) usage ;; - *) break ;; - esac - shift -done - -if [ $# -ne 1 ]; then - usage -fi - -if [ `$idprog -u` -ne 0 ]; then - err "You must be root to run this script." -fi - -if [ ! -d $pkgsrc ]; then - err "pkgsrc directory $pkgsrc does not exist." -fi - -if [ ! -d $src -a "$need_src" = "yes" ]; then - err "source directory $src does not exist." -fi - -if [ ! -d $xsrc -a "$need_xsrc" = "yes" ]; then - err "xsrc directory $xsrc does not exist." -fi - -sandbox=$1 -sandbox_script="$sandbox/sandbox" - -packages=`(cd $pkgsrc/pkgtools/pkglint; $bmakeprog show-var VARNAME=PACKAGES)` -distfiles=`(cd $pkgsrc/pkgtools/pkglint; $bmakeprog show-var VARNAME=DISTDIR)` -localbase=`(cd $pkgsrc/pkgtools/pkglint; $bmakeprog show-var VARNAME=LOCALBASE)` -pkg_dbdir=`(cd $pkgsrc/pkgtools/pkglint; $bmakeprog show-var VARNAME=PKG_DBDIR)` -localpatches=`(cd $pkgsrc/pkgtools/pkglint; $bmakeprog show-var VARNAME=LOCALPATCHES)` - -test -d "$localpatches" || echo "WARNING: LOCALPATCHES directory does not exist - ignoring" - -$mkdirprog $sandbox -cat > $sandbox_script <<EOS -#! /bin/sh - -sandbox=$sandbox - -r3() { - _R= - while [ \$# -ge 3 ] - do - _R="\$1 \$2 \$3 \$_R" - shift; shift; shift - done - echo "\$_R" -} - -fses="\\ -EOS - -if [ ! -z "$kernel" ]; then - echo "Copying the kernel" - $cpprog $kernel $sandbox -fi - -echo "Checking package hierarchy in $localbase and package database in $pkg_dbdir exist" -$mkdirprog $sandbox/$localbase $sandbox/$pkg_dbdir - -echo "Make and populate $sandbox/dev" -$mkdirprog $sandbox/dev - -case "$opsys" in -SunOS) - /usr/sbin/devfsadm -r $sandbox - if [ -f /usr/ucblib/ucblinks.awk -a -x /usr/ucb/ucblinks ]; then - /usr/ucb/ucblinks -r $sandbox - fi - ;; -Linux) - $cpprog /dev/MAKEDEV* $sandbox/dev - (cd $sandbox/dev; ./MAKEDEV generic) - ;; -*) - $cpprog /dev/MAKEDEV* $sandbox/dev - (cd $sandbox/dev; ./MAKEDEV all) -esac - -echo "Make and populate $sandbox/etc" -$mkdirprog $sandbox/etc -case "$paxprog" in -"") (cd /etc; $gtarprog -cf - . | (cd $sandbox/etc; $gtarprog xf - )) ;; -*) (cd /etc; $paxprog -rwpe . $sandbox/etc) ;; -esac -rm -f $sandbox/etc/localtime - -case "$opsys" in -SunOS) - $cpprog /etc/TIMEZONE $sandbox/etc/TIMEZONE - ;; -*) - $cpprog /usr/share/zoneinfo/GMT $sandbox/etc/localtime - ;; -esac - -echo "Make empty dirs upon which to mount the null mounts" -for d in $sandboxMountDirs; do - test -d $d || continue; - $mkdirprog $sandbox$d; - $mountprog $mountflags -r $d $sandbox$d; - echo "$d $d ro \\" >> $sandbox_script -done - -echo "Making /tmp in $sandbox" -$mkdirprog $sandbox/tmp $sandbox/var/tmp -$chmodprog 1777 $sandbox/tmp $sandbox/var/tmp - -echo "Making /var/games in $sandbox" -$mkdirprog $sandbox/var/games -$chownprog games:games $sandbox/var/games -$chmodprog 2775 $sandbox/var/games - -for d in $sandboxEmptyDirs; do - test -d $d || continue - echo "Making $d in $sandbox" - $mkdirprog $sandbox$d -done - -for f in $sandboxEmptyFiles; do - test -f $f || continue - echo "Making $f in $sandbox" - $cpprog /dev/null $sandbox$f -done - -if [ "$need_src" = "yes" ]; then - echo "Mount $src from $sandbox" - $mkdirprog $sandbox/usr/src - $mountprog $mountflags -r $src $sandbox/usr/src - echo "$src /usr/src ro \\" >> $sandbox_script -fi - -echo "Mount $pkgsrc from $sandbox" -$mkdirprog $sandbox/usr/pkgsrc -$mountprog $mountflags $pkgsrc $sandbox/usr/pkgsrc -echo "$pkgsrc /usr/pkgsrc rw \\" >> $sandbox_script - -if [ "$need_xsrc" = "yes" ]; then - echo "Mount $xsrc from $sandbox" - $mkdirprog $sandbox/usr/xsrc - $mountprog $mountflags -r $xsrc $sandbox/usr/xsrc - echo "$xsrc /usr/xsrc ro \\" >> $sandbox_script -fi - -echo "Mounting $packages and $distfiles from $sandbox" -$mkdirprog $sandbox/$packages $sandbox/$distfiles -$mkdirprog $packages $distfiles -$mountprog $mountflags $packages $sandbox/$packages -$mountprog $mountflags $distfiles $sandbox/$distfiles -echo "$packages $packages rw \\" >> $sandbox_script -echo "$distfiles $distfiles rw \\" >> $sandbox_script - -if [ -n "$localpatches" ] && [ -d "$localpatches" ]; then - echo "Mounting $localpatches from $sandbox" - $mkdirprog $sandbox/$localpatches - $mountprog $mountflags $localpatches $sandbox/$localpatches - echo "$localpatches $localpatches rw \\" >> $sandbox_script -fi - -cat >> $sandbox_script <<EOS -" -case x\$1 in - xmount) - set dummy \$fses - shift - while [ \$# -ge 3 ]; do - fs=\$1; shift - mntpoint=\$1; shift - rwro=\$1; shift - case \$rwro in - ro) mount $mountflags -r \$fs \$sandbox/\$mntpoint || exit 1 ;; - rw) mount $mountflags \$fs \$sandbox/\$mntpoint || exit 1 ;; - esac - done - ;; - xumount) - set dummy \`r3 \$fses\` - shift - while [ \$# -ge 3 ]; do - fs=\$1; shift - mntpoint=\$1; shift - dummy=\$1; shift - umount \$sandbox/\$mntpoint - done - ;; - *) - if [ \$# -eq 0 ]; then - set dummy /bin/sh - shift - fi - chroot \$sandbox "\$@" - ;; -esac -EOS - -chmod +x $sandbox_script - -case $opsys in -SunOS) - $cpprog /etc/mnttab $sandbox/etc/mnttab - ;; -*) -esac - -echo "Sandbox creation is now complete" - -exit 0 diff --git a/mk/bulk/parallel.txt b/mk/bulk/parallel.txt deleted file mode 100644 index ffd844bfe17..00000000000 --- a/mk/bulk/parallel.txt +++ /dev/null @@ -1,207 +0,0 @@ -# $Id: parallel.txt,v 1.5 2004/07/10 14:37:36 grant Exp $ -# - -These are my (<dmcmahill>) thoughts on how one would want a parallel -bulk build to work. - - -==================================================================== -Single Machine Build Process -==================================================================== - -The current (as of 2003-03-16) bulk build system works in the -following manner: - -1) All installed packages are removed. - -2) Packages listed in the BULK_PREREQ variable are installed. This - must be done before step 2 as some packages (like xpkgwedge) can - affect the dependencies of other packages when installed. - -3) Each package directory is visited and its explicitly listed - dependencies are extracted and put in a 'dependstree' file. The - mk/bulk/tflat script is used to generate flattened dependencies - for all packages from this dependstree file in both the up and - down directions. The result is a file 'dependsfile' which has one - line per package that lists all build dependencies. Additionally, - a 'supportsfile' is created which has one line for each package - and lists all packages which depend upon the listed pacakge. - Finally, tsort(1) is applied to the 'dependstree' file to - determine the correct build order for the bulk build. The build - order is stored in a 'buildorder' file. This is all achieved via - the 'bulk-cache' top level target. By extracting dependencies in - this fashion, we avoid highly redundant recursive make calls. For - example, we no longer need to use a recursive make to find the - dependencies for libtool literally thousands and thousands of - times throughout the build. - -4) During the build, the 'buildorder' file is consulted to figure out - which package should be built next. Then to build the package, - the following steps are taken: - - a) Check for the existance of a '.broken' file in the package - directory. If this file exists, then the package is already - broken for some reason so move on to the next package. - - b) Remove all packages which are not needed to build the current - package. This dependency list is obtained from the 'dependsfile' - created in step 3 and the BULK_PREREQ variable. - - c) Install via pkg_add all packages which are needed to build the - current package. We are able to do this because we have been - building our packages in a bottom up order so all dependencies - should have been built. - - d) Build and package the package. - - e) If the package build fails, then we copy over the build log to - a .broken file and in addition, we consult the 'supportsfile' and - mark all packages which depend upon this one as broken by adding a - line to their .broken files (creating them if needed). By going - ahead and marking these packages as broken, we avoid wasting time - on them later. - - f) Append the package directory name to the top level pkgsrc - '.make' file to indicate that we have processed this package. - -5) Run the mk/bulk/post-build script to collect the summary and - generate html pages and the email we've all seen. - -==================================================================== -Single Machine Build Comments -==================================================================== - -There are several features of this approach that are worth mentioning -explicitly. - -1) Packages are built in the correct order. We don't want to rebuild - the gnome meta-pkg and then rebuild gnome-libs for example. - -2) Restarting the build is a cheap operation. Remember that this - build can take weeks or more. In fact the 1.6 build took nearly 6 - weeks on a sparc 20! If for some reason, the build needs to be - interrupted, it can be easily restarted because in step 4f we keep - track of what has been built in a file. The lines in the build - script which control this are: - - for pkgdir in `cat $ORDERFILE` ; do - if ! grep -q "^${pkgdir}\$" $BUILDLOG ; then - (cd $pkgdir && \ - nice -n 20 ${BMAKE} USE_BULK_CACHE=yes bulk-package) - fi - done - - In addition to storing the progress to disk, the bulk cache files - (the 'dependstreefile', 'dependsfile', 'supportsfile', and - 'orderfile') are stored on disk so they do not need to be - recreated if a build is stopped and then restarted. - -3) By leaving packages installed and only deleting the ones which are - not needed before each build, we reduce the amount of installing - and deinstalling needed during the build. For example, it is - quite common to build several packages in a row which all need GNU - make or perl. - -4) Using the 'supportsfile' to mark all packages which depend upon a - package which has just failed to build can greatly reduce the time - wasted on trying to build packages which known broken dependencies. - -==================================================================== -Parallel Build Thoughts -==================================================================== - -To exploit multiple machines in an attempt to reduce the build time, -many of the same ideas used in the single machine build can still be -used. My view of how a parallel build should work is detailed here. - -master == master machine. This machine is in charge of directing - the build and may or may not actively participate in it. - In addition, this machine might not be of the same - architecture or operating system as the slaves (unless it - is to be used as a slave as well). - -slave#x == slave machine #x. All slave machines are of the same - MACHINE_ARCH and have the same operating system and access - the same pkgsrc tree via NFS and access the same binary - packages directory. - - If the master machine is also to be used as a build - machine, then it is also considered a slave. - -Prior to starting the build, the master directs one of the slaves to -extract the dependency information per steps 1-3 in the single machine -case. - -The actually build should progress as follows: - -1) For each slave which needs a job, the master assigns a package to - build based on the rule that only packages that have had all their - dependencies built will be sent to slaves for compilation. - -2) When a slave finishes, the master either notes that the binary - package is now available for use as a depends _or_ notes failure - and marks all packages which depend upon it as broken as in step - 4e of the single machine build. - - -Each slave builds a package in the same way as it would in a single -machine build (steps 4a-d). - -==================================================================== -Important Parallel Build Considerations -==================================================================== - - -1) Security. Packages are installed as root prior to packaging. - -2) All state kept by the master should be stored to disk to - facilitate restarting a build. Remember this could take weeks so - we don't want to have to start over. - -3) The master needs to be able to monitor all slaves for signs of - life. I.e., if a slave machine is simply shut off, the master - should detect that it's no longer there and re-assign that slaves - current job. - -3a) The master must be able to distinguish between a slave failing to - compile a package due to the package failing vs a - network/power/disk/etc. failure. The former causes the package to - be marked as broken, the latter causes the slave to be marked as - broken. - -4) Ability to add and remove slaves from the cluster during a build. - Again, a build may take a long time so we want to add/remove - slaves while the build is in progress. - -==================================================================== -Additional Thoughts -==================================================================== - -This is mostly related to using slaves which are not on a local -network. - -- maybe a hook could be put in place which rsync's the binary package - tree between the binary package repository machine and the slave - machine before and after each package is built? - -- security - -- support for Kerberos? - -==================================================================== -Implementation Thoughts -==================================================================== - -- Can this all be written around using ssh to send out tasks? How do - we monitor slaves for signs of life? How do we indicate 'build - failed/build succeeded/slave failed' conditions? - -- Maybe we could have a file listing slaves and the master consults - this each time it needs a slave. That would make adding/removing - slaves easy. There would need to be another file to keep track of - which slaves are busy (and with what). - -- Do we want to use something like pvm instead? There is a - p5-Parallel-Pvm package and perl nicely deals with parsing some of - these files and sorting dependencies although I hate to add any - extra dependencies to the build system. diff --git a/mk/bulk/post-build b/mk/bulk/post-build deleted file mode 100644 index c01e9105817..00000000000 --- a/mk/bulk/post-build +++ /dev/null @@ -1,591 +0,0 @@ -#!/usr/pkg/bin/perl -# $NetBSD: post-build,v 1.47.2.4 2005/05/17 18:29:45 tv Exp $ -# -# Collect stuff after a pkg bulk build -# -# (c) 2000 Hubert Feyrer, All Rights Reserved. -# - -use File::Basename; -use POSIX qw(strftime); -use strict; -use warnings; - -my %vars; -my $verbose = 1; # set to 2 to see more command execution detail - -sub my_system (@) { - print STDERR '> '.join(' ', @_)."\n" if ($verbose >= 2); - system(@_); -} - -# Where config vars are stored (/bin/sh syntax) - -my $BULK_BUILD_CONF = $ENV{BULK_BUILD_CONF} || (dirname($0).'/build.conf'); -$BULK_BUILD_CONF = "./$BULK_BUILD_CONF" if ($BULK_BUILD_CONF !~ m:^/:); - -# Dig given variable out of config file, and set it -sub getconf (@) { - open(I, ". $BULK_BUILD_CONF; for var in ".join(' ', @_)."; do eval echo \\\${\$var}; done |") || die 'cannot open pipe'; - - foreach my $var (@_) { - $vars{$var} = <I>; - chomp $vars{$var}; - die "\$$var not defined by $BULK_BUILD_CONF" if ($vars{$var} eq ''); - - print STDERR "> $var=$vars{$var}\n" if ($verbose >= 2); - } -} - -getconf( - 'ADMINSIG', # "-Your Name" - 'FTPURL', # "pub/NetBSD/pkgstat/`date +%Y%m%d.%H%M`" - 'FTP', # "/disk1/ftp/${FTPURL}" - 'FTPHOST', # ftp://ftp.machi.ne/ - 'REPORT', # "broken.html" - 'USR_PKGSRC', # "/usr/pkgsrc" - 'arch', # `uname -m` - 'osrev', # `uname -r` -); - -my $reportf = basename($vars{REPORT}); - -my $os = `uname -s`; -chomp $os; - -my $BMAKE = $ENV{BMAKE} || die '$BMAKE not defined in environment'; - -sub getmakevars (@) { - open(I, "cd $vars{USR_PKGSRC}/pkgtools/pkglint && $BMAKE show-vars BATCH=1 VARNAMES='".join(' ', @_)."' |") || die 'cannot open pipe'; - - foreach my $var (@_) { - $vars{$var} = <I>; - chomp $vars{$var}; - die "\${$var} not defined by $BMAKE" if ($vars{$var} eq ''); - - print STDERR "> $var=$vars{$var}\n" if ($verbose >= 2); - } -} - -# Extract the names of the files used for the build log and broken build logs. -# These have defaults set by bsd.bulk-pkg.mk and may be overridden in -# /etc/mk.conf -getmakevars(qw( - BROKENFILE - BROKENWRKLOG - BULK_DBFILE - DEPENDSFILE - DEPENDSTREEFILE - FIND - GTAR - INDEXFILE - LOCALBASE - ORDERFILE - PKG_DBDIR - STARTFILE - SUPPORTSFILE - X11BASE -)); - -my $bulkdbfile = basename($vars{BULK_DBFILE}); -my $dtfile = basename($vars{DEPENDSTREEFILE}); -my $depfile = basename($vars{DEPENDSFILE}); -my $supfile = basename($vars{SUPPORTSFILE}); -my $indfile = basename($vars{INDEXFILE}); -my $ordfile = basename($vars{ORDERFILE}); - -my $startdate = (stat($vars{STARTFILE}))[9]; -my $enddate = ''; -if ($startdate == 0) { - $startdate = "unknown"; -} else { - local $ENV{TZ} = "UTC"; - $startdate = strftime("%c %Z", gmtime($startdate)); - $enddate = strftime("%c %Z", gmtime(time())); -} - -my $report_head = <<EOF; -Package Breaks Maintainer --------------------------------------------------------------- -EOF -my $report_form = <<EOF; -@<<<<<<<<<<<<<<<<<<<<<<<<< @<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<< -EOF -# $pkg, $nbrokenby, $maintainer - - -my_system("mkdir -p $vars{FTP}"); - -# Copy over the output from the build process -chdir($vars{USR_PKGSRC}); -my_system("/bin/ls -1 $vars{BROKENFILE} $vars{BROKENWRKLOG} */*/$vars{BROKENFILE} */*/$vars{BROKENWRKLOG} 2>/dev/null | $vars{GTAR} -T - -plcf - | (cd $vars{FTP}; $vars{GTAR} -plxf -)"); - -# Copy over the cache files used during the build -foreach my $f qw(BULK_DBFILE DEPENDSTREEFILE DEPENDSFILE SUPPORTSFILE INDEXFILE ORDERFILE) { - my_system("cp $vars{$f} $vars{FTP}") if -f $vars{$f}; -} - -chdir($vars{FTP}); -writeReport(); - -# -# Adjust "last" symlink -# -{ - my ($base, $dir) = ($vars{FTP} =~ m|^(.*)/([^/]*)$|); - - unlink("$base/last"); - symlink($dir, "$base/last"); -} - -# -# Generate leftovers-$vars{arch}.html: files not deleted -# Leftover files are copied to leftovers-$vars{arch} dir, and linked from -# leftovers-$vars{arch}.html -# -{ - chdir($vars{FTP}); - my_system("mkdir -p leftovers-$vars{arch}"); - - # Find files since last build: - my $leftovers_txt = "leftovers-$vars{arch}.txt"; - my $leftovers_html = "leftovers-$vars{arch}.html"; - - my_system("$vars{FIND} $vars{LOCALBASE}/ -newer $vars{STARTFILE} -type f -print >>$leftovers_txt"); - my_system("$vars{FIND} $vars{X11BASE}/ -newer $vars{STARTFILE} -type f -print >>$leftovers_txt"); - - # Strip perl-files: - my $perlfiles; - { - local $/; - undef $/; - $perlfiles = `pkg_info -qL perl*`; - } - - my $perlfiles_pattern = $perlfiles; - $perlfiles_pattern =~ s/\n/|/g; - $perlfiles_pattern =~ s/|$//; - - open (LEFT, $leftovers_txt) or die "can't read $leftovers_txt: $!"; - my @left = <LEFT>; - close (LEFT); - my @leftovers = grep(!/^(?:${perlfiles_pattern})$/, @left); - - if (index($vars{PKG_DBDIR}, $vars{LOCALBASE}) == 0) { - # If PKG_DBDIR is inside LOCALBASE, exclude it from the leftovers. - @leftovers = grep { index($_, $vars{PKG_DBDIR}) != 0 } @leftovers; - } - - open (LEFT, ">$leftovers_txt") or die "can't write $leftovers_txt: $!"; - print LEFT @leftovers; - close (LEFT); - - if (scalar(@leftovers)) { - # Store leftovers, for easier identification: - my_system("$vars{GTAR} -T $leftovers_txt -plcf - | (cd leftovers-$vars{arch}; $vars{GTAR} -plxf -)"); - } - - # Add links to leftover list: - open (OUT, "> $leftovers_html") - or die "can't write $leftovers_html"; - print OUT <<EOOUT; -<html> -<body> -<pre> -EOOUT - foreach (@leftovers) { - chomp; - print OUT "<a href=\"$vars{FTPHOST}/$vars{FTPURL}/leftovers-$vars{arch}$_\">$_</a>\n"; - } - print OUT <<EOOUT2; -</pre> -</body> -</html> -EOOUT2 - close(OUT); -} - -# print the result of a single broken package -sub pkgResult ($$) { - my ($pinfo, $state) = @_; - my $pkg = $pinfo->{pkg}; - my $nbrokenby = $pinfo->{nbrokenby}; - my $nerrors = $pinfo->{nerrors}; - - my @idents = `$vars{FIND} $vars{USR_PKGSRC}/$pkg -type f -print | xargs grep \\\$NetBSD`; - my $datetime = ""; - my $file = ""; - my $ver = ""; - foreach my $ident (@idents) { - $ident =~ /\$[N]etBSD: ([^ ]*),v [^ ]* ([^ ]*) ([^ ]*) [^ ]* Exp \$/; - if (defined($2) && defined($3) && ("$2 $3" gt $datetime)) { - $datetime = "$2 $3"; - $file = $1; - $ver = $1; - } - } - - my $maintainer = `grep ^MAINTAINER $vars{USR_PKGSRC}/$pkg/Makefile`; - $maintainer =~ s/MAINTAINER=[ \t]*//; - if (! $maintainer) { - $maintainer = `cd $vars{USR_PKGSRC}/$pkg ; $BMAKE show-var VARNAME=MAINTAINER`; - } - $maintainer =~ s/</</g; - $maintainer =~ s/>/>/g; - chomp($maintainer); - - (my $state_style = $state) =~ s/ //g; - - my $nbrokenby_html = '<td> </td>'; - $nbrokenby_html = - '<td align="right" class="pkg-'.$state_style.'">'.$nbrokenby.'</td>' - if $nbrokenby > 0; - - if ($pinfo->{nerrors} != 0 && $verbose && ($state eq "broken" || $state eq "topten")) { - print swrite($report_form, $pkg, $nbrokenby > 0 ? $nbrokenby : "", $maintainer); - } - - return <<EOHTML; -<tr> - <td><a class="pkg-$state_style" href="$pinfo->{bf}" title="build log for $pkg">$pkg</a></td> - $nbrokenby_html - <td>$file</td> - <td>$maintainer</td> -</tr> - -EOHTML -} - -# write the build report -sub writeReport { - my $broken = getBroken(); - my $nbroken = scalar(@{$broken->{"broken"}}); - my $nbrokendep = scalar(@{$broken->{"broken depends"}}); - my $nunpackaged = scalar(@{$broken->{"not packaged"}}); - my $nbrokentot = $nbroken + $nbrokendep; - my $ntotal = $nunpackaged + $nbroken + $nbrokendep; - - # determine the number of packages attempted, and then successful - open(ORDER, $vars{ORDERFILE}) || die "can't open $vars{ORDERFILE}: $!"; - my @order = <ORDER>; - close(ORDER); - my $nattempted = scalar(@order); - my $nsuccessful = $nattempted - $ntotal; - - if ($verbose) { - print <<EOF; -pkgsrc bulk build results -$os $vars{osrev}/$vars{arch} - -Summary: - -EOF - my $summary_form = <<EOF; - @<<<<<<<<<<<<<<<<<<<<<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<< -EOF - print swrite($summary_form, "Build started:", $startdate); - print swrite($summary_form, "Build ended:", $enddate); - print swrite($summary_form, '', ''); - print swrite($summary_form, "Successfully packaged:", $nsuccessful); - print swrite($summary_form, "Packages really broken:", $nbroken); - print swrite($summary_form, "Pkgs broken due to them:", $nbrokendep); - print swrite($summary_form, "Total broken:", $nbrokentot); - print swrite($summary_form, "Not packaged:", $nunpackaged); - print swrite($summary_form, "Total:", $ntotal); - print <<EOF; - -Packages not listed here resulted in a binary package. The build -report, including logs of failed/not-packaged is available from: - -$vars{FTPHOST}/$vars{FTPURL}/$reportf -EOF - } - - open(HTML, ">$vars{REPORT}") or die "Can't write $vars{REPORT}: $!\n"; - print HTML <<EOHTML; -<html> -<head> -<title>$os $vars{osrev}/$vars{arch} bulk package build</title> -<style type="text/css"> -<!-- - -body { - Font-Family: Tahoma, Verdana, sans-serif; - Line-Height: 1.3em; - Text-Decoration: None; - Color: black; - Background-Color: white; - Border-Width: 0; -} - -table { - Border-Width: 0; -} - -table td { - Font-Family: Tahoma, Verdana, sans-serif; - line-height: 1em; -} - -a:link { - Color: #3535c5; -} - -a:visited { - Color: #700080; -} - -a:hover { - Color: #6565e5; - Text-Decoration: underline; -} - -tr { - Vertical-Align: top; -} - -td { - Vertical-Align: top; -} - -h1 { - Font-Size: 3.5ex; - Line-Height: 1em; - Color: #000066; -} - -h2 { - Font-Size: 2.5ex; - Line-Height: 1em; - Color: #660000; -} - -h3 { - Font-Size: 2ex; - Color: #660066; -} - -h4 { - Font-Size: 1.8ex; - Color: #006600; -} - -tt.filename { - Line-Height: 1.3em; - Color: #AA0000; -} - -.pkgname { - Font-Family: Arial, Helvetica, Courier, fixed; - Font-Style: Italic; - Text-Decoration: none; - Line-Height: 1.3em; -} - -.pkg-broken { - Color: red; -} - -.pkg-brokendepends { - Color: orange; -} - -.pkg-notpackaged { - Color: blue; -} ---> -</style> -</head> - -<body bgcolor="white" text="black" link="#3535c5" vlink="#700080" alink="#3535c5"> - -<a name="top"></a> -<h1>pkgsrc bulk build results</h1> -<h2>$os $vars{osrev}/$vars{arch}</h2> - -<h3>Summary</h3> - -<table> -<tr> - <td>Build started: <td align="right">$startdate</td> -</tr> -<tr> - <td>Build ended: <td align="right">$enddate</td> -</tr> -<tr> - <td> </td> <td> </td> -</tr> -<tr> - <td>Successfully packaged:</td> <td align="right">$nsuccessful</td> -</tr> -<tr class="pkg-broken"> - <td>Packages really broken:</td> <td align="right">$nbroken</td> -</tr> -<tr class="pkg-brokendepends"> - <td>Packages broken due to them:</td> <td align="right">$nbrokendep</td> -</tr> -<tr> - <td>Total broken:</td> <td align="right">$nbrokentot</td> -</tr> -<tr class="pkg-notpackaged"> - <td>Not packaged:</td> <td align="right">$nunpackaged</td> -</tr> -<tr> - <td>Total:</td> <td align="right">$ntotal</td> -</tr> -</table> - -<p> - Packages not listed here resulted in a <a - href="../../packages/" title="binary packages for $os $vars{osrev}/$vars{arch}">binary - package</a>. Results of failed packages are available below. -</p> - -<p> - Files leftover from the build (because of broken PLISTs, etc.) can be - found in <a href="leftovers-$vars{arch}.html" title="leftover files">this - list</a>. -</p> - -<p> - Jump to:<br/> - <ul> - <li><a href="#topten">Top Ten Offenders</a></li> - <li><a href="#broken">Broken packages</a></li> - <li><a href="#broken depends">Broken dependencies</a></li> - <li><a href="#not packaged">Not packaged</a></li> - </ul> -</p> - -EOHTML - - my %state_head = ( - "topten" => "Top Ten Offenders", - "broken" => "Broken packages", - "broken depends" => "Broken dependencies", - "not packaged" => "Not packaged" - ); - - foreach my $state ("topten", "broken", "broken depends", "not packaged") { - next unless scalar(@{$broken->{$state}}); - - if ($verbose && ($state eq "topten" || $state eq "broken")) { - print "\n\n$state_head{$state}\n\n"; - print $report_head; - } - - - print HTML <<EOHTML; - -<a name="$state"></a> -<h2>$state_head{$state}</h2> -<table width="100%"> -<tr align="left"> - <th width="30%">Package</th> - <th>Breaks</th> - <th>File touched last</th> - <th>Maintainer</th> -</tr> - -EOHTML - foreach my $pinfo (@{$broken->{$state}}) { - print HTML pkgResult($pinfo, $state); - } - - print HTML <<EOHTML; -</table> -<hr> -<a href="#top">Up to top</a><br/> -<hr> -EOHTML - } - - print HTML <<EOHTML; -<hr> -<p> -The following cache files were used during the build: -</p> -<ul> -<li>The <a href="$bulkdbfile">SPECIFIC_PKGS bulk database file</a>.</li> -<li>The <a href="$dtfile">depends tree file</a>.</li> -<li>The <a href="$depfile">depends file</a>.</li> -<li>The <a href="$supfile">supports file</a>.</li> -<li>The <a href="$indfile">index file</a>.</li> -<li>The <a href="$ordfile">build order file</a>.</li> -</ul> -<hr> - -<p> -<ul> -<!-- <li>See the list of <a href="../index.html">all log files</a>. --> -<li>Visit the <a href="http://www.NetBSD.org">NetBSD web site</a>. -<li>Learn more about - <a href="http://www.NetBSD.org/Documentation/software/packages.html"> - The NetBSD Packages Collection</a>. -</ul> -</p> -</body> -</html> -EOHTML - close(HTML); - - if ($verbose) { - print "\n\n$vars{ADMINSIG}\n\n"; - print "[* This message was created by the Packages Collection bulk build software *]\n"; - } -} - -# get and sort the broken packages -sub getBroken { - my $res = { - 'broken' => [], - 'broken depends' => [], - 'not packaged' => [], - }; - - open (BF, $vars{BROKENFILE}) || return $res; - my @in = <BF>; - close (BF); - - foreach (@in) { - chomp; - my ($nerrors, $bf, $nbrokenby) = split; - my $pkg = $bf; - $pkg =~ s,/$vars{BROKENFILE},,; - my %tmp = ( - bf => $bf, - pkg => $pkg, - nbrokenby => $nbrokenby, - nerrors => $nerrors, - ); - - if ($nerrors > 0) { - push(@{$res->{"broken"}}, \%tmp); - } elsif ($nerrors == -1) { - push(@{$res->{"broken depends"}}, \%tmp); - } else { - push(@{$res->{"not packaged"}}, \%tmp); - } - } - - # sort pkgs in each state - foreach my $state ("broken", "broken depends", "not packaged") { - $res->{$state} = [ sort { $a->{pkg} cmp $b->{pkg} } @{$res->{$state}} ]; - } - - $res->{"topten"} = [ sort { $b->{nbrokenby} <=> $a->{nbrokenby} } @{$res->{"broken"}} ]; - - for (my $count = $#{$res->{"topten"}}; $count >= 10; $count--) { - pop(@{$res->{"topten"}}); - } - - return $res; -} - -sub swrite { - my $format = shift; - $^A = ""; - formline($format, @_); - $^A =~ s/\n\n/\n/; - return $^A; -} diff --git a/mk/bulk/pre-build b/mk/bulk/pre-build deleted file mode 100644 index 37534dc7044..00000000000 --- a/mk/bulk/pre-build +++ /dev/null @@ -1,286 +0,0 @@ -#!/bin/sh -# $NetBSD: pre-build,v 1.40.2.3 2005/05/17 18:29:45 tv Exp $ -# -# Clean up system to be ready for bulk pkg build -# -# (c) 2000 Hubert Feyrer, All Rights Reserved. -# - -#set -v # Debug - -# Pull in USR_PKGSRC, CVS_USER: -if [ -f "$BULK_BUILD_CONF" ]; then - . $BULK_BUILD_CONF -else - . `dirname $0`/build.conf -fi - -PRUNEDISTFILES=${PRUNEDISTFILES:-"no"} -PRUNEPACKAGES=${PRUNEPACKAGES:-"yes"} -PRUNELINKS=${PRUNEPACKAGES:-"yes"} - -PKGLINT_PKG_DIR=${USR_PKGSRC}/pkgtools/pkglint - -# Extract the name of the files used for the build log and broken build log. -# These have defaults set by bsd.bulk-pkg.mk and may be overridden in -# /etc/mk.conf -BROKENF=`( cd ${PKGLINT_PKG_DIR} ; ${BMAKE} show-var VARNAME=BROKENFILE )`; -export BROKENF -if [ "$BROKENF" = "" ]; then - echo "Had problems determining the name of the .broken files" - exit 1 -fi -BRKWRKLOG=`( cd ${PKGLINT_PKG_DIR} ; ${BMAKE} show-var VARNAME=BROKENWRKLOG )`; -if [ "$BRKWRKLOG" = "" ]; then - echo "Had problems determining the name of the .broken.work files" - exit 1 -fi -BLDLOG=`( cd ${PKGLINT_PKG_DIR} ; ${BMAKE} show-var VARNAME=BUILDLOG )`; -if [ "$BLDLOG" = "" ]; then - echo "Had problems determining the name of the .make files" - exit 1 -fi -LINTPKGSRC_DB=`( cd ${PKGLINT_PKG_DIR} ; ${BMAKE} show-var VARNAME=LINTPKGSRC_DB )`; -if [ "$LINTPKGSRC_DB" = "" ]; then - echo "Had problems determining the name of the .lintpkgsrc.db file" - exit 1 -fi -STARTFILE=`( cd ${PKGLINT_PKG_DIR} ; ${BMAKE} show-var VARNAME=STARTFILE )`; -if [ "$STARTFILE" = "" ]; then - echo "Had problems determining the name of the .start file" - exit 1 -fi - -LOCALBASE=`( cd ${PKGLINT_PKG_DIR} ; ${BMAKE} show-var VARNAME=LOCALBASE )`; -X11BASE=`( cd ${PKGLINT_PKG_DIR} ; ${BMAKE} show-var VARNAME=X11BASE )`; -DISTDIR=`( cd ${PKGLINT_PKG_DIR} ; ${BMAKE} show-var VARNAME=DISTDIR )`; - -# Make sure the pkg-vulnerabilities file is up to date -echo "Making sure vulnerability-list is upto date:" -if [ -z "$UPDATE_VULNERABILITY_LIST" -o "$UPDATE_VULNERABILITY_LIST" = yes ]; then - ( cd ${USR_PKGSRC}/security/audit-packages \ - && ${BMAKE} bulk-install \ - && env PKGVULNDIR=${DISTDIR} download-vulnerability-list ) - echo 'done.' -else - echo '(skipped)' -fi - -# On non-NetBSD platforms we need to keep the bootstrap-files! -if [ x"$BMAKE" = x"bmake" ]; then - ( cd $LOCALBASE && tar cf /tmp/$$.tar \ - sbin/mtree sbin/pkg_add sbin/pkg_admin sbin/pkg_create \ - sbin/pkg_delete sbin/pkg_info sbin/pkg_view sbin/linkfarm \ - share/mk/sys.mk share/mk/bsd.dep.mk share/mk/bsd.depall.mk \ - share/mk/bsd.files.mk share/mk/bsd.hostprog.mk \ - share/mk/bsd.doc.mk share/mk/bsd.inc.mk share/mk/bsd.info.mk \ - share/mk/bsd.kernobj.mk share/mk/bsd.kinc.mk \ - share/mk/bsd.kmod.mk share/mk/bsd.lib.mk \ - share/mk/bsd.links.mk share/mk/bsd.man.mk \ - share/mk/bsd.nls.mk share/mk/bsd.obj.mk \ - share/mk/bsd.prog.mk share/mk/bsd.subdir.mk \ - share/mk/bsd.sys.mk share/mk/bsd.own.mk \ - bin/bmake bin/ftp bin/digest bin/nbsed bin/install-sh \ - bin/pax bin/tar bin/cpio man/cat1/ftp.1 man/cat1/pax.0 \ - man/cat1/tar.0 man/cat1/cpio.0 man/cat1/pkg_add.0 \ - man/cat1/pkg_admin.0 man/cat1/pkg_create.0 \ - man/cat1/pkg_delete.0 man/cat1/pkg_info.0 \ - man/cat1/pkg_view.0 man/cat1/linkfarm.0 \ - man/man1/digest.1 man/man1/pax.1 man/man1/cpio.1 \ - man/man1/tar.1 man/man1/nbsed.1 man/man1/pkg_add.1 \ - man/man1/pkg_admin.1 man/man1/pkg_create.1 \ - man/man1/pkg_delete.1 man/man1/pkg_info.1 \ - man/man1/pkg_view.1 man/man1/linkfarm.1 man/cat8/mtree.0 \ - man/man8/mtree.8 man/cat7/packages.0 ) -fi - -# -# Clean out everything and its mother -# -PKG_DBDIR=`(cd ${PKGLINT_PKG_DIR}; ${BMAKE} show-var VARNAME=PKG_DBDIR)` -DEPOTBASE=`(cd ${PKGLINT_PKG_DIR}; ${BMAKE} show-var VARNAME=DEPOTBASE)` - -for dbdir in ${PKG_DBDIR} ${DEPOTBASE}; do - echo Removing all installed packages in $dbdir - if [ -d $dbdir ]; then - cd $dbdir - for pkg in * - do - if `pkg_info -K $dbdir -qe $pkg`; then - echo pkg_delete -r $pkg - pkg_delete -K $dbdir -r $pkg - fi - done - # - # this should have removed everything. - # Now force any broken pkgs to deinstall. - # - for pkg in * - do - if `pkg_info -K $dbdir -qe $pkg`; then - echo pkg_delete -f $pkg - pkg_delete -K $dbdir -f $pkg - fi - done - - # We've tried our best to get rid of the pkgs, now do it - # the hard way. If it wasn't for stuff in $X11BASE, I - # wouldn't have hard feelings about this! - rm -fr * - fi -done - -cd $LOCALBASE && rm -fr * && rm -fr .??*; -if [ x"$BMAKE" = x"bmake" ]; then - ( cd $LOCALBASE && tar xf /tmp/$$.tar ) -fi - -# Stuff known to be broken -rm -fr $X11BASE/share/gimp -rm -fr $X11BASE/share/gnome -rm -fr $X11BASE/share/kde -rm -fr $X11BASE/share/netscape -rm -fr /var/tmp/inst* -rm -fr $X11BASE/lib/libgimp* # gimp doesn't build with old libs around -rm -fr /nonexistent # broken useradd on pop3d -rm -fr /home/majordom # mail/majordomo pkg -rm -fr /home/nut # sysutils/ups-net -rm -fr /var/wwwoffle # www/wwwoffle -rm -fr /tmp/config-guess-* -rm -fr /tmp/mod* - - -# Clean up state files -cd ${USR_PKGSRC} -echo Cleaning up leftover state files from previous runs -rm -f */*/$BROKENF */*/$BRKWRKLOG */*/$BLDLOG -rm -f $BROKENF $BRKWRKLOG $BLDLOG $STARTFILE - - - - -# -# Install cvs package and do a cvs update here -# -if [ "$CVS_USER" != "" ]; then - if [ ! -f /usr/bin/cvs ]; then - echo Installing required cvs pkgs for CVS update - ( cd ${USR_PKGSRC}/devel/cvs ; ${BMAKE} bulk-install ) - fi - if [ ! -f /usr/bin/ssh ]; then - echo Installing required ssh pkgs for CVS update - ( cd ${USR_PKGSRC}/security/ssh ; ${BMAKE} bulk-install ) - fi - echo Performing CVS update - this will take some time - su - ${CVS_USER} -c "stty sane ; cd ${USR_PKGSRC} ; \ - env CVS_RSH=ssh cvs -q update -Pd ${CVS_FLAGS} || exit 0" - if [ $? != 0 ] - then - echo CVS update not successful, aborting. - exit 1 - fi - echo CVS update done. -fi - -# -# Remove old/broken distfiles and binary packages -# -PACKAGES=`( cd ${PKGLINT_PKG_DIR} ; ${BMAKE} show-var VARNAME=PACKAGES )`; - -case "$LINTPKGSRC_CACHE$PRUNEDISTFILES$PRUNEPACKAGES" in -*yes*|*YES*) - # install pkglint to support any of these options - ( cd ${PKGLINT_PKG_DIR} ; ${BMAKE} PRECLEAN=yes bulk-install ) - ;; -esac - -case "$LINTPKGSRC_CACHE" in -yes|YES) - echo "Creating lintpkgsrc cache database." - lintpkgsrc -K $PACKAGES -P $USR_PKGSRC -M $DISTDIR -E $LINTPKGSRC_DB - lintpkgsrc_cache="-I $LINTPKGSRC_DB" - echo done. - ;; -*) - lintpkgsrc_cache='' - ;; -esac - -case "$PRUNEDISTFILES" in -yes|YES) - echo "Removing old/broken distfiles" - lintpkgsrc $lintpkgsrc_cache -K $PACKAGES -P $USR_PKGSRC -M $DISTDIR -o -m -r - echo done. - ;; -*) - echo "Skipping distfile pruning." - ;; -esac - -case "$PRUNEPACKAGES" in -yes|YES) - echo "Removing old (out of date) binary packages" - lintpkgsrc $lintpkgsrc_cache -K $PACKAGES -P $USR_PKGSRC -M $DISTDIR -p -r - echo done. - ;; -*) - echo "Skipping packages pruning." - ;; -esac - -case "$PRUNELINKS" in -yes|YES) - echo "Checking for and removing orphaned packages links" - find $PACKAGES -type l -print | \ - while read f - do - if [ ! -d $f -a ! -f $f ]; then - echo "Removing orphaned link: \"$f\"" - rm $f - fi - done - echo done. - ;; -*) - echo "Skipping pruning of packages links." - ;; -esac - -# Clean out the packages added above so that the real bulk build -# is started in a consistent state. -for dbdir in ${PKG_DBDIR} ${DEPOTBASE}; do - echo Removing all installed packages in $dbdir - if [ -d $dbdir ]; then - cd $dbdir - for pkg in * - do - if `pkg_info -K $dbdir -qe $pkg`; then - echo pkg_delete -r $pkg - pkg_delete -K $dbdir -r $pkg - fi - done - fi -done - -cd ${USR_PKGSRC} - -# on non-NetBSD, we don't want these to build as they overwrite -# bootstrap-pkgsrc generated files and thus would break the following builds. - -if [ x"$BMAKE" = x"bmake" ]; then - for pkg in archivers/pax devel/bmake devel/mk-files net/tnftp \ - security/kth-krb4 textproc/nbsed; do - echo "Don't blow away bootstrap-pkgsrc stuff!" > \ - ${pkg}/${BROKENF} - done -fi - -if [ -f mk/bulk/pre-build.local ]; then - . mk/bulk/pre-build.local -fi - -if [ ! -d ${PKG_DBDIR} ]; then - mkdir ${PKG_DBDIR} -fi - -touch $STARTFILE diff --git a/mk/bulk/printdepends b/mk/bulk/printdepends deleted file mode 100644 index 4a650b09afe..00000000000 --- a/mk/bulk/printdepends +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/sh -# $NetBSD: printdepends,v 1.15.8.1 2005/05/17 18:29:45 tv Exp $ - -# -# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org> -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. All advertising materials mentioning features or use of this software -# must display the following acknowledgement: -# This product includes software developed by Hubert Feyrer for -# the NetBSD Project. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# - -# -# Print list of pkg dependencies suitable for tsort(1). -# Start in $USR_PKGSRC. -# -# If an argument is given, use it as a file name -# to contain any failure messages in each package directory. -# - -# Pull in PKGLIST -if [ -f "$BULK_BUILD_CONF" ]; then - . $BULK_BUILD_CONF -else - . `dirname $0`/build.conf -fi - -if [ ! -z "$1" ]; then - brokenfile=$1 -else - brokenfile=/dev/null -fi - -opsys=`uname -s` -case "$opsys" in -NetBSD) BMAKE=make ;; -*) BMAKE=bmake ;; -esac - -export BMAKE - -# $USR_PKGSRC -cwd=$PWD - -# get some initial variables -cd $cwd/pkgtools/pkglint -GREP=`${BMAKE} show-var VARNAME=GREP USE_TOOLS=grep` -SED=`${BMAKE} show-var VARNAME=SED USE_TOOLS=sed` -cd $cwd - -if [ -n "${PKGLIST}" ]; then - list="${PKGLIST}" -else - # List of all pkgs, from pkgsrc/*/Makefile - list=`${GREP} '^[[:space:]]*'SUBDIR */Makefile | ${GREP} -v regress/ | ${SED} 's,/Makefile.*=[[:space:]]*,/,'` -fi - -# cache the package list for printindex -echo list='"'${list}'"' > .pkglist - -for pkgdir in $list -do - if [ ! -d $pkgdir ]; then - echo "WARNING: the package directory $pkgdir is listed in" > /dev/stderr - echo $pkgdir | ${SED} 's;/.*;/Makefile;g' > /dev/stderr - echo "but the directory does not exist. Please fix this!" > /dev/stderr - else - cd $pkgdir - - l=`${BMAKE} show-depends-dirs` - if [ $? != 0 ]; then - echo "WARNING (printdepends): the package in $pkgdir had problem with" > /dev/stderr - echo " ${BMAKE} show-depends-dirs" > /dev/stderr - echo " dependency information in the cache will" > /dev/stderr - echo " be dropped for $pkgdir" > /dev/stderr - echo "${BMAKE} show-depends-dirs failed:" > $brokenfile - ${BMAKE} show-depends-dirs >> $brokenfile 2>&1 - echo "$pkgdir $pkgdir" - else - if [ "$l" = "" ]; then - # No dependencies - echo "$pkgdir $pkgdir" - else - for depdir in $l - do - echo "$depdir $pkgdir" - done - fi - fi - fi - cd $cwd -done diff --git a/mk/bulk/printindex b/mk/bulk/printindex deleted file mode 100644 index 909c08f2064..00000000000 --- a/mk/bulk/printindex +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/sh -# $NetBSD: printindex,v 1.15.6.1 2005/05/17 18:29:45 tv Exp $ -# -# -# Copyright (c) 2001 The NetBSD Foundation, Inc. -# All rights reserved. -# -# This code is derived from software contributed to The NetBSD Foundation -# by -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. All advertising materials mentioning features or use of this software -# must display the following acknowledgement: -# This product includes software developed by the NetBSD -# Foundation, Inc. and its contributors. -# 4. Neither the name of The NetBSD Foundation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS -# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# - - -# Generate package index. If an argument is given, use it as a file name -# to contain any failure messages in each package directory. -# -# generates a pkgsrc directory <-> package name database. eg. -# foo/bar bar-2.0 -# foo/baz baz-1.0.1 -# -# Start in $USR_PKGSRC. -# - -if [ ! -z "$1" ]; then - brokenfile=$1 -else - brokenfile=/dev/null -fi - -# $USR_PKGSRC -cwd=$PWD - -# get some initial variables -cd $cwd/pkgtools/pkglint -GREP=`${BMAKE} show-var VARNAME=GREP USE_TOOLS=grep` -SED=`${BMAKE} show-var VARNAME=SED USE_TOOLS=sed` -cd $cwd - -if [ -r ${cwd}/.pkglist ]; then - . ${cwd}/.pkglist -else - # fall back to all packages. - list=`${GREP} '^[[:space:]]*'SUBDIR */Makefile | ${SED} 's,/Makefile.*=[[:space:]]*,/,'` -fi - -for pkgdir in $list -do - if [ ! -d $pkgdir ]; then - echo "WARNING: the package directory $pkgdir is listed in" > /dev/stderr - echo $pkgdir | ${SED} 's;/.*;/Makefile;g' > /dev/stderr - echo "but the directory does not exist. Please fix this!" > /dev/stderr - else - cd $pkgdir - pkgname=`${BMAKE} show-var VARNAME=PKGNAME` - if [ $? != 0 ]; then - echo "ERROR: printindex could not extract PKGNAME for $pkgdir" > /dev/stderr - echo "${BMAKE} show-var VARNAME=PKGNAME failed" > $brokenfile - ${BMAKE} show-var VARNAME=PKGNAME >> $brokenfile 2>&1 - #exit 1 - fi - echo "$pkgdir $pkgname " - fi - cd $cwd -done - -# Get the list of packages which should always be installed during the build. -# Make sure these ended up in the index file. For example, xpkgwedge, might -# not be enabled for builds in pkgtools/xpkgwedge, but we may want to have -# it listed in the index file. - -cd $cwd/pkgtools/pkglint && BULK_PREREQ=`${BMAKE} show-var VARNAME=BULK_PREREQ` -cd $cwd - -for pkgdir in $BULK_PREREQ -do - case $list in - *$pkgdir*) - # its already listed, do nothing - ;; - *) - cd $cwd/$pkgdir - pkgname=`${BMAKE} show-var VARNAME=PKGNAME` - if [ $? != 0 ]; then - echo "ERROR: printindex could not extract PKNAME for $pkgdir" > /dev/stderr - echo "${BMAKE} show-var VARNAME=PKGNAME failed" > $brokenfile - ${BMAKE} show-var VARNAME=PKGNAME >> $brokenfile 2>&1 - #exit 1 - fi - echo "$pkgdir $pkgname " - cd $cwd - ;; - esac -done - diff --git a/mk/bulk/tflat b/mk/bulk/tflat deleted file mode 100755 index 2ed94a0e58e..00000000000 --- a/mk/bulk/tflat +++ /dev/null @@ -1,204 +0,0 @@ -# $NetBSD: tflat,v 1.13.6.1 2005/05/17 18:29:45 tv Exp $ -# -# Copyright (c) 2001 The NetBSD Foundation, Inc. -# All rights reserved. -# -# This code is derived from software contributed to The NetBSD Foundation -# by Dan McMahill. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# 3. All advertising materials mentioning features or use of this software -# must display the following acknowledgement: -# This product includes software developed by the NetBSD -# Foundation, Inc. and its contributors. -# 4. Neither the name of The NetBSD Foundation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS -# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS -# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# - -BEGIN { - if (ARGC != 3){ - printf("tflat: wrong number of arguments\n"); - usage(); - exit(1); - } - - if ( ARGV[1] == "up" ) { - up=1; - } - else{ - if ( ARGV[1] == "down" ) { up=0; } - else{ - printf("tflat: unknown option \"%s\"\n",ARGV[1]); - usage(); - exit(1); - } - } - - InFile = ARGV[2]; - - # - # Read in the entire depends tree - # - if (up){ - while(getline < InFile > 0){ - if ($1 in topdepends) - topdepends[$1] = topdepends[$1] " " $2 " " ; - else{ - topdepends[$1] = " " $2 " "; - } - - # Note that it is possible for a package "foo/bar" to - # never appear in $1. In fact if foo/bar is not - # depended upon by anything and it has depends then it - # will not show up in $1 however, we need to make sure - # we get a topdepends[foo/bar] entry so add it here if - # its not already there. - if (!($2 in topdepends)) - topdepends[$2] = ""; - } - depstr = " is depended on by: "; - } - else{ - while(getline < InFile > 0){ - if ($2 in topdepends) - topdepends[$2] = topdepends[$2] " " $1 " " ; - else - topdepends[$2] = " " $1 " " ; - } - depstr = " depends on: "; - } - - close(InFile); - # - # Now recurse the tree to give a flattened depends list for each pkg - # - - - for (toppkg in topdepends){ - find_all_depends(toppkg); - } - - for (x in alldepends){ - print x depstr alldepends[x] | "${SORT-sort}"; - } - - printf("\n"); - exit 0 -} - -function find_all_depends(pkg,pkgreg,i,deps){ - # If we find the package already has been fully depended - # then return the depends list. - if (pkg in alldepends){ - return(alldepends[pkg]); - } - - # If we find the package listed in its own depends list, then - # return an empty list if we're going down the depends tree. - # When a package lists itself in the depends tree file, it simply - # is a place holder and means the package has no depends. However - # other packages may still depend upon it, so we need to keep looking. - if ( (!up) && (topdepends[pkg]~reg2str(pkg)) ){ - alldepends[pkg] = " "; - return(alldepends[pkg]); - } - - # Otherwise recursively gather depends that each of the depends - # has. - pkgreg=reg2str(pkg); - split(topdepends[pkg],deps); - i=1; - alldepends[pkg] = " "; - while ( i in deps ){ - # Don't add ourselves to the list (a possibility when going up - # the tree). - if (" "deps[i]" "!~pkgreg){ - alldepends[pkg] = alldepends[pkg] " " deps[i] " " find_all_depends(deps[i]); - } - i=i+1; - } - alldepends[pkg] = uniq(alldepends[pkg]); - return(alldepends[pkg]); -} - -# -# Take a string which has special characters like '+' in it and -# escape them. Also put a space before and after since that's how -# we'll distinguish things like gnome from gnome-libs -# -function reg2str(reg){ - gsub(/\+/,"\\+",reg); - reg = " "reg" "; - return(reg); -} - -# -# Take the depends lists and uniq them. -# -function uniq(list,deps,i,ulist){ - - # split out the depends - split(list,deps); - - i=1; - ulist = " "; - while (i in deps){ - if (ulist !~reg2str(deps[i])){ - ulist = ulist deps[i]" "; - } - i++; - } - return(ulist); -} - -# -# show usage -# -function usage(){ - printf("tflat -- flatten a depends tree. tflat is used to show all\n"); - printf(" packages which depend upon a given package or alternatively\n"); - printf(" all packages which are depend upon by a given package.\n"); - printf("\n"); - printf("Usage:\ttflat up|down depfile\n"); - printf("\n"); - printf("Options:\tdown\tgo down the depends tree (ie \"foo depends on:\")\n"); - printf(" \tup\tgo up the depends tree (ie \"foo is depended on by:\")\n"); - printf("\n"); - printf("Input file format is in the form\n"); - printf("foo bar\n"); - printf("foo baz\n"); - printf("libbar bar\n"); - printf("\n"); - printf("meaning \"foo is depended upon by bar,\n"); - printf(" foo is depended upon by baz,\n"); - printf(" libbar is depended upon by bar\"\n"); - printf("\n"); - printf("The typical use is:\n"); - printf("cd /usr/pkgsrc\n"); - printf("./mk/bulk/printdepends > .depends\n"); - printf("./mk/bulk/tflat up .depends > .supports\n"); - printf("./mk/bulk/tflat down .depends > .requires\n"); - printf("\n"); - -} - diff --git a/mk/bulk/upload b/mk/bulk/upload deleted file mode 100644 index fc8c56d20cc..00000000000 --- a/mk/bulk/upload +++ /dev/null @@ -1,204 +0,0 @@ -#!/bin/sh -# $NetBSD: upload,v 1.18.6.2 2005/05/17 18:29:45 tv Exp $ - -# -# Upload non-restricted binary pkgs to ftp server -# - -MD5="digest md5"; -SHA1="digest sha1"; - -opsys=`uname -s` -case "$opsys" in -NetBSD) BMAKE=make; - BSDSUM="sum -o 1"; - CKSUM="cksum"; - SYSVSUM="sum -o 2"; - ;; -IRIX*) BMAKE=bmake; - BSDSUM="sum -r"; - CKSUM="cksum"; - SYSVSUM="sum"; - ;; -*) BMAKE=bmake ;; -esac - -export BMAKE - -# Bring in variables for bulk-install -BATCH=1 -DEPENDS_TARGET=bulk-install -export BATCH DEPENDS_TARGET - -# Pull in RSYNC_DST, RSYNC_OPTS: -if [ -f "$BULK_BUILD_CONF" ]; then - . $BULK_BUILD_CONF -else - . `dirname $0`/build.conf -fi - -cd $USR_PKGSRC - -if [ -z "$RSYNC_DST" ]; then - echo "You must set the variable RSYNC_DST, see build.conf-example." - exit 1 -fi - -# -# Some temp files -# - -umask 022 -TMPDIR="${TMPDIR:-/tmp}" -TMP="${TMPDIR}"/pkg_upload.$$ -(umask 077 && mkdir "${TMP}") -if [ $? -ne 0 ] -then - echo $0: cannot create temporary directory \""${TMP}"\" >&2 - exit 1 -fi - -exf="$TMP"/exclude -vf="$TMP"/vulnerable -upload="$TMP"/upload -upload_general="$TMP"/upload_general -upload_vulnerable="$TMP"/upload_vulnerable - -# May be different than $USR_PKGSRC: -pkgsrcdir=`cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=_PKGSRCDIR` -packages=`cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=PACKAGES` -distdir=`cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=DISTDIR` - -# Pull in some pkgs needed -( cd pkgtools/pkglint ; ${BMAKE} bulk-install ) -( cd net/rsync ; ${BMAKE} bulk-install ) -( cd security/audit-packages ; ${BMAKE} bulk-install ) - -echo "Making sure vulnerability-list is up-to-date:" -if [ -z "$UPDATE_VULNERABILITY_LIST" -o "$UPDATE_VULNERABILITY_LIST" = "yes" ] -then - env PKGVULNDIR=${distdir} download-vulnerability-list -else - echo '(skipped)' -fi - -case $LINTPKGSRC_CACHE in -yes|YES) - lintpkgsrc_cache="-I `cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=LINTPKGSRC_DB`" - ;; -*) - lintpkgsrc_cache='' - ;; -esac - -echo "Checking for restricted and out of date packages:" -# -p = report old versions of packages -# -R = report restricted packages -lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -pR | sed 's@'$packages'/@@' > "$exf" - -echo "Checking for vulnerable packages:" -lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -V | sed 's@'$packages'/@@' > "$vf" - -RSFLAGS="-vap --progress $RSYNC_OPTS" - -failed=no -cd $packages - -if [ "${MKSUMS}" = "yes" -o "${MKSUMS}" = "YES" ]; then - - echo "Calculating checksum files..." - - SUMFILES="BSDSUM CKSUM MD5 SHA1 SYSVSUM" - - rm -f ${SUMFILES} - - if [ x"${SIGN_AS}" != x"" ]; then - ( cd ${pkgsrcdir}/security/gnupg; ${BMAKE} bulk-install ) - for i in ${SUMFILES}; do - echo > $i - echo "This file is signed with ${SIGN_AS}'s PGP key." >> $i - echo >> $i - done - fi - - ( cd ${pkgsrcdir}/pkgtools/digest; ${BMAKE} bulk-install ) - - [ -z "${BSDSUM}" ] && BSDSUM="echo" - [ -z "${CKSUM}" ] && CKSUM="echo" - [ -z "${SYSVSUM}" ] && SYSVSUM="echo" - - for i in All/*; do - if ! grep -q $i $exf; then - ${BSDSUM} $i >> BSDSUM - ${CKSUM} $i >> CKSUM - ${MD5} $i >> MD5 - ${SHA1} $i >> SHA1 - ${SYSVSUM} $i >> SYSVSUM - fi - done - - [ "${BSDSUM}" = "echo" ] && rm BSDSUM - [ "${CKSUM}" = "echo" ] && rm CKSUM - [ "${SYSVSUM}" = "echo" ] && rm SYSVSUM - - if [ x"${SIGN_AS}" != x"" ]; then - for i in ${SUMFILES}; do - if [ -s $i ]; then - echo "Signing $i" - gpg --clearsign $i && rm $i - fi - done - else - echo "Checksum files not PGP-signed. Please do so manually!" - echo "(Run 'gpg --clearsign' on all of them)" - fi -fi - - -echo "#!/bin/sh" > "$upload" -echo "packages=$packages" >> "$upload" -echo "if ! cd $packages ; then" >> "$upload" -echo " echo \"could not cd to $packages\"" >> "$upload" -echo " exit 1" >> "$upload" -echo "fi" >> "$upload" - -echo "Uploading non-vulnerable pkgs" -cmd="rsync $RSFLAGS --exclude-from=\"$exf\" --exclude-from=\"$vf\" . \"$RSYNC_DST\"" -cp -f "$upload" "$upload_general" -echo "$cmd" >> "$upload_general" -chmod 755 "$upload_general" -echo "$cmd" -sh "$upload_general" -if [ $? != 0 ]; then - echo "--------------------------------------------------" - echo " " - echo "WARNING: rsync failed. To retry later, you can run" - echo " $upload_general" - echo " " - echo "--------------------------------------------------" - failed=yes -fi - -echo "Uploading vulnerable pkgs" -sed -n "s@All/@@p" "$exf" > "$exf.new" -sed -n "s@All/@@p" "$vf" > "$vf.new" -cmd="rsync $RSFLAGS --exclude-from=\"$exf.new\" --include-from=\"$vf.new\" --exclude='*' All/ \"$RSYNC_DST/vulnerable/\"" -cp -f "$upload" "$upload_vulnerable" -echo "$cmd" >> "$upload_vulnerable" -chmod 755 "$upload_vulnerable" -echo "$cmd" -sh "$upload_vulnerable" -if [ $? != 0 ]; then - echo "--------------------------------------------------" - echo " " - echo "WARNING: rsync failed. To retry later, you can run" - echo " $upload_vulnerable" - echo " " - echo "--------------------------------------------------" - failed=yes -fi - -# clean up temp files -if [ "$failed" = "no" ]; then - rm -fr "$TMP" -fi |