diff options
author | dmcmahill <dmcmahill> | 2001-02-13 23:02:23 +0000 |
---|---|---|
committer | dmcmahill <dmcmahill> | 2001-02-13 23:02:23 +0000 |
commit | 241584a3b1a090851802a0c3d55e2e3b84e7aeda (patch) | |
tree | c39cfad2d1ee1f56f69f931ff5bacba929c2ea0b | |
parent | 90cc152825f53a19f80b83c3f9fea950f8818c10 (diff) | |
download | pkgsrc-241584a3b1a090851802a0c3d55e2e3b84e7aeda.tar.gz |
only use the depends cache files during a bulk build, otherwise revert to
the old behaviour. This is done because unless we're in the middle of a
bulk build, we don't really know that the cache files are up to date. These
are fairly time consuming (relative to a single smallish package build) to
generate and depend on all of the pkgsrc makefiles. During a bulk build, the
overhead is far outweighed by the savings.
In particular, 'make bulk-install' will now work correctly outside of a bulk
build (useful for debugging broken packages).
Thanks to Hubert who noted the 'make bulk-install' problem in a private email.
-rw-r--r-- | mk/bulk/bsd.bulk-pkg.mk | 108 | ||||
-rw-r--r-- | mk/bulk/build | 3 |
2 files changed, 67 insertions, 44 deletions
diff --git a/mk/bulk/bsd.bulk-pkg.mk b/mk/bulk/bsd.bulk-pkg.mk index 83131625a05..795fc4c30f8 100644 --- a/mk/bulk/bsd.bulk-pkg.mk +++ b/mk/bulk/bsd.bulk-pkg.mk @@ -1,4 +1,4 @@ -# $NetBSD: bsd.bulk-pkg.mk,v 1.18 2001/02/08 19:17:38 dmcmahill Exp $ +# $NetBSD: bsd.bulk-pkg.mk,v 1.19 2001/02/13 23:02:23 dmcmahill Exp $ # # Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@netbsd.org> @@ -45,6 +45,14 @@ LS?= ls WC?= wc +# This variable is set to 'yes' by the pkgsrc/mk/bulk/build script. It enables +# the use of several cache files (DEPENDSTREEFILE, DEPENDSFILE, SUPPORTSFILE, +# and INDEXFILE) for speeding up the processing of dependencies. If we're not +# actively in the middle of a bulk build, however, we don't know that these files +# exist or are up to date and they take quite a while to rebuild. So unless +# they're known to exist and be up to date, don't use them. +USE_BULK_CACHE?= no + # Shall we remove any packages which are installed, but not required # to build this package PRECLEAN?= yes @@ -176,14 +184,19 @@ bulk-package: ${ECHO_MSG} ${RM} -f ${PKGFILE} ; \ ${DO} ${RM} -f ${PKGFILE} ; \ fi; \ - ${ECHO_MSG} "BULK> Full rebuild in progress..." ; \ - ${ECHO_MSG} "BULK> Cleaning package and its depends" ;\ + ${ECHO_MSG} "BULK> Full rebuild in progress..." ; \ + ${ECHO_MSG} "BULK> Cleaning package and its depends" ;\ + if [ "${USE_BULK_CACHE}" = "yes" ]; then \ thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\ for pkgdir in $$thisdir `${GREP} "^$$thisdir " ${DEPENDSFILE} | ${SED} -e 's;^.*:;;g'`; do \ ${ECHO_MSG} "(cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} clean)" ; \ ${DO} (cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} clean) ; \ done ;\ - if [ "${PRECLEAN}" = "yes" ]; then \ + else \ + ${ECHO_MSG} ${MAKE} clean CLEANDEPENDS=YES;\ + ${DO} ${MAKE} clean CLEANDEPENDS=YES;\ + fi; \ + if [ "${PRECLEAN}" = "yes" -a "${USE_BULK_CACHE}" = "yes" ]; then \ ${ECHO_MSG} "BULK> Removing installed packages which are not needed to build ${PKGNAME}" ; \ thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\ for pkgname in `${PKG_INFO} | ${AWK} '{print $$1}'` ; \ @@ -209,18 +222,20 @@ bulk-package: fi ;\ done ; \ fi ;\ - thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\ - ${ECHO_MSG} "BULK> Installing packages which are required to build ${PKGNAME}." ;\ - for pkgdir in `${GREP} "^$$thisdir " ${DEPENDSFILE} | ${SED} -e 's;^.*:;;g'`; do \ - pkgname=`${GREP} "^$$pkgdir " ${INDEXFILE} | ${AWK} '{print $$2}'` ; \ - pkgfile=${PACKAGES}/All/$${pkgname}.tgz ;\ - if ! `${PKG_INFO} -qe $$pkgname` ; then \ - ${ECHO_MSG} "BULK> " ${PKG_ADD} $$pkgfile ; \ - ${DO} ${PKG_ADD} $$pkgfile ; \ - else \ - ${ECHO_MSG} "BULK> Required package $$pkgname ($$pkgdir) is already installed" ; \ - fi ;\ - done ;\ + if [ "${USE_BULK_CACHE}" = "yes" ]; then \ + thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\ + ${ECHO_MSG} "BULK> Installing packages which are required to build ${PKGNAME}." ;\ + for pkgdir in `${GREP} "^$$thisdir " ${DEPENDSFILE} | ${SED} -e 's;^.*:;;g'`; do \ + pkgname=`${GREP} "^$$pkgdir " ${INDEXFILE} | ${AWK} '{print $$2}'` ; \ + pkgfile=${PACKAGES}/All/$${pkgname}.tgz ;\ + if ! `${PKG_INFO} -qe $$pkgname` ; then \ + ${ECHO_MSG} "BULK> " ${PKG_ADD} $$pkgfile ; \ + ${DO} ${PKG_ADD} $$pkgfile ; \ + else \ + ${ECHO_MSG} "BULK> Required package $$pkgname ($$pkgdir) is already installed" ; \ + fi ;\ + done ;\ + fi ;\ ${ECHO_MSG} ${MAKE} package '(${PKGNAME})' 2>&1 ; \ ${DO} ( ${MAKE} package 2>&1 ); \ ) 2>&1 | tee -a ${BUILDLOG} ; \ @@ -233,40 +248,47 @@ bulk-package: ${LS} -la ${BROKENFILE} ; \ ${ECHO_MSG} ${MAKE} deinstall ; \ ${DO} ${MAKE} deinstall ; \ - ${ECHO_MSG} "BULK> Marking all packages which depend upon ${PKGNAME} as broken:"; \ - thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\ nbrokenby=0;\ - for pkgdir in `${GREP} "^$$thisdir " ${SUPPORTSFILE} | ${SED} -e 's;^.*:;;g'`; do \ - pkgname=`${GREP} "^$$pkgdir " ${INDEXFILE} | ${AWK} '{print $$2}'` ;\ - ${ECHO_MSG} "BULK> marking package that requires ${PKGNAME} as broken: $$pkgname ($$pkgdir)";\ - pkgerr="-1"; \ - pkgignore=`(cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} show-var VARNAME=IGNORE)`; \ - if [ ! -z "$$pkgignore" -a ! -f ${PKGSRCDIR}/$$pkgdir/${BROKENFILE} ]; then \ - ${ECHO_MSG} "BULK> $$pkgname ($$pkgdir) may not be packaged because:" >> ${PKGSRCDIR}/$$pkgdir/${BROKENFILE};\ - ${ECHO_MSG} "BULK> $$pkgignore" >> ${PKGSRCDIR}/$$pkgdir/${BROKENFILE};\ - if [ -z "`(cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} show-var VARNAME=BROKEN)`" ]; then \ - pkgerr="0"; \ - else \ - pkgerr="1"; \ + if [ "${USE_BULK_CACHE}" = "yes" ]; then \ + ${ECHO_MSG} "BULK> Marking all packages which depend upon ${PKGNAME} as broken:"; \ + thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\ + for pkgdir in `${GREP} "^$$thisdir " ${SUPPORTSFILE} | ${SED} -e 's;^.*:;;g'`; do \ + pkgname=`${GREP} "^$$pkgdir " ${INDEXFILE} | ${AWK} '{print $$2}'` ;\ + ${ECHO_MSG} "BULK> marking package that requires ${PKGNAME} as broken: $$pkgname ($$pkgdir)";\ + pkgerr="-1"; \ + pkgignore=`(cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} show-var VARNAME=IGNORE)`; \ + if [ ! -z "$$pkgignore" -a ! -f ${PKGSRCDIR}/$$pkgdir/${BROKENFILE} ]; then \ + ${ECHO_MSG} "BULK> $$pkgname ($$pkgdir) may not be packaged because:" >> ${PKGSRCDIR}/$$pkgdir/${BROKENFILE};\ + ${ECHO_MSG} "BULK> $$pkgignore" >> ${PKGSRCDIR}/$$pkgdir/${BROKENFILE};\ + if [ -z "`(cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} show-var VARNAME=BROKEN)`" ]; then \ + pkgerr="0"; \ + else \ + pkgerr="1"; \ + fi; \ fi; \ - fi; \ - ${ECHO_MSG} "BULK> $$pkgname ($$pkgdir) is broken because it depends upon ${PKGNAME} ($$thisdir) which is broken." \ - >> ${PKGSRCDIR}/$$pkgdir/${BROKENFILE};\ - nbrokenby=`expr $$nbrokenby + 1`;\ - if ! `${GREP} " $$pkgdir/${BROKENFILE}" ${PKGSRCDIR}/${BROKENFILE} >/dev/null 2>&1` ; then \ - ${ECHO} " $$pkgerr $$pkgdir/${BROKENFILE} 0 " >> ${PKGSRCDIR}/${BROKENFILE} ;\ - fi ;\ - done ;\ + ${ECHO_MSG} "BULK> $$pkgname ($$pkgdir) is broken because it depends upon ${PKGNAME} ($$thisdir) which is broken." \ + >> ${PKGSRCDIR}/$$pkgdir/${BROKENFILE};\ + nbrokenby=`expr $$nbrokenby + 1`;\ + if ! `${GREP} " $$pkgdir/${BROKENFILE}" ${PKGSRCDIR}/${BROKENFILE} >/dev/null 2>&1` ; then \ + ${ECHO} " $$pkgerr $$pkgdir/${BROKENFILE} 0 " >> ${PKGSRCDIR}/${BROKENFILE} ;\ + fi ;\ + done ;\ + fi ;\ nerrors=`${GREP} -c '^\*\*\* Error code' ${BROKENFILE} || true`; \ ${ECHO_MSG} " $$nerrors ${PKGPATH}/${BROKENFILE} $$nbrokenby " >> ${PKGSRCDIR}/${BROKENFILE} \ ) 2>&1 | tee -a ${BROKENFILE}; \ fi ; \ ${ECHO_MSG} "BULK> Cleaning packages and its depends" ;\ - thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\ - for pkgdir in $$thisdir `${GREP} "^$$thisdir " ${DEPENDSFILE} | ${SED} -e 's;^.*:;;g'`; do \ - ${ECHO_MSG} "(cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} clean)" ; \ - ${DO} (cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} clean) ; \ - done ;\ + if [ "${USE_BULK_CACHE}" = "yes" ]; then \ + thisdir=`${GREP} " ${PKGNAME} " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\ + for pkgdir in $$thisdir `${GREP} "^$$thisdir " ${DEPENDSFILE} | ${SED} -e 's;^.*:;;g'`; do \ + ${ECHO_MSG} "(cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} clean)" ; \ + ${DO} (cd ${PKGSRCDIR}/$$pkgdir && ${MAKE} clean) ; \ + done ;\ + else \ + ${ECHO_MSG} ${MAKE} clean CLEANDEPENDS=YES;\ + ${DO} ${MAKE} clean CLEANDEPENDS=YES;\ + fi ;\ fi @if [ ! -f ${PKGFILE} ]; then \ ${ECHO_MSG} "BULK>" Build for ${PKGNAME} was not successful, aborting. | tee -a ${BROKENFILE} ; \ diff --git a/mk/bulk/build b/mk/bulk/build index f831e50143a..b0e861c384c 100644 --- a/mk/bulk/build +++ b/mk/bulk/build @@ -1,5 +1,5 @@ #!/bin/sh -# $NetBSD: build,v 1.8 2001/02/01 08:47:29 dmcmahill Exp $ +# $NetBSD: build,v 1.9 2001/02/13 23:02:23 dmcmahill Exp $ # # Do builk build # @@ -69,6 +69,7 @@ fi echo "Starting actual build using the order specified in $ORDERFILE..." nice -n 20 make \ + USE_BULK_CACHE=yes \ SPECIFIC_PKGS=1 \ GROUP_SPECIFIC_PKGS="`cat $ORDERFILE | tr '\012' ' '`" \ bulk-package | sed 's/^/'`uname -p`'> /g' |