diff options
author | gdt <gdt@pkgsrc.org> | 2006-12-05 18:23:52 +0000 |
---|---|---|
committer | gdt <gdt@pkgsrc.org> | 2006-12-05 18:23:52 +0000 |
commit | d2d7243e5b9f1961b59b943408a655d13eeafee2 (patch) | |
tree | 088c86b4d47d344524e5f2ba25e4c08f45e36cce | |
parent | ced12c3e3d04d31d1b691e37771a9ef31f83babe (diff) | |
download | pkgsrc-d2d7243e5b9f1961b59b943408a655d13eeafee2.tar.gz |
pkg_rolling-replace does 'make replace' on one package at a time,
tsorting the packages being replaced according to their
interdependencies, which avoids most duplicate rebuilds.
-rw-r--r-- | pkgtools/pkg_rolling-replace/DESCR | 3 | ||||
-rw-r--r-- | pkgtools/pkg_rolling-replace/Makefile | 53 | ||||
-rw-r--r-- | pkgtools/pkg_rolling-replace/PLIST | 3 | ||||
-rw-r--r-- | pkgtools/pkg_rolling-replace/files/pkg_rolling-replace.8 | 91 | ||||
-rwxr-xr-x | pkgtools/pkg_rolling-replace/files/pkg_rolling-replace.sh | 402 |
5 files changed, 552 insertions, 0 deletions
diff --git a/pkgtools/pkg_rolling-replace/DESCR b/pkgtools/pkg_rolling-replace/DESCR new file mode 100644 index 00000000000..1f04a63715a --- /dev/null +++ b/pkgtools/pkg_rolling-replace/DESCR @@ -0,0 +1,3 @@ +pkg_rolling-replace does 'make replace' on one package at a time, +tsorting the packages being replaced according to their +interdependencies, which avoids most duplicate rebuilds. diff --git a/pkgtools/pkg_rolling-replace/Makefile b/pkgtools/pkg_rolling-replace/Makefile new file mode 100644 index 00000000000..06e8abed82b --- /dev/null +++ b/pkgtools/pkg_rolling-replace/Makefile @@ -0,0 +1,53 @@ +# $NetBSD: Makefile,v 1.1.1.1 2006/12/05 18:23:52 gdt Exp $ + +DISTNAME= pkg_rolling-replace-0.04 +PKGREVISION= 2 +CATEGORIES= pkgtools +MASTER_SITES= # empty +DISTFILES= # empty + +MAINTAINER= ngoffee@bbn.com +COMMENT= Replace/upgrade packages in-place + +DEPENDS+= pkg_chk>=1.76:../../pkgtools/pkg_chk +DEPENDS+= pkg_tarup>=1.6.6:../../pkgtools/pkg_tarup + +PKG_INSTALLATION_TYPES= overwrite pkgviews + +USE_TOOLS+= patch + +WRKSRC= ${WRKDIR} +NO_CHECKSUM= yes +NO_BUILD= yes +USE_LANGUAGES= # none + +INSTALLATION_DIRS= sbin man/man8 + +.include "../../mk/bsd.prefs.mk" + +# target exists only to force check-pkgsrc-patch prior to install +do-extract: check-pkgsrc-patch + cp ${FILESDIR}/pkg_rolling-replace.sh ${WRKSRC} + +do-install: check-pkgsrc-patch + ${INSTALL_SCRIPT} ${WRKSRC}/pkg_rolling-replace.sh ${PREFIX}/sbin/pkg_rolling-replace + ${INSTALL_SCRIPT} ${FILESDIR}/pkg_rolling-replace.8 ${PREFIX}/${PKGMANDIR}/man8/pkg_rolling-replace.8 + +check-pkgsrc-patch: + @${ECHO_N} "=> Checking for unsafe_depends support in pkgsrc..."; \ + REPLACE_MK="${PKGSRCDIR}/mk/flavor/pkg/replace.mk"; \ + if [ -f "$$REPLACE_MK" ]; then \ + if [ "`${EGREP} unsafe_depends=YES $$REPLACE_MK`" ]; then \ + ${ECHO} "found."; \ + ${TRUE}; \ + else \ + ${ECHO}; \ + ${ECHO} "*** pkg_rolling replace requires pkgsrc>=2006Q3" \ + ${FALSE}; \ + fi; \ + else \ + ${ECHO} "*** pkg_rolling replace requires pkgsrc>=2006Q3"; \ + ${FALSE}; \ + fi + +.include "../../mk/bsd.pkg.mk" diff --git a/pkgtools/pkg_rolling-replace/PLIST b/pkgtools/pkg_rolling-replace/PLIST new file mode 100644 index 00000000000..dd7d1172d9e --- /dev/null +++ b/pkgtools/pkg_rolling-replace/PLIST @@ -0,0 +1,3 @@ +@comment $NetBSD: PLIST,v 1.1.1.1 2006/12/05 18:23:52 gdt Exp $ +sbin/pkg_rolling-replace +man/man8/pkg_rolling-replace.8 diff --git a/pkgtools/pkg_rolling-replace/files/pkg_rolling-replace.8 b/pkgtools/pkg_rolling-replace/files/pkg_rolling-replace.8 new file mode 100644 index 00000000000..6e6fdf66bb3 --- /dev/null +++ b/pkgtools/pkg_rolling-replace/files/pkg_rolling-replace.8 @@ -0,0 +1,91 @@ +.\" $NetBSD: pkg_rolling-replace.8,v 1.1.1.1 2006/12/05 18:23:52 gdt Exp $ +.Dd August 7, 2006 +.Dt PKG_ROLLING-REPLACE 8 +.Sh NAME +.Nm pkg_rolling-replace +.Nd rebuild or update packages using 'make replace' in tsorted order +.Sh SYNOPSIS +.Nm +.Op Fl hnuvx +.Op Fl x Ar pkgs +.Sh DESCRIPTION +.Nm +runs 'make replace' on packages that have been marked with the + 'unsafe_depends=YES' flag or the 'rebuild=YES' flag, optionally +replacing any outdated packages (as reported by +.Xr pkg_chk 8 ) +as well. +.Pp +Whenever 'make replace' is run on a package, all installed packages +that depend on it are marked with the 'unsafe_depends=YES' flag, which +indicates that they may no longer be ABI-compatible with the replaced +package. Running +.Nm +with no arguments will restore such packages to a safe state by +building a dependency graph, running +.Xr tsort 1 +on it, and running 'make replace' in the resulting topological order. +The dependency graph and topological sort are re-calculated after each +'make replace', in case it causes additional +packages to be marked unsafe. +.Pp +The user can mark additional packages to be rebuilt by setting the + 'rebuild=YES' flag on them using the command +.Dl % pkg_admin set rebuild=YES <pkg> +.Pp +If the +.Fl u +option is given, +.Nm +will use +.Xl pkg_chk 8 +to check for packages for which the version in pkgsrc is different +than the installed version, and run 'make replace' on those packages +as well (tsorting them along with the unsafe/rebuild packages). +(Usually this results in packages being upgraded to newer versions, +assuming the pkgsrc tree is more recent than the currently installed +packages.) +.Pp +.Ss Options +.Bl -tag -width xxxxxxxx +.It Fl h +Brief help. +.It Fl n +Display actions that would be taken, but don't actually run 'make +replace' or change any flags on installed packages. This option +attempts to calculate the new packages that would be marked unsafe +after each 'make replace'. +.It Fl u +Use +.Xl pkg_chk 8 +to check for outdated packages, and replace those too. +.It Fl v +Verbose output. +.It Fl x Ar pkgs +Exclude the comma-separated list of packages from the update check +when +.Fl u +is given (does not exclude unsafe or rebuild-requested packages). +.El +.Sh ENVIRONMENT +.Nm +uses the following environment variables. +.Bl -tag -width xxxx +.It Ev PKGSRCDIR +Base of pkgsrc tree. Defaults to +.Pa /usr/pkgsrc +.It Ev PKG_DBDIR +pkgsrc database directory. +If not set in environment then defaults to +.Pa /var/db/pkg +.El +.Sh AUTHORS +.An Nick Goffee +.Aq ngoffee@bbn.com , +with initial idea and many suggestions from +.An Greg Troxel +.Aq gdt@ir.bbn.com . +.Sh BUGS +.Nm +does not run fully automatically when any significant number of +packages need to be replaced. diff --git a/pkgtools/pkg_rolling-replace/files/pkg_rolling-replace.sh b/pkgtools/pkg_rolling-replace/files/pkg_rolling-replace.sh new file mode 100755 index 00000000000..3023ff14a9d --- /dev/null +++ b/pkgtools/pkg_rolling-replace/files/pkg_rolling-replace.sh @@ -0,0 +1,402 @@ +#!/bin/sh + +# $NetBSD: pkg_rolling-replace.sh,v 1.1.1.1 2006/12/05 18:23:52 gdt Exp $ +#<license> +# Copyright (c) 2006 BBN Technologies Corp. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. Neither the name of BBN Technologies nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY BBN TECHNOLOGIES AND CONTRIBUTORS ``AS IS'' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL BBN TECHNOLOGIES OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# Effort sponsored in part by the Defense Advanced Research Projects +# Agency (DARPA) and the Department of the Interior National Business +# Center under agreement number NBCHC050166. +#</license> + +# REVIEW NOTATION LEGEND +# XXX marks things that are broken/kludgy +# TODO marks things that need to be done +# FIXED marks items fixed, needs review and deletion of FIXED comment + +# Items to be completed for initial release: + +# TODO: move bulk of help into a real man page. + +# Items for future work (some imply changes in pkgsrc, not necessarily here) +# +# Make a pass over all packages for build depends, and rememeber +# which packages have been so checked. Check newly unsafe packages. +# The current lazy evaluation can perhaps fail to do things in the +# right order. Alternatively, explain why the current scheme is safe +# and preferred. +# +# Consider expansion to update via binary packages. This raises +# interesting questions about ABIs and binary builds +# +# Deal with conflicts in a smarter way, perhaps more automatically. +# This is hard because just removing packages is not ok. +# +# Handle foo being split into foo and foo-share. Now, make replace +# of foo pulls in foo-share which conflicts with foo. This needlessly +# loses. +# +# Parse mk.conf for PKGSRCDIR, etc. + +test -n "$PKG_DBDIR" || PKG_DBDIR=/var/db/pkg +test -n "$PKGSRCDIR" || PKGSRCDIR=/usr/pkgsrc + +unset PKG_PATH || true #or pkgsrc makefiles will complain + +MAKE=make +# XXX Find BSD make at package build time. +UNAME=`uname` +case "$UNAME" in + Darwin) + MAKE=bmake + ;; +esac + +usage() +{ + echo "Usage: pkg_rolling-replace [opts] + -h This help + -n Don't actually do make replace + -u Update mismatched packages + -v Verbose + -x <pkg> exclude <pkg> from update check + +pkg_rolling-replace does 'make replace' on one package at a time, +tsorting the packages being replaced according to their +interdependencies, which avoids most duplicate rebuilds. + +pkg_rolling-replace can be used in one of two ways: + + - 'make replace' is unsafe in that, if the replaced package's ABI + changes, its dependent packages may break. If this happens, run + 'pkg_rolling-replace' (no arguments) to rebuild them against the + new version. + + - 'pkg_chk -u' will delete all your mismatched (outdated) + packages, then reinstall them one at a time, leaving you without + those packages in the meantime. 'pkg_rolling-replace -u' will + instead upgrade them in place, allowing you to keep using your + system in the meantime (maybe...if you're lucky...because + pkg_rolling-replace replaces the \"deepest\" dependency first, + things could still break if that happens to be a fundamental + library whose ABI has changed). +" + exit 1 +} + +# pkg_rolling-replace's output is intermingled with pkgsrc make +# output. Thus, we use a different prefix. pkgsrc uses '===>' for +# major steps, and '=>' for minor ones. Thus, we use 'RR>'. + +OPI='RR>' +OPC='rr>' # continuation + + +# Echo the names of packages needing updates, versions stripped. This +# has been tested with pkg_chk 1.76. Older versions are not +# supported. Newer versions may or may not work (patches welcome). +check_packages_mismatched() +{ + pkg_chk -u -q | while read line; do + # duplicate output of pkg_chk to stderr (bypass $(...) or `...`) + echo "${OPC} $line" > /dev/stderr + # Look for the first thing that looks like pkg-version rather + # than category/pkg and remove the version. + for word in $line; do + if [ "$(echo $word | egrep '^[^/]+-[0-9][^/]*$')" ]; then + echo $word | sed 's/-[0-9].*//' + break #done with this line + fi + done + done +} + +# usage: check_packages_w_flag flag +# echo packages with flag=YES +check_packages_w_flag() +{ + _flag=$1; shift + for pkgver in $(pkg_info -e '*'); do + if pkg_info -Bq $pkgver \ + | egrep "^$_flag=[Yy][Ee][Ss]" > /dev/null; then + echo $pkgver | sed 's/-[0-9].*//' + fi + done +} + +# echo dep->pkg edges for all installed packages +depgraph_installed() +{ + for pkgver in $(pkg_info -e '*'); do + pkg=$(echo $pkgver | sed 's/-[0-9].*//') + # Include $pkg as a node without dependencies in case it has none. + echo $pkg $pkg + for depver in $(pkg_info -Nq $pkg); do + dep=$(echo $depver | sed 's/-[0-9].*//') + echo $dep $pkg + done + done +} + +# usage: who_requires pkg --in-graph DEPGRAPH +who_requires() +{ + _target=$1; shift + test "$1" = '--in-graph' && shift || return $((false)) + + while [ $# -gt 0 ]; do + _dep=$1; _pkg=$2; shift 2; + if [ $_dep = $_target -a $_dep != $_pkg ]; then + echo $_pkg + fi + done +} + +# usage: is_member x list +# return true (0) if x appears in list, false (1) otherwise +is_member() +{ + _x="$1"; shift; + while [ $# -gt 0 -a "$1" != "$_x" ]; do + shift + done + + test $# -gt 0 +} + +# usage: exclude list1 --from list2 +# echo list2 with items from list1 excluded +exclude() +{ + _excl= + while [ $# -gt 0 -a $1 != "--from" ]; do + _excl="$_excl $1" + shift + done + if [ $# -gt 0 ]; then + shift #discard '--from' + fi + while [ $# -gt 0 ]; do + is_member $1 $_excl || echo $1 + shift + done +} + +# usage: uniquify list +# just does sort -u on list (equiv. to sort | uniq) +# CAUTION: reorders list +uniqify() +{ + for _x in $*; do echo $_x; done | sort -u +} + +verbose() +{ + if [ -n "$opt_v" ]; then + echo "$@" + fi +} + +vsleep() +{ + if [ -n "$opt_v" ]; then + sleep $1 + fi +} + +###################################################################### +## +## main() +## + +EXCLUDE= + +args=$(getopt hnuvx: $*) +if [ $? -ne 0 ]; then + opt_h=1 +fi +set -- $args +while [ $# -gt 0 ]; do + case "$1" in + -h) opt_h=1 ;; + -n) opt_n=1 ;; + -u) opt_u=1 ;; + -v) opt_v=1 ;; + -x) EXCLUDE="$EXCLUDE $(echo $2 | sed 's/,/ /g')" ; shift ;; + --) shift; break ;; + esac + shift +done + +if [ -n "$opt_h" ]; then + usage +fi + +MISMATCH_TODO= +if [ -n "$opt_u" ]; then + echo "${OPI} Checking for mismatched installed packages using pkg_chk" + MISMATCH_TODO=$(check_packages_mismatched) + echo "${OPI} Excluding the following mismatched packages:" + echo "${OPC} EXCLUDE=[$EXCLUDE]" + MISMATCH_TODO=$(exclude $EXCLUDE --from $MISMATCH_TODO) +fi + +echo "${OPI} Checking for rebuild-requested installed packages (rebuild=YES)" +REBUILD_TODO=$(check_packages_w_flag 'rebuild') + +echo "${OPI} Checking for unsafe installed packages (unsafe_depends=YES)" +UNSAFE_TODO=$(check_packages_w_flag 'unsafe_depends') + +# DEPGRAPH_INSTALLED is rebuilt each round. DEPGRAPH_SRC will collect +# edges that we discover using 'make show-depends', but that weren't +# listed as depends by the installed version of a package, and +# DEPENDS_CHECKED lists packages for which we've already done that +# check. +DEPGRAPH_INSTALLED= +DEPGRAPH_SRC= +DEPENDS_CHECKED= + +verbose "${OPI} Packages to rebuild:" +verbose "${OPC} MISMATCH_TODO=[$(echo $MISMATCH_TODO)]" #strip newlines +verbose "${OPC} REBUILD_TODO=[$(echo $REBUILD_TODO)]" +verbose "${OPC} UNSAFE_TODO=[$(echo $UNSAFE_TODO)]" +vsleep 2 + +REPLACE_TODO=$(uniqify $MISMATCH_TODO $REBUILD_TODO $UNSAFE_TODO) +depgraph_built=0 + +while [ -n "$REPLACE_TODO" ]; do + # don't rebuild depgraph if we continued from new-depends step below + if [ "$depgraph_built" -eq 0 ]; then + echo "${OPI} Building dependency graph for installed packages" + DEPGRAPH_INSTALLED=$(depgraph_installed) + depgraph_built=1 + fi + + echo "${OPI} Tsorting dependency graph" + TSORTED=$(echo $DEPGRAPH_INSTALLED $DEPGRAPH_SRC | tsort) + for pkg in $TSORTED; do + if is_member $pkg $REPLACE_TODO; then + break; + fi + done + pkgdir=$(pkg_info -Bq $pkg | awk -F= '/PKGPATH=/{print $2}') + echo "${OPI} Selecting $pkg ($pkgdir) as next package to replace" + sleep 1 + + # Newer versions in pkgsrc sometimes have more depends than are + # recorded for the installed version (this is entirely to be + # expected as packages are split, renamed, and add new features). + # When this happens, add the new edges to the graph and re-tsort. + # Don't re-tsort if the new edges were already installed once + # (otherwise we'll go into a loop). + + # XXX I'm not sure that DEPENDS_CHECKED is necessary, or whether + # simply ignoring new deps that aren't in $REPLACE_TODO would have + # been sufficient. The DEPENDS_CHECKED approach causes one extra + # tsort in some cases, which isn't overly expensive. + + # XXX After tsorting and choosing what to 'make replace', we + # filter out packages that aren't already installed and marked for + # replacement. This is safe, since uninstalled depends will be + # pulled in automatically by 'make replace'; but in rare cases it + # might be non-optimal, leading to a duplicate rebuild that we + # would have avoided had we selected the uninstalled package in + # tsorted order and run 'make install' on it. This seems like + # such a rare case that the added complexity isn't worth it. + + if ! is_member $pkg $DEPENDS_CHECKED; then + echo "${OPI} Checking if $pkg has new depends..." + OLD_DEPENDS=$(pkg_info -Nq $pkg | sed 's/-[0-9].*[[:>:]]//g') + NEW_DEPENDS= + cd "$PKGSRCDIR/$pkgdir" + bdeps=$(${MAKE} show-depends VARNAME=BUILD_DEPENDS) + rdeps=$(${MAKE} show-depends) + for depver in $bdeps $rdeps; do + dep=$(echo $depver | sed -e 's/-[0-9].*//' -e 's/[<>]=[0-9].*//' \ + -e 's/-[][].*//') + if ! is_member $dep $OLD_DEPENDS $NEW_DEPENDS; then + NEW_DEPENDS="$NEW_DEPENDS $dep" + DEPGRAPH_SRC="$DEPGRAPH_SRC $dep $pkg" + fi + done + DEPENDS_CHECKED="$DEPENDS_CHECKED $pkg" + if [ -n "$NEW_DEPENDS" ]; then + echo "${OPI} $pkg has the following new depends (need to re-tsort):" + echo "${OPC} [$(echo $NEW_DEPENDS)]" + sleep 2 + continue + fi + fi + + # Do make replace, with clean before, and package and clean afterwards. + echo "${OPI} Replacing $(pkg_info -e $pkg)" + FAIL= + if [ -z "$opt_n" ]; then + cd "$PKGSRCDIR/$pkgdir" \ + && ${MAKE} clean && ${MAKE} replace && ${MAKE} package && ${MAKE} clean \ + || FAIL=1 + else + echo "cd $PKGSRCDIR/$pkgdir && ${MAKE} clean && ${MAKE} replace && ${MAKE} package && ${MAKE} clean" + fi + if [ -n "$FAIL" ]; then + echo "*** 'make replace' failed for package $pkg." + echo "*** Please read the errors listed above, fix the problem," + echo "*** then re-run pkg_rolling-replace to continue." + exit 1 + fi + sleep 1 + + # remove just-replaced package from all *_TODO lists + MISMATCH_TODO=$(exclude $pkg --from $MISMATCH_TODO) + REBUILD_TODO=$(exclude $pkg --from $REBUILD_TODO) + UNSAFE_TODO=$(exclude $pkg --from $UNSAFE_TODO) + + echo "${OPI} Re-checking for unsafe installed packages (unsafe_depends=YES)" + if [ -n "$opt_n" ]; then + # With -n, the replace didn't happen, and thus the packages that would + # have been marked unsafe_depends=YES were not. Add the set that + # would have been marked so we can watch what pkg_rolling-replace + # would have done (approximately). + UNSAFE_TODO=$(uniqify $UNSAFE_TODO \ + $(who_requires $pkg --in-graph $DEPGRAPH_INSTALLED)) + sleep 1 + else + UNSAFE_TODO=$(check_packages_w_flag 'unsafe_depends') + fi + + verbose "${OPI} Packages to rebuild:" + verbose "${OPC} MISMATCH_TODO=[$(echo $MISMATCH_TODO)]" #strips newlines + verbose "${OPC} REBUILD_TODO=[$(echo $REBUILD_TODO)]" + verbose "${OPC} UNSAFE_TODO=[$(echo $UNSAFE_TODO)]" + vsleep 4 + + REPLACE_TODO=$(uniqify $MISMATCH_TODO $REBUILD_TODO $UNSAFE_TODO) + depgraph_built=0 +done + +echo "${OPI} No more packages to replace; done." |