summaryrefslogtreecommitdiff
path: root/mk
diff options
context:
space:
mode:
authorhubertf <hubertf@pkgsrc.org>2000-09-15 22:05:46 +0000
committerhubertf <hubertf@pkgsrc.org>2000-09-15 22:05:46 +0000
commit0e7843f045abaeb7f5b7978a900e8e74f06a3cf6 (patch)
tree67ed6b156f65a3fdbaaabab3bdd241af82ecf6eb /mk
parenta638bc51076f2592a0dfaeb2581e9047d3000fff (diff)
downloadpkgsrc-0e7843f045abaeb7f5b7978a900e8e74f06a3cf6.tar.gz
Bulk build framework
Diffstat (limited to 'mk')
-rw-r--r--mk/bulk/TODO1
-rw-r--r--mk/bulk/build51
-rw-r--r--mk/bulk/build.conf-example35
-rw-r--r--mk/bulk/post-build231
-rw-r--r--mk/bulk/pre-build84
-rw-r--r--mk/bulk/printdepends44
6 files changed, 446 insertions, 0 deletions
diff --git a/mk/bulk/TODO b/mk/bulk/TODO
new file mode 100644
index 00000000000..6721987cfe9
--- /dev/null
+++ b/mk/bulk/TODO
@@ -0,0 +1 @@
+ * allow using non-/etc mk.conf
diff --git a/mk/bulk/build b/mk/bulk/build
new file mode 100644
index 00000000000..d28f9e90b49
--- /dev/null
+++ b/mk/bulk/build
@@ -0,0 +1,51 @@
+#!/bin/sh
+# $Id: build,v 1.1 2000/09/15 22:05:47 hubertf Exp $
+#
+# Do builk build
+#
+# (c) 2000 Hubert Feyrer, All Rights Reserved.
+#
+
+echo Bulk build started: `date`
+echo ""
+
+# Pull in ADMIN:
+if [ -f "$BULK_BUILD_CONF" ]; then
+ . $BULK_BUILD_CONF
+else
+ conf=`dirname $0`/build.conf
+ if [ -f "$conf" ]; then
+ . $conf
+ else
+ echo "$0: Cannot find config file $conf, aborting."
+ exit 1
+ fi
+fi
+
+unset DISPLAY # allow sane failure for gimp, xlispstat
+
+cd ${USR_PKGSRC}
+
+if [ "$1" = "restart" ]; then
+ echo Restarting - skipping pre-build script
+else
+ sh mk/bulk/pre-build # make veryveryclean :)
+fi
+
+# Figure out optimal build order, and build
+if [ "$1" != "restart" ]; then
+ sh mk/bulk/printdepends | tee .l
+fi
+nice -n 20 make \
+ SPECIFIC_PKGS=1 \
+ GROUP_SPECIFIC_PKGS="`tsort .l | tr '\012' ' '`" \
+ bulk-package
+rm .l
+
+# Perl was wiped, reinstall it!
+( cd lang/perl5-base ; make bulk-install )
+perl mk/bulk/post-build | mail -s "pkgsrc bulk build results" $ADMIN
+
+# Done!
+echo ""
+echo Bulk build ended: `date`
diff --git a/mk/bulk/build.conf-example b/mk/bulk/build.conf-example
new file mode 100644
index 00000000000..a4780eb5bd5
--- /dev/null
+++ b/mk/bulk/build.conf-example
@@ -0,0 +1,35 @@
+# build.conf
+# $Id: build.conf-example,v 1.1 2000/09/15 22:05:47 hubertf Exp $
+#
+# config file in /bin/sh syntax for {,pre,post}-build
+#
+
+
+# Some paths for output files and paths to log files
+FTPx=`date +%Y%m%d.%H%M`
+FTPURL="pub/NetBSD/pkgstat/${FTPx}" # relative to ~ftp !
+FTP="/home/ftp/${FTPURL}" # absolute base path
+FTPHOST="ftp://ftp.machi.ne" # host for broken.html
+
+# Where build will mail the report
+ADMIN="you@some.whe.re"
+
+# The OS release this bulk build is happening on
+osrev=`uname -r`
+
+# The machine architecture this is happening on
+arch=`uname -m`
+
+# Where our pkgsrc is located
+USR_PKGSRC="/usr/pkgsrc"
+
+# As which user to run "cvs updates"
+CVS_USER="yourlogin"
+
+###########################################################################
+### No changes should be needed below this line !!!
+###########################################################################
+
+# Name of the (generated) HTML file which lists all broken pkgs
+REPORT="$FTP/broken.html"
+
diff --git a/mk/bulk/post-build b/mk/bulk/post-build
new file mode 100644
index 00000000000..d3bee13b6d6
--- /dev/null
+++ b/mk/bulk/post-build
@@ -0,0 +1,231 @@
+#!/usr/pkg/bin/perl
+# $Id: post-build,v 1.1 2000/09/15 22:05:47 hubertf Exp $
+#
+# Collect stuff after a pkg bulk build
+#
+# (c) 2000 Hubert Feyrer, All Rights Reserved.
+#
+
+use File::Basename;
+
+# Where config vars are stored (/bin/sh syntax)
+
+if (-f $ENV{"BULK_BUILD_CONF"}) {
+ $BULK_BUILD_CONF=$ENV{"BULK_BUILD_CONF"};
+} else {
+ $BULK_BUILD_CONF=dirname("$0")."/build.conf";
+}
+
+# Dig given variable out of config file, and set it
+sub getconf
+{
+ local($var)=@_;
+ local($val);
+
+ chomp($val=`. ./$BULK_BUILD_CONF ; echo \$$var`);
+ eval "\$$var=\"$val\";";
+}
+
+
+getconf("FTPURL"); # "pub/NetBSD/pkgstat/`date +%Y%m%d.%H%M`"
+getconf("FTP"); # "/disk1/ftp/${FTPURL}"
+getconf("REPORT"); # "broken.html"
+getconf("USR_PKGSRC"); # "/usr/pkgsrc"
+getconf("osrev"); # `uname -r`
+getconf("arch"); # `uname -m`
+chomp($date=`date`);
+
+$verbose=1;
+
+
+if ($verbose) {
+ print "\n";
+ print "*** NetBSD $osrev/$arch\n";
+ print "*** Result of bulk pkgsrc build as of $date:\n";
+ print "\n";
+}
+
+
+system("mkdir -p ${FTP}");
+
+# Copy over the output from the build process
+chdir("$USR_PKGSRC");
+system("tar plcf - .broken* */*/.broken* | ( cd ${FTP} ; tar plxf - )");
+
+
+open(HTML,">$REPORT") or die;
+print HTML <<EOHTML
+<HTML>
+<HEAD>
+<TITLE>NetBSD-$osrev/$arch bulk package build</TITLE>
+</HEAD>
+
+<BODY BGCOLOR=#cccccc>
+
+<H1> NetBSD $osrev/$arch </H1>
+<H2>Output of the pkgsrc bulk build<BR>
+ as of $date</H2>
+
+ Packages not listed here resulted in a <A
+ HREF="../../packages/">binary package</A>. To see the output of
+ the (failed) build, select the package below.
+ <P>
+
+ Files not listed in PLISTs can be found in <A HREF="leftovers-${arch}.html">this list</A>.
+ <P>
+
+<TABLE>
+<TR><TH> Package <TH> State <TH> Last touched by <TH> File touched last <TH> Maintainer
+EOHTML
+;
+
+# some nifty postprocessing of $FTP/.broken* -> HTML here
+$nunpackaged = $nbroken = 0;
+chdir("$FTP");
+$bf=".broken.$arch";
+open(BF,"sort +1 $bf |") or die "can't open .broken-file '$bf'";
+while (<BF>) {
+ ($nerrors, $bf) = split;
+ ( $pkg = $bf ) =~ s,/.broken.*$,,;
+
+# next
+# if $pkg!~/^a/; # DEBUG! HF
+
+ if ($nerrors > 0) {
+ $color = "red";
+ $state = "build broken";
+ $nbroken++;
+ } else {
+ $color = "yellow";
+ $state = "not packaged";
+ $nunpackaged++;
+ }
+
+ # The idea of this is stolen from bsd.pkg.mk's
+ # show-pkgtools-version target - thanks Al! :-)
+ $whofile = `find ${USR_PKGSRC}/${pkg} -type f -print | xargs ident 2>/dev/null | awk '\$1 ~ /\\\$NetBSD/ { gsub("/", "", \$4); print \$4 " " \$6 " " \$2 }' | sort -n | tail -n 1 | awk '{ print \$2 " " \$3 }' `;
+ $whofile=~s/,v$//;
+ chomp($whofile);
+ ($who,$file) = split(/\s+/, $whofile);
+
+ # $maintainer=`( cd $USR_PKGSRC/$pkg ; ( echo '.include "Makefile"' ; echo 'bla: ; \@echo ${MAINTAINER}' ) | make -f - bla )`;
+ $maintainer=`grep ^MAINTAINER $USR_PKGSRC/$pkg/Makefile | awk '{ print \$2\$3\$4\$5\$6}' | sed -e 's/\</&lt;/g' -e 's/\>/&gt;/g'`;
+ chomp($maintainer);
+
+ print HTML "<TR><TD><A HREF=\"$bf\">$pkg</A>";
+ print HTML " <TD><FONT COLOR=$color>$state</FONT>";
+ print HTML " <TD> $who <TD> $file";
+ print HTML " <TD> $maintainer\n";
+
+ ($category, $dbfeed_pkg) = split('/', $pkg);
+
+ if ($nerrors > 0) {
+ printf("%-23s $state\t($who, $maintainer)\n", "$pkg:")
+ if $verbose;
+ }
+}
+close(BF);
+
+
+$ntotal=$nunpackaged+$nbroken;
+print HTML <<EOHTML
+</TABLE>
+
+<HR>
+
+<TABLE>
+<TR><TD> Packages not packaged: <TD ALIGHT=RIGHT> <FONT COLOR=yellow>$nunpackaged</FONT>
+<TR><TD> Packages broken: <TD ALIGHT=RIGHT> <FONT COLOR=red>$nbroken</FONT>
+<TR><TD> Total: <TD ALIGHT=RIGHT> $ntotal
+</TABLE>
+
+<HR>
+<P>
+<UL>
+<!-- <LI>View the <A HREF="$PROG">progress</A> of the build. -->
+<!-- <LI>See the list of <A HREF="../index.html">all log files</A>. -->
+<LI>Visit the <A HREF="http://www.netbsd.org">NetBSD Homepage</A>.
+<LI>Learn about the
+ <A HREF="http://www.netbsd.org/Documentation/software/packages.html">
+ NetBSD Package System</A>.
+</UL>
+</P>
+<HR>
+Hubert Feyrer, 1999-2000.
+</BODY>
+</HTML>
+EOHTML
+;
+close(HTML);
+
+#
+# Adjust "last" symlink
+#
+{
+ local($base, $dir) = $FTP=~m|^(.*)/([^/]*)$|;
+
+ unlink("$base/last");
+ symlink($dir, "$base/last");
+}
+
+#
+# Generate leftovers-${arch}.html: files not deleted
+# Leftover files are copied to leftovers-${arch} dir, and linked from
+# leftovers-${arch}.html
+#
+{
+ chdir("${FTP}");
+ system("mkdir -p leftovers-${arch}");
+
+ # Find files since last build:
+ system("find /usr/pkg -newer ${USR_PKGSRC}/.start.${arch} -type f >>leftovers-${arch}.txt");
+ system("find /usr/X11R6 -newer ${USR_PKGSRC}/.start.${arch} -type f >>leftovers-${arch}.txt");
+
+ # Strip perl-files:
+ system("pkg_info -qL perl-base > .p.${arch}");
+ system("fgrep -v -x -f .p.${arch} leftovers-${arch}.txt >leftovers-${arch}.txt.new");
+ rename("leftovers-${arch}.txt.new", "leftovers-${arch}.txt");
+ unlink(".p.${arch}");
+
+ # Store leftovers, for easier identification:
+ system("tar plcf - `cat leftovers-${arch}.txt` | ( cd leftovers-${arch} ; tar plxf - )");
+
+ # Add links to leftover list:
+ open (IN, "< leftovers-${arch}.txt")
+ or die "can't read leftovers-${arch}.txt";
+ open (OUT, "> leftovers-${arch}.html")
+ or die "can't write leftovers-${arch}.html";
+ print OUT <<EOOUT
+<HTML>
+<BODY>
+<PRE>
+EOOUT
+;
+ while (<IN>) {
+ print OUT "<A HREF=\"/${FTPURL}/leftovers-${arch}$_\">$_</A>\n";
+ }
+ print OUT <<EOOUT2
+</PRE>
+</BODY>
+</HTML>
+EOOUT2
+;
+ close(IN);
+ close(OUT);
+}
+
+if ($verbose) {
+ print "\n";
+ print "Packages not packaged: $nunpackaged\n";
+ print "Packages broken: $nbroken\n";
+ print "Total: $ntotal\n";
+ print "\n";
+ print "See $FTPHOST/$FTPURL/broken.html\n";
+ print "for logs of builds broken or not resulting in a binary pkg.\n";
+ print "\n";
+ print "\n";
+ print " - Hubert\n";
+ print "\n";
+ print "[* This message was created automatically! *]\n";
+ print "\n";
+}
diff --git a/mk/bulk/pre-build b/mk/bulk/pre-build
new file mode 100644
index 00000000000..ba4a7e6e287
--- /dev/null
+++ b/mk/bulk/pre-build
@@ -0,0 +1,84 @@
+#!/bin/sh
+# $Id: pre-build,v 1.1 2000/09/15 22:05:47 hubertf Exp $
+#
+# Clean up system to be ready for bulk pkg build
+#
+# (c) 2000 Hubert Feyrer, All Rights Reserved.
+#
+
+#set -v # Debug
+
+# Pull in USR_PKGSRC, CVS_USER:
+if [ -f "$BULK_BUILD_CONF" ]; then
+ . $BULK_BUILD_CONF
+else
+ . `dirname $0`/build.conf
+fi
+
+
+#
+# Install cvs package and do a cvs update here
+#
+echo Installing required pkgs for CVS update
+( cd ${USR_PKGSRC}/devel/cvs ; make bulk-install )
+( cd ${USR_PKGSRC}/security/ssh ; make bulk-install )
+echo Performing CVS update - this will take some time
+su - ${CVS_USER} -c 'stty sane ; cd '${USR_PKGSRC}' ; cvs -q update -Pd'
+echo CVS update done.
+
+#
+# Remove old/broken distfiles and binary packages
+#
+echo Removing old/broken distfiles and binary packages
+( cd ${USR_PKGSRC}/pkgtools/pkglint ; make bulk-install )
+lintpkgsrc -r
+echo done.
+
+#
+# Clean out everything and it's mother
+#
+echo Removing all installed packages
+mount -o async -u /usr
+
+if [ -d /var/db/pkg ]; then
+ cd /var/db/pkg
+ for pkg in *
+ do
+ echo pkg_delete -rR $pkg
+ pkg_delete -rR $pkg
+ done
+ for pkg in *
+ do
+ echo pkg_delete -rR $pkg
+ pkg_delete -rR -f $pkg
+ done
+
+ # We've tried our best to get rid of the pkgs, now do it the hard way
+ # If it wasn't for stuff in $X11BASE, I wouldn't have hard feelings
+ # about this!
+ rm -fr *
+fi
+
+# General cleanout - easy!
+cd /usr/pkg
+rm -fr *
+rm -fr .??*
+
+# Stuff known to be broken
+rm -fr /usr/X11R6/share/gimp
+rm -fr /usr/X11R6/share/gnome
+rm -fr /usr/X11R6/share/kde
+rm -fr /usr/X11R6/share/netscape
+rm -fr /var/tmp/inst*
+rm -fr /usr/X11R6/lib/libgimp* # gimp doesn't build with old libs around
+
+
+# Clean up state files
+cd ${USR_PKGSRC}
+rm -f .broken* */*/.broken*
+rm -f .make* */*/.make*
+rm -f .start*
+
+mount -o noasync -u /usr
+
+touch .start.${arch}
diff --git a/mk/bulk/printdepends b/mk/bulk/printdepends
new file mode 100644
index 00000000000..d6dcd1f4ba1
--- /dev/null
+++ b/mk/bulk/printdepends
@@ -0,0 +1,44 @@
+#!/bin/sh
+# $NetBSD: printdepends,v 1.1 2000/09/15 22:05:47 hubertf Exp $
+#
+# Print list of pkg dependencies suitable for tsort(1).
+# Start in /usr/pkgsrc.
+#
+# (c) Copyright 2000 Hubert Feyrer <hubertf@netbsd.org>.
+# All rights reserved.
+#
+
+# /usr/pkgsrc
+cwd=$PWD
+
+# List of all pkgs, from pkgsrc/*/Makefile
+ list=`grep '^[^#].*'SUBDIR */Makefile | sed 's,/Makefile.*= *,/,'`
+#list=`grep '^[^#].*'SUBDIR ma*/Makefile | sed 's,/Makefile.*= *,/,'`
+#list=x11/gnome
+
+
+# Makefile to print the DEPENDS' and BUILD_DEPENDS' directory parts
+# bsd.pkg.mk is pulled in via the Makefile
+cat >$cwd/.m <<EOF
+bla:
+ @echo \${DEPENDS:C/^[^:]*://:C/:.*$//} \${BUILD_DEPENDS:C/^[^:]*://:C/:.*$//}
+.include "Makefile"
+EOF
+
+
+for pkgdir in $list
+do
+ cd $pkgdir
+
+ for reldir in `make -f - <$cwd/.m bla` # XXX
+ do
+ cd $reldir
+ d=`dirname $PWD`
+ absdir=`basename $d`/`basename $PWD`
+ cd -
+ echo "$absdir $pkgdir"
+ done
+ cd $cwd
+done
+
+rm -f $cwd/.m