Bulk build framework

6 files changed, 446 insertions, 0 deletions

diff --git a/mk/bulk/TODO b/mk/bulk/TODO
new file mode 100644
index 00000000000..6721987cfe9
--- /dev/null
+++ b/mk/bulk/TODO
@@ -0,0 +1 @@
+ * allow using non-/etc mk.conf
diff --git a/mk/bulk/build b/mk/bulk/build
new file mode 100644
index 00000000000..d28f9e90b49
--- /dev/null
+++ b/mk/bulk/build
@@ -0,0 +1,51 @@
+#!/bin/sh
+# $Id: build,v 1.1 2000/09/15 22:05:47 hubertf Exp $
+#
+# Do builk build
+#
+# (c) 2000 Hubert Feyrer, All Rights Reserved.
+#
+
+echo Bulk build started: `date`
+echo ""
+
+# Pull in ADMIN:
+if [ -f "$BULK_BUILD_CONF" ]; then
+    . $BULK_BUILD_CONF
+else
+	conf=`dirname $0`/build.conf
+	if [ -f "$conf" ]; then
+		. $conf
+	else
+		echo "$0: Cannot find config file $conf, aborting."
+		exit 1
+	fi
+fi
+
+unset DISPLAY		# allow sane failure for gimp, xlispstat
+
+cd ${USR_PKGSRC}
+
+if [ "$1" = "restart" ]; then
+	echo Restarting - skipping pre-build script
+else
+	sh mk/bulk/pre-build	# make veryveryclean :)
+fi
+
+# Figure out optimal build order, and build
+if [ "$1" != "restart" ]; then
+	sh mk/bulk/printdepends | tee .l
+fi
+nice -n 20 make \
+		SPECIFIC_PKGS=1 \
+		GROUP_SPECIFIC_PKGS="`tsort .l | tr '\012' ' '`" \
+		bulk-package
+rm .l
+
+# Perl was wiped, reinstall it!
+( cd lang/perl5-base ; make bulk-install )
+perl mk/bulk/post-build | mail -s "pkgsrc bulk build results" $ADMIN
+
+# Done!
+echo ""
+echo Bulk build ended: `date`
diff --git a/mk/bulk/build.conf-example b/mk/bulk/build.conf-example
new file mode 100644
index 00000000000..a4780eb5bd5
--- /dev/null
+++ b/mk/bulk/build.conf-example
@@ -0,0 +1,35 @@
+# build.conf
+# $Id: build.conf-example,v 1.1 2000/09/15 22:05:47 hubertf Exp $
+#
+# config file in /bin/sh syntax for {,pre,post}-build
+#
+
+
+# Some paths for output files and paths to log files
+FTPx=`date +%Y%m%d.%H%M`
+FTPURL="pub/NetBSD/pkgstat/${FTPx}"		# relative to ~ftp !
+FTP="/home/ftp/${FTPURL}"			# absolute base path
+FTPHOST="ftp://ftp.machi.ne"                    # host for broken.html
+
+# Where build will mail the report
+ADMIN="you@some.whe.re"
+
+# The OS release this bulk build is happening on
+osrev=`uname -r`
+
+# The machine architecture this is happening on
+arch=`uname -m`
+
+# Where our pkgsrc is located
+USR_PKGSRC="/usr/pkgsrc"
+
+# As which user to run "cvs updates"
+CVS_USER="yourlogin"
+
+###########################################################################
+### No changes should be needed below this line !!!
+###########################################################################
+
+# Name of the (generated) HTML file which lists all broken pkgs
+REPORT="$FTP/broken.html"
+
diff --git a/mk/bulk/post-build b/mk/bulk/post-build
new file mode 100644
index 00000000000..d3bee13b6d6
--- /dev/null
+++ b/mk/bulk/post-build
@@ -0,0 +1,231 @@
+#!/usr/pkg/bin/perl
+# $Id: post-build,v 1.1 2000/09/15 22:05:47 hubertf Exp $
+#
+# Collect stuff after a pkg bulk build
+#
+# (c) 2000 Hubert Feyrer, All Rights Reserved.
+#
+
+use File::Basename;
+
+# Where config vars are stored (/bin/sh syntax)
+
+if (-f $ENV{"BULK_BUILD_CONF"}) {
+    $BULK_BUILD_CONF=$ENV{"BULK_BUILD_CONF"};
+} else {
+    $BULK_BUILD_CONF=dirname("$0")."/build.conf";
+}
+
+# Dig given variable out of config file, and set it
+sub getconf
+{
+    local($var)=@_;
+    local($val);
+
+    chomp($val=`. ./$BULK_BUILD_CONF ; echo \$$var`);
+    eval "\$$var=\"$val\";";
+}
+
+
+getconf("FTPURL");		# "pub/NetBSD/pkgstat/`date +%Y%m%d.%H%M`"
+getconf("FTP");			# "/disk1/ftp/${FTPURL}"
+getconf("REPORT");		# "broken.html"
+getconf("USR_PKGSRC");		# "/usr/pkgsrc"
+getconf("osrev");	       	# `uname -r`
+getconf("arch"); 		# `uname -m`
+chomp($date=`date`);
+
+$verbose=1;
+
+
+if ($verbose) {
+	print "\n";
+	print "*** NetBSD $osrev/$arch\n";
+	print "*** Result of bulk pkgsrc build as of $date:\n";
+	print "\n";
+}
+
+
+system("mkdir -p ${FTP}");
+
+# Copy over the output from the build process
+chdir("$USR_PKGSRC");
+system("tar plcf - .broken* */*/.broken* | ( cd ${FTP} ; tar plxf - )");
+
+
+open(HTML,">$REPORT") or die;
+print HTML <<EOHTML
+<HTML>
+<HEAD>
+<TITLE>NetBSD-$osrev/$arch bulk package build</TITLE>
+</HEAD>
+
+<BODY BGCOLOR=#cccccc>
+
+<H1> NetBSD $osrev/$arch </H1>
+<H2>Output of the pkgsrc bulk build<BR>
+    as of $date</H2>
+
+    Packages not listed here resulted in a <A
+    HREF="../../packages/">binary package</A>. To see the output of
+    the (failed) build, select the package below.
+    <P>
+
+    Files not listed in PLISTs can be found in <A HREF="leftovers-${arch}.html">this list</A>.
+    <P>
+    
+<TABLE>
+<TR><TH> Package <TH> State <TH> Last touched by <TH> File touched last <TH> Maintainer
+EOHTML
+;
+
+# some nifty postprocessing of $FTP/.broken* -> HTML here
+$nunpackaged = $nbroken = 0;
+chdir("$FTP");
+$bf=".broken.$arch";
+open(BF,"sort +1 $bf |") or die "can't open .broken-file '$bf'";
+while (<BF>) {
+    ($nerrors, $bf) = split;
+    ( $pkg = $bf ) =~ s,/.broken.*$,,;
+
+#    next
+#	if $pkg!~/^a/; # DEBUG! HF
+    
+    if ($nerrors > 0) {
+	$color = "red";
+	$state = "build broken";
+	$nbroken++;
+    } else {
+	$color = "yellow";
+	$state = "not packaged";
+	$nunpackaged++;
+    }
+
+    # The idea of this is stolen from bsd.pkg.mk's
+    # show-pkgtools-version target - thanks Al! :-)
+    $whofile = `find ${USR_PKGSRC}/${pkg} -type f -print  | xargs ident 2>/dev/null | awk '\$1 ~ /\\\$NetBSD/ { gsub("/", "", \$4); print \$4 " " \$6 " " \$2 }' | sort -n | tail -n 1 | awk '{ print \$2 " " \$3 }' `;
+    $whofile=~s/,v$//;
+    chomp($whofile);
+    ($who,$file) = split(/\s+/, $whofile);
+
+    # $maintainer=`( cd $USR_PKGSRC/$pkg ; ( echo '.include "Makefile"' ; echo 'bla: ; \@echo ${MAINTAINER}' ) | make -f - bla )`;
+    $maintainer=`grep ^MAINTAINER $USR_PKGSRC/$pkg/Makefile | awk '{ print \$2\$3\$4\$5\$6}' | sed -e 's/\</&lt;/g' -e 's/\>/&gt;/g'`;
+    chomp($maintainer);
+    
+    print HTML "<TR><TD><A HREF=\"$bf\">$pkg</A>";
+    print HTML "    <TD><FONT COLOR=$color>$state</FONT>";
+    print HTML "    <TD> $who <TD> $file";
+    print HTML "    <TD> $maintainer\n";
+
+    ($category, $dbfeed_pkg) = split('/', $pkg);
+
+    if ($nerrors > 0) {
+    	printf("%-23s $state\t($who, $maintainer)\n", "$pkg:")
+	    if $verbose;
+    }
+}
+close(BF);
+
+
+$ntotal=$nunpackaged+$nbroken;
+print HTML <<EOHTML
+</TABLE>
+
+<HR>
+
+<TABLE>
+<TR><TD> Packages not packaged:	<TD ALIGHT=RIGHT> <FONT COLOR=yellow>$nunpackaged</FONT>
+<TR><TD> Packages broken:	<TD ALIGHT=RIGHT> <FONT COLOR=red>$nbroken</FONT>
+<TR><TD> Total:			<TD ALIGHT=RIGHT> $ntotal
+</TABLE>
+
+<HR>
+<P>     
+<UL>
+<!-- <LI>View the <A HREF="$PROG">progress</A> of the build. -->
+<!-- <LI>See the list of <A HREF="../index.html">all log files</A>. -->
+<LI>Visit the <A HREF="http://www.netbsd.org">NetBSD Homepage</A>.
+<LI>Learn about the
+    <A HREF="http://www.netbsd.org/Documentation/software/packages.html">
+    NetBSD Package System</A>.
+</UL>
+</P>
+<HR>
+Hubert Feyrer, 1999-2000.
+</BODY>
+</HTML>
+EOHTML
+;
+close(HTML);
+
+#
+# Adjust "last" symlink
+#
+{
+	local($base, $dir) = $FTP=~m|^(.*)/([^/]*)$|;
+
+	unlink("$base/last");
+	symlink($dir, "$base/last");
+}
+
+#
+# Generate leftovers-${arch}.html: files not deleted
+# Leftover files are copied to leftovers-${arch} dir, and linked from
+# leftovers-${arch}.html
+#
+{
+	chdir("${FTP}");
+	system("mkdir -p leftovers-${arch}");
+
+	# Find files since last build:
+	system("find /usr/pkg   -newer ${USR_PKGSRC}/.start.${arch} -type f >>leftovers-${arch}.txt");
+	system("find /usr/X11R6 -newer ${USR_PKGSRC}/.start.${arch} -type f >>leftovers-${arch}.txt");
+
+	# Strip perl-files:
+	system("pkg_info -qL perl-base > .p.${arch}");
+	system("fgrep -v -x -f .p.${arch} leftovers-${arch}.txt >leftovers-${arch}.txt.new");
+	rename("leftovers-${arch}.txt.new", "leftovers-${arch}.txt");
+	unlink(".p.${arch}");
+
+	# Store leftovers, for easier identification:
+	system("tar plcf - `cat leftovers-${arch}.txt` | ( cd leftovers-${arch} ; tar plxf - )");
+
+	# Add links to leftover list:
+	open (IN, "< leftovers-${arch}.txt") 
+		or die "can't read leftovers-${arch}.txt";
+	open (OUT, "> leftovers-${arch}.html")
+		or die "can't write leftovers-${arch}.html";
+	print OUT <<EOOUT
+<HTML>
+<BODY>
+<PRE>
+EOOUT
+;
+	while (<IN>) {
+	    	print OUT "<A HREF=\"/${FTPURL}/leftovers-${arch}$_\">$_</A>\n";
+	}
+	print OUT <<EOOUT2
+</PRE>
+</BODY>
+</HTML>
+EOOUT2
+;
+    	close(IN);
+    	close(OUT);
+}
+
+if ($verbose) {
+	print "\n";
+	print "Packages not packaged:  $nunpackaged\n";
+	print "Packages broken:        $nbroken\n";
+	print "Total:                  $ntotal\n";
+	print "\n";
+	print "See $FTPHOST/$FTPURL/broken.html\n";
+	print "for logs of builds broken or not resulting in a binary pkg.\n";
+	print "\n";
+	print "\n";
+	print " - Hubert\n";
+	print "\n";
+	print "[* This message was created automatically! *]\n";
+	print "\n";
+}
diff --git a/mk/bulk/pre-build b/mk/bulk/pre-build
new file mode 100644
index 00000000000..ba4a7e6e287
--- /dev/null
+++ b/mk/bulk/pre-build
@@ -0,0 +1,84 @@
+#!/bin/sh
+# $Id: pre-build,v 1.1 2000/09/15 22:05:47 hubertf Exp $
+#
+# Clean up system to be ready for bulk pkg build
+#
+# (c) 2000 Hubert Feyrer, All Rights Reserved.
+#
+
+#set -v		# Debug
+
+# Pull in USR_PKGSRC, CVS_USER:
+if [ -f "$BULK_BUILD_CONF" ]; then
+    . $BULK_BUILD_CONF
+else
+    . `dirname $0`/build.conf
+fi
+
+
+#
+# Install cvs package and do a cvs update here
+#
+echo Installing required pkgs for CVS update
+( cd ${USR_PKGSRC}/devel/cvs ; make bulk-install )
+( cd ${USR_PKGSRC}/security/ssh ; make bulk-install )
+echo Performing CVS update - this will take some time
+su - ${CVS_USER} -c 'stty sane ; cd '${USR_PKGSRC}' ; cvs -q update -Pd'
+echo CVS update done.
+
+#
+# Remove old/broken distfiles and binary packages
+#
+echo Removing old/broken distfiles and binary packages
+( cd ${USR_PKGSRC}/pkgtools/pkglint ; make bulk-install )
+lintpkgsrc -r
+echo done.
+
+#
+# Clean out everything and it's mother
+#
+echo Removing all installed packages 
+mount -o async -u /usr
+
+if [ -d /var/db/pkg ]; then
+	cd /var/db/pkg
+	for pkg in *
+	do
+		echo pkg_delete -rR $pkg
+		pkg_delete -rR $pkg
+	done
+	for pkg in *
+	do
+		echo pkg_delete -rR $pkg
+		pkg_delete -rR -f $pkg
+	done
+	
+	# We've tried our best to get rid of the pkgs, now do it the hard way
+	# If it wasn't for stuff in $X11BASE, I wouldn't have hard feelings
+	# about this!
+	rm -fr *
+fi
+	
+# General cleanout - easy!
+cd /usr/pkg
+rm -fr *
+rm -fr .??*
+
+# Stuff known to be broken
+rm -fr /usr/X11R6/share/gimp
+rm -fr /usr/X11R6/share/gnome
+rm -fr /usr/X11R6/share/kde
+rm -fr /usr/X11R6/share/netscape
+rm -fr /var/tmp/inst*
+rm -fr /usr/X11R6/lib/libgimp*	# gimp doesn't build with old libs around
+
+
+# Clean up state files
+cd ${USR_PKGSRC}
+rm -f .broken* */*/.broken*
+rm -f .make* */*/.make*
+rm -f .start*
+
+mount -o noasync -u /usr
+
+touch .start.${arch}
diff --git a/mk/bulk/printdepends b/mk/bulk/printdepends
new file mode 100644
index 00000000000..d6dcd1f4ba1
--- /dev/null
+++ b/mk/bulk/printdepends
@@ -0,0 +1,44 @@
+#!/bin/sh
+# $NetBSD: printdepends,v 1.1 2000/09/15 22:05:47 hubertf Exp $
+#
+# Print list of pkg dependencies suitable for tsort(1).
+# Start in /usr/pkgsrc.
+#
+# (c) Copyright 2000 Hubert Feyrer <hubertf@netbsd.org>.
+#     All rights reserved.
+#
+
+# /usr/pkgsrc
+cwd=$PWD
+
+# List of all pkgs, from pkgsrc/*/Makefile
+ list=`grep '^[^#].*'SUBDIR */Makefile | sed 's,/Makefile.*= *,/,'`
+#list=`grep '^[^#].*'SUBDIR ma*/Makefile | sed 's,/Makefile.*= *,/,'`
+#list=x11/gnome
+
+
+# Makefile to print the DEPENDS' and BUILD_DEPENDS' directory parts
+# bsd.pkg.mk is pulled in via the Makefile
+cat >$cwd/.m <<EOF
+bla:
+	@echo \${DEPENDS:C/^[^:]*://:C/:.*$//} \${BUILD_DEPENDS:C/^[^:]*://:C/:.*$//}
+.include "Makefile"
+EOF
+
+
+for pkgdir in $list
+do
+	cd $pkgdir
+
+	for reldir in `make -f - <$cwd/.m bla`	# XXX
+	do
+		cd $reldir
+		d=`dirname $PWD`
+		absdir=`basename $d`/`basename $PWD`
+		cd -
+		echo "$absdir $pkgdir"
+	done
+	cd $cwd
+done
+
+rm -f $cwd/.m