pbulk-0.63:

Introduce a separate client-prepare script. The default configuration
runs the renamed client_prepare_action for each scan/build client once,
dropping duplicates in the list. Clients are prepared in parallel, but
the master waits for all before starting the actual scan. Make
bulkbuild-restart re-prepare the clients, i.e. in case they have a
pkgsrc tree in a local tmpfs.

7 files changed, 64 insertions, 12 deletions

diff --git a/pkgtools/pbulk/Makefile b/pkgtools/pbulk/Makefile
index 2db34b1e8e3..98cb22c6c5f 100644
--- a/pkgtools/pbulk/Makefile
+++ b/pkgtools/pbulk/Makefile
@@ -1,6 +1,6 @@
-# $NetBSD: Makefile,v 1.78 2016/12/18 09:36:38 joerg Exp $
+# $NetBSD: Makefile,v 1.79 2016/12/18 22:38:16 joerg Exp $
 
-PKGNAME=	pbulk-0.62
+PKGNAME=	pbulk-0.63
 COMMENT=	Modular bulk build framework
 
 .include "../../pkgtools/pbulk/Makefile.common"
@@ -25,7 +25,7 @@ SUBST_STAGE.tools=	post-patch
 SUBST_MESSAGE.tools=	Fixing references to tools
 SUBST_FILES.tools=	pbulk.conf scripts/build scripts/build-client-start \
 	scripts/bulkbuild scripts/bulkbuild-rebuild scripts/bulkbuild-restart \
-	scripts/client-clean \
+	scripts/client-clean scripts/client-prepare \
 	scripts/pkg-build scripts/pkg-up-to-date scripts/pre-build \
 	scripts/report scripts/scan scripts/scan-client-start scripts/upload \
 	scripts/compute-packages.awk scripts/create-broken-graph.awk \
@@ -39,7 +39,7 @@ SUBST_VARS.tools=	AWK BZIP2 CHOWN DIGEST GZIP_CMD ID MAIL_CMD NEATO \
 CONF_FILES+=	share/examples/pbulk/pbulk.conf ${PKG_SYSCONFDIR}/pbulk.conf
 
 PBULK_CONFIG=		${PKG_SYSCONFDIR}/pbulk.conf
-PBULK_CONFIG_VERSION=	0.54
+PBULK_CONFIG_VERSION=	0.63
 
 INSTALLATION_DIRS=	bin libexec/pbulk share/examples/pbulk
 USE_BSD_MAKEFILE=	yes
diff --git a/pkgtools/pbulk/PLIST b/pkgtools/pbulk/PLIST
index f3b4139a21c..66495445584 100644
--- a/pkgtools/pbulk/PLIST
+++ b/pkgtools/pbulk/PLIST
@@ -1,10 +1,11 @@
-@comment $NetBSD: PLIST,v 1.5 2009/06/14 18:11:02 joerg Exp $
+@comment $NetBSD: PLIST,v 1.6 2016/12/18 22:38:16 joerg Exp $
 bin/bulkbuild
 bin/bulkbuild-rebuild
 bin/bulkbuild-restart
 libexec/pbulk/build
 libexec/pbulk/build-client-start
 libexec/pbulk/client-clean
+libexec/pbulk/client-prepare
 libexec/pbulk/compute-packages
 libexec/pbulk/create-broken-graph
 libexec/pbulk/create-report
diff --git a/pkgtools/pbulk/files/pbulk/pbulk.conf b/pkgtools/pbulk/files/pbulk/pbulk.conf
index 0c6f79f9f12..dbf77c81d4c 100644
--- a/pkgtools/pbulk/files/pbulk/pbulk.conf
+++ b/pkgtools/pbulk/files/pbulk/pbulk.conf
@@ -1,4 +1,4 @@
-# $NetBSD: pbulk.conf,v 1.22 2016/10/07 13:22:49 joerg Exp $
+# $NetBSD: pbulk.conf,v 1.23 2016/12/18 22:38:16 joerg Exp $
 
 # Version of the configuration file. This is bumped whenever the default
 # config changes to notify the administrator about updates.
@@ -115,10 +115,11 @@ pscan=@PREFIX@/bin/pbulk-scan
 keep_wrkdir=no
 keep_prefix=no
 
+client_prepare=@PREFIX@/libexec/pbulk/client-prepare
+client_prepare_action=@PREFIX@/libexec/pbulk/client-clean
 pkg_up_to_date_script=@PREFIX@/libexec/pbulk/pkg-up-to-date
 pbuild_script=@PREFIX@/libexec/pbulk/pkg-build
 pbuild_start_script=@PREFIX@/libexec/pbulk/build-client-start
-pscan_prepare=@PREFIX@/libexec/pbulk/client-clean
 pscan_start_script=@PREFIX@/libexec/pbulk/scan-client-start
 report_script=@PREFIX@/libexec/pbulk/create-report
 report_html_script=@PREFIX@/libexec/pbulk/create-report-html
diff --git a/pkgtools/pbulk/files/pbulk/scripts/Makefile b/pkgtools/pbulk/files/pbulk/scripts/Makefile
index 09467460c48..057e5b9455e 100644
--- a/pkgtools/pbulk/files/pbulk/scripts/Makefile
+++ b/pkgtools/pbulk/files/pbulk/scripts/Makefile
@@ -1,8 +1,9 @@
-# $NetBSD: Makefile,v 1.3 2008/04/04 17:58:59 joerg Exp $
+# $NetBSD: Makefile,v 1.4 2016/12/18 22:38:16 joerg Exp $
 
 SCRIPTS=	build build-client-start bulkbuild bulkbuild-rebuild \
 		bulkbuild-restart \
-		client-clean pkg-build pkg-up-to-date pre-build report \
+		client-clean client-prepare \
+		pkg-build pkg-up-to-date pre-build report \
 		scan scan-client-start upload \
 		compute-packages.awk create-broken-graph.awk \
 		create-report-html.awk create-report-txt.awk \
diff --git a/pkgtools/pbulk/files/pbulk/scripts/bulkbuild-restart b/pkgtools/pbulk/files/pbulk/scripts/bulkbuild-restart
index 49a024464c8..85424890db0 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/bulkbuild-restart
+++ b/pkgtools/pbulk/files/pbulk/scripts/bulkbuild-restart
@@ -1,5 +1,5 @@
 #!@SH@
-# $NetBSD: bulkbuild-restart,v 1.3 2008/10/08 16:43:34 joerg Exp $
+# $NetBSD: bulkbuild-restart,v 1.4 2016/12/18 22:38:16 joerg Exp $
 
 . @PBULK_CONFIG@
 
@@ -36,6 +36,12 @@ fi
 if [ "$restart_build" = "no" ]; then
 	${script_phase_pre_build}
 	${script_phase_scan}
+else
+	case "${master_mode}" in
+	[yY][eE][sS])
+		${client_prepare}
+		;;
+	esac
 fi
 ${script_phase_build}
 ${script_phase_report}
diff --git a/pkgtools/pbulk/files/pbulk/scripts/client-prepare b/pkgtools/pbulk/files/pbulk/scripts/client-prepare
new file mode 100755
index 00000000000..e28c3300bc2
--- /dev/null
+++ b/pkgtools/pbulk/files/pbulk/scripts/client-prepare
@@ -0,0 +1,41 @@
+#!@SH@
+# $NetBSD: client-prepare,v 1.1 2016/12/18 22:38:16 joerg Exp $
+
+. @PBULK_CONFIG@
+
+set -e
+
+if [ "${config_version}" != "@PBULK_CONFIG_VERSION@" ]; then
+	echo "Your configuration has version ${config_version}."
+	echo "This version of pbulk expects version @PBULK_CONFIG_VERSION@."
+	exit 1
+fi
+
+unique_clients=
+for client in ${scan_clients} ${build_clients}; do
+	case ${unique_clients} in
+	"${client}" | "${client} "* | *" ${client} "* | *" ${client}" )
+		: # exists already
+		;;
+	*)
+		if [ -n "$unique_clients" ]; then
+			unique_clients="$unique_clients "
+		fi
+		unique_clients="$unique_clients ${client}"
+		;;
+	esac
+done
+
+for client in ${unique_clients}; do
+	case ${client} in 
+	*:*)
+		port="-p ${client##*:}"
+		client=${client%%:*}
+		;;
+	*)
+		;;
+	esac
+	ssh $port $client "${client_prepare_action}" &
+done
+
+wait
diff --git a/pkgtools/pbulk/files/pbulk/scripts/scan-client-start b/pkgtools/pbulk/files/pbulk/scripts/scan-client-start
index 700a82e1cc5..4e241ab891e 100755
--- a/pkgtools/pbulk/files/pbulk/scripts/scan-client-start
+++ b/pkgtools/pbulk/files/pbulk/scripts/scan-client-start
@@ -1,5 +1,5 @@
 #!@SH@
-# $NetBSD: scan-client-start,v 1.4 2015/06/07 22:49:04 joerg Exp $
+# $NetBSD: scan-client-start,v 1.5 2016/12/18 22:38:16 joerg Exp $
 
 . @PBULK_CONFIG@
 
@@ -17,6 +17,8 @@ else
 	extra_pscan_args=""
 fi
 
+${client_prepare}
+
 for client in ${scan_clients}; do
 	case ${client} in 
 	*:*)
@@ -26,5 +28,5 @@ for client in ${scan_clients}; do
 	*)
 		;;
 	esac
-	ssh $port $client "${pscan_prepare} && ${pscan} -c ${master_port_scan} -M ${make} ${extra_pscan_args} ${pkgsrc}" &
+	ssh $port $client "${pscan} -c ${master_port_scan} -M ${make} ${extra_pscan_args} ${pkgsrc}" &
 done