Add coarse-grained locking to pkgsrc builds, by means of a beefed

up version of the bare-bones code in PR 7590, from David Maxwell.

The definition governing the type of locking used is PKGSRC_LOCKTYPE,
which can take any of the values "none", "sleep", and "once".  The
default is "none".  If "sleep" locking is used, and process A is
building a package, when process B attempts to build the same package,
process B will sleep for PKGSRC_SLEEPSECS seconds, and attempt to grab
the lock again.

Coarse-grained locking uses the OBJHOSTNAME definition to ensure that
the PID space is regular for shlock(1) to do its work.  The
pkgsrc/pkgtools/shlock package has been provided for environments
where shlock is not standard.

6 files changed, 110 insertions, 17 deletions

diff --git a/mk/bsd.pkg.defaults.mk b/mk/bsd.pkg.defaults.mk
index 3fae30d90f2..d2d24876fbc 100644
--- a/mk/bsd.pkg.defaults.mk
+++ b/mk/bsd.pkg.defaults.mk
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.pkg.defaults.mk,v 1.32 2002/01/11 14:41:42 agc Exp $
+# $NetBSD: bsd.pkg.defaults.mk,v 1.33 2002/01/19 21:29:41 agc Exp $
 #
 
 # A file providing defaults for pkgsrc and the packages collection.
@@ -31,6 +31,24 @@
 # Possible: defined, not defined
 # Default: not defined
 
+PKGSRC_LOCKTYPE?= none
+# The type of locking which will be done if competing processes attempt to
+# do work on one package directory simultaneously.
+# + Locking requires that OBJHOSTNAME is set.
+# + Locking may require the pkgsrc/pkgtools/shlock package to be installed
+#   on certain OS types.
+# + Sleep locking means that the process will sleep for ${PKGSRC_SLEEPSECS}
+#   seconds, then try to acquire the lock, and sleeping again if it's not
+#   available.
+# + Once locking will try once to acquire the lock, and then fail.
+# Possible: none, sleep, once
+# Default: none
+
+PKGSRC_SLEEPSECS?= 5
+# The number of seconds to wait when PKGSRC_LOCKTYPE is sleep
+# Possible: a positive integer
+# Default: 5
+
 #USETBL=
 # Run raw manual pages through tbl(1) before piping through troff(1)
 # when formatting manual pages.
diff --git a/mk/bsd.pkg.mk b/mk/bsd.pkg.mk
index f9cc42ebe9d..471c6669794 100644
--- a/mk/bsd.pkg.mk
+++ b/mk/bsd.pkg.mk
@@ -1,4 +1,4 @@
-#	$NetBSD: bsd.pkg.mk,v 1.906 2002/01/18 22:14:09 tron Exp $
+#	$NetBSD: bsd.pkg.mk,v 1.907 2002/01/19 21:29:41 agc Exp $
 #
 # This file is in the public domain.
 #
@@ -900,6 +900,20 @@ EXTRACT_ONLY?=	${DISTFILES}
 	@${FALSE}
 .endif
 
+.if (${PKGSRC_LOCKTYPE} == "sleep" || ${PKGSRC_LOCKTYPE} == "once") && !defined(OBJHOSTNAME) 
+.BEGIN:
+	@${ECHO_MSG} 'PKGSRC_LOCKTYPE needs OBJHOSTNAME defined.'
+	@${FALSE}
+.endif
+
+.if (${PKGSRC_LOCKTYPE} == "sleep" || ${PKGSRC_LOCKTYPE} == "once") && !exists(${SHLOCK}) 
+.BEGIN:
+	@${ECHO_MSG} 'The ${SHLOCK} utility does not exist, and is necessary for locking.'
+	@${ECHO_MSG} 'Please go to the ${.CURDIR}/../../pkgtools/shlock directory, and type'
+	@${ECHO_MSG} 'make install'
+	@${FALSE}
+.endif
+
 PKGREPOSITORYSUBDIR?=	All
 PKGREPOSITORY?=		${PACKAGES}/${PKGREPOSITORYSUBDIR}
 PKGFILE?=		${PKGREPOSITORY}/${PKGNAME}${PKG_SUFX}
@@ -1372,6 +1386,31 @@ show-pkgsrc-dir:
 .  endif
 .endif
 
+# pkgsrc coarse-grained locking definitions and targets
+.if ${PKGSRC_LOCKTYPE} == "none"
+_ACQUIRE_LOCK=	${_PKG_SILENT}${_PKG_DEBUG}${DO_NADA}
+_RELEASE_LOCK=	${_PKG_SILENT}${_PKG_DEBUG}${DO_NADA}
+.else
+LOCKFILE=	${WRKDIR}/.lockfile
+
+_ACQUIRE_LOCK=								\
+	${_PKG_SILENT}${_PKG_DEBUG}					\
+	ppid=`${PS} -p $$$$ -o ppid | ${AWK} 'NR == 2 { print $$0 }'`;	\
+	while true; do							\
+		${SHLOCK} -f ${LOCKFILE} -p $$ppid && break;		\
+		${ECHO} "=> Lock is held by pid `cat ${LOCKFILE}`";	\
+		case "${PKGSRC_LOCKTYPE}" in				\
+		once)	exit 1 ;;					\
+		sleep)	sleep ${PKGSRC_SLEEPSECS} ;;			\
+		esac							\
+	done;								\
+	${ECHO_MSG} "=> Lock acquired on behalf of process $$ppid"
+
+_RELEASE_LOCK=								\
+	${_PKG_SILENT}${_PKG_DEBUG}					\
+	${ECHO_MSG} "=> Lock released on behalf of process `${CAT} ${LOCKFILE}`"; \
+	${RM} ${LOCKFILE}
+.endif # PKGSRC_LOCKTYPE
 
 # Extract
 
@@ -1439,19 +1478,30 @@ EXTRACT_CMD${__suffix__}?=	${DECOMPRESS_CMD${__suffix__}} ${DOWNLOADED_DISTFILE}
 .  endif
 .endfor
 
-.if !target(do-extract)
-do-extract:
-.  ifndef KEEP_WRKDIR
+${WRKDIR}:
+.if !defined(KEEP_WRKDIR)
+.  if ${PKGSRC_LOCKTYPE} == "sleep" || ${PKGSRC_LOCKTYPE} == "once"
+.    if !exists(${LOCKFILE})
 	${_PKG_SILENT}${_PKG_DEBUG}${RM} -rf ${WRKDIR}
+.    endif
 .  endif
+.endif
 	${_PKG_SILENT}${_PKG_DEBUG}${MKDIR} ${WRKDIR}
-.  ifdef WRKOBJDIR
+.ifdef WRKOBJDIR
+.  if ${PKGSRC_LOCKTYPE} == "sleep" || ${PKGSRC_LOCKTYPE} == "once"
+.    if !exists(${LOCKFILE})
+	${_PKG_SILENT}${_PKG_DEBUG}					\
+	${RM} -f ${WRKDIR_BASENAME} || ${TRUE}
+.    endif
+.  endif
 	${_PKG_SILENT}${_PKG_DEBUG}					\
-	${RM} -f ${WRKDIR_BASENAME} || ${TRUE};				\
 	if ${LN} -s ${WRKDIR} ${WRKDIR_BASENAME} 2>/dev/null; then	\
 		${ECHO} "${WRKDIR_BASENAME} -> ${WRKDIR}";		\
 	fi
-.  endif # WRKOBJDIR
+.endif # WRKOBJDIR
+
+.if !target(do-extract)
+do-extract: ${WRKDIR}
 .  if defined(EXTRACT_CMD) && !empty(EXTRACT_CMD)
 	${_PKG_SILENT}${_PKG_DEBUG}					\
 	for file in "" ${EXTRACT_ONLY}; do				\
@@ -1809,7 +1859,7 @@ real-su-install: ${MESSAGE}
 	if [ X"$$found" != X"" ]; then					\
 		${ECHO} "$$found" >> ${WRKDIR}/.CONFLICTS;		\
 	fi
-.    endfor
+.     endfor
 	${_PKG_SILENT}${_PKG_DEBUG}					\
 	if [ -s ${WRKDIR}/.CONFLICTS ]; then \
 		found=`${SED} -e s'|${PKG_DBDIR}/||g' ${WRKDIR}/.CONFLICTS | tr '\012' ' '`; \
@@ -2129,6 +2179,23 @@ show-shlib-type:
 .  endif # libc.sylib
 .endif
 
+acquire-extract-lock:
+	${_ACQUIRE_LOCK}
+acquire-patch-lock:
+	${_ACQUIRE_LOCK}
+acquire-configure-lock:
+	${_ACQUIRE_LOCK}
+acquire-build-lock:
+	${_ACQUIRE_LOCK}
+
+release-extract-lock:
+	${_RELEASE_LOCK}
+release-patch-lock:
+	${_RELEASE_LOCK}
+release-configure-lock:
+	${_RELEASE_LOCK}
+release-build-lock:
+	${_RELEASE_LOCK}
 
 ################################################################
 # Skeleton targets start here
@@ -2145,19 +2212,19 @@ fetch:
 .endif
 
 .if !target(extract)
-extract: checksum ${EXTRACT_COOKIE}
+extract: checksum ${WRKDIR} acquire-extract-lock ${EXTRACT_COOKIE} release-extract-lock
 .endif
 
 .if !target(patch)
-patch: extract ${PATCH_COOKIE}
+patch: extract acquire-patch-lock ${PATCH_COOKIE} release-patch-lock
 .endif
 
 .if !target(configure)
-configure: patch ${CONFIGURE_COOKIE}
+configure: patch acquire-configure-lock ${CONFIGURE_COOKIE} release-configure-lock
 .endif
 
 .if !target(build)
-build: configure ${BUILD_COOKIE}
+build: configure acquire-build-lock ${BUILD_COOKIE} release-build-lock
 .endif
 
 .if !target(install)
diff --git a/mk/defs.Darwin.mk b/mk/defs.Darwin.mk
index ea4a5c042d1..32ddf3c01e6 100644
--- a/mk/defs.Darwin.mk
+++ b/mk/defs.Darwin.mk
@@ -1,4 +1,4 @@
-# $NetBSD: defs.Darwin.mk,v 1.16 2001/12/27 21:27:07 agc Exp $
+# $NetBSD: defs.Darwin.mk,v 1.17 2002/01/19 21:29:41 agc Exp $
 #
 # Variable definitions for the Darwin operating system.
 
@@ -36,11 +36,13 @@ MV?=		/bin/mv
 PATCH?=		/usr/bin/patch
 PAX?=		/bin/pax
 PKGLOCALEDIR?=	share
+PS?=		/bin/ps
 RM?=		/bin/rm
 RMDIR?=		/bin/rmdir
 SED?=		/usr/bin/sed
 SETENV?=	/usr/bin/env
 SH?=		/bin/sh
+SHLOCK=		/usr/bin/shlock
 SORT?=		/usr/bin/sort
 SU?=		/usr/bin/su
 TAIL?=		/usr/bin/tail
diff --git a/mk/defs.Linux.mk b/mk/defs.Linux.mk
index d69770fac3e..9d1975a1c70 100644
--- a/mk/defs.Linux.mk
+++ b/mk/defs.Linux.mk
@@ -1,4 +1,4 @@
-# $NetBSD: defs.Linux.mk,v 1.21 2001/12/19 10:29:11 agc Exp $
+# $NetBSD: defs.Linux.mk,v 1.22 2002/01/19 21:29:41 agc Exp $
 #
 # Variable definitions for the Linux operating system.
 
@@ -36,11 +36,13 @@ MV?=		/bin/mv
 PATCH?=		/usr/bin/patch
 PAX?=		${ZOULARISBASE}/bin/pax
 PKGLOCALEDIR?=	share
+PS?=		/bin/ps
 RM?=		/bin/rm
 RMDIR?=		/bin/rmdir
 SED?=		/bin/sed
 SETENV?=	/usr/bin/env
 SH?=		/bin/sh
+SHLOCK=		${LOCALBASE}/bin/shlock
 .if exists(/bin/sort)
 SORT?=		/bin/sort
 .else
diff --git a/mk/defs.NetBSD.mk b/mk/defs.NetBSD.mk
index ae691cff5bc..397d4b1b927 100644
--- a/mk/defs.NetBSD.mk
+++ b/mk/defs.NetBSD.mk
@@ -1,4 +1,4 @@
-# $NetBSD: defs.NetBSD.mk,v 1.19 2001/12/19 10:29:11 agc Exp $
+# $NetBSD: defs.NetBSD.mk,v 1.20 2002/01/19 21:29:41 agc Exp $
 #
 # Variable definitions for the NetBSD operating system.
 
@@ -36,11 +36,13 @@ MV?=		/bin/mv
 PATCH?=		/usr/bin/patch
 PAX?=		/bin/pax
 PKGLOCALEDIR?=	share
+PS?=		/bin/ps
 RM?=		/bin/rm
 RMDIR?=		/bin/rmdir
 SED?=		/usr/bin/sed
 SETENV?=	/usr/bin/env
 SH?=		/bin/sh
+SHLOCK=		/usr/bin/shlock
 SORT?=		/usr/bin/sort
 SU?=		/usr/bin/su
 TAIL?=		/usr/bin/tail
diff --git a/mk/defs.SunOS.mk b/mk/defs.SunOS.mk
index 6d2a6616dac..ed705c49ce0 100644
--- a/mk/defs.SunOS.mk
+++ b/mk/defs.SunOS.mk
@@ -1,4 +1,4 @@
-# $NetBSD: defs.SunOS.mk,v 1.19 2001/12/19 10:29:11 agc Exp $
+# $NetBSD: defs.SunOS.mk,v 1.20 2002/01/19 21:29:41 agc Exp $
 #
 # Variable definitions for the SunOS/Solaris operating system.
 
@@ -47,11 +47,13 @@ PATCH?=		${LOCALBASE}/bin/patch -b
 .endif
 PAX?=		/bin/pax
 PKGLOCALEDIR?=	lib
+PS?=		/bin/ps
 RM?=		/usr/bin/rm
 RMDIR?=		/usr/bin/rmdir
 SED?=		/usr/xpg4/bin/sed
 SETENV?=	/usr/bin/env
 SH?=		/bin/ksh
+SHLOCK=		${LOCALBASE}/bin/shlock
 SORT?=		/usr/bin/sort
 SU?=		/usr/bin/su
 TAIL?=		/usr/xpg4/bin/tail