The buildlink2 problems on non-NetBSD platform seem to be related to

correct manipulation of variables containing values with escaped
metacharacters by the shell.  The new code only does partial caching, which
doesn't cache "special" arguments that contain escaped metacharacters or
other oddly quoted values.  On certain platforms listed in _BLNK_CACHE_ALL
in bsd.buildlink2.mk, we do full caching, which has enormous speed benefits
since it avoids an extra sed invocation per "special" argument.

Known working platforms:

	MacOS X 10.1.5 (schmonz)
	MacOS X 10.2 (grant)
	NetBSD 1.6 [full caching] (jlam)
	OpenBSD (cjep)
	Solaris 8 (grant)
	Solaris 9 (salo)

10 files changed, 129 insertions, 94 deletions

diff --git a/mk/buildlink2/NOTES b/mk/buildlink2/NOTES
new file mode 100644
index 00000000000..06112108b44
--- /dev/null
+++ b/mk/buildlink2/NOTES
@@ -0,0 +1,36 @@
+$NetBSD: NOTES,v 1.1 2002/12/22 19:02:43 jlam Exp $
+
+ Caching
+ =======
+
+The caching problems seem to be related to correct manipulation of
+variables containing values with escaped metacharacters by the shell.  The
+current code only does partial caching, which doesn't cache "special"
+arguments that contain escaped metacharacters or other oddly quoted values.
+On certain platforms listed in _BLNK_CACHE_ALL in bsd.buildlink2.mk, we do
+full caching, which has enormous speed benefits since it avoids an extra
+sed invocation per "special" argument.
+
+The test process for buildlink2 consists of building and installing:
+
+	shells/tcsh	tests handling of specially-quoted arguments
+	devel/gettext	same as above, but uses libtool
+
+Full caching can be tested be adding ${OPSYS} to _BLNK_CACHE_ALL and
+performing the tests above.
+
+Known working platforms:
+
+	MacOS X 10.1.5 (schmonz)
+	MacOS X 10.2 (grant)
+	NetBSD 1.6 [full caching] (jlam)
+	OpenBSD (cjep)
+	Solaris 8 (grant)
+	Solaris 9 (salo)
+
+Unknown status:
+
+	IRIX
+	NetBSD 1.5.x
+	Solaris 7
+	Linux
diff --git a/mk/buildlink2/bsd.buildlink2.mk b/mk/buildlink2/bsd.buildlink2.mk
index 758abe75630..f11fb8ce576 100644
--- a/mk/buildlink2/bsd.buildlink2.mk
+++ b/mk/buildlink2/bsd.buildlink2.mk
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.buildlink2.mk,v 1.62 2002/12/03 08:31:41 jlam Exp $
+# $NetBSD: bsd.buildlink2.mk,v 1.63 2002/12/22 19:02:43 jlam Exp $
 #
 # An example package buildlink2.mk file:
 #
@@ -64,11 +64,22 @@ _BLNK_LDFLAGS=		-L${LOCALBASE}/lib -Wl,-R${LOCALBASE}/lib
 .endif
 _BLNK_OPSYS=		${OPSYS}
 
+BUILDLINK_SHELL?=	${SH}
+
 # The configure process usually tests for outlandish or missing things
 # that we don't want polluting the argument cache.
 #
 CONFIGURE_ENV+=		BUILDLINK_UPDATE_CACHE=no
 
+# The caching code, which greatly speeds up the build process, works only
+# on certain platforms.
+#
+_BLNK_CACHE_ALL=	NetBSD
+.if !empty(_BLNK_CACHE_ALL:M${OPSYS})
+CONFIGURE_ENV+=		BUILDLINK_CACHE_ALL=yes
+MAKE_ENV+=		BUILDLINK_CACHE_ALL=yes
+.endif
+
 .if defined(USE_X11)
 USE_X11_LINKS?=		YES
 .  if !empty(USE_X11_LINKS:M[nN][oO])
@@ -520,18 +531,6 @@ _ALIASES.CPP=		cpp
 _ALIASES.FC=		f77 g77
 _ALIASES.LD=		ld
 
-# On Darwin, protect against using /bin/sh if it's zsh.
-.if ${_BLNK_OPSYS} == "Darwin"
-.  if exists(/bin/bash)
-BUILDLINK_SHELL?=	/bin/bash
-.  else
-BUILD_DEPENDS+=		bash-[0-9]*:../../shells/bash2
-BUILDLINK_SHELL?=	${LOCALBASE}/bin/bash
-.  endif
-.else
-BUILDLINK_SHELL?=	${SH}
-.endif
-
 # _BLNK_WRAP_*.<wrappee> variables represent "template methods" of the
 
 # wrapper script that may be customized per wrapper:
@@ -652,6 +651,7 @@ _BLNK_WRAPPER_TRANSFORM_SED.${_wrappee_}=				\
 	-e "s|@CAT@|${CAT:Q}|g"						\
 	-e "s|@ECHO@|${ECHO:Q}|g"					\
 	-e "s|@SED@|${SED:Q}|g"						\
+	-e "s|@TEST@|${TEST:Q}|g"					\
 	-e "s|@TOUCH@|${TOUCH:Q}|g"					\
 	-e "s|@_BLNK_LIBTOOL_FIX_LA@|${_BLNK_LIBTOOL_FIX_LA:Q}|g"	\
 	-e "s|@_BLNK_WRAP_LOG@|${_BLNK_WRAP_LOG:Q}|g"			\
@@ -767,6 +767,7 @@ ${_BLNK_WRAP_POST_CACHE}: ${.CURDIR}/../../mk/buildlink2/post-cache
 	${_PKG_SILENT}${_PKG_DEBUG}${SED}				\
 		-e "s|@LOCALBASE@|${LOCALBASE}|g"			\
 		-e "s|@X11BASE@|${X11BASE}|g"				\
+		-e 's|@CAT@|${CAT}|g'					\
 		-e 's|@ECHO@|${ECHO}|g'					\
 		-e 's|@_BLNK_TRANSFORM_SED@||g'				\
 		${.ALLSRC} > ${.TARGET}.tmp
@@ -779,6 +780,7 @@ ${_BLNK_WRAP_POST_CACHE_TRANSFORM}:					\
 	${_PKG_SILENT}${_PKG_DEBUG}${SED}				\
 		-e "s|@LOCALBASE@|${LOCALBASE}|g"			\
 		-e "s|@X11BASE@|${X11BASE}|g"				\
+		-e 's|@CAT@|${CAT}|g'					\
 		-e 's|@ECHO@|${ECHO}|g'					\
 		-e 's|@_BLNK_TRANSFORM_SED@|${_BLNK_TRANSFORM_SED:Q}|g'	\
 		${.CURDIR}/../../mk/buildlink2/post-cache > ${.TARGET}.tmp
@@ -830,6 +832,7 @@ ${_BLNK_LIBTOOL_FIX_LA}:						\
 		-e "s|@MV@|${MV:Q}|g"					\
 		-e "s|@RM@|${RM:Q}|g"					\
 		-e "s|@SED@|${SED:Q}|g"					\
+		-e "s|@TEST@|${TEST:Q}|g"				\
 		-e "s|@TOUCH@|${TOUCH:Q}|g"				\
 		-e 's|@_BLNK_WRAP_LT_UNTRANSFORM_SED@|${_BLNK_WRAP_LT_UNTRANSFORM_SED:Q}|g' \
 		-e 's|@_BLNK_UNTRANSFORM_SED@|${_BLNK_UNTRANSFORM_SED:Q}|g' \
@@ -859,6 +862,7 @@ ${_BLNK_FAKE_LA}: ${.CURDIR}/../../mk/buildlink2/fake-la
 		-e "s|@MV@|${MV:Q}|g"					\
 		-e "s|@RM@|${RM:Q}|g"					\
 		-e "s|@SED@|${SED:Q}|g"					\
+		-e "s|@TEST@|${TEST:Q}|g"				\
 		${.ALLSRC} > ${.TARGET}.tmp
 	${_PKG_SILENT}${_PKG_DEBUG}${CHMOD} +x ${.TARGET}.tmp
 	${_PKG_SILENT}${_PKG_DEBUG}${MV} -f ${.TARGET}.tmp ${.TARGET}
diff --git a/mk/buildlink2/fake-la b/mk/buildlink2/fake-la
index 84b2bb5d0e8..9959047e050 100644
--- a/mk/buildlink2/fake-la
+++ b/mk/buildlink2/fake-la
@@ -1,6 +1,6 @@
 #!@BUILDLINK_SHELL@
 #
-# $NetBSD: fake-la,v 1.2 2002/10/18 15:41:08 jlam Exp $
+# $NetBSD: fake-la,v 1.3 2002/12/22 19:02:44 jlam Exp $
 
 BASENAME="@BASENAME@"
 CC="@CC@"
@@ -12,6 +12,7 @@ MKDIR="@MKDIR@"
 MV="@MV@"
 RM="@RM@"
 SED="@SED@"
+TEST="@TEST@"
 
 BUILDLINK_DIR="@BUILDLINK_DIR@"
 
@@ -20,7 +21,7 @@ realdir=`${DIRNAME} $reallib`
 libname=`${BASENAME} $reallib`
 tmpdir=${BUILDLINK_DIR}/.tmp
 
-if [ ! -f $tmpdir/inst/$libbase.la ]; then
+if ! ${TEST} -f $tmpdir/inst/$libbase.la; then
 	case $libname in
 	*.so.[0-9]*)
 		libbase=`${ECHO} $libname | ${SED} -e "s/\.so\.[0-9.]*$//"`
diff --git a/mk/buildlink2/gen-transform.sh b/mk/buildlink2/gen-transform.sh
index 756fd55483d..49197dae89f 100644
--- a/mk/buildlink2/gen-transform.sh
+++ b/mk/buildlink2/gen-transform.sh
@@ -125,7 +125,7 @@ EOF
 	esac
 }
 
-for arg; do
+for arg do
 	gen transform "$arg"
 	gen untransform "$arg"
 done
diff --git a/mk/buildlink2/ld-logic b/mk/buildlink2/ld-logic
index 1ebab74863e..91c0a998091 100644
--- a/mk/buildlink2/ld-logic
+++ b/mk/buildlink2/ld-logic
@@ -1,10 +1,7 @@
-# $NetBSD: ld-logic,v 1.1 2002/09/23 01:11:39 jlam Exp $
+# $NetBSD: ld-logic,v 1.2 2002/12/22 19:02:44 jlam Exp $
 
 case $arg in
 -Wl,*)
-	newarg=`@ECHO@ "X$arg" | $Xsed \
-		-e "s|^-Wl,||g" -e "s|,| |g" \
-	`
-	arg="$newarg"
+	arg=`@ECHO@ "X$arg" | $Xsed -e "s|^-Wl,||g" -e "s|,| |g"`
 	;;
 esac
diff --git a/mk/buildlink2/libtool-fix-la b/mk/buildlink2/libtool-fix-la
index 0fe9d8cb69d..1ddbabf4ade 100644
--- a/mk/buildlink2/libtool-fix-la
+++ b/mk/buildlink2/libtool-fix-la
@@ -1,4 +1,4 @@
-# $NetBSD: libtool-fix-la,v 1.12 2002/10/30 00:23:00 jlam Exp $
+# $NetBSD: libtool-fix-la,v 1.13 2002/12/22 19:02:44 jlam Exp $
 #
 # For *.la files, in the "relink_command" line, we make the following
 # replacements:
@@ -16,7 +16,7 @@ labase=`@BASENAME@ $lafile .la`
 ladir=`@DIRNAME@ $lafile`
 latimestamp=${ladir}/.${labase}.la.timestamp
 laifile=${ladir}/.libs/${labase}.lai
-if [ -f $lafile ]; then
+if @TEST@ -f $lafile; then
 	. ${ladir}/${labase}.la
 
 	deps=
@@ -44,7 +44,7 @@ if [ -f $lafile ]; then
 				fixla=1
 				;;
 			esac
-			if [ $fixla -gt 0 ]; then
+			if @TEST@ $fixla -gt 0; then
 				case "$deps" in
 				*"-L$ltlibsdir "*) ;;
 				*"-L$ltlibsdir") ;;
@@ -89,7 +89,7 @@ if [ -f $lafile ]; then
 				fixla=1
 				;;
 			esac
-			if [ $fixla -gt 0 ]; then
+			if @TEST@ $fixla -gt 0; then
 				case "$relink" in
 				*"-L$ltlibsdir "*) ;;
 				*"-L$ltlibsdir") ;;
@@ -103,10 +103,10 @@ if [ -f $lafile ]; then
 			;;
 		esac
 	done
-	if [ -n "$deps" -o -n "$relink" ] && \
+	if @TEST@ -n "$deps" || @TEST@ -n "$relink" && \
 	   ! @EGREP@ "^#.*modified by buildlink2" $lafile >/dev/null 2>&1; then
 		@MV@ -f $lafile ${lafile}.blsav
-		if [ -n "$deps" ]; then
+		if @TEST@ -n "$deps"; then
 			(
 			@CAT@ ${lafile}.blsav | @SED@ -e '/^dependency_libs=/,$d'
 			@ECHO@ "dependency_libs='$deps'"
@@ -115,7 +115,7 @@ if [ -f $lafile ]; then
 		else
 			@CAT@ ${lafile}.blsav > ${lafile}.tmp
 		fi
-		if [ -n "$relink" ]; then
+		if @TEST@ -n "$relink"; then
 			(
 			@CAT@ ${lafile}.tmp | @SED@ -e '/^relink_command=/,$d'
 			@ECHO@ "relink_command='$relink'"
@@ -129,7 +129,7 @@ if [ -f $lafile ]; then
 		@RM@ ${lafile}.tmp
 		@ECHO@ "==> Fixed $lafile" >> $wrapperlog
 	fi
-	if [ -n "$laifile" ] && [ -f "$laifile" ] && \
+	if @TEST@ -n "$laifile" && @TEST@ -f "$laifile" && \
 	   ! @EGREP@ "^#.*modified by buildlink2" $laifile >/dev/null 2>&1; then
 		@SED@	@_BLNK_WRAP_LT_UNTRANSFORM_SED@ \
 			@_BLNK_UNTRANSFORM_SED@ \
@@ -163,7 +163,7 @@ if [ -f $lafile ]; then
 			esac
 			prev="$i"
 		done
-		if [ -z "$lexist" ]; then
+		if @TEST@ -z "$lexist"; then
 			L=
 		fi
 		deps="$L$l"
diff --git a/mk/buildlink2/libtool.sh b/mk/buildlink2/libtool.sh
index 4ad3b6094bf..a005bb62216 100644
--- a/mk/buildlink2/libtool.sh
+++ b/mk/buildlink2/libtool.sh
@@ -1,6 +1,6 @@
 #!@BUILDLINK_SHELL@
 #
-# $NetBSD: libtool.sh,v 1.6 2002/11/14 21:45:21 jlam Exp $
+# $NetBSD: libtool.sh,v 1.7 2002/12/22 19:02:44 jlam Exp $
 
 Xsed='@SED@ -e 1s/^X//'
 sed_quote_subst='s/\([\\`\\"$\\\\]\)/\\\1/g'
@@ -13,6 +13,7 @@ specificlogic="@_BLNK_WRAP_SPECIFIC_LOGIC@"
 libtool_fix_la="@_BLNK_LIBTOOL_FIX_LA@"
 wrapperlog="@_BLNK_WRAP_LOG@"
 updatecache=${BUILDLINK_UPDATE_CACHE-yes}
+cacheall=${BUILDLINK_CACHE_ALL-no}
 
 cmd="@WRAPPEE@"
 lafile=
@@ -23,7 +24,7 @@ case "$1" in
 	doinstall=yes
 	;;
 *)
-	while [ $# -gt 0 ]; do
+	while @TEST@ $# -gt 0; do
 		arg="$1"; shift
 		case $arg in
 		--fix-la)
@@ -36,8 +37,8 @@ case "$1" in
 			esac
 			;;
 		--mode|--mode=install)
-			if [ "$arg" = "--mode=install" ] || \
-			   [ "$arg" = "--mode" -a "$1" = "install" ]; then
+			if @TEST@ "$arg" = "--mode=install" || \
+			   @TEST@ "$arg" = "--mode" -a "$1" = "install"; then
 				doinstall=yes
 				break
 			fi
@@ -56,19 +57,17 @@ case "$1" in
 			;;
 		esac
 		. $specificlogic
-		args="$args $arg"
+		cmd="$cmd $arg"
 	done
 	;;
 esac
-if [ -n "$doinstall" ]; then
-	args="$args $arg"
-	while [ $# -gt 0 ]; do
+if @TEST@ -n "$doinstall"; then
+	cmd="$cmd $arg"
+	while @TEST@ $# -gt 0; do
 		arg="$1"; shift
 		case $arg in
 		*[\`\"\$\\]*)
-			arg=`@ECHO@ X$arg | \
-				$Xsed -e "$sed_quote_subst" \
-			`
+			arg=`@ECHO@ "X$arg" | $Xsed -e "$sed_quote_subst"`
 			;;
 		esac
 		case $arg in
@@ -76,10 +75,9 @@ if [ -n "$doinstall" ]; then
 			arg="\"$arg\""
 			;;
 		esac
-		args="$args $arg"
+		cmd="$cmd $arg"
 	done
 fi
-cmd="$cmd $args"
 
 @_BLNK_WRAP_ENV@
 @_BLNK_WRAP_SANITIZE_PATH@
@@ -88,7 +86,7 @@ cmd="$cmd $args"
 eval $cmd
 wrapper_result=$?
 
-if [ -n "$lafile" ] && [ -f "$lafile" ]; then
+if @TEST@ -n "$lafile" && @TEST@ -f "$lafile"; then
 	. $libtool_fix_la
 fi
 
diff --git a/mk/buildlink2/post-cache b/mk/buildlink2/post-cache
index e730e9adf0d..64e455b33b2 100644
--- a/mk/buildlink2/post-cache
+++ b/mk/buildlink2/post-cache
@@ -1,58 +1,54 @@
-# $NetBSD: post-cache,v 1.7 2002/10/30 00:23:00 jlam Exp $
+# $NetBSD: post-cache,v 1.8 2002/12/22 19:02:44 jlam Exp $
 
 *)
+	# $qarg contains a correctly-quoted $arg
 	case $arg in
 	*[\`\"\$\\]*)
-		arg=`@ECHO@ X$arg | $Xsed -e "$sed_quote_subst"`
-		case $arg in
-		*[\[\~\#\^\&\*\(\)\{\}\|\;\<\>\?\'\ \	]*|*]*|"")
-			arg="\"$arg\""
-			;;
-		esac
-		case $updatecache in
-		yes)
-			newarg=`@ECHO@ X$arg | $Xsed -e "$sed_quote_subst"`
-			@ECHO@ "$arg) arg=\"$newarg\" ;; #1" >> $cache
-			cacheupdated=yes
-			;;
-		esac
+		qarg=`@ECHO@ "X$arg" | $Xsed -e "$sed_quote_subst"`
+		quotedarg=yes
 		;;
-	*[\[\~\#\^\&\*\(\)\{\}\|\;\<\>\?\'\ \	]*|*]*|"")
-		arg="\"$arg\""
-		case $updatecache in
-		yes)
-			newarg=`@ECHO@ X$arg | $Xsed -e "$sed_quote_subst"`
-			@ECHO@ "$arg) arg=\"$newarg\" ;; #2" >> $cache
-			cacheupdated=yes
-			;;
-		esac
+	*)
+		qarg="$arg"
 		;;
-	-D*)
-		arg=`@ECHO@ X$arg | $Xsed -e "$sed_quote_subst"`
-		case $arg in
+	esac
+	case $qarg in
 		*[\[\~\#\^\&\*\(\)\{\}\|\;\<\>\?\'\ \	]*|*]*|"")
-			arg="\"$arg\""
-			;;
-		esac
-		case $updatecache in
-		yes)
-			newarg=`@ECHO@ X$arg | $Xsed -e "$sed_quote_subst"`
-			@ECHO@ "$arg) arg=\"$newarg\" ;; #3" >> $cache
-			cacheupdated=yes
-			;;
-		esac
+		qarg="\"$qarg\""
+		quotedarg=yes
+		;;
+	esac
+	searcharg="$qarg"
+
+	case $arg in
+	-D*)
+		arg="$qarg"
+		addtocache=yes
 		;;
         -[ILRl]*|-Wl,*|/usr/lib/lib*|@LOCALBASE@/*/lib*|@X11BASE@/*/lib*)
-		newarg=`@ECHO@ "X$arg" | $Xsed \
-			@_BLNK_TRANSFORM_SED@ \
-		`
-		case $updatecache in
-		yes)
-			@ECHO@ "$arg) arg=\"$newarg\" ;; #4" >> $cache
-			cacheupdated=yes
+		arg=`@ECHO@ "X$qarg" | $Xsed @_BLNK_TRANSFORM_SED@`
+		addtocache=yes
+		;;
+	*)
+		arg="$qarg"
+		addtocache=no
+		;;
+	esac
+	case $updatecache,$addtocache,$cacheall,$quotedarg in
+	yes,yes,yes,*|yes,yes,no,no)
+		case $arg in
+		*[\`\"\$\\]*)
+			cachearg=`@ECHO@ "X$arg" | $Xsed -e "$sed_quote_subst"`
+			;;
+		*)
+			cachearg="$arg"
 			;;
 		esac
-		arg="$newarg"
+		@CAT@ >> $cache << EOF
+$searcharg)
+	arg="$cachearg"
+	;;
+EOF
+		cacheupdated=yes
 		;;
 	esac
 	;;
diff --git a/mk/buildlink2/pre-cache b/mk/buildlink2/pre-cache
index 9b530ab0174..0247abdc193 100644
--- a/mk/buildlink2/pre-cache
+++ b/mk/buildlink2/pre-cache
@@ -1,7 +1,10 @@
-# $NetBSD: pre-cache,v 1.5 2002/10/31 22:14:09 jlam Exp $
+# $NetBSD: pre-cache,v 1.6 2002/12/22 19:02:45 jlam Exp $
+
+quotedarg=no
+addtocache=no
 
 case $arg in
 -[IL]@WRKDIR@*|-[IL]@BUILDLINK_DIR@*|-[IL]@BUILDLINK_X11_DIR@*)
 	;;
--[IL].|-[IL]./..*|-[IL]..*)
+-[IL].|-[IL]./*|-[IL]..*)
 	;;
diff --git a/mk/buildlink2/wrapper.sh b/mk/buildlink2/wrapper.sh
index d9737169552..e1f02a3f634 100644
--- a/mk/buildlink2/wrapper.sh
+++ b/mk/buildlink2/wrapper.sh
@@ -1,6 +1,6 @@
 #!@BUILDLINK_SHELL@
 #
-# $NetBSD: wrapper.sh,v 1.4 2002/09/28 23:46:42 jlam Exp $
+# $NetBSD: wrapper.sh,v 1.5 2002/12/22 19:02:45 jlam Exp $
 
 Xsed='@SED@ -e 1s/^X//'
 sed_quote_subst='s/\([\\`\\"$\\\\]\)/\\\1/g'
@@ -12,18 +12,18 @@ logic="@_BLNK_WRAP_LOGIC@"
 specificlogic="@_BLNK_WRAP_SPECIFIC_LOGIC@"
 wrapperlog="@_BLNK_WRAP_LOG@"
 updatecache=${BUILDLINK_UPDATE_CACHE-yes}
+cacheall=${BUILDLINK_CACHE_ALL-no}
 
 cmd="@WRAPPEE@"
-for arg; do
+for arg do
 	cacheupdated=no
 	. $logic
 	case "$cacheupdated" in
 	yes) @CAT@ $pre_cache $cache $post_cache > $logic ;;
 	esac
 	. $specificlogic
-	args="$args $arg"
+	cmd="$cmd $arg"
 done
-cmd="$cmd $args"
 
 @_BLNK_WRAP_ENV@
 @_BLNK_WRAP_SANITIZE_PATH@