6793120 pkill fails on native and sn1 branded zones

6800929 snv_106 ksh93 update breaks Install(1M)
Contributed by Roland Mainz <roland.mainz@nrubsig.org>

6 files changed, 221 insertions, 4 deletions

diff --git a/usr/src/lib/libshell/Makefile.demo b/usr/src/lib/libshell/Makefile.demo
index fa8235af6e..f32a2c75b1 100644
--- a/usr/src/lib/libshell/Makefile.demo
+++ b/usr/src/lib/libshell/Makefile.demo
@@ -20,7 +20,7 @@
 #
 
 #
-# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
 
@@ -102,6 +102,7 @@ DEMOFILES= \
 	tests/sun_solaris_cr_6754020_weird_square_bracket_expansion.sh \
 	tests/sun_solaris_cr_6763594_command_failure_execs_twice.sh \
 	tests/sun_solaris_cr_6766246_pattern_matching_bug.sh \
+	tests/sun_solaris_cr_6800929_large_command_substitution_hang.sh \
 	tests/sun_solaris_getconf.sh \
 	tests/sun_solaris_local_compound_nameref001.sh \
 	tests/sun_solaris_staticvariables.sh \
diff --git a/usr/src/lib/libshell/common/include/defs.h b/usr/src/lib/libshell/common/include/defs.h
index 78263fefb8..f5e03fd98e 100644
--- a/usr/src/lib/libshell/common/include/defs.h
+++ b/usr/src/lib/libshell/common/include/defs.h
@@ -166,6 +166,7 @@ struct limits
 	char		winch; \
 	char		indebug; 	/* set when in debug trap */ \
 	unsigned char	lastsig;	/* last signal received */ \
+	char		subshare;	/* set when in ${..} comsub */ \
 	char		*readscript;	/* set before reading a script */ \
 	int		*inpipe;	/* input pipe pointer */ \
 	int		*outpipe;	/* output pipe pointer */ \
diff --git a/usr/src/lib/libshell/common/sh/subshell.c b/usr/src/lib/libshell/common/sh/subshell.c
index 75dfc1f97d..4d54449c27 100644
--- a/usr/src/lib/libshell/common/sh/subshell.c
+++ b/usr/src/lib/libshell/common/sh/subshell.c
@@ -89,6 +89,7 @@ static struct subshell
 	int		coutpipe;
 	int		cpipe;
 	int		nofork;
+	char		subshare;
 } *subshell_data;
 
 static int subenv;
@@ -477,7 +478,9 @@ Sfio_t *sh_subshell(Shnode_t *t, int flags, int comsub)
 	sp->bckpid = shp->bckpid;
 	if(comsub)
 		sh_stats(STAT_COMSUB);
-	if(!comsub || (comsub==1 && !sh_isoption(SH_SUBSHARE)))
+	sp->subshare = shp->subshare;
+	shp->subshare = comsub==2 ||  (comsub==1 && sh_isoption(SH_SUBSHARE));
+	if(!comsub || !shp->subshare)
 	{
 		sp->shpwd = shp->pwd;
 		sp->pwd = (shp->pwd?strdup(shp->pwd):0);
@@ -677,6 +680,7 @@ Sfio_t *sh_subshell(Shnode_t *t, int flags, int comsub)
 		shp->cpipe[1] = sp->cpipe;
 		shp->coutpipe = sp->coutpipe;
 	}
+	shp->subshare = sp->subshare;
 	if(shp->subshell)
 		SH_SUBSHELLNOD->nvalue.s = --shp->subshell;
 	if(sp->sig)
diff --git a/usr/src/lib/libshell/common/sh/xec.c b/usr/src/lib/libshell/common/sh/xec.c
index a930c18460..1c93fa9dab 100644
--- a/usr/src/lib/libshell/common/sh/xec.c
+++ b/usr/src/lib/libshell/common/sh/xec.c
@@ -1406,7 +1406,12 @@ int sh_exec(register const Shnode_t *t, int flags)
 			pid_t	savepgid = job.curpgid;
 			job.curpgid = 0;
 			if(shp->subshell)
-				sh_subtmpfile(1);
+			{
+				if(shp->subshare)
+					sh_subtmpfile(0);
+				else
+					sh_subfork();
+			}
 			shp->inpipe = pvo;
 			shp->outpipe = pvn;
 			pvo[1] = -1;
diff --git a/usr/src/lib/libshell/common/tests/sun_solaris_cr_6800929_large_command_substitution_hang.sh b/usr/src/lib/libshell/common/tests/sun_solaris_cr_6800929_large_command_substitution_hang.sh
new file mode 100644
index 0000000000..396095ee63
--- /dev/null
+++ b/usr/src/lib/libshell/common/tests/sun_solaris_cr_6800929_large_command_substitution_hang.sh
@@ -0,0 +1,137 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+# Use is subject to license terms.
+#
+
+#
+# Test whether CR #6800929 ("snv_106 ksh93 update breaks Install(1M)") has been fixed.
+# 
+# Quote from CR #6800929: 
+# ---- snip ----
+# so i just upgraded this morning from snv_105 to snv_106.  now
+# Install(1M) is hanging whenever i run it.  i'm running it as follows:
+#         Install -o debug -k i86xpv -T domu-219:/tmp
+# 
+# and here's where it's hung:
+# ---8<---
+#  xxxxx@xxxxx $ pstack 204600
+# 204600: /bin/ksh /opt/onbld/bin/Install -o debug -k i86xpv -T domu-219:/tmp
+#  fffffd7fff2e3d1a write    (1, 4154c0, 64)
+#  fffffd7ffefdafc8 sfwr () + 2d0
+#  fffffd7ffefc0f6f _sfflsbuf () + 217
+#  fffffd7ffefcb9f7 sfsync () + 17f
+#  fffffd7ffefc5c58 _sfphead () + 188
+#  fffffd7ffefc5ef5 _sfpmove () + 55
+#  fffffd7ffefc2595 _sfmode () + 22d
+#  fffffd7ffefc5fb1 sfpool () + 99
+#  fffffd7fff15eb8e sh_exec () + 2f56
+#  fffffd7fff15f78c sh_exec () + 3b54
+#  fffffd7fff15d9c8 sh_exec () + 1d90
+#  fffffd7fff15788e sh_subshell () + 646
+#  fffffd7fff134562 comsubst () + 8a2
+#  fffffd7fff12f61f copyto () + bcf
+#  fffffd7fff12df79 sh_macexpand () + 1f1
+#  fffffd7fff1129f5 arg_expand () + a5
+#  fffffd7fff112812 sh_argbuild () + 9a
+#  fffffd7fff15dbe2 sh_exec () + 1faa
+#  fffffd7fff15d854 sh_exec () + 1c1c
+#  fffffd7fff0f22ef b_dot_cmd () + 507
+#  fffffd7fff161559 sh_funct () + 199
+#  fffffd7fff15ef35 sh_exec () + 32fd
+#  fffffd7fff136e86 exfile () + 786
+#  fffffd7fff136676 sh_main () + 7fe
+#  0000000000400e72 main () + 52
+#  0000000000400ccc ????
+# ---8<---
+# 
+# there is only one place where Install(1M) invokes "uniq":
+#         set -- `grep "^CONF" $modlist | sort | uniq`;
+# 
+# as it turns out, i can easily reproduce this problem as follows:
+# ---8<---
+#  xxxxx@xxxxx $ ksh93
+#  xxxxx@xxxxx $ set -- `cat /etc/termcap | sort | uniq`
+# <hang>
+# ---8<---
+# ---- snip ----
+
+
+function err_exit
+{
+	print -u2 -n "\t"
+	print -u2 -r ${Command}[$1]: "${@:2}"
+	(( Errors+=1 ))
+}
+
+alias err_exit='err_exit $LINENO'
+
+integer Errors=0
+
+integer i j d
+typeset tmpfile
+
+# test 1: run loop and check various temp filesizes
+tmpfile="$(mktemp "/tmp/sun_solaris_cr_6800929_large_command_substitution_hang.${PPID}.$$.XXXXXX")" || err_exit "Cannot create temporary file."
+
+for (( i=1*1024 ; i <= 512*1024 ; i*=2 )) ; do
+	# Create temp file
+	{
+		for ((j=0 ; j < i ; j+=16 )) ; do
+			print "0123456789abcde"
+		done
+	} >"${tmpfile}"
+	
+	# Run test child
+	${SHELL} -c "builtin cat ; print -- \"\$(cat \"${tmpfile}\" | cat)\" ; true" >/dev/null &
+	(( childpid=$! ))
+
+	# wait up to log2(i) seconds for the child to terminate
+	# (this is 10 seconds for 1KB and 19 seconds for 512KB)
+	(( d=log2(i) ))
+	for (( j=0 ; j < d ; j++ )) ; do
+		kill -0 ${childpid} 2>/dev/null || break
+		sleep 0.5
+	done
+
+	if kill -0 ${childpid} 2>/dev/null ; then
+		err_exit "test1: child (pid=${childpid}) still busy, filesize=${i}."
+		kill -KILL ${childpid} 2>/dev/null
+	fi
+	wait # wait for child (and/or avoid zombies/slime)
+	rm "${tmpfile}"
+done
+
+
+# test 2: Edward's Solaris-specific testcase
+${SHELL} -c 'builtin uniq ; set -- `cat /etc/termcap | sort | uniq` ; true' >/dev/null &
+(( childpid=$! ))
+sleep 5
+if kill -0 ${childpid} 2>/dev/null ; then
+	err_exit "test2: child (pid=${childpid}) still busy."
+	kill -KILL ${childpid} 2>/dev/null
+fi
+wait # wait for child (and/or avoid zombies/slime)
+
+# tests done
+exit $((Errors))
diff --git a/usr/src/lib/libshell/misc/ERRATA.txt b/usr/src/lib/libshell/misc/ERRATA.txt
index f94d327dd3..f14a66a950 100644
--- a/usr/src/lib/libshell/misc/ERRATA.txt
+++ b/usr/src/lib/libshell/misc/ERRATA.txt
@@ -20,7 +20,7 @@
 #
 
 #
-# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
  
@@ -239,4 +239,73 @@ Index: src/lib/libshell/common/sh/jobs.c
  			/* check to see whether job status has been saved */
 -- snip --
 
+
+######## Errata #004: ########
+A fix was backported to cure a hang in a command substitution when the
+amount of data exceeds a certain amount of data (this was causing
+the hang in CR #6800929 ("snv_106 ksh93 update breaks Install(1M)")).
+The following files have been changed:
+-- snip --
+Index: src/lib/libshell/common/include/defs.h
+===================================================================
+--- src/lib/libshell/common/include/defs.h	(revision 1391)
++++ src/lib/libshell/common/include/defs.h	(working copy)
+@@ -166,6 +166,7 @@
+ 	char		winch; \
+ 	char		indebug; 	/* set when in debug trap */ \
+ 	unsigned char	lastsig;	/* last signal received */ \
++	char		subshare;	/* set when in ${..} comsub */ \
+ 	char		*readscript;	/* set before reading a script */ \
+ 	int		*inpipe;	/* input pipe pointer */ \
+ 	int		*outpipe;	/* output pipe pointer */ \
+Index: src/lib/libshell/common/sh/subshell.c
+===================================================================
+--- src/lib/libshell/common/sh/subshell.c	(revision 1391)
++++ src/lib/libshell/common/sh/subshell.c	(working copy)
+@@ -89,6 +89,7 @@
+ 	int		coutpipe;
+ 	int		cpipe;
+ 	int		nofork;
++	char		subshare;
+ } *subshell_data;
+ 
+ static int subenv;
+@@ -477,7 +478,9 @@
+ 	sp->bckpid = shp->bckpid;
+ 	if(comsub)
+ 		sh_stats(STAT_COMSUB);
+-	if(!comsub || (comsub==1 && !sh_isoption(SH_SUBSHARE)))
++	sp->subshare = shp->subshare;
++	shp->subshare = comsub==2 ||  (comsub==1 && sh_isoption(SH_SUBSHARE));
++	if(!comsub || !shp->subshare)
+ 	{
+ 		sp->shpwd = shp->pwd;
+ 		sp->pwd = (shp->pwd?strdup(shp->pwd):0);
+@@ -677,6 +680,7 @@
+ 		shp->cpipe[1] = sp->cpipe;
+ 		shp->coutpipe = sp->coutpipe;
+ 	}
++	shp->subshare = sp->subshare;
+ 	if(shp->subshell)
+ 		SH_SUBSHELLNOD->nvalue.s = --shp->subshell;
+ 	if(sp->sig)
+Index: src/lib/libshell/common/sh/xec.c
+===================================================================
+--- src/lib/libshell/common/sh/xec.c	(revision 1391)
++++ src/lib/libshell/common/sh/xec.c	(working copy)
+@@ -1406,7 +1406,12 @@
+ 			pid_t	savepgid = job.curpgid;
+ 			job.curpgid = 0;
+ 			if(shp->subshell)
+-				sh_subtmpfile(1);
++			{
++				if(shp->subshare)
++					sh_subtmpfile(0);
++				else
++					sh_subfork();
++			}
+ 			shp->inpipe = pvo;
+ 			shp->outpipe = pvn;
+ 			pvo[1] = -1;
+-- snip --
 #EOF.