9056 dtrace_unregister()/fasttrap_pid_disable()/prgetmap() deadlock

9057 unloadable modules should use NO_UNLOAD stubs
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Tim Kordas <tim.kordas@joyent.com>
Approved by: Richard Lowe <richlowe@richlowe.net>

4 files changed, 206 insertions, 44 deletions

diff --git a/usr/src/cmd/dtrace/test/tst/common/usdt/tst.deadstub.ksh b/usr/src/cmd/dtrace/test/tst/common/usdt/tst.deadstub.ksh
new file mode 100644
index 0000000000..3543ec091d
--- /dev/null
+++ b/usr/src/cmd/dtrace/test/tst/common/usdt/tst.deadstub.ksh
@@ -0,0 +1,161 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright (c) 2017, Joyent, Inc. All rights reserved.
+#
+
+#
+# This test attempts to reproduce a three-way deadlock between mod_lock,
+# dtrace_lock and P_PR_LOCK that is induced by shmsys having to go through
+# mod_hold_stub.
+#
+if [ $# != 1 ]; then
+	echo expected one argument: '<'dtrace-path'>'
+	exit 2
+fi
+
+dtrace=$1
+DIR=/var/tmp/dtest.$$
+
+mkdir $DIR
+cd $DIR
+
+cat > prov.d <<EOF
+provider test_prov {
+	probe ripraf();
+};
+EOF
+
+$dtrace -h -s prov.d
+if [ $? -ne 0 ]; then
+	print -u2 "failed to generate header file"
+	exit 1
+fi
+
+cat > test.c <<EOF
+#include <unistd.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "prov.h"
+
+void
+main(int argc)
+{
+	void *addr;
+	int shmid;
+
+	if (argc > 1) {
+		TEST_PROV_RIPRAF();
+		exit(0);
+	}
+
+	shmid = shmget(IPC_PRIVATE, sizeof (int), IPC_CREAT | 0666);
+
+	if (shmid == -1) {
+		perror("shmget: ");
+		exit(1);
+	}
+
+	if ((addr = shmat(shmid, NULL, 0)) == (void *)-1) {
+		perror("shmat: ");
+		exit(1);
+	}
+
+	printf("%p\n", addr);
+
+	for (;;) {
+		TEST_PROV_RIPRAF();
+		sleep(1);
+	}
+}
+EOF
+
+gcc -m32 -c test.c
+if [ $? -ne 0 ]; then
+	print -u2 "failed to compile test.c"
+	exit 1
+fi
+
+$dtrace -G -32 -s prov.d test.o
+
+if [ $? -ne 0 ]; then
+	print -u2 "failed to create DOF"
+	exit 1
+fi
+
+gcc -m32 -o test test.o prov.o
+
+if [ $? -ne 0 ]; then
+	print -u2 "failed to link final executable"
+	exit 1
+fi
+
+#
+# Kick off the victim program.
+#
+./test &
+
+victim=$!
+
+#
+# Kick off a shell that will do nothing but read our victim's /proc map
+#
+( while true ; do read foo < /proc/$victim/map ; done ) &
+stubby=$!
+
+#
+# Kick off a shell that will do nothing but instrument (and de-instrument)
+# the victim
+#
+( while true; do \
+    $dtrace -q -P test_prov$victim -n BEGIN'{exit(0)}' > /dev/null ; done ) &
+inst=$!
+
+#
+# Finally, kick off a shell that will cause lots of provider registration and
+# (importantly) de-registration
+#
+( while true; do ./test foo ; done) &
+reg=$!
+
+echo $DIR
+echo victim: $victim
+echo stubby: $stubby
+echo inst: $inst
+echo reg: $reg
+
+sleep 120
+
+kill $reg
+sleep 1
+kill $inst
+sleep 1
+kill $stubby
+sleep 1
+kill $victim
+
+#
+# If we're deadlocked, this DTrace enabling won't work (if we even make it this
+# far, which seems unlikely).  In the spirit of the deadlock, we denote our
+# success by emiting a classic Faulknerism.
+#
+raf="Maybe you're not so worthless!"
+dtrace -qn BEGIN"{printf(\"$raf\"); exit(0)}"
+
+cd /
+/usr/bin/rm -rf $DIR
+
+exit 0
diff --git a/usr/src/pkg/manifests/system-dtrace-tests.mf b/usr/src/pkg/manifests/system-dtrace-tests.mf
index 33ed8b8b4d..3a5d1e8e05 100644
--- a/usr/src/pkg/manifests/system-dtrace-tests.mf
+++ b/usr/src/pkg/manifests/system-dtrace-tests.mf
@@ -2071,6 +2071,7 @@ file path=opt/SUNWdtrt/tst/common/usdt/tst.args.d mode=0444
 file path=opt/SUNWdtrt/tst/common/usdt/tst.args.exe mode=0555
 file path=opt/SUNWdtrt/tst/common/usdt/tst.badguess.ksh mode=0444
 file path=opt/SUNWdtrt/tst/common/usdt/tst.corruptenv.ksh mode=0444
+file path=opt/SUNWdtrt/tst/common/usdt/tst.deadstub.ksh mode=0444
 file path=opt/SUNWdtrt/tst/common/usdt/tst.dlclose1.ksh mode=0444
 file path=opt/SUNWdtrt/tst/common/usdt/tst.dlclose1.ksh.out mode=0444
 file path=opt/SUNWdtrt/tst/common/usdt/tst.dlclose2.ksh mode=0444
diff --git a/usr/src/uts/intel/ia32/ml/modstubs.s b/usr/src/uts/intel/ia32/ml/modstubs.s
index 76570f8855..d38b4e87c9 100644
--- a/usr/src/uts/intel/ia32/ml/modstubs.s
+++ b/usr/src/uts/intel/ia32/ml/modstubs.s
@@ -143,7 +143,7 @@ fcnname/**/_info:							\
 	.long	weak;			/* 0x20 */			\
 	SET_SIZE(fcnname/**/_info)
 
-#define STUB_UNLOADABLE(module, fcnname, install_fcn, retfcn, weak)	\
+#define STUB_NO_UNLOADABLE(module, fcnname, install_fcn, retfcn, weak)	\
 	ENTRY(fcnname);							\
 	leaq	fcnname/**/_info(%rip), %rax;				\
 	testb	$MODS_INSTALLED, MODS_FLAG(%rax); /* installed? */	\
@@ -294,7 +294,7 @@ fcnname/**/_info:							\
 	.long   weak;							\
 	SET_SIZE(fcnname/**/_info)
 	
-#define STUB_UNLOADABLE(module, fcnname, install_fcn, retfcn, weak)	\
+#define STUB_NO_UNLOADABLE(module, fcnname, install_fcn, retfcn, weak)	\
 	ENTRY(fcnname);							\
 	leal	fcnname/**/_info, %eax;					\
 	testb	$MODS_INSTALLED, MODS_FLAG(%eax); /* installed? */	\
@@ -377,13 +377,13 @@ fcnname/**/_info:							\
  * User *MUST* guarantee the module is not unloadable (no _fini routine).
  */
 #define NO_UNLOAD_STUB(module, fcnname, retfcn) \
-    STUB_UNLOADABLE(module, fcnname,  retfcn, retfcn, MODS_NOUNLOAD)
+    STUB_NO_UNLOADABLE(module, fcnname,  retfcn, retfcn, MODS_NOUNLOAD)
 
 /*
  * "weak stub" for non-unloadable module, don't load on account of this call
  */
 #define NO_UNLOAD_WSTUB(module, fcnname, retfcn) \
-    STUB_UNLOADABLE(module, fcnname, retfcn, retfcn, MODS_NOUNLOAD|MODS_WEAK)
+    STUB_NO_UNLOADABLE(module, fcnname, retfcn, retfcn, MODS_NOUNLOAD|MODS_WEAK)
 
 /*
  * this is just a marker for the beginning area of text that contains stubs
@@ -720,9 +720,9 @@ fcnname/**/_info:							\
  */
 #ifndef FIFO_MODULE
 	MODULE(fifofs,fs);
-	STUB(fifofs, fifovp,      	0);
-	STUB(fifofs, fifo_getinfo,	0);
-	STUB(fifofs, fifo_vfastoff,	0);
+	NO_UNLOAD_STUB(fifofs, fifovp,      	nomod_zero);
+	NO_UNLOAD_STUB(fifofs, fifo_getinfo,	nomod_zero);
+	NO_UNLOAD_STUB(fifofs, fifo_vfastoff,	nomod_zero);
 	END_MODULE(fifofs);
 #endif
 
@@ -868,8 +868,8 @@ fcnname/**/_info:							\
  */
 #ifndef SYSACCT_MODULE
 	MODULE(sysacct,sys);
-	WSTUB(sysacct, acct,  		nomod_zero);
-	WSTUB(sysacct, acct_fs_in_use, 	nomod_zero);
+	NO_UNLOAD_WSTUB(sysacct, acct,  		nomod_zero);
+	NO_UNLOAD_WSTUB(sysacct, acct_fs_in_use, 	nomod_zero);
 	END_MODULE(sysacct);
 #endif
 
@@ -878,7 +878,7 @@ fcnname/**/_info:							\
  */
 #ifndef SEMSYS_MODULE
 	MODULE(semsys,sys);
-	WSTUB(semsys, semexit,		nomod_zero);
+	NO_UNLOAD_WSTUB(semsys, semexit,		nomod_zero);
 	END_MODULE(semsys);
 #endif
 
@@ -887,9 +887,9 @@ fcnname/**/_info:							\
  */
 #ifndef SHMSYS_MODULE
 	MODULE(shmsys,sys);
-	WSTUB(shmsys, shmexit,		nomod_zero);
-	WSTUB(shmsys, shmfork,		nomod_zero);
-	WSTUB(shmsys, shmgetid,		nomod_minus_one);
+	NO_UNLOAD_WSTUB(shmsys, shmexit,		nomod_zero);
+	NO_UNLOAD_WSTUB(shmsys, shmfork,		nomod_zero);
+	NO_UNLOAD_WSTUB(shmsys, shmgetid,		nomod_minus_one);
 	END_MODULE(shmsys);
 #endif
 
@@ -898,19 +898,19 @@ fcnname/**/_info:							\
  */
 #ifndef DOOR_MODULE
 	MODULE(doorfs,sys);
-	WSTUB(doorfs, door_slam,			nomod_zero);
-	WSTUB(doorfs, door_exit,			nomod_zero);
-	WSTUB(doorfs, door_revoke_all,			nomod_zero);
-	WSTUB(doorfs, door_fork,			nomod_zero);
+	NO_UNLOAD_WSTUB(doorfs, door_slam,		nomod_zero);
+	NO_UNLOAD_WSTUB(doorfs, door_exit,		nomod_zero);
+	NO_UNLOAD_WSTUB(doorfs, door_revoke_all,	nomod_zero);
+	NO_UNLOAD_WSTUB(doorfs, door_fork,		nomod_zero);
 	NO_UNLOAD_STUB(doorfs, door_upcall,		nomod_einval);
 	NO_UNLOAD_STUB(doorfs, door_ki_create,		nomod_einval);
 	NO_UNLOAD_STUB(doorfs, door_ki_open,		nomod_einval);
 	NO_UNLOAD_STUB(doorfs, door_ki_lookup,		nomod_zero);
-	WSTUB(doorfs, door_ki_upcall,			nomod_einval);
-	WSTUB(doorfs, door_ki_upcall_limited,		nomod_einval);
-	WSTUB(doorfs, door_ki_hold,			nomod_zero);
-	WSTUB(doorfs, door_ki_rele,			nomod_zero);
-	WSTUB(doorfs, door_ki_info,			nomod_einval);
+	NO_UNLOAD_WSTUB(doorfs, door_ki_upcall,		nomod_einval);
+	NO_UNLOAD_WSTUB(doorfs, door_ki_upcall_limited,	nomod_einval);
+	NO_UNLOAD_WSTUB(doorfs, door_ki_hold,		nomod_zero);
+	NO_UNLOAD_WSTUB(doorfs, door_ki_rele,		nomod_zero);
+	NO_UNLOAD_WSTUB(doorfs, door_ki_info,		nomod_einval);
 	END_MODULE(doorfs);
 #endif
 
diff --git a/usr/src/uts/sparc/ml/modstubs.s b/usr/src/uts/sparc/ml/modstubs.s
index 700dcf92cc..1641c734da 100644
--- a/usr/src/uts/sparc/ml/modstubs.s
+++ b/usr/src/uts/sparc/ml/modstubs.s
@@ -121,7 +121,7 @@ module/**/_modinfo:			\
  * User *MUST* guarantee the module is not unloadable (no _fini routine).
  */
 #define NO_UNLOAD_STUB(module, fcnname, retfcn)	\
-    STUB_UNLOADABLE(module, fcnname, retfcn, retfcn, MODS_NOUNLOAD)
+    STUB_NO_UNLOADABLE(module, fcnname, retfcn, retfcn, MODS_NOUNLOAD)
 
 /*
  * Macro for modstubbed system calls whose modules are not unloadable.
@@ -130,10 +130,10 @@ module/**/_modinfo:			\
  * the modstub is a system call, because %fp comes from user frame.
  */
 #define	SCALL_NU_STUB(module, fcnname, retfcn)	\
-    SCALL_UNLOADABLE(module, fcnname, retfcn, retfcn, MODS_NOUNLOAD)
+    SCALL_NO_UNLOADABLE(module, fcnname, retfcn, retfcn, MODS_NOUNLOAD)
 /* "weak stub" for non-unloadable module, don't load on account of this call */
 #define NO_UNLOAD_WSTUB(module, fcnname, retfcn) \
-    STUB_UNLOADABLE(module, fcnname, retfcn, retfcn, MODS_NOUNLOAD|MODS_WEAK)
+    STUB_NO_UNLOADABLE(module, fcnname, retfcn, retfcn, MODS_NOUNLOAD|MODS_WEAK)
 
 #define	STUB_DATA(module, fcnname, install_fcn, retfcn, weak)		\
 	.seg	".data";						\
@@ -181,7 +181,7 @@ fcnname/**/_info:							\
 	SET_SIZE(fcnname);						\
 	STUB_DATA(module, fcnname, install_fcn, retfcn, weak)
 
-#define STUB_UNLOADABLE(module, fcnname, install_fcn, retfcn, weak)	\
+#define STUB_NO_UNLOADABLE(module, fcnname, install_fcn, retfcn, weak)	\
 	ENTRY_NP(fcnname);						\
 	save	%sp, -SA(MINFRAME), %sp;	/* new window */	\
 	set	fcnname/**/_info, %l5;					\
@@ -609,9 +609,9 @@ stubs_base:
  */
 #ifndef FIFO_MODULE
 	MODULE(fifofs,fs);
-	STUB(fifofs, fifovp,      	0);
-	STUB(fifofs, fifo_getinfo,	0);
-	STUB(fifofs, fifo_vfastoff,	0);
+	NO_UNLOAD_STUB(fifofs, fifovp,      	nomod_zero);
+	NO_UNLOAD_STUB(fifofs, fifo_getinfo,	nomod_zero);
+	NO_UNLOAD_STUB(fifofs, fifo_vfastoff,	nomod_zero);
 	END_MODULE(fifofs);
 #endif
 
@@ -778,8 +778,8 @@ stubs_base:
  */
 #ifndef SYSACCT_MODULE
 	MODULE(sysacct,sys);
-	WSTUB(sysacct, acct,  		nomod_zero);
-	WSTUB(sysacct, acct_fs_in_use,	nomod_zero);
+	NO_UNLOAD_WSTUB(sysacct, acct,  		nomod_zero);
+	NO_UNLOAD_WSTUB(sysacct, acct_fs_in_use,	nomod_zero);
 	END_MODULE(sysacct);
 #endif
 
@@ -788,7 +788,7 @@ stubs_base:
  */
 #ifndef SEMSYS_MODULE
 	MODULE(semsys,sys);
-	WSTUB(semsys, semexit,		nomod_zero);
+	NO_UNLOAD_WSTUB(semsys, semexit,		nomod_zero);
 	END_MODULE(semsys);
 #endif
 
@@ -797,9 +797,9 @@ stubs_base:
  */
 #ifndef SHMSYS_MODULE
 	MODULE(shmsys,sys);
-	WSTUB(shmsys, shmexit,		nomod_zero);
-	WSTUB(shmsys, shmfork,		nomod_zero);
-	WSTUB(shmsys, shmgetid,		nomod_minus_one);
+	NO_UNLOAD_WSTUB(shmsys, shmexit,		nomod_zero);
+	NO_UNLOAD_WSTUB(shmsys, shmfork,		nomod_zero);
+	NO_UNLOAD_WSTUB(shmsys, shmgetid,		nomod_minus_one);
 	END_MODULE(shmsys);
 #endif
 
@@ -808,19 +808,19 @@ stubs_base:
  */
 #ifndef DOORFS_MODULE
 	MODULE(doorfs,sys);
-	WSTUB(doorfs, door_slam,			nomod_zero);
-	WSTUB(doorfs, door_exit,			nomod_zero);
-	WSTUB(doorfs, door_revoke_all,			nomod_zero);
-	WSTUB(doorfs, door_fork,			nomod_zero);
+	NO_UNLOAD_WSTUB(doorfs, door_slam,		nomod_zero);
+	NO_UNLOAD_WSTUB(doorfs, door_exit,		nomod_zero);
+	NO_UNLOAD_WSTUB(doorfs, door_revoke_all,	nomod_zero);
+	NO_UNLOAD_WSTUB(doorfs, door_fork,		nomod_zero);
 	NO_UNLOAD_STUB(doorfs, door_upcall,		nomod_einval);
 	NO_UNLOAD_STUB(doorfs, door_ki_create,		nomod_einval);
 	NO_UNLOAD_STUB(doorfs, door_ki_open,		nomod_einval);
 	NO_UNLOAD_STUB(doorfs, door_ki_lookup,		nomod_zero);
-	WSTUB(doorfs, door_ki_upcall,			nomod_einval);
-	WSTUB(doorfs, door_ki_upcall_limited,		nomod_einval);
-	WSTUB(doorfs, door_ki_hold,			nomod_zero);
-	WSTUB(doorfs, door_ki_rele,			nomod_zero);
-	WSTUB(doorfs, door_ki_info,			nomod_einval);
+	NO_UNLOAD_WSTUB(doorfs, door_ki_upcall,		nomod_einval);
+	NO_UNLOAD_WSTUB(doorfs, door_ki_upcall_limited,	nomod_einval);
+	NO_UNLOAD_WSTUB(doorfs, door_ki_hold,		nomod_zero);
+	NO_UNLOAD_WSTUB(doorfs, door_ki_rele,		nomod_zero);
+	NO_UNLOAD_WSTUB(doorfs, door_ki_info,		nomod_einval);
 	END_MODULE(doorfs);
 #endif