7505 dtrace helpers leaked during fork when lwp_create fails

Reviewed by: Bryan Cantrill <bryan@joyent.com>
Reviewed by: Ryan Zezeski <rpz@joyent.com>
Reviewed by: Adam Leventhal <adam.leventhal@gmail.com>
Reviewed by: Sanjay Nadkarni <sanjay.nadkarni@nexenta.com>
Reviewed by: Alek Pinchuk <pinchuk.alek@gmail.com>
Approved by: Richard Lowe <richlowe@richlowe.net>

6 files changed, 25 insertions, 10 deletions

diff --git a/usr/src/uts/common/dtrace/dtrace.c b/usr/src/uts/common/dtrace/dtrace.c
index 3aeeef27f7..28ad7ec09b 100644
--- a/usr/src/uts/common/dtrace/dtrace.c
+++ b/usr/src/uts/common/dtrace/dtrace.c
@@ -15440,11 +15440,10 @@ dtrace_helpers_create(proc_t *p)
 }
 
 static void
-dtrace_helpers_destroy(void)
+dtrace_helpers_destroy(proc_t *p)
 {
 	dtrace_helpers_t *help;
 	dtrace_vstate_t *vstate;
-	proc_t *p = curproc;
 	int i;
 
 	mutex_enter(&dtrace_lock);
@@ -15459,10 +15458,21 @@ dtrace_helpers_destroy(void)
 	 * We're now going to lose the help from this process.
 	 */
 	p->p_dtrace_helpers = NULL;
-	dtrace_sync();
+	if (p == curproc) {
+		dtrace_sync();
+	} else {
+		/*
+		 * It is sometimes necessary to clean up dtrace helpers from a
+		 * an incomplete child process as part of a failed fork
+		 * operation.  In such situations, a dtrace_sync() call should
+		 * be unnecessary as the process should be devoid of threads,
+		 * much less any in probe context.
+		 */
+		VERIFY(p->p_stat == SIDL);
+	}
 
 	/*
-	 * Destory the helper actions.
+	 * Destroy the helper actions.
 	 */
 	for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
 		dtrace_helper_action_t *h, *next;
diff --git a/usr/src/uts/common/os/dtrace_subr.c b/usr/src/uts/common/os/dtrace_subr.c
index 44cfddad97..680f04a958 100644
--- a/usr/src/uts/common/os/dtrace_subr.c
+++ b/usr/src/uts/common/os/dtrace_subr.c
@@ -22,6 +22,7 @@
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2016 Joyent, Inc.
  */
 
 #include <sys/dtrace.h>
@@ -39,7 +40,7 @@
 void (*dtrace_cpu_init)(processorid_t);
 void (*dtrace_modload)(struct modctl *);
 void (*dtrace_modunload)(struct modctl *);
-void (*dtrace_helpers_cleanup)(void);
+void (*dtrace_helpers_cleanup)(proc_t *);
 void (*dtrace_helpers_fork)(proc_t *, proc_t *);
 void (*dtrace_cpustart_init)(void);
 void (*dtrace_cpustart_fini)(void);
diff --git a/usr/src/uts/common/os/exec.c b/usr/src/uts/common/os/exec.c
index 2aaa6a9076..0065b4945b 100644
--- a/usr/src/uts/common/os/exec.c
+++ b/usr/src/uts/common/os/exec.c
@@ -26,7 +26,7 @@
 /*	Copyright (c) 1988 AT&T	*/
 /*	  All Rights Reserved  	*/
 /*
- * Copyright 2014, Joyent, Inc.  All rights reserved.
+ * Copyright 2016 Joyent, Inc.
  */
 
 #include <sys/types.h>
@@ -1997,7 +1997,7 @@ exec_args(execa_t *uap, uarg_t *args, intpdata_t *intp, void **auxvpp)
 	 */
 	if (p->p_dtrace_helpers != NULL) {
 		ASSERT(dtrace_helpers_cleanup != NULL);
-		(*dtrace_helpers_cleanup)();
+		(*dtrace_helpers_cleanup)(p);
 	}
 
 	mutex_enter(&p->p_lock);
diff --git a/usr/src/uts/common/os/exit.c b/usr/src/uts/common/os/exit.c
index f0c0983a3a..1b9359da47 100644
--- a/usr/src/uts/common/os/exit.c
+++ b/usr/src/uts/common/os/exit.c
@@ -452,7 +452,7 @@ proc_exit(int why, int what)
 	 */
 	if (p->p_dtrace_helpers != NULL) {
 		ASSERT(dtrace_helpers_cleanup != NULL);
-		(*dtrace_helpers_cleanup)();
+		(*dtrace_helpers_cleanup)(p);
 	}
 
 	/*
diff --git a/usr/src/uts/common/os/fork.c b/usr/src/uts/common/os/fork.c
index ce0913a52a..a63931459f 100644
--- a/usr/src/uts/common/os/fork.c
+++ b/usr/src/uts/common/os/fork.c
@@ -641,6 +641,10 @@ forklwperr:
 
 	forklwp_fail(cp);
 	fork_fail(cp);
+	if (cp->p_dtrace_helpers != NULL) {
+		ASSERT(dtrace_helpers_cleanup != NULL);
+		(*dtrace_helpers_cleanup)(cp);
+	}
 	rctl_set_free(cp->p_rctls);
 	mutex_enter(&pidlock);
 
diff --git a/usr/src/uts/common/sys/dtrace.h b/usr/src/uts/common/sys/dtrace.h
index b1bb6d8dd2..4ee1a94510 100644
--- a/usr/src/uts/common/sys/dtrace.h
+++ b/usr/src/uts/common/sys/dtrace.h
@@ -25,7 +25,7 @@
  */
 
 /*
- * Copyright (c) 2013, Joyent, Inc. All rights reserved.
+ * Copyright 2016 Joyent, Inc.
  * Copyright (c) 2013 by Delphix. All rights reserved.
  */
 
@@ -2286,7 +2286,7 @@ extern void dtrace_membar_consumer(void);
 extern void (*dtrace_cpu_init)(processorid_t);
 extern void (*dtrace_modload)(struct modctl *);
 extern void (*dtrace_modunload)(struct modctl *);
-extern void (*dtrace_helpers_cleanup)();
+extern void (*dtrace_helpers_cleanup)(proc_t *);
 extern void (*dtrace_helpers_fork)(proc_t *parent, proc_t *child);
 extern void (*dtrace_cpustart_init)();
 extern void (*dtrace_cpustart_fini)();