6590401 fixes for 6418491, 6518780 and 6570016 conspire to break java

10 files changed, 145 insertions, 175 deletions

diff --git a/usr/src/lib/libc/inc/libc.h b/usr/src/lib/libc/inc/libc.h
index 020c311eab..50526e96fa 100644
--- a/usr/src/lib/libc/inc/libc.h
+++ b/usr/src/lib/libc/inc/libc.h
@@ -68,7 +68,7 @@ extern int thr_kill(thread_t tid, int sig);
 extern thread_t thr_self(void);
 extern int mutex_lock(mutex_t *mp);
 extern int mutex_unlock(mutex_t *mp);
-extern int fork_lock_enter(const char *);
+extern void fork_lock_enter(void);
 extern void fork_lock_exit(void);
 extern void *lmalloc(size_t);
 extern void lfree(void *, size_t);
diff --git a/usr/src/lib/libc/inc/thr_uberdata.h b/usr/src/lib/libc/inc/thr_uberdata.h
index ce30f25ef1..b8dd212875 100644
--- a/usr/src/lib/libc/inc/thr_uberdata.h
+++ b/usr/src/lib/libc/inc/thr_uberdata.h
@@ -653,19 +653,21 @@ typedef struct robust {
 
 /*
  * Make our hot locks reside on private cache lines (64 bytes).
- * pad_owner and pad_count (aka fork_owner and fork_count)
- * are used only in fork_lock_enter() and fork_lock_exit()
- * to implement the special form of mutual exclusion therein.
  */
 typedef struct {
 	mutex_t	pad_lock;
-	ulwp_t	*pad_owner;
-	size_t	pad_count;
-	char	pad_pad[64 -
-		(sizeof (mutex_t) + sizeof (ulwp_t *) + sizeof (size_t))];
+	char	pad_pad[64 - sizeof (mutex_t)];
 } pad_lock_t;
 
 /*
+ * Make our semi-hot locks reside on semi-private cache lines (32 bytes).
+ */
+typedef struct {
+	mutex_t	pad_lock;
+	char	pad_pad[32 - sizeof (mutex_t)];
+} pad32_lock_t;
+
+/*
  * The threads hash table is used for fast lookup and locking of an active
  * thread structure (ulwp_t) given a thread-id.  It is an N-element array of
  * thr_hash_table_t structures, where N == 1 before the main thread creates
@@ -772,7 +774,8 @@ typedef struct {
  */
 typedef struct uberdata {
 	pad_lock_t	_link_lock;
-	pad_lock_t	_fork_lock;
+	pad32_lock_t	_fork_lock;
+	pad32_lock_t	_atfork_lock;
 	pad_lock_t	_tdb_hash_lock;
 	tdb_sync_stats_t tdb_hash_lock_stats;
 	siguaction_t	siguaction[NSIG];
@@ -816,8 +819,7 @@ typedef struct uberdata {
 
 #define	link_lock	_link_lock.pad_lock
 #define	fork_lock	_fork_lock.pad_lock
-#define	fork_owner	_fork_lock.pad_owner
-#define	fork_count	_fork_lock.pad_count
+#define	atfork_lock	_atfork_lock.pad_lock
 #define	tdb_hash_lock	_tdb_hash_lock.pad_lock
 
 #pragma align 64(__uberdata)
@@ -976,7 +978,8 @@ typedef struct ulwp32 {
 
 typedef struct uberdata32 {
 	pad_lock_t	_link_lock;
-	pad_lock_t	_fork_lock;
+	pad32_lock_t	_fork_lock;
+	pad32_lock_t	_atfork_lock;
 	pad_lock_t	_tdb_hash_lock;
 	tdb_sync_stats_t tdb_hash_lock_stats;
 	siguaction32_t	siguaction[NSIG];
@@ -1221,7 +1224,7 @@ extern	void	postfork1_child_sigev_aio(void);
 extern	void	postfork1_child_sigev_mq(void);
 extern	void	postfork1_child_sigev_timer(void);
 extern	void	postfork1_child_tpool(void);
-extern	int	fork_lock_enter(const char *);
+extern	void	fork_lock_enter(void);
 extern	void	fork_lock_exit(void);
 extern	void	suspend_fork(void);
 extern	void	continue_fork(int);
diff --git a/usr/src/lib/libc/port/gen/atexit.c b/usr/src/lib/libc/port/gen/atexit.c
index b4dc369c5f..69de21f524 100644
--- a/usr/src/lib/libc/port/gen/atexit.c
+++ b/usr/src/lib/libc/port/gen/atexit.c
@@ -196,15 +196,15 @@ again:
 static void
 _preexec_atfork_unload(Lc_addr_range_t range[], uint_t count)
 {
-	uberdata_t *udp = curthread->ul_uberdata;
+	ulwp_t *self = curthread;
+	uberdata_t *udp = self->ul_uberdata;
 	atfork_t *atfork_q;
 	atfork_t *atfp;
 	atfork_t *next;
 	void (*func)(void);
 	int start_again;
-	int error;
 
-	error = fork_lock_enter(NULL);
+	(void) _private_mutex_lock(&udp->atfork_lock);
 	if ((atfork_q = udp->atforklist) != NULL) {
 		atfp = atfork_q;
 		do {
@@ -217,7 +217,7 @@ _preexec_atfork_unload(Lc_addr_range_t range[], uint_t count)
 			    in_range(func, range, count)) ||
 			    ((func = atfp->child) != NULL &&
 			    in_range(func, range, count))) {
-				if (error) {
+				if (self->ul_fork) {
 					/*
 					 * dlclose() called from a fork handler.
 					 * Deleting the entry would wreak havoc.
@@ -245,7 +245,7 @@ _preexec_atfork_unload(Lc_addr_range_t range[], uint_t count)
 			}
 		} while ((atfp = next) != atfork_q || start_again);
 	}
-	fork_lock_exit();
+	(void) _private_mutex_unlock(&udp->atfork_lock);
 }
 
 /*
diff --git a/usr/src/lib/libc/port/gen/atfork.c b/usr/src/lib/libc/port/gen/atfork.c
index b0b3c192a4..4db5bd0912 100644
--- a/usr/src/lib/libc/port/gen/atfork.c
+++ b/usr/src/lib/libc/port/gen/atfork.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -55,33 +55,33 @@ _pthread_atfork(void (*prepare)(void),
 	uberdata_t *udp = self->ul_uberdata;
 	atfork_t *atfp;
 	atfork_t *head;
-	int error;
+	int error = 0;
 
-	if ((error = fork_lock_enter("pthread_atfork")) != 0) {
+	(void) _private_mutex_lock(&udp->atfork_lock);
+	if (self->ul_fork) {
 		/*
 		 * Cannot call pthread_atfork() from a fork handler.
 		 */
-		fork_lock_exit();
-		return (error);
-	}
-	if ((atfp = lmalloc(sizeof (atfork_t))) == NULL) {
-		fork_lock_exit();
-		return (ENOMEM);
-	}
-	atfp->prepare = prepare;
-	atfp->parent = parent;
-	atfp->child = child;
-	if ((head = udp->atforklist) == NULL) {
-		udp->atforklist = atfp;
-		atfp->forw = atfp->back = atfp;
+		error = EDEADLK;
+	} else if ((atfp = lmalloc(sizeof (atfork_t))) == NULL) {
+		error = ENOMEM;
 	} else {
-		head->back->forw = atfp;
-		atfp->forw = head;
-		atfp->back = head->back;
-		head->back = atfp;
+		atfp->prepare = prepare;
+		atfp->parent = parent;
+		atfp->child = child;
+		if ((head = udp->atforklist) == NULL) {
+			udp->atforklist = atfp;
+			atfp->forw = atfp->back = atfp;
+		} else {
+			head->back->forw = atfp;
+			atfp->forw = head;
+			atfp->back = head->back;
+			head->back = atfp;
+		}
 	}
-	fork_lock_exit();
-	return (0);
+
+	(void) _private_mutex_unlock(&udp->atfork_lock);
+	return (error);
 }
 
 /*
@@ -95,6 +95,7 @@ _prefork_handler(void)
 	atfork_t *atfork_q;
 	atfork_t *atfp;
 
+	ASSERT(MUTEX_OWNED(&udp->atfork_lock, curthread));
 	if ((atfork_q = udp->atforklist) != NULL) {
 		atfp = atfork_q = atfork_q->back;
 		do {
@@ -115,6 +116,7 @@ _postfork_parent_handler(void)
 	atfork_t *atfork_q;
 	atfork_t *atfp;
 
+	ASSERT(MUTEX_OWNED(&udp->atfork_lock, curthread));
 	if ((atfork_q = udp->atforklist) != NULL) {
 		atfp = atfork_q;
 		do {
@@ -135,6 +137,7 @@ _postfork_child_handler(void)
 	atfork_t *atfork_q;
 	atfork_t *atfp;
 
+	ASSERT(MUTEX_OWNED(&udp->atfork_lock, curthread));
 	if ((atfork_q = udp->atforklist) != NULL) {
 		atfp = atfork_q;
 		do {
diff --git a/usr/src/lib/libc/port/gen/getnetgrent.c b/usr/src/lib/libc/port/gen/getnetgrent.c
index d8d111e818..83c9ebbe31 100644
--- a/usr/src/lib/libc/port/gen/getnetgrent.c
+++ b/usr/src/lib/libc/port/gen/getnetgrent.c
@@ -18,8 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -150,7 +151,7 @@ setnetgrent(const char *netgroup)
 		netgroup = "";
 	}
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	be = getnetgrent_backend;
 	if (be != NULL && NSS_INVOKE_DBOP(be, NSS_DBOP_SETENT,
 	    (void *)netgroup) != NSS_SUCCESS) {
@@ -185,7 +186,7 @@ getnetgrent_r(machinep, namep, domainp, buffer, buflen)
 	args.buflen	= buflen;
 	args.status	= NSS_NETGR_NO;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	if (getnetgrent_backend != 0) {
 		(void) NSS_INVOKE_DBOP(getnetgrent_backend,
 			NSS_DBOP_GETENT, &args);
@@ -218,10 +219,10 @@ getnetgrent(machinep, namep, domainp)
 int
 endnetgrent()
 {
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	if (getnetgrent_backend != 0) {
 		(void) NSS_INVOKE_DBOP(getnetgrent_backend,
-			NSS_DBOP_DESTRUCTOR, 0);
+		    NSS_DBOP_DESTRUCTOR, 0);
 		getnetgrent_backend = 0;
 	}
 	fork_lock_exit();
diff --git a/usr/src/lib/libc/port/gen/ttyname.c b/usr/src/lib/libc/port/gen/ttyname.c
index 8754c219dd..bddc9f7419 100644
--- a/usr/src/lib/libc/port/gen/ttyname.c
+++ b/usr/src/lib/libc/port/gen/ttyname.c
@@ -215,7 +215,7 @@ _ttyname_common(struct stat64 *fsp, char *buffer, uint_t match_mask)
 	 * We can't use lmutex_lock() here because we call malloc()/free()
 	 * and _libc_gettext().  Use the brute-force fork_lock_enter().
 	 */
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 
 	/*
 	 * match special cases
diff --git a/usr/src/lib/libc/port/i18n/gettext.c b/usr/src/lib/libc/port/i18n/gettext.c
index 2bea032af3..8614265425 100644
--- a/usr/src/lib/libc/port/i18n/gettext.c
+++ b/usr/src/lib/libc/port/i18n/gettext.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -73,7 +73,7 @@ _bindtextdomain(const char *domain, const char *binding)
 {
 	char	*res;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	INIT_GT(NULL);
 	res = _real_bindtextdomain_u(domain, binding, TP_BINDING);
 	fork_lock_exit();
@@ -85,7 +85,7 @@ _bind_textdomain_codeset(const char *domain, const char *codeset)
 {
 	char	*res;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	INIT_GT(NULL);
 	res = _real_bindtextdomain_u(domain, codeset, TP_CODESET);
 	fork_lock_exit();
@@ -102,7 +102,7 @@ _textdomain(const char *domain)
 	char	*res;
 	char	tmp_domain[TEXTDOMAINMAX + 1];
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	INIT_GT(NULL);
 	res = _textdomain_u(domain, tmp_domain);
 	if (res == NULL) {
@@ -123,7 +123,7 @@ _gettext(const char *msg_id)
 	char	*res;
 	int	errno_save = errno;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	INIT_GT((char *)msg_id);
 	res = _real_gettext_u(NULL, msg_id, NULL, 0, LC_MESSAGES, 0);
 	fork_lock_exit();
@@ -141,7 +141,7 @@ _dgettext(const char *domain, const char *msg_id)
 	char	*res;
 	int	errno_save = errno;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	INIT_GT((char *)msg_id);
 	res = _real_gettext_u(domain, msg_id, NULL, 0, LC_MESSAGES, 0);
 	fork_lock_exit();
@@ -155,7 +155,7 @@ _dcgettext(const char *domain, const char *msg_id, const int category)
 	char	*res;
 	int	errno_save = errno;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	INIT_GT((char *)msg_id);
 	res = _real_gettext_u(domain, msg_id, NULL, 0, category, 0);
 	fork_lock_exit();
@@ -169,7 +169,7 @@ _ngettext(const char *msgid1, const char *msgid2, unsigned long int n)
 	char	*res;
 	int	errno_save = errno;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	INIT_GT((char *)msgid1);
 	res = _real_gettext_u(NULL, msgid1, msgid2, n, LC_MESSAGES, 1);
 	fork_lock_exit();
@@ -184,7 +184,7 @@ _dngettext(const char *domain, const char *msgid1, const char *msgid2,
 	char	*res;
 	int	errno_save = errno;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	INIT_GT((char *)msgid1);
 	res = _real_gettext_u(domain, msgid1, msgid2, n, LC_MESSAGES, 1);
 	fork_lock_exit();
@@ -199,7 +199,7 @@ _dcngettext(const char *domain, const char *msgid1, const char *msgid2,
 	char	*res;
 	int	errno_save = errno;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	INIT_GT((char *)msgid1);
 	res = _real_gettext_u(domain, msgid1, msgid2, n, category, 1);
 	fork_lock_exit();
diff --git a/usr/src/lib/libc/port/i18n/wdresolve.c b/usr/src/lib/libc/port/i18n/wdresolve.c
index aaae177acf..9f21528385 100644
--- a/usr/src/lib/libc/port/i18n/wdresolve.c
+++ b/usr/src/lib/libc/port/i18n/wdresolve.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -87,7 +87,7 @@ _wdinitialize(void)
 		if (wdchknd == NULL)
 			wdchknd = wdchkind_C;
 		wdbdg = (int(*)(wchar_t, wchar_t, int))dlsym(modhandle,
-			"_wdbindf_");
+		    "_wdbindf_");
 		if (wdbdg == NULL)
 			wdbdg = wdbindf_C;
 		wddlm = (wchar_t *(*)(wchar_t, wchar_t, int))
@@ -120,7 +120,7 @@ wdinit()
 {
 	int res;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	res = _wdinitialize();
 	fork_lock_exit();
 	return (res);
@@ -135,7 +135,7 @@ wdchkind(wchar_t wc)
 {
 	int i;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	if (!initialized)
 		(void) _wdinitialize();
 	i = (*wdchknd)(wc);
@@ -170,7 +170,7 @@ wdbindf(wchar_t wc1, wchar_t wc2, int type)
 {
 	int i;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	if (!initialized)
 		(void) _wdinitialize();
 	if (!iswprint(wc1) || !iswprint(wc2)) {
@@ -203,7 +203,7 @@ wddelim(wchar_t wc1, wchar_t wc2, int type)
 {
 	wchar_t *i;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	if (!initialized)
 		(void) _wdinitialize();
 	if (!iswprint(wc1) || !iswprint(wc2)) {
@@ -230,7 +230,7 @@ mcfiller(void)
 {
 	wchar_t fillerchar;
 
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	if (!initialized)
 		(void) _wdinitialize();
 	if (mcfllr) {
@@ -254,7 +254,7 @@ mcfiller(void)
 int
 mcwrap(void)
 {
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 	if (!initialized)
 		(void) _wdinitialize();
 	if (mcwrp)
diff --git a/usr/src/lib/libc/port/threads/scalls.c b/usr/src/lib/libc/port/threads/scalls.c
index 1b59f8e3a5..1513203fc5 100644
--- a/usr/src/lib/libc/port/threads/scalls.c
+++ b/usr/src/lib/libc/port/threads/scalls.c
@@ -35,81 +35,30 @@
 #include <sys/uio.h>
 
 /*
- * fork_lock is special -- We can't use lmutex_lock() (and thereby enter
- * a critical region) because the second thread to reach this point would
- * become unstoppable and the first thread would hang waiting for the
- * second thread to stop itself.  Therefore we don't use lmutex_lock() in
- * fork_lock_enter(), but we do defer signals (the other form of concurrency).
- *
- * fork_lock_enter() does triple-duty.  Not only does it serialize
- * calls to fork() and forkall(), but it also serializes calls to
- * thr_suspend() (fork() and forkall() also suspend other threads),
- * and furthermore it serializes I18N calls to functions in other
+ * fork_lock_enter() does triple-duty.  Not only does it (and atfork_lock)
+ * serialize calls to fork() and forkall(), but it also serializes calls to
+ * thr_suspend() and thr_continue() (fork() and forkall() also suspend other
+ * threads), and furthermore it serializes I18N calls to functions in other
  * dlopen()ed L10N objects that might be calling malloc()/free().
  */
-
-static void
-fork_lock_error(const char *who)
-{
-	char msg[200];
-
-	(void) strlcpy(msg, "deadlock condition: ", sizeof (msg));
-	(void) strlcat(msg, who, sizeof (msg));
-	(void) strlcat(msg, "() called from a fork handler", sizeof (msg));
-	thread_error(msg);
-}
-
-int
-fork_lock_enter(const char *who)
+void
+fork_lock_enter(void)
 {
 	ulwp_t *self = curthread;
-	uberdata_t *udp = self->ul_uberdata;
-	int error = 0;
 
 	ASSERT(self->ul_critical == 0);
-	sigoff(self);
-	(void) _private_mutex_lock(&udp->fork_lock);
-	if (udp->fork_count) {
-		ASSERT(udp->fork_owner == self);
-		/*
-		 * This is a simple recursive lock except that we
-		 * inform the caller if we have been called from
-		 * a fork handler and let it deal with that fact.
-		 */
-		if (self->ul_fork) {
-			/*
-			 * We have been called from a fork handler.
-			 */
-			if (who != NULL &&
-			    udp->uberflags.uf_thread_error_detection)
-				fork_lock_error(who);
-			error = EDEADLK;
-		}
-	}
-	udp->fork_owner = self;
-	udp->fork_count++;
-	return (error);
+	(void) _private_mutex_lock(&self->ul_uberdata->fork_lock);
 }
 
 void
 fork_lock_exit(void)
 {
 	ulwp_t *self = curthread;
-	uberdata_t *udp = self->ul_uberdata;
 
 	ASSERT(self->ul_critical == 0);
-	ASSERT(udp->fork_count != 0 && udp->fork_owner == self);
-	if (--udp->fork_count == 0)
-		udp->fork_owner = NULL;
-	(void) _private_mutex_unlock(&udp->fork_lock);
-	sigon(self);
+	(void) _private_mutex_unlock(&self->ul_uberdata->fork_lock);
 }
 
-/*
- * Note: Instead of making this function static, we reduce it to local
- * scope in the mapfile. That allows the linker to prevent it from
- * appearing in the .SUNW_dynsymsort section.
- */
 #pragma weak forkx = _private_forkx
 #pragma weak _forkx = _private_forkx
 pid_t
@@ -118,7 +67,6 @@ _private_forkx(int flags)
 	ulwp_t *self = curthread;
 	uberdata_t *udp = self->ul_uberdata;
 	pid_t pid;
-	int error;
 
 	if (self->ul_vfork) {
 		/*
@@ -139,34 +87,42 @@ _private_forkx(int flags)
 		return (pid);
 	}
 
-	if ((error = fork_lock_enter("fork")) != 0) {
+	sigoff(self);
+	if (self->ul_fork) {
 		/*
 		 * Cannot call fork() from a fork handler.
 		 */
-		fork_lock_exit();
-		errno = error;
+		sigon(self);
+		errno = EDEADLK;
 		return (-1);
 	}
 	self->ul_fork = 1;
+	(void) _private_mutex_lock(&udp->atfork_lock);
 
 	/*
 	 * The functions registered by pthread_atfork() are defined by
 	 * the application and its libraries and we must not hold any
-	 * internal libc locks while invoking them.  The fork_lock_enter()
-	 * function serializes fork(), thr_suspend(), pthread_atfork() and
-	 * dlclose() (which destroys whatever pthread_atfork() functions
-	 * the library may have set up).  If one of these pthread_atfork()
-	 * functions attempts to fork or suspend another thread or call
-	 * pthread_atfork() or dlclose a library, it will detect a deadlock
-	 * in fork_lock_enter().  Otherwise, the pthread_atfork() functions
+	 * internal lmutex_lock()-acquired locks while invoking them.
+	 * We hold only udp->atfork_lock to protect the atfork linkages.
+	 * If one of these pthread_atfork() functions attempts to fork
+	 * or to call pthread_atfork(), it will detect the error and
+	 * fail with EDEADLK.  Otherwise, the pthread_atfork() functions
 	 * are free to do anything they please (except they will not
 	 * receive any signals).
 	 */
 	_prefork_handler();
 
 	/*
+	 * Block every other thread attempting thr_suspend() or thr_continue().
+	 * This also blocks every other thread attempting calls to I18N
+	 * functions in dlopen()ed L10N objects, but this is benign;
+	 * the other threads will soon be suspended anyway.
+	 */
+	fork_lock_enter();
+
+	/*
 	 * Block all signals.
-	 * Just deferring them via sigon() is not enough.
+	 * Just deferring them via sigoff() is not enough.
 	 * We have to avoid taking a deferred signal in the child
 	 * that was actually sent to the parent before __forkx().
 	 */
@@ -198,16 +154,19 @@ _private_forkx(int flags)
 		unregister_locks();
 		postfork1_child();
 		restore_signals(self);
+		fork_lock_exit();
 		_postfork_child_handler();
 	} else {
 		/* restart all threads that were suspended for fork() */
 		continue_fork(0);
 		restore_signals(self);
+		fork_lock_exit();
 		_postfork_parent_handler();
 	}
 
+	(void) _private_mutex_unlock(&udp->atfork_lock);
 	self->ul_fork = 0;
-	fork_lock_exit();
+	sigon(self);
 
 	return (pid);
 }
@@ -238,7 +197,6 @@ _private_forkallx(int flags)
 	ulwp_t *self = curthread;
 	uberdata_t *udp = self->ul_uberdata;
 	pid_t pid;
-	int error;
 
 	if (self->ul_vfork) {
 		if (udp->uberflags.uf_mt) {
@@ -253,12 +211,15 @@ _private_forkallx(int flags)
 		return (pid);
 	}
 
-	if ((error = fork_lock_enter("forkall")) != 0) {
-		fork_lock_exit();
-		errno = error;
+	sigoff(self);
+	if (self->ul_fork) {
+		sigon(self);
+		errno = EDEADLK;
 		return (-1);
 	}
 	self->ul_fork = 1;
+
+	fork_lock_enter();
 	block_all_signals(self);
 	suspend_fork();
 
@@ -276,8 +237,9 @@ _private_forkallx(int flags)
 		continue_fork(0);
 	}
 	restore_signals(self);
-	self->ul_fork = 0;
 	fork_lock_exit();
+	self->ul_fork = 0;
+	sigon(self);
 
 	return (pid);
 }
@@ -467,7 +429,7 @@ getpmsg(int fd, struct strbuf *ctlptr, struct strbuf *dataptr,
 	int *bandp, int *flagsp)
 {
 	extern int _getpmsg(int, struct strbuf *, struct strbuf *,
-		int *, int *);
+	    int *, int *);
 	int rv;
 
 	PERFORM(_getpmsg(fd, ctlptr, dataptr, bandp, flagsp))
@@ -478,7 +440,7 @@ putmsg(int fd, const struct strbuf *ctlptr,
 	const struct strbuf *dataptr, int flags)
 {
 	extern int _putmsg(int, const struct strbuf *,
-		const struct strbuf *, int);
+	    const struct strbuf *, int);
 	int rv;
 
 	PERFORM(_putmsg(fd, ctlptr, dataptr, flags))
@@ -489,7 +451,7 @@ __xpg4_putmsg(int fd, const struct strbuf *ctlptr,
 	const struct strbuf *dataptr, int flags)
 {
 	extern int _putmsg(int, const struct strbuf *,
-		const struct strbuf *, int);
+	    const struct strbuf *, int);
 	int rv;
 
 	PERFORM(_putmsg(fd, ctlptr, dataptr, flags|MSG_XPG4))
@@ -500,7 +462,7 @@ putpmsg(int fd, const struct strbuf *ctlptr,
 	const struct strbuf *dataptr, int band, int flags)
 {
 	extern int _putpmsg(int, const struct strbuf *,
-		const struct strbuf *, int, int);
+	    const struct strbuf *, int, int);
 	int rv;
 
 	PERFORM(_putpmsg(fd, ctlptr, dataptr, band, flags))
@@ -511,7 +473,7 @@ __xpg4_putpmsg(int fd, const struct strbuf *ctlptr,
 	const struct strbuf *dataptr, int band, int flags)
 {
 	extern int _putpmsg(int, const struct strbuf *,
-		const struct strbuf *, int, int);
+	    const struct strbuf *, int, int);
 	int rv;
 
 	PERFORM(_putpmsg(fd, ctlptr, dataptr, band, flags|MSG_XPG4))
@@ -581,7 +543,7 @@ restart:
 			}
 		} else {
 			rqlapse = (hrtime_t)(uint32_t)rqtp->tv_sec * NANOSEC +
-				rqtp->tv_nsec;
+			    rqtp->tv_nsec;
 			lapse = gethrtime() - start;
 			if (rqlapse > lapse) {
 				hrt2ts(rqlapse - lapse, &reltime);
@@ -873,7 +835,7 @@ _pollsys(struct pollfd *fds, nfds_t nfd, const timespec_t *timeout,
 	const sigset_t *sigmask)
 {
 	extern int __pollsys(struct pollfd *, nfds_t, const timespec_t *,
-		const sigset_t *);
+	    const sigset_t *);
 	int rv;
 
 	PROLOGUE_MASK(sigmask)
@@ -887,7 +849,7 @@ int
 _sigtimedwait(const sigset_t *set, siginfo_t *infop, const timespec_t *timeout)
 {
 	extern int __sigtimedwait(const sigset_t *, siginfo_t *,
-		const timespec_t *);
+	    const timespec_t *);
 	siginfo_t info;
 	int sig;
 
@@ -924,7 +886,7 @@ int
 _sigqueue(pid_t pid, int signo, const union sigval value)
 {
 	extern int __sigqueue(pid_t pid, int signo,
-		/* const union sigval */ void *value, int si_code, int block);
+	    /* const union sigval */ void *value, int si_code, int block);
 	return (__sigqueue(pid, signo, value.sival_ptr, SI_QUEUE, 0));
 }
 
diff --git a/usr/src/lib/libc/port/threads/thr.c b/usr/src/lib/libc/port/threads/thr.c
index 6df0608e1c..ad6ba56227 100644
--- a/usr/src/lib/libc/port/threads/thr.c
+++ b/usr/src/lib/libc/port/threads/thr.c
@@ -75,6 +75,7 @@ extern const Lc_interface rtld_funcs[];
 uberdata_t __uberdata = {
 	{ DEFAULTMUTEX, NULL, 0 },	/* link_lock */
 	{ RECURSIVEMUTEX, NULL, 0 },	/* fork_lock */
+	{ RECURSIVEMUTEX, NULL, 0 },	/* atfork_lock */
 	{ DEFAULTMUTEX, NULL, 0 },	/* tdb_hash_lock */
 	{ 0, },				/* tdb_hash_lock_stats */
 	{ { 0 }, },			/* siguaction[NSIG] */
@@ -238,7 +239,7 @@ ulwp_clean(ulwp_t *ulwp)
 	ulwp->ul_schedctl = NULL;
 	ulwp->ul_bindflags = 0;
 	(void) _private_memset(&ulwp->ul_td_evbuf, 0,
-		sizeof (ulwp->ul_td_evbuf));
+	    sizeof (ulwp->ul_td_evbuf));
 	ulwp->ul_td_events_enable = 0;
 	ulwp->ul_qtype = 0;
 	ulwp->ul_usropts = 0;
@@ -255,10 +256,10 @@ ulwp_clean(ulwp_t *ulwp)
 	/* PROBE_SUPPORT end */
 	ulwp->ul_siglink = NULL;
 	(void) _private_memset(ulwp->ul_ftsd, 0,
-		sizeof (void *) * TSD_NFAST);
+	    sizeof (void *) * TSD_NFAST);
 	ulwp->ul_stsd = NULL;
 	(void) _private_memset(&ulwp->ul_spinlock, 0,
-		sizeof (ulwp->ul_spinlock));
+	    sizeof (ulwp->ul_spinlock));
 	ulwp->ul_spin_lock_spin = 0;
 	ulwp->ul_spin_lock_spin2 = 0;
 	ulwp->ul_spin_lock_sleep = 0;
@@ -432,7 +433,7 @@ find_stack(size_t stksize, size_t guardsize)
 			ulwp->ul_ix = -1;
 			if (guardsize)	/* protect the extra red zone */
 				(void) _private_mprotect(stk,
-					guardsize, PROT_NONE);
+				    guardsize, PROT_NONE);
 		}
 	}
 	return (ulwp);
@@ -746,7 +747,7 @@ _thr_create(void *stk, size_t stksize, void *(*func)(void *), void *arg,
 	long flags, thread_t *new_thread)
 {
 	return (_thrp_create(stk, stksize, func, arg, flags, new_thread,
-		curthread->ul_pri, curthread->ul_policy, 0));
+	    curthread->ul_pri, curthread->ul_policy, 0));
 }
 
 /*
@@ -1049,7 +1050,7 @@ _thrp_join(thread_t tid, thread_t *departed, void **status, int do_cancel)
 		ulwp->ul_forw = ulwp->ul_back = NULL;
 		udp->nzombies--;
 		ASSERT(ulwp->ul_dead && !ulwp->ul_detached &&
-			!(ulwp->ul_usropts & (THR_DETACHED|THR_DAEMON)));
+		    !(ulwp->ul_usropts & (THR_DETACHED|THR_DAEMON)));
 		/*
 		 * We can't call ulwp_unlock(ulwp) after we set
 		 * ulwp->ul_ix = -1 so we have to get a pointer to the
@@ -1065,7 +1066,7 @@ _thrp_join(thread_t tid, thread_t *departed, void **status, int do_cancel)
 			ulwp->ul_next = NULL;
 			if (udp->ulwp_replace_free == NULL)
 				udp->ulwp_replace_free =
-					udp->ulwp_replace_last = ulwp;
+				    udp->ulwp_replace_last = ulwp;
 			else {
 				udp->ulwp_replace_last->ul_next = ulwp;
 				udp->ulwp_replace_last = ulwp;
@@ -1345,7 +1346,7 @@ libc_init(void)
 #endif
 
 	self->ul_stktop =
-		(uintptr_t)uc.uc_stack.ss_sp + uc.uc_stack.ss_size;
+	    (uintptr_t)uc.uc_stack.ss_sp + uc.uc_stack.ss_size;
 	(void) _private_getrlimit(RLIMIT_STACK, &rl);
 	self->ul_stksiz = rl.rlim_cur;
 	self->ul_stk = (caddr_t)(self->ul_stktop - self->ul_stksiz);
@@ -1632,7 +1633,7 @@ postfork1_child()
 	/* no one in the child is on a sleep queue; reinitialize */
 	if (udp->queue_head) {
 		(void) _private_memset(udp->queue_head, 0,
-			2 * QHASHSIZE * sizeof (queue_head_t));
+		    2 * QHASHSIZE * sizeof (queue_head_t));
 		for (i = 0; i < 2 * QHASHSIZE; i++) {
 			mp = &udp->queue_head[i].qh_lock;
 			mp->mutex_flag = LOCK_INITED;
@@ -1674,7 +1675,7 @@ postfork1_child()
 				ulwp->ul_next = NULL;
 				if (udp->ulwp_replace_free == NULL) {
 					udp->ulwp_replace_free =
-						udp->ulwp_replace_last = ulwp;
+					    udp->ulwp_replace_last = ulwp;
 				} else {
 					udp->ulwp_replace_last->ul_next = ulwp;
 					udp->ulwp_replace_last = ulwp;
@@ -1932,7 +1933,7 @@ _thrp_suspend(thread_t tid, uchar_t whystopped)
 	 * This also allows only one suspension at a time.
 	 */
 	if (tid != self->ul_lwpid)
-		(void) fork_lock_enter(NULL);
+		fork_lock_enter();
 
 	if ((ulwp = find_lwp(tid)) == NULL)
 		error = ESRCH;
@@ -2077,7 +2078,7 @@ continue_fork(int child)
 	if (child) {
 		for (ulwp = self->ul_forw; ulwp != self; ulwp = ulwp->ul_forw) {
 			ulwp->ul_schedctl_called =
-				ulwp->ul_dead? &udp->uberflags : NULL;
+			    ulwp->ul_dead? &udp->uberflags : NULL;
 			ulwp->ul_schedctl = NULL;
 		}
 	}
@@ -2113,7 +2114,7 @@ _thrp_continue(thread_t tid, uchar_t whystopped)
 	/*
 	 * We single-thread the entire thread suspend/continue mechanism.
 	 */
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 
 	if ((ulwp = find_lwp(tid)) == NULL) {
 		fork_lock_exit();
@@ -2200,7 +2201,7 @@ do_exit_critical()
 			unsleep_self();
 			set_parking_flag(self, 0);
 			(void) _thrp_suspend(self->ul_lwpid,
-				self->ul_pleasestop);
+			    self->ul_pleasestop);
 		}
 		self->ul_critical--;
 
@@ -2635,7 +2636,7 @@ _thr_suspend_allmutators(void)
 	/*
 	 * We single-thread the entire thread suspend/continue mechanism.
 	 */
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 
 top:
 	lmutex_lock(&udp->link_lock);
@@ -2705,7 +2706,7 @@ _thr_continue_allmutators()
 	/*
 	 * We single-thread the entire thread suspend/continue mechanism.
 	 */
-	(void) fork_lock_enter(NULL);
+	fork_lock_enter();
 
 	lmutex_lock(&udp->link_lock);
 	if (!suspendedallmutators) {