PSARC 2007/285 robust locks revisited

6296770 process robust mutexes should be much faster

13 files changed, 950 insertions, 644 deletions

diff --git a/usr/src/lib/libc/amd64/sys/_lwp_mutex_unlock.s b/usr/src/lib/libc/amd64/sys/_lwp_mutex_unlock.s
index abf8442dee..b9f30fc2d1 100644
--- a/usr/src/lib/libc/amd64/sys/_lwp_mutex_unlock.s
+++ b/usr/src/lib/libc/amd64/sys/_lwp_mutex_unlock.s
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -44,7 +44,8 @@
 	xchgl	(%rax), %ecx	/* clear lock and get old lock into %ecx */
 	andl	$WAITER_MASK, %ecx	/* was anyone waiting on it? */
 	je	1f
-	SYSTRAP_RVAL1(lwp_mutex_wakeup)
+	xorl	%esi, %esi
+	SYSTRAP_RVAL1(lwp_mutex_wakeup)	/* lwp_mutex_wakeup(mp, 0) */
 	SYSLWPERR
 	RET
 1:
diff --git a/usr/src/lib/libc/common/sys/syslwp.s b/usr/src/lib/libc/common/sys/syslwp.s
index 1b0cca505f..fda8f2625a 100644
--- a/usr/src/lib/libc/common/sys/syslwp.s
+++ b/usr/src/lib/libc/common/sys/syslwp.s
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -130,7 +130,7 @@
 
 /*
  * int
- * ___lwp_mutex_wakeup(lwp_mutex_t *mp)
+ * ___lwp_mutex_wakeup(lwp_mutex_t *mp, int)
  */
 	ENTRY(___lwp_mutex_wakeup)
 	SYSTRAP_RVAL1(lwp_mutex_wakeup)
@@ -238,10 +238,10 @@
 
 /*
  * int
- * ___lwp_mutex_init(lwp_mutex_t *mp, int type)
+ * ___lwp_mutex_register(lwp_mutex_t *mp)
  */
-	ENTRY(___lwp_mutex_init)
-	SYSTRAP_RVAL1(lwp_mutex_init)
+	ENTRY(___lwp_mutex_register)
+	SYSTRAP_RVAL1(lwp_mutex_register)
 	SYSLWPERR
 	RET
-	SET_SIZE(___lwp_mutex_init)
+	SET_SIZE(___lwp_mutex_register)
diff --git a/usr/src/lib/libc/i386/sys/_lwp_mutex_unlock.s b/usr/src/lib/libc/i386/sys/_lwp_mutex_unlock.s
index 3afafcab1b..543e194d82 100644
--- a/usr/src/lib/libc/i386/sys/_lwp_mutex_unlock.s
+++ b/usr/src/lib/libc/i386/sys/_lwp_mutex_unlock.s
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -44,7 +44,16 @@
 	xchgl	(%eax), %ecx	/ clear lock and get old lock into %ecx
 	andl	$WAITER_MASK, %ecx / was anyone waiting on it?
 	je	1f
-	SYSTRAP_RVAL1(lwp_mutex_wakeup)
+	movl	0(%esp), %ecx	/ prepare to call lwp_mutex_wakeup()
+	movl	4(%esp), %edx
+	movl	$0, 4(%esp)
+	movl	%edx, 0(%esp)
+	pushl	%ecx
+	SYSTRAP_RVAL1(lwp_mutex_wakeup)	/ lwp_mutex_wakeup(mp, 0)
+	popl	%ecx		/ reconstruct the stack frame
+	movl	0(%esp), %edx
+	movl	%edx, 4(%esp)
+	movl	%ecx, 0(%esp)
 	SYSLWPERR
 	RET
 1:
diff --git a/usr/src/lib/libc/inc/synonyms.h b/usr/src/lib/libc/inc/synonyms.h
index 81e03fcc82..47fc223452 100644
--- a/usr/src/lib/libc/inc/synonyms.h
+++ b/usr/src/lib/libc/inc/synonyms.h
@@ -600,6 +600,8 @@ extern "C" {
 #define	munlockall		_munlockall
 #define	munlock			_munlock
 #define	munmap			_munmap
+#define	_mutex_consistent	__mutex_consistent
+#define	mutex_consistent	__mutex_consistent
 #define	_mutex_destroy		__mutex_destroy
 #define	mutex_destroy		__mutex_destroy
 #define	_mutex_held		__mutex_held
diff --git a/usr/src/lib/libc/inc/thr_uberdata.h b/usr/src/lib/libc/inc/thr_uberdata.h
index 7dbe4d7aab..ce30f25ef1 100644
--- a/usr/src/lib/libc/inc/thr_uberdata.h
+++ b/usr/src/lib/libc/inc/thr_uberdata.h
@@ -127,8 +127,8 @@
 #define	MUTEX_OWNER(mp)	((ulwp_t *)(uintptr_t)(mp)->mutex_owner)
 
 /*
- * Test if a thread owns a USYNC_THREAD mutex.  This is inappropriate
- * for a process-shared (USYNC_PROCESS | USYNC_PROCESS_ROBUST) mutex.
+ * Test if a thread owns a process-private (USYNC_THREAD) mutex.
+ * This is inappropriate for a process-shared (USYNC_PROCESS) mutex.
  * The 'mp' argument must not have side-effects since it is evaluated twice.
  */
 #define	MUTEX_OWNED(mp, thrp)	\
@@ -368,9 +368,11 @@ typedef union {
 #define	MX	0
 #define	CV	1
 #define	FIFOQ	0x10	/* or'ing with FIFOQ asks for FIFO queueing */
-#define	QHASHSIZE		512
-#define	QUEUE_HASH(wchan, type)						\
-	((uint_t)((((uintptr_t)(wchan) >> 3) ^ ((uintptr_t)(wchan) >> 12)) \
+#define	QHASHSHIFT	9			/* number of hashing bits */
+#define	QHASHSIZE	(1 << QHASHSHIFT)	/* power of 2 (1<<9 == 512) */
+#define	QUEUE_HASH(wchan, type)	((uint_t)			\
+	((((uintptr_t)(wchan) >> 3)				\
+	^ ((uintptr_t)(wchan) >> (QHASHSHIFT + 3)))		\
 	& (QHASHSIZE - 1)) + (((type) == MX)? 0 : QHASHSIZE))
 
 extern	queue_head_t	*queue_lock(void *, int);
@@ -542,12 +544,18 @@ typedef struct ulwp {
 	mxchain_t	*ul_mxchain;	/* chain of owned ceiling mutexes */
 	pri_t		ul_epri;	/* effective scheduling priority */
 	pri_t		ul_emappedpri;	/* effective mapped priority */
-	uint_t		ul_rdlocks;	/* # of entries in ul_readlock array */
-					/* 0 means there is but a single lock */
-	union {				/* single rwlock or pointer to array */
+	uint_t		ul_rdlockcnt;	/* # entries in ul_readlock array */
+				/* 0 means there is but a single entry */
+	union {				/* single entry or pointer to array */
 		readlock_t	single;
 		readlock_t	*array;
 	} ul_readlock;
+	uint_t		ul_heldlockcnt;	/* # entries in ul_heldlocks array */
+				/* 0 means there is but a single entry */
+	union {				/* single entry or pointer to array */
+		mutex_t		*single;
+		mutex_t		**array;
+	} ul_heldlocks;
 	/* PROBE_SUPPORT begin */
 	void		*ul_tpdp;
 	/* PROBE_SUPPORT end */
@@ -624,6 +632,26 @@ typedef struct atfork {
 } atfork_t;
 
 /*
+ * Element in the table of registered process robust locks.
+ * We keep track of these to make sure that we only call
+ * ___lwp_mutex_register() once for each such lock.
+ */
+typedef struct robust {
+	struct robust	*robust_next;
+	mutex_t		*robust_lock;
+} robust_t;
+
+/*
+ * Parameters of the lock registration hash table.
+ */
+#define	LOCKSHIFT	9			/* number of hashing bits */
+#define	LOCKHASHSZ	(1 << LOCKSHIFT)	/* power of 2 (1<<9 == 512) */
+#define	LOCK_HASH(addr)	(uint_t)			\
+	((((uintptr_t)(addr) >> 3)			\
+	^ ((uintptr_t)(addr) >> (LOCKSHIFT + 3)))	\
+	& (LOCKHASHSZ - 1))
+
+/*
  * Make our hot locks reside on private cache lines (64 bytes).
  * pad_owner and pad_count (aka fork_owner and fork_count)
  * are used only in fork_lock_enter() and fork_lock_exit()
@@ -781,6 +809,7 @@ typedef struct uberdata {
 	ulwp_t	*ulwp_replace_free;
 	ulwp_t	*ulwp_replace_last;
 	atfork_t	*atforklist;	/* circular Q for fork handlers */
+	robust_t	**robustlocks;	/* table of registered robust locks */
 	struct uberdata **tdb_bootstrap;
 	tdb_t	tdb;		/* thread debug interfaces (for libc_db) */
 } uberdata_t;
@@ -910,12 +939,18 @@ typedef struct ulwp32 {
 	caddr32_t	ul_mxchain;	/* chain of owned ceiling mutexes */
 	pri_t		ul_epri;	/* effective scheduling priority */
 	pri_t		ul_emappedpri;	/* effective mapped priority */
-	uint_t		ul_rdlocks;	/* # of entries in ul_readlock array */
-					/* 0 means there is but a single lock */
-	union {				/* single rwlock or pointer to array */
+	uint_t		ul_rdlockcnt;	/* # entries in ul_readlock array */
+				/* 0 means there is but a single entry */
+	union {				/* single entry or pointer to array */
 		readlock32_t	single;
 		caddr32_t	array;
 	} ul_readlock;
+	uint_t		ul_heldlockcnt;	/* # entries in ul_heldlocks array */
+				/* 0 means there is but a single entry */
+	union {				/* single entry or pointer to array */
+		caddr32_t	single;
+		caddr32_t	array;
+	} ul_heldlocks;
 	/* PROBE_SUPPORT begin */
 	caddr32_t	ul_tpdp;
 	/* PROBE_SUPPORT end */
@@ -974,6 +1009,7 @@ typedef struct uberdata32 {
 	caddr32_t	ulwp_replace_free;
 	caddr32_t	ulwp_replace_last;
 	caddr32_t	atforklist;
+	caddr32_t	robustlocks;
 	caddr32_t	tdb_bootstrap;
 	tdb32_t		tdb;
 } uberdata32_t;
@@ -1053,6 +1089,8 @@ extern	void	tls_setup(void);
 extern	void	tls_exit(void);
 extern	void	tls_free(ulwp_t *);
 extern	void	rwl_free(ulwp_t *);
+extern	void	heldlock_exit(void);
+extern	void	heldlock_free(ulwp_t *);
 extern	void	sigacthandler(int, siginfo_t *, void *);
 extern	void	signal_init(void);
 extern	int	sigequalset(const sigset_t *, const sigset_t *);
@@ -1075,6 +1113,10 @@ extern	void	grab_assert_lock(void);
 extern	void	dump_queue_statistics(void);
 extern	void	collect_queue_statistics(void);
 extern	void	record_spin_locks(ulwp_t *);
+extern	void	remember_lock(mutex_t *);
+extern	void	forget_lock(mutex_t *);
+extern	void	register_lock(mutex_t *);
+extern	void	unregister_locks(void);
 #if defined(__sparc)
 extern	void	_flush_windows(void);
 #else
@@ -1083,8 +1125,8 @@ extern	void	_flush_windows(void);
 extern	void	set_curthread(void *);
 
 /*
- * Utility function used by cond_broadcast() and rw_unlock()
- * when waking up many threads (more than MAXLWPS) all at once.
+ * Utility function used when waking up many threads (more than MAXLWPS)
+ * all at once.  See mutex_wakeup_all(), cond_broadcast(), and rw_unlock().
  */
 #define	MAXLWPS	128	/* max remembered lwpids before overflow */
 #define	NEWLWPS	2048	/* max remembered lwpids at first overflow */
@@ -1271,20 +1313,17 @@ extern	int	_private_mutex_unlock(mutex_t *);
 
 extern	int	_mutex_init(mutex_t *, int, void *);
 extern	int	_mutex_destroy(mutex_t *);
+extern	int	_mutex_consistent(mutex_t *);
 extern	int	_mutex_lock(mutex_t *);
 extern	int	_mutex_trylock(mutex_t *);
 extern	int	_mutex_unlock(mutex_t *);
-extern	void	_mutex_set_typeattr(mutex_t *, int);
 extern	int	__mutex_init(mutex_t *, int, void *);
 extern	int	__mutex_destroy(mutex_t *);
+extern	int	__mutex_consistent(mutex_t *);
 extern	int	__mutex_lock(mutex_t *);
 extern	int	__mutex_trylock(mutex_t *);
 extern	int	__mutex_unlock(mutex_t *);
 extern	int	mutex_is_held(mutex_t *);
-extern	int	mutex_lock_internal(mutex_t *, timespec_t *, int);
-extern	int	mutex_trylock_adaptive(mutex_t *);
-extern	int	mutex_queuelock_adaptive(mutex_t *);
-extern	int	mutex_lock_impl(mutex_t *mp, timespec_t *tsp);
 
 extern	int	_cond_init(cond_t *, int, void *);
 extern	int	_cond_wait(cond_t *, mutex_t *);
@@ -1293,8 +1332,6 @@ extern	int	_cond_reltimedwait(cond_t *, mutex_t *, const timespec_t *);
 extern	int	_cond_signal(cond_t *);
 extern	int	_cond_broadcast(cond_t *);
 extern	int	_cond_destroy(cond_t *);
-extern	int	cond_sleep_queue(cond_t *, mutex_t *, timespec_t *);
-extern	int	cond_sleep_kernel(cond_t *, mutex_t *, timespec_t *);
 extern	int	cond_signal_internal(cond_t *);
 extern	int	cond_broadcast_internal(cond_t *);
 
@@ -1344,11 +1381,11 @@ extern	int	get_info_by_policy(int);
 /*
  * System call wrappers (direct interfaces to the kernel)
  */
-extern	int	___lwp_mutex_init(mutex_t *, int);
+extern	int	___lwp_mutex_register(mutex_t *);
 extern	int	___lwp_mutex_trylock(mutex_t *);
 extern	int	___lwp_mutex_timedlock(mutex_t *, timespec_t *);
 extern	int	___lwp_mutex_unlock(mutex_t *);
-extern	int	___lwp_mutex_wakeup(mutex_t *);
+extern	int	___lwp_mutex_wakeup(mutex_t *, int);
 extern	int	___lwp_cond_wait(cond_t *, mutex_t *, timespec_t *, int);
 extern	int	__lwp_cond_signal(lwp_cond_t *);
 extern	int	__lwp_cond_broadcast(lwp_cond_t *);
diff --git a/usr/src/lib/libc/port/mapfile-vers b/usr/src/lib/libc/port/mapfile-vers
index 932505d279..d32f776934 100644
--- a/usr/src/lib/libc/port/mapfile-vers
+++ b/usr/src/lib/libc/port/mapfile-vers
@@ -82,6 +82,7 @@ SUNW_1.23 {		# SunOS 5.11 (Solaris 11)
 	mq_timedreceive;
 	mq_timedsend;
 	mq_unlink;
+	mutex_consistent;
 	nanosleep;
 	ntohl;
 	ntohs;
@@ -370,7 +371,7 @@ SUNW_1.22 {		# SunOS 5.10 (Solaris 10)
 	pthread_condattr_setclock;
 	pthread_mutexattr_getrobust_np;
 	pthread_mutexattr_setrobust_np;
-	pthread_mutex_consistent_np;
+	pthread_mutex_consistent_np =	NODYNSORT;
 	pthread_mutex_reltimedlock_np;
 	pthread_mutex_timedlock;
 	pthread_rwlock_reltimedrdlock_np;
@@ -1724,6 +1725,8 @@ SUNWprivate_1.1 {
 	_msgctl64;
 	__multi_innetgr;
 	_munlockall;
+	_mutex_consistent =		NODYNSORT;
+	__mutex_consistent =		NODYNSORT;
 	_mutex_destroy =		NODYNSORT;
 	__mutex_destroy;
 	mutex_held;
@@ -1887,7 +1890,7 @@ SUNWprivate_1.1 {
 	_pthread_mutexattr_setpshared;
 	_pthread_mutexattr_setrobust_np;
 	_pthread_mutexattr_settype;
-	_pthread_mutex_consistent_np;
+	_pthread_mutex_consistent_np =	NODYNSORT;
 	_pthread_mutex_destroy =	NODYNSORT;
 	_pthread_mutex_getprioceiling;
 	_pthread_mutex_init;
diff --git a/usr/src/lib/libc/port/threads/assfail.c b/usr/src/lib/libc/port/threads/assfail.c
index 0ffd9a3219..ac3e6a068f 100644
--- a/usr/src/lib/libc/port/threads/assfail.c
+++ b/usr/src/lib/libc/port/threads/assfail.c
@@ -216,7 +216,7 @@ lock_error(const mutex_t *mp, const char *who, void *cv, const char *msg)
 		/* EMPTY */;
 	else if (mcopy.mutex_lockw == 0)
 		(void) strcat(buf, "\nthe lock is unowned");
-	else if (!(mcopy.mutex_type & (USYNC_PROCESS|USYNC_PROCESS_ROBUST))) {
+	else if (!(mcopy.mutex_type & USYNC_PROCESS)) {
 		(void) strcat(buf, "\nthe lock owner is ");
 		ultos((uint64_t)mcopy.mutex_owner, 16, buf + strlen(buf));
 	} else {
diff --git a/usr/src/lib/libc/port/threads/pthr_mutex.c b/usr/src/lib/libc/port/threads/pthr_mutex.c
index 208cce06ac..3eabd3de61 100644
--- a/usr/src/lib/libc/port/threads/pthr_mutex.c
+++ b/usr/src/lib/libc/port/threads/pthr_mutex.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 1999-2003 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -66,9 +66,10 @@ _pthread_mutexattr_destroy(pthread_mutexattr_t *attr)
 }
 
 /*
- * pthread_mutexattr_setpshared: sets the shared attr to PRIVATE or SHARED.
- * This is equivalent to setting USYNC_PROCESS/USYNC_THREAD flag in
- * mutex_init().
+ * pthread_mutexattr_setpshared: sets the shared attribute
+ * to PTHREAD_PROCESS_PRIVATE or PTHREAD_PROCESS_SHARED.
+ * This is equivalent to setting the USYNC_THREAD/USYNC_PROCESS
+ * flag in mutex_init().
  */
 #pragma weak pthread_mutexattr_setpshared =  _pthread_mutexattr_setpshared
 int
@@ -85,7 +86,7 @@ _pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared)
 }
 
 /*
- * pthread_mutexattr_getpshared: gets the shared attr.
+ * pthread_mutexattr_getpshared: gets the shared attribute.
  */
 #pragma weak pthread_mutexattr_getpshared =  _pthread_mutexattr_getpshared
 int
@@ -101,7 +102,7 @@ _pthread_mutexattr_getpshared(const pthread_mutexattr_t *attr, int *pshared)
 }
 
 /*
- * pthread_mutexattr_setprioceiling: sets the prioceiling attr.
+ * pthread_mutexattr_setprioceiling: sets the prioceiling attribute.
  */
 #pragma weak pthread_mutexattr_setprioceiling = \
 					_pthread_mutexattr_setprioceiling
@@ -118,8 +119,7 @@ _pthread_mutexattr_setprioceiling(pthread_mutexattr_t *attr, int prioceiling)
 }
 
 /*
- * pthread_mutexattr_getprioceiling: gets the prioceiling attr.
- * Currently unsupported.
+ * pthread_mutexattr_getprioceiling: gets the prioceiling attribute.
  */
 #pragma weak pthread_mutexattr_getprioceiling = \
 					_pthread_mutexattr_getprioceiling
@@ -137,7 +137,6 @@ _pthread_mutexattr_getprioceiling(const pthread_mutexattr_t *attr, int *ceiling)
 
 /*
  * pthread_mutexattr_setprotocol: sets the protocol attribute.
- * Currently unsupported.
  */
 #pragma weak pthread_mutexattr_setprotocol =  _pthread_mutexattr_setprotocol
 int
@@ -157,7 +156,6 @@ _pthread_mutexattr_setprotocol(pthread_mutexattr_t *attr, int protocol)
 
 /*
  * pthread_mutexattr_getprotocol: gets the protocol attribute.
- * Currently unsupported.
  */
 #pragma weak pthread_mutexattr_getprotocol =  _pthread_mutexattr_getprotocol
 int
@@ -173,7 +171,8 @@ _pthread_mutexattr_getprotocol(const pthread_mutexattr_t *attr, int *protocol)
 }
 
 /*
- * pthread_mutexattr_setrobust_np: sets the robustness attr to ROBUST or STALL.
+ * pthread_mutexattr_setrobust_np: sets the robustness attribute
+ * to PTHREAD_MUTEX_ROBUST_NP or PTHREAD_MUTEX_STALL_NP.
  */
 #pragma weak pthread_mutexattr_setrobust_np = \
 					_pthread_mutexattr_setrobust_np
@@ -191,7 +190,7 @@ _pthread_mutexattr_setrobust_np(pthread_mutexattr_t *attr, int robust)
 }
 
 /*
- * pthread_mutexattr_getrobust_np: gets the robustness attr.
+ * pthread_mutexattr_getrobust_np: gets the robustness attribute.
  */
 #pragma weak pthread_mutexattr_getrobust_np = \
 					_pthread_mutexattr_getrobust_np
@@ -208,95 +207,33 @@ _pthread_mutexattr_getrobust_np(const pthread_mutexattr_t *attr, int *robust)
 }
 
 /*
- * pthread_mutex_consistent_np: make an inconsistent mutex consistent.
- * The mutex must have been made inconsistent due to the last owner of it
- * having died. Currently, no validation is done to check if:
- *      - the caller owns the mutex
- * Since this function is supported only for PI/robust locks, to check
- * if the caller owns the mutex, one needs to call the kernel. For now,
- * such extra validation does not seem necessary.
- */
-#pragma weak pthread_mutex_consistent_np = _pthread_mutex_consistent_np
-int
-_pthread_mutex_consistent_np(pthread_mutex_t *pmp)
-{
-	mutex_t *mp = (mutex_t *)pmp;
-
-	/*
-	 * Do this only for an inconsistent, initialized, PI, Robust lock.
-	 * For all other cases, return EINVAL.
-	 */
-	if ((mp->mutex_type & PTHREAD_PRIO_INHERIT) &&
-	    (mp->mutex_type & PTHREAD_MUTEX_ROBUST_NP) &&
-	    (mp->mutex_flag & LOCK_INITED) &&
-	    (mp->mutex_flag & LOCK_OWNERDEAD)) {
-		mp->mutex_flag &= ~LOCK_OWNERDEAD;
-		return (0);
-	}
-	return (EINVAL);
-}
-
-/*
- * pthread_mutex_init: Initializes the mutex object. It copies the
- * pshared attr into type argument and calls mutex_init().
+ * pthread_mutex_init: Initializes the mutex object.  It copies the
+ * various attributes into one type argument and calls mutex_init().
  */
 #pragma weak pthread_mutex_init = _pthread_mutex_init
 int
 _pthread_mutex_init(pthread_mutex_t *mutex, pthread_mutexattr_t *attr)
 {
-	mutex_t *mp = (mutex_t *)mutex;
+	mattr_t *ap;
 	int	type;
-	int	pshared;
-	int	protocol;
 	int	prioceiling = 0;
-	int	robust;
-	int	error;
-	mattr_t *ap;
 
+	/*
+	 * All of the pshared, type, protocol, robust attributes
+	 * translate to bits in the mutex_type field.
+	 */
 	if (attr != NULL) {
 		if ((ap = attr->__pthread_mutexattrp) == NULL)
 			return (EINVAL);
-		pshared = ap->pshared;
-		type = ap->type;
-		protocol = ap->protocol;
-		if (protocol == PTHREAD_PRIO_PROTECT)
+		type = ap->pshared | ap->type | ap->protocol | ap->robustness;
+		if (ap->protocol == PTHREAD_PRIO_PROTECT)
 			prioceiling = ap->prioceiling;
-		robust = ap->robustness;
-		/*
-		 * Support robust mutexes only for PI locks.
-		 */
-		if (robust == PTHREAD_MUTEX_ROBUST_NP &&
-		    protocol != PTHREAD_PRIO_INHERIT)
-			return (EINVAL);
 	} else {
-		pshared = DEFAULT_TYPE;
-		type = PTHREAD_MUTEX_DEFAULT;
-		protocol = PTHREAD_PRIO_NONE;
-		robust = PTHREAD_MUTEX_STALL_NP;
+		type = DEFAULT_TYPE | PTHREAD_MUTEX_DEFAULT |
+		    PTHREAD_PRIO_NONE | PTHREAD_MUTEX_STALL_NP;
 	}
 
-	error = _private_mutex_init(mp, pshared, NULL);
-	if (error == 0) {
-		/*
-		 * Use the same routine to set the protocol, and robustness
-		 * attributes, as that used to set the type attribute, since
-		 * all of these attributes translate to bits in the mutex_type
-		 * field.
-		 *
-		 * Note that robustness is a new bit, not the Solaris robust
-		 * bit - the latter implies USYNC_PROCESS_ROBUST, or
-		 * SHARED,ROBUST together. For POSIX, since robustness is an
-		 * orthogonal attribute, both SHARED,ROBUST and PRIVATE,ROBUST
-		 * should be valid combinations for the future. Hence,
-		 * introduce a new bit in the mutex type field. See
-		 * sys/synch.h or pthread.h. In the future, if we ever
-		 * introduce a USYNC_THREAD_ROBUST, the latter could use this
-		 * new bit...
-		 */
-		_mutex_set_typeattr(mp, type|protocol|robust);
-		mp->mutex_ceiling = (uint8_t)prioceiling;
-	}
-	return (error);
+	return (_private_mutex_init((mutex_t *)mutex, type, &prioceiling));
 }
 
 /*
@@ -353,7 +290,7 @@ _pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
 		type = LOCK_ERRORCHECK;
 		break;
 	case PTHREAD_MUTEX_RECURSIVE:
-		type = LOCK_RECURSIVE|LOCK_ERRORCHECK;
+		type = LOCK_RECURSIVE | LOCK_ERRORCHECK;
 		break;
 	default:
 		return (EINVAL);
@@ -364,7 +301,7 @@ _pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type)
 
 /*
  * UNIX98
- * pthread_mutexattr_gettype: gets the type attr.
+ * pthread_mutexattr_gettype: gets the type attribute.
  */
 #pragma weak pthread_mutexattr_gettype =  _pthread_mutexattr_gettype
 int
@@ -383,7 +320,7 @@ _pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *typep)
 	case LOCK_ERRORCHECK:
 		type = PTHREAD_MUTEX_ERRORCHECK;
 		break;
-	case LOCK_RECURSIVE|LOCK_ERRORCHECK:
+	case LOCK_RECURSIVE | LOCK_ERRORCHECK:
 		type = PTHREAD_MUTEX_RECURSIVE;
 		break;
 	default:
diff --git a/usr/src/lib/libc/port/threads/rwlock.c b/usr/src/lib/libc/port/threads/rwlock.c
index 408f74489e..0f58b3a230 100644
--- a/usr/src/lib/libc/port/threads/rwlock.c
+++ b/usr/src/lib/libc/port/threads/rwlock.c
@@ -45,7 +45,7 @@
 /*
  * Find/allocate an entry for rwlp in our array of rwlocks held for reading.
  * We must be deferring signals for this to be safe.
- * Else if we are returning an entry with ul_rdlocks == 0,
+ * Else if we are returning an entry with ul_rdlockcnt == 0,
  * it could be reassigned behind our back in a signal handler.
  */
 static readlock_t *
@@ -59,7 +59,7 @@ rwl_entry(rwlock_t *rwlp)
 	/* we must be deferring signals */
 	ASSERT((self->ul_critical + self->ul_sigdefer) != 0);
 
-	if ((nlocks = self->ul_rdlocks) != 0)
+	if ((nlocks = self->ul_rdlockcnt) != 0)
 		readlockp = self->ul_readlock.array;
 	else {
 		nlocks = 1;
@@ -81,12 +81,12 @@ rwl_entry(rwlock_t *rwlp)
 	 * No entry available.  Allocate more space, converting the single
 	 * readlock_t entry into an array of readlock_t entries if necessary.
 	 */
-	if ((nlocks = self->ul_rdlocks) == 0) {
+	if ((nlocks = self->ul_rdlockcnt) == 0) {
 		/*
 		 * Initial allocation of the readlock_t array.
 		 * Convert the single entry into an array.
 		 */
-		self->ul_rdlocks = nlocks = NLOCKS;
+		self->ul_rdlockcnt = nlocks = NLOCKS;
 		readlockp = lmalloc(nlocks * sizeof (readlock_t));
 		/*
 		 * The single readlock_t becomes the first entry in the array.
@@ -108,7 +108,7 @@ rwl_entry(rwlock_t *rwlp)
 		nlocks * sizeof (readlock_t));
 	lfree(self->ul_readlock.array, nlocks * sizeof (readlock_t));
 	self->ul_readlock.array = readlockp;
-	self->ul_rdlocks *= 2;
+	self->ul_rdlockcnt *= 2;
 	/*
 	 * Return the next available entry in the newly allocated array.
 	 */
@@ -124,9 +124,9 @@ rwl_free(ulwp_t *ulwp)
 {
 	uint_t nlocks;
 
-	if ((nlocks = ulwp->ul_rdlocks) != 0)
+	if ((nlocks = ulwp->ul_rdlockcnt) != 0)
 		lfree(ulwp->ul_readlock.array, nlocks * sizeof (readlock_t));
-	ulwp->ul_rdlocks = 0;
+	ulwp->ul_rdlockcnt = 0;
 	ulwp->ul_readlock.single.rd_rwlock = NULL;
 	ulwp->ul_readlock.single.rd_count = 0;
 }
@@ -157,7 +157,7 @@ _rw_read_held(rwlock_t *rwlp)
 		 * The lock is held for reading by some thread.
 		 * Search our array of rwlocks held for reading for a match.
 		 */
-		if ((nlocks = self->ul_rdlocks) != 0)
+		if ((nlocks = self->ul_rdlockcnt) != 0)
 			readlockp = self->ul_readlock.array;
 		else {
 			nlocks = 1;
diff --git a/usr/src/lib/libc/port/threads/scalls.c b/usr/src/lib/libc/port/threads/scalls.c
index 34540e0d05..29da638f13 100644
--- a/usr/src/lib/libc/port/threads/scalls.c
+++ b/usr/src/lib/libc/port/threads/scalls.c
@@ -195,6 +195,7 @@ _private_forkx(int flags)
 		self->ul_siginfo.si_signo = 0;
 		udp->pid = _private_getpid();
 		/* reset the library's data structures to reflect one thread */
+		unregister_locks();
 		postfork1_child();
 		restore_signals(self);
 		_postfork_child_handler();
@@ -269,6 +270,7 @@ _private_forkallx(int flags)
 		self->ul_cursig = 0;
 		self->ul_siginfo.si_signo = 0;
 		udp->pid = _private_getpid();
+		unregister_locks();
 		continue_fork(1);
 	} else {
 		continue_fork(0);
diff --git a/usr/src/lib/libc/port/threads/synch.c b/usr/src/lib/libc/port/threads/synch.c
index ec9fabc226..d3f0e70da3 100644
--- a/usr/src/lib/libc/port/threads/synch.c
+++ b/usr/src/lib/libc/port/threads/synch.c
@@ -34,11 +34,14 @@
 /*
  * This mutex is initialized to be held by lwp#1.
  * It is used to block a thread that has returned from a mutex_lock()
- * of a PTHREAD_PRIO_INHERIT mutex with an unrecoverable error.
+ * of a LOCK_PRIO_INHERIT mutex with an unrecoverable error.
  */
 mutex_t	stall_mutex = DEFAULTMUTEX;
 
 static int shared_mutex_held(mutex_t *);
+static int mutex_unlock_internal(mutex_t *, int);
+static int mutex_queuelock_adaptive(mutex_t *);
+static void mutex_wakeup_all(mutex_t *);
 
 /*
  * Lock statistics support functions.
@@ -102,16 +105,19 @@ int	thread_queue_spin = 1000;
  */
 #define	mutex_spinners	mutex_ownerpid
 
-void
-_mutex_set_typeattr(mutex_t *mp, int attr)
-{
-	mp->mutex_type |= (uint8_t)attr;
-}
+#define	ALL_ATTRIBUTES				\
+	(LOCK_RECURSIVE | LOCK_ERRORCHECK |	\
+	LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT |	\
+	LOCK_ROBUST)
 
 /*
- * 'type' can be one of USYNC_THREAD or USYNC_PROCESS, possibly
- * augmented by the flags LOCK_RECURSIVE and/or LOCK_ERRORCHECK,
- * or it can be USYNC_PROCESS_ROBUST with no extra flags.
+ * 'type' can be one of USYNC_THREAD, USYNC_PROCESS, or USYNC_PROCESS_ROBUST,
+ * augmented by zero or more the flags:
+ *	LOCK_RECURSIVE
+ *	LOCK_ERRORCHECK
+ *	LOCK_PRIO_INHERIT
+ *	LOCK_PRIO_PROTECT
+ *	LOCK_ROBUST
  */
 #pragma weak _private_mutex_init = __mutex_init
 #pragma weak mutex_init = __mutex_init
@@ -120,28 +126,62 @@ _mutex_set_typeattr(mutex_t *mp, int attr)
 int
 __mutex_init(mutex_t *mp, int type, void *arg)
 {
-	int error;
+	int basetype = (type & ~ALL_ATTRIBUTES);
+	int error = 0;
+
+	if (basetype == USYNC_PROCESS_ROBUST) {
+		/*
+		 * USYNC_PROCESS_ROBUST is a deprecated historical type.
+		 * We change it into (USYNC_PROCESS | LOCK_ROBUST) but
+		 * retain the USYNC_PROCESS_ROBUST flag so we can return
+		 * ELOCKUNMAPPED when necessary (only USYNC_PROCESS_ROBUST
+		 * mutexes will ever draw ELOCKUNMAPPED).
+		 */
+		type |= (USYNC_PROCESS | LOCK_ROBUST);
+		basetype = USYNC_PROCESS;
+	}
 
-	switch (type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) {
-	case USYNC_THREAD:
-	case USYNC_PROCESS:
+	if (!(basetype == USYNC_THREAD || basetype == USYNC_PROCESS) ||
+	    (type & (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT))
+	    == (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT)) {
+		error = EINVAL;
+	} else if (type & LOCK_ROBUST) {
+		/*
+		 * Callers of mutex_init() with the LOCK_ROBUST attribute
+		 * are required to pass an initially all-zero mutex.
+		 * Multiple calls to mutex_init() are allowed; all but
+		 * the first return EBUSY.  A call to mutex_init() is
+		 * allowed to make an inconsistent robust lock consistent
+		 * (for historical usage, even though the proper interface
+		 * for this is mutex_consistent()).  Note that we use
+		 * atomic_or_16() to set the LOCK_INITED flag so as
+		 * not to disturb surrounding bits (LOCK_OWNERDEAD, etc).
+		 */
+		extern void _atomic_or_16(volatile uint16_t *, uint16_t);
+		if (!(mp->mutex_flag & LOCK_INITED)) {
+			mp->mutex_type = (uint8_t)type;
+			_atomic_or_16(&mp->mutex_flag, LOCK_INITED);
+			mp->mutex_magic = MUTEX_MAGIC;
+		} else if (type != mp->mutex_type ||
+		    ((type & LOCK_PRIO_PROTECT) &&
+		    mp->mutex_ceiling != (*(int *)arg))) {
+			error = EINVAL;
+		} else if (__mutex_consistent(mp) != 0) {
+			error = EBUSY;
+		}
+		/* register a process robust mutex with the kernel */
+		if (basetype == USYNC_PROCESS)
+			register_lock(mp);
+	} else {
 		(void) _memset(mp, 0, sizeof (*mp));
 		mp->mutex_type = (uint8_t)type;
 		mp->mutex_flag = LOCK_INITED;
-		error = 0;
-		break;
-	case USYNC_PROCESS_ROBUST:
-		if (type & (LOCK_RECURSIVE|LOCK_ERRORCHECK))
-			error = EINVAL;
-		else
-			error = ___lwp_mutex_init(mp, type);
-		break;
-	default:
-		error = EINVAL;
-		break;
-	}
-	if (error == 0)
 		mp->mutex_magic = MUTEX_MAGIC;
+	}
+
+	if (error == 0 && (type & LOCK_PRIO_PROTECT))
+		mp->mutex_ceiling = (uint8_t)(*(int *)arg);
+
 	return (error);
 }
 
@@ -293,7 +333,7 @@ spin_lock_clear(mutex_t *mp)
 
 	mp->mutex_owner = 0;
 	if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK) {
-		(void) ___lwp_mutex_wakeup(mp);
+		(void) ___lwp_mutex_wakeup(mp, 0);
 		if (self->ul_spin_lock_wakeup != UINT_MAX)
 			self->ul_spin_lock_wakeup++;
 	}
@@ -308,6 +348,7 @@ queue_alloc(void)
 {
 	ulwp_t *self = curthread;
 	uberdata_t *udp = self->ul_uberdata;
+	mutex_t *mp;
 	void *data;
 	int i;
 
@@ -321,8 +362,11 @@ queue_alloc(void)
 	    == MAP_FAILED)
 		thr_panic("cannot allocate thread queue_head table");
 	udp->queue_head = (queue_head_t *)data;
-	for (i = 0; i < 2 * QHASHSIZE; i++)
-		udp->queue_head[i].qh_lock.mutex_magic = MUTEX_MAGIC;
+	for (i = 0; i < 2 * QHASHSIZE; i++) {
+		mp = &udp->queue_head[i].qh_lock;
+		mp->mutex_flag = LOCK_INITED;
+		mp->mutex_magic = MUTEX_MAGIC;
+	}
 }
 
 #if defined(THREAD_DEBUG)
@@ -688,12 +732,14 @@ unsleep_self(void)
  * Common code for calling the the ___lwp_mutex_timedlock() system call.
  * Returns with mutex_owner and mutex_ownerpid set correctly.
  */
-int
+static int
 mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp)
 {
 	ulwp_t *self = curthread;
 	uberdata_t *udp = self->ul_uberdata;
+	int mtype = mp->mutex_type;
 	hrtime_t begin_sleep;
+	int acquired;
 	int error;
 
 	self->ul_sp = stkptr();
@@ -711,13 +757,17 @@ mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp)
 	DTRACE_PROBE1(plockstat, mutex__block, mp);
 
 	for (;;) {
-		if ((error = ___lwp_mutex_timedlock(mp, tsp)) != 0) {
-			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
-			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
+		/*
+		 * A return value of EOWNERDEAD or ELOCKUNMAPPED
+		 * means we successfully acquired the lock.
+		 */
+		if ((error = ___lwp_mutex_timedlock(mp, tsp)) != 0 &&
+		    error != EOWNERDEAD && error != ELOCKUNMAPPED) {
+			acquired = 0;
 			break;
 		}
 
-		if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) {
+		if (mtype & USYNC_PROCESS) {
 			/*
 			 * Defend against forkall().  We may be the child,
 			 * in which case we don't actually own the mutex.
@@ -726,16 +776,13 @@ mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp)
 			if (mp->mutex_ownerpid == udp->pid) {
 				mp->mutex_owner = (uintptr_t)self;
 				exit_critical(self);
-				DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
-				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
-				    0, 0);
+				acquired = 1;
 				break;
 			}
 			exit_critical(self);
 		} else {
 			mp->mutex_owner = (uintptr_t)self;
-			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
-			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
+			acquired = 1;
 			break;
 		}
 	}
@@ -744,6 +791,14 @@ mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp)
 	self->ul_wchan = NULL;
 	self->ul_sp = 0;
 
+	if (acquired) {
+		DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
+		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
+	} else {
+		DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
+		DTRACE_PROBE2(plockstat, mutex__error, mp, error);
+	}
+
 	return (error);
 }
 
@@ -756,18 +811,22 @@ mutex_trylock_kernel(mutex_t *mp)
 {
 	ulwp_t *self = curthread;
 	uberdata_t *udp = self->ul_uberdata;
+	int mtype = mp->mutex_type;
 	int error;
+	int acquired;
 
 	for (;;) {
-		if ((error = ___lwp_mutex_trylock(mp)) != 0) {
-			if (error != EBUSY) {
-				DTRACE_PROBE2(plockstat, mutex__error, mp,
-				    error);
-			}
+		/*
+		 * A return value of EOWNERDEAD or ELOCKUNMAPPED
+		 * means we successfully acquired the lock.
+		 */
+		if ((error = ___lwp_mutex_trylock(mp)) != 0 &&
+		    error != EOWNERDEAD && error != ELOCKUNMAPPED) {
+			acquired = 0;
 			break;
 		}
 
-		if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) {
+		if (mtype & USYNC_PROCESS) {
 			/*
 			 * Defend against forkall().  We may be the child,
 			 * in which case we don't actually own the mutex.
@@ -776,18 +835,23 @@ mutex_trylock_kernel(mutex_t *mp)
 			if (mp->mutex_ownerpid == udp->pid) {
 				mp->mutex_owner = (uintptr_t)self;
 				exit_critical(self);
-				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
-				    0, 0);
+				acquired = 1;
 				break;
 			}
 			exit_critical(self);
 		} else {
 			mp->mutex_owner = (uintptr_t)self;
-			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
+			acquired = 1;
 			break;
 		}
 	}
 
+	if (acquired) {
+		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
+	} else if (error != EBUSY) {
+		DTRACE_PROBE2(plockstat, mutex__error, mp, error);
+	}
+
 	return (error);
 }
 
@@ -932,31 +996,42 @@ preempt_unpark(ulwp_t *self, lwpid_t lwpid)
 }
 
 /*
- * Spin for a while, trying to grab the lock.  We know that we
- * failed set_lock_byte(&mp->mutex_lockw) once before coming here.
+ * Spin for a while, trying to grab the lock.
  * If this fails, return EBUSY and let the caller deal with it.
  * If this succeeds, return 0 with mutex_owner set to curthread.
  */
-int
+static int
 mutex_trylock_adaptive(mutex_t *mp)
 {
 	ulwp_t *self = curthread;
+	int error = EBUSY;
 	ulwp_t *ulwp;
 	volatile sc_shared_t *scp;
 	volatile uint8_t *lockp;
 	volatile uint64_t *ownerp;
-	int count, max = self->ul_adaptive_spin;
+	int count;
+	int max;
 
-	ASSERT(!(mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)));
+	ASSERT(!(mp->mutex_type & USYNC_PROCESS));
 
-	if (max == 0 || (mp->mutex_spinners >= self->ul_max_spinners))
+	if (MUTEX_OWNER(mp) == self)
 		return (EBUSY);
 
-	lockp = (volatile uint8_t *)&mp->mutex_lockw;
-	ownerp = (volatile uint64_t *)&mp->mutex_owner;
+	/* short-cut, not definitive (see below) */
+	if (mp->mutex_flag & LOCK_NOTRECOVERABLE) {
+		ASSERT(mp->mutex_type & LOCK_ROBUST);
+		DTRACE_PROBE2(plockstat, mutex__error, mp, ENOTRECOVERABLE);
+		return (ENOTRECOVERABLE);
+	}
+
+	if ((max = self->ul_adaptive_spin) == 0 ||
+	    mp->mutex_spinners >= self->ul_max_spinners)
+		max = 1;	/* try at least once */
 
 	DTRACE_PROBE1(plockstat, mutex__spin, mp);
 
+	lockp = (volatile uint8_t *)&mp->mutex_lockw;
+	ownerp = (volatile uint64_t *)&mp->mutex_owner;
 	/*
 	 * This spin loop is unfair to lwps that have already dropped into
 	 * the kernel to sleep.  They will starve on a highly-contended mutex.
@@ -968,14 +1043,11 @@ mutex_trylock_adaptive(mutex_t *mp)
 	 */
 	enter_critical(self);		/* protects ul_schedctl */
 	atomic_inc_32(&mp->mutex_spinners);
-	for (count = 0; count < max; count++) {
+	for (count = 1; count <= max; count++) {
 		if (*lockp == 0 && set_lock_byte(lockp) == 0) {
 			*ownerp = (uintptr_t)self;
-			atomic_dec_32(&mp->mutex_spinners);
-			exit_critical(self);
-			DTRACE_PROBE2(plockstat, mutex__spun, 1, count);
-			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
-			return (0);
+			error = 0;
+			break;
 		}
 		SMT_PAUSE();
 		/*
@@ -1004,16 +1076,39 @@ mutex_trylock_adaptive(mutex_t *mp)
 	atomic_dec_32(&mp->mutex_spinners);
 	exit_critical(self);
 
-	DTRACE_PROBE2(plockstat, mutex__spun, 0, count);
+	if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) {
+		ASSERT(mp->mutex_type & LOCK_ROBUST);
+		/*
+		 * We shouldn't own the mutex; clear the lock.
+		 */
+		mp->mutex_owner = 0;
+		if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK)
+			mutex_wakeup_all(mp);
+		error = ENOTRECOVERABLE;
+	}
 
-	return (EBUSY);
+	if (error) {
+		DTRACE_PROBE2(plockstat, mutex__spun, 0, count);
+		if (error != EBUSY) {
+			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
+		}
+	} else {
+		DTRACE_PROBE2(plockstat, mutex__spun, 1, count);
+		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
+		if (mp->mutex_flag & LOCK_OWNERDEAD) {
+			ASSERT(mp->mutex_type & LOCK_ROBUST);
+			error = EOWNERDEAD;
+		}
+	}
+
+	return (error);
 }
 
 /*
  * Same as mutex_trylock_adaptive(), except specifically for queue locks.
  * The owner field is not set here; the caller (spin_lock_set()) sets it.
  */
-int
+static int
 mutex_queuelock_adaptive(mutex_t *mp)
 {
 	ulwp_t *ulwp;
@@ -1044,71 +1139,93 @@ mutex_queuelock_adaptive(mutex_t *mp)
 
 /*
  * Like mutex_trylock_adaptive(), but for process-shared mutexes.
- * Spin for a while, trying to grab the lock.  We know that we
- * failed set_lock_byte(&mp->mutex_lockw) once before coming here.
+ * Spin for a while, trying to grab the lock.
  * If this fails, return EBUSY and let the caller deal with it.
  * If this succeeds, return 0 with mutex_owner set to curthread
  * and mutex_ownerpid set to the current pid.
  */
-int
+static int
 mutex_trylock_process(mutex_t *mp)
 {
 	ulwp_t *self = curthread;
-	uberdata_t *udp = self->ul_uberdata;
-	int count;
+	int error = EBUSY;
 	volatile uint8_t *lockp;
-	volatile uint64_t *ownerp;
-	volatile int32_t *pidp;
-	pid_t pid, newpid;
-	uint64_t owner, newowner;
-
-	if ((count = ncpus) == 0)
-		count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
-	count = (count > 1)? self->ul_adaptive_spin : 0;
+	int count;
+	int max;
 
-	ASSERT((mp->mutex_type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) ==
-		USYNC_PROCESS);
+	ASSERT(mp->mutex_type & USYNC_PROCESS);
 
-	if (count == 0)
+	if (shared_mutex_held(mp))
 		return (EBUSY);
 
+	/* short-cut, not definitive (see below) */
+	if (mp->mutex_flag & LOCK_NOTRECOVERABLE) {
+		ASSERT(mp->mutex_type & LOCK_ROBUST);
+		DTRACE_PROBE2(plockstat, mutex__error, mp, ENOTRECOVERABLE);
+		return (ENOTRECOVERABLE);
+	}
+
+	if (ncpus == 0)
+		ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
+	max = (ncpus > 1)? self->ul_adaptive_spin : 1;
+	if (max == 0)
+		max = 1;	/* try at least once */
+
+	DTRACE_PROBE1(plockstat, mutex__spin, mp);
+
 	lockp = (volatile uint8_t *)&mp->mutex_lockw;
-	ownerp = (volatile uint64_t *)&mp->mutex_owner;
-	pidp = (volatile int32_t *)&mp->mutex_ownerpid;
-	owner = *ownerp;
-	pid = *pidp;
 	/*
 	 * This is a process-shared mutex.
 	 * We cannot know if the owner is running on a processor.
 	 * We just spin and hope that it is on a processor.
 	 */
-	while (--count >= 0) {
-		if (*lockp == 0) {
-			enter_critical(self);
-			if (set_lock_byte(lockp) == 0) {
-				*ownerp = (uintptr_t)self;
-				*pidp = udp->pid;
-				exit_critical(self);
-				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
-				    0, 0);
-				return (0);
-			}
-			exit_critical(self);
-		} else if ((newowner = *ownerp) == owner &&
-		    (newpid = *pidp) == pid) {
-			SMT_PAUSE();
-			continue;
+	enter_critical(self);
+	for (count = 1; count <= max; count++) {
+		if (*lockp == 0 && set_lock_byte(lockp) == 0) {
+			mp->mutex_owner = (uintptr_t)self;
+			mp->mutex_ownerpid = self->ul_uberdata->pid;
+			error = 0;
+			break;
 		}
+		SMT_PAUSE();
+	}
+	exit_critical(self);
+
+	if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) {
+		ASSERT(mp->mutex_type & LOCK_ROBUST);
 		/*
-		 * The owner of the lock changed; start the count over again.
-		 * This may be too aggressive; it needs testing.
+		 * We shouldn't own the mutex; clear the lock.
 		 */
-		owner = newowner;
-		pid = newpid;
-		count = self->ul_adaptive_spin;
+		mp->mutex_owner = 0;
+		mp->mutex_ownerpid = 0;
+		if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK) {
+			no_preempt(self);
+			(void) ___lwp_mutex_wakeup(mp, 1);
+			preempt(self);
+		}
+		error = ENOTRECOVERABLE;
 	}
 
-	return (EBUSY);
+	if (error) {
+		DTRACE_PROBE2(plockstat, mutex__spun, 0, count);
+		if (error != EBUSY) {
+			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
+		}
+	} else {
+		DTRACE_PROBE2(plockstat, mutex__spun, 1, count);
+		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
+		if (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
+			ASSERT(mp->mutex_type & LOCK_ROBUST);
+			if (mp->mutex_flag & LOCK_OWNERDEAD)
+				error = EOWNERDEAD;
+			else if (mp->mutex_type & USYNC_PROCESS_ROBUST)
+				error = ELOCKUNMAPPED;
+			else
+				error = EOWNERDEAD;
+		}
+	}
+
+	return (error);
 }
 
 /*
@@ -1117,7 +1234,7 @@ mutex_trylock_process(mutex_t *mp)
  * The caller of mutex_wakeup() must call __lwp_unpark(lwpid)
  * to wake up the specified lwp.
  */
-lwpid_t
+static lwpid_t
 mutex_wakeup(mutex_t *mp)
 {
 	lwpid_t lwpid = 0;
@@ -1140,11 +1257,73 @@ mutex_wakeup(mutex_t *mp)
 }
 
 /*
+ * Mutex wakeup code for releasing all waiters on a USYNC_THREAD mutex.
+ */
+static void
+mutex_wakeup_all(mutex_t *mp)
+{
+	queue_head_t *qp;
+	int nlwpid = 0;
+	int maxlwps = MAXLWPS;
+	ulwp_t **ulwpp;
+	ulwp_t *ulwp;
+	ulwp_t *prev = NULL;
+	lwpid_t buffer[MAXLWPS];
+	lwpid_t *lwpid = buffer;
+
+	/*
+	 * Walk the list of waiters and prepare to wake up all of them.
+	 * The waiters flag has already been cleared from the mutex.
+	 *
+	 * We keep track of lwpids that are to be unparked in lwpid[].
+	 * __lwp_unpark_all() is called to unpark all of them after
+	 * they have been removed from the sleep queue and the sleep
+	 * queue lock has been dropped.  If we run out of space in our
+	 * on-stack buffer, we need to allocate more but we can't call
+	 * lmalloc() because we are holding a queue lock when the overflow
+	 * occurs and lmalloc() acquires a lock.  We can't use alloca()
+	 * either because the application may have allocated a small
+	 * stack and we don't want to overrun the stack.  So we call
+	 * alloc_lwpids() to allocate a bigger buffer using the mmap()
+	 * system call directly since that path acquires no locks.
+	 */
+	qp = queue_lock(mp, MX);
+	ulwpp = &qp->qh_head;
+	while ((ulwp = *ulwpp) != NULL) {
+		if (ulwp->ul_wchan != mp) {
+			prev = ulwp;
+			ulwpp = &ulwp->ul_link;
+		} else {
+			if (nlwpid == maxlwps)
+				lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps);
+			(void) queue_unlink(qp, ulwpp, prev);
+			lwpid[nlwpid++] = ulwp->ul_lwpid;
+		}
+	}
+	mp->mutex_waiters = 0;
+
+	if (nlwpid == 0) {
+		queue_unlock(qp);
+	} else {
+		no_preempt(curthread);
+		queue_unlock(qp);
+		if (nlwpid == 1)
+			(void) __lwp_unpark(lwpid[0]);
+		else
+			(void) __lwp_unpark_all(lwpid, nlwpid);
+		preempt(curthread);
+	}
+
+	if (lwpid != buffer)
+		(void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t));
+}
+
+/*
  * Spin for a while, testing to see if the lock has been grabbed.
  * If this fails, call mutex_wakeup() to release a waiter.
  */
-lwpid_t
-mutex_unlock_queue(mutex_t *mp)
+static lwpid_t
+mutex_unlock_queue(mutex_t *mp, int release_all)
 {
 	ulwp_t *self = curthread;
 	uint32_t *lockw = &mp->mutex_lockword;
@@ -1168,13 +1347,12 @@ mutex_unlock_queue(mutex_t *mp)
 	 * any of the adaptive code because the waiter bit has been cleared
 	 * and the adaptive code is unreliable in this case.
 	 */
-	if (!(*lockw & WAITERMASK)) {	/* no waiter exists right now */
+	if (release_all || !(*lockw & WAITERMASK)) {
 		mp->mutex_owner = 0;
 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
 		if (!(atomic_swap_32(lockw, 0) & WAITERMASK))
-			return (0);	/* still no waiters */
+			return (0);	/* no waiters */
 		no_preempt(self);	/* ensure a prompt wakeup */
-		lwpid = mutex_wakeup(mp);
 	} else {
 		no_preempt(self);	/* ensure a prompt wakeup */
 		lockp = (volatile uint8_t *)&mp->mutex_lockw;
@@ -1217,9 +1395,14 @@ mutex_unlock_queue(mutex_t *mp)
 		 * Wake up some lwp that is waiting for it.
 		 */
 		mp->mutex_waiters = 0;
-		lwpid = mutex_wakeup(mp);
 	}
 
+	if (release_all) {
+		mutex_wakeup_all(mp);
+		lwpid = 0;
+	} else {
+		lwpid = mutex_wakeup(mp);
+	}
 	if (lwpid == 0)
 		preempt(self);
 	return (lwpid);
@@ -1229,8 +1412,8 @@ mutex_unlock_queue(mutex_t *mp)
  * Like mutex_unlock_queue(), but for process-shared mutexes.
  * We tested the waiters field before calling here and it was non-zero.
  */
-void
-mutex_unlock_process(mutex_t *mp)
+static void
+mutex_unlock_process(mutex_t *mp, int release_all)
 {
 	ulwp_t *self = curthread;
 	int count;
@@ -1239,14 +1422,14 @@ mutex_unlock_process(mutex_t *mp)
 	/*
 	 * See the comments in mutex_unlock_queue(), above.
 	 */
-	if ((count = ncpus) == 0)
-		count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
-	count = (count > 1)? self->ul_release_spin : 0;
+	if (ncpus == 0)
+		ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
+	count = (ncpus > 1)? self->ul_release_spin : 0;
 	no_preempt(self);
 	mp->mutex_owner = 0;
 	mp->mutex_ownerpid = 0;
 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
-	if (count == 0) {
+	if (release_all || count == 0) {
 		/* clear lock, test waiter */
 		if (!(atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK)) {
 			/* no waiters now */
@@ -1271,7 +1454,7 @@ mutex_unlock_process(mutex_t *mp)
 		 */
 		mp->mutex_waiters = 0;
 	}
-	(void) ___lwp_mutex_wakeup(mp);
+	(void) ___lwp_mutex_wakeup(mp, release_all);
 	preempt(self);
 }
 
@@ -1296,7 +1479,7 @@ stall(void)
 /*
  * Acquire a USYNC_THREAD mutex via user-level sleep queues.
  * We failed set_lock_byte(&mp->mutex_lockw) before coming here.
- * Returns with mutex_owner set correctly.
+ * If successful, returns with mutex_owner set correctly.
  */
 int
 mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp,
@@ -1333,8 +1516,6 @@ mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp,
 	for (;;) {
 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
 			mp->mutex_owner = (uintptr_t)self;
-			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
-			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
 			mp->mutex_waiters = dequeue_self(qp, mp);
 			break;
 		}
@@ -1357,17 +1538,10 @@ mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp,
 		 */
 		qp = queue_lock(mp, MX);
 		if (self->ul_sleepq == NULL) {
-			if (error) {
-				DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
-				DTRACE_PROBE2(plockstat, mutex__error, mp,
-				    error);
+			if (error)
 				break;
-			}
 			if (set_lock_byte(&mp->mutex_lockw) == 0) {
 				mp->mutex_owner = (uintptr_t)self;
-				DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
-				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
-				    0, 0);
 				break;
 			}
 			enqueue(qp, self, mp, MX);
@@ -1378,28 +1552,164 @@ mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp,
 		    self->ul_wchan == mp);
 		if (error) {
 			mp->mutex_waiters = dequeue_self(qp, mp);
-			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
-			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
 			break;
 		}
 	}
-
 	ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
 	    self->ul_wchan == NULL);
 	self->ul_sp = 0;
-
 	queue_unlock(qp);
+
 	if (msp)
 		msp->mutex_sleep_time += gethrtime() - begin_sleep;
 
 	ASSERT(error == 0 || error == EINVAL || error == ETIME);
+
+	if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) {
+		ASSERT(mp->mutex_type & LOCK_ROBUST);
+		/*
+		 * We shouldn't own the mutex; clear the lock.
+		 */
+		mp->mutex_owner = 0;
+		if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK)
+			mutex_wakeup_all(mp);
+		error = ENOTRECOVERABLE;
+	}
+
+	if (error) {
+		DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
+		DTRACE_PROBE2(plockstat, mutex__error, mp, error);
+	} else {
+		DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
+		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
+		if (mp->mutex_flag & LOCK_OWNERDEAD) {
+			ASSERT(mp->mutex_type & LOCK_ROBUST);
+			error = EOWNERDEAD;
+		}
+	}
+
 	return (error);
 }
 
+static int
+mutex_recursion(mutex_t *mp, int mtype, int try)
+{
+	ASSERT(mutex_is_held(mp));
+	ASSERT(mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK));
+	ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK);
+
+	if (mtype & LOCK_RECURSIVE) {
+		if (mp->mutex_rcount == RECURSION_MAX) {
+			DTRACE_PROBE2(plockstat, mutex__error, mp, EAGAIN);
+			return (EAGAIN);
+		}
+		mp->mutex_rcount++;
+		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0);
+		return (0);
+	}
+	if (try == MUTEX_LOCK) {
+		DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
+		return (EDEADLK);
+	}
+	return (EBUSY);
+}
+
+/*
+ * Register this USYNC_PROCESS|LOCK_ROBUST mutex with the kernel so
+ * it can apply LOCK_OWNERDEAD|LOCK_UNMAPPED if it becomes necessary.
+ * We use tdb_hash_lock here and in the synch object tracking code in
+ * the tdb_agent.c file.  There is no conflict between these two usages.
+ */
+void
+register_lock(mutex_t *mp)
+{
+	uberdata_t *udp = curthread->ul_uberdata;
+	uint_t hash = LOCK_HASH(mp);
+	robust_t *rlp;
+	robust_t **rlpp;
+	robust_t **table;
+
+	if ((table = udp->robustlocks) == NULL) {
+		lmutex_lock(&udp->tdb_hash_lock);
+		if ((table = udp->robustlocks) == NULL) {
+			table = lmalloc(LOCKHASHSZ * sizeof (robust_t *));
+			_membar_producer();
+			udp->robustlocks = table;
+		}
+		lmutex_unlock(&udp->tdb_hash_lock);
+	}
+	_membar_consumer();
+
+	/*
+	 * First search the registered table with no locks held.
+	 * This is safe because the table never shrinks
+	 * and we can only get a false negative.
+	 */
+	for (rlp = table[hash]; rlp != NULL; rlp = rlp->robust_next) {
+		if (rlp->robust_lock == mp)	/* already registered */
+			return;
+	}
+
+	/*
+	 * The lock was not found.
+	 * Repeat the operation with tdb_hash_lock held.
+	 */
+	lmutex_lock(&udp->tdb_hash_lock);
+
+	for (rlpp = &table[hash];
+	    (rlp = *rlpp) != NULL;
+	    rlpp = &rlp->robust_next) {
+		if (rlp->robust_lock == mp) {	/* already registered */
+			lmutex_unlock(&udp->tdb_hash_lock);
+			return;
+		}
+	}
+
+	/*
+	 * The lock has never been registered.
+	 * Register it now and add it to the table.
+	 */
+	(void) ___lwp_mutex_register(mp);
+	rlp = lmalloc(sizeof (*rlp));
+	rlp->robust_lock = mp;
+	_membar_producer();
+	*rlpp = rlp;
+
+	lmutex_unlock(&udp->tdb_hash_lock);
+}
+
+/*
+ * This is called in the child of fork()/forkall() to start over
+ * with a clean slate.  (Each process must register its own locks.)
+ * No locks are needed because all other threads are suspended or gone.
+ */
+void
+unregister_locks(void)
+{
+	uberdata_t *udp = curthread->ul_uberdata;
+	uint_t hash;
+	robust_t **table;
+	robust_t *rlp;
+	robust_t *next;
+
+	if ((table = udp->robustlocks) != NULL) {
+		for (hash = 0; hash < LOCKHASHSZ; hash++) {
+			rlp = table[hash];
+			while (rlp != NULL) {
+				next = rlp->robust_next;
+				lfree(rlp, sizeof (*rlp));
+				rlp = next;
+			}
+		}
+		lfree(table, LOCKHASHSZ * sizeof (robust_t *));
+		udp->robustlocks = NULL;
+	}
+}
+
 /*
  * Returns with mutex_owner set correctly.
  */
-int
+static int
 mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try)
 {
 	ulwp_t *self = curthread;
@@ -1407,6 +1717,8 @@ mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try)
 	int mtype = mp->mutex_type;
 	tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
 	int error = 0;
+	uint8_t ceil;
+	int myprio;
 
 	ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK);
 
@@ -1416,184 +1728,86 @@ mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try)
 	if (msp && try == MUTEX_TRY)
 		tdb_incr(msp->mutex_try);
 
-	if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_is_held(mp)) {
-		if (mtype & LOCK_RECURSIVE) {
-			if (mp->mutex_rcount == RECURSION_MAX) {
-				error = EAGAIN;
-			} else {
-				mp->mutex_rcount++;
-				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
-				    1, 0);
-				return (0);
-			}
-		} else if (try == MUTEX_TRY) {
-			return (EBUSY);
-		} else {
-			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
-			return (EDEADLK);
-		}
-	}
+	if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_is_held(mp))
+		return (mutex_recursion(mp, mtype, try));
 
 	if (self->ul_error_detection && try == MUTEX_LOCK &&
 	    tsp == NULL && mutex_is_held(mp))
 		lock_error(mp, "mutex_lock", NULL, NULL);
 
-	if (mtype &
-	    (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) {
-		uint8_t ceil;
-		int myprio;
-
-		if (mtype & PTHREAD_PRIO_PROTECT) {
-			ceil = mp->mutex_ceiling;
-			ASSERT(_validate_rt_prio(SCHED_FIFO, ceil) == 0);
-			myprio = real_priority(self);
-			if (myprio > ceil) {
-				DTRACE_PROBE2(plockstat, mutex__error, mp,
-				    EINVAL);
-				return (EINVAL);
-			}
-			if ((error = _ceil_mylist_add(mp)) != 0) {
-				DTRACE_PROBE2(plockstat, mutex__error, mp,
-				    error);
-				return (error);
-			}
-			if (myprio < ceil)
-				_ceil_prio_inherit(ceil);
+	if (mtype & LOCK_PRIO_PROTECT) {
+		ceil = mp->mutex_ceiling;
+		ASSERT(_validate_rt_prio(SCHED_FIFO, ceil) == 0);
+		myprio = real_priority(self);
+		if (myprio > ceil) {
+			DTRACE_PROBE2(plockstat, mutex__error, mp, EINVAL);
+			return (EINVAL);
 		}
-
-		if (mtype & PTHREAD_PRIO_INHERIT) {
-			/* go straight to the kernel */
-			if (try == MUTEX_TRY)
-				error = mutex_trylock_kernel(mp);
-			else	/* MUTEX_LOCK */
-				error = mutex_lock_kernel(mp, tsp, msp);
-			/*
-			 * The kernel never sets or clears the lock byte
-			 * for PTHREAD_PRIO_INHERIT mutexes.
-			 * Set it here for debugging consistency.
-			 */
-			switch (error) {
-			case 0:
-			case EOWNERDEAD:
-				mp->mutex_lockw = LOCKSET;
-				break;
-			}
-		} else if (mtype & USYNC_PROCESS_ROBUST) {
-			/* go straight to the kernel */
-			if (try == MUTEX_TRY)
-				error = mutex_trylock_kernel(mp);
-			else	/* MUTEX_LOCK */
-				error = mutex_lock_kernel(mp, tsp, msp);
-		} else {	/* PTHREAD_PRIO_PROTECT */
-			/*
-			 * Try once at user level before going to the kernel.
-			 * If this is a process shared mutex then protect
-			 * against forkall() while setting mp->mutex_ownerpid.
-			 */
-			if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) {
-				enter_critical(self);
-				if (set_lock_byte(&mp->mutex_lockw) == 0) {
-					mp->mutex_owner = (uintptr_t)self;
-					mp->mutex_ownerpid = udp->pid;
-					exit_critical(self);
-					DTRACE_PROBE3(plockstat,
-					    mutex__acquire, mp, 0, 0);
-				} else {
-					exit_critical(self);
-					error = EBUSY;
-				}
-			} else {
-				if (set_lock_byte(&mp->mutex_lockw) == 0) {
-					mp->mutex_owner = (uintptr_t)self;
-					DTRACE_PROBE3(plockstat,
-					    mutex__acquire, mp, 0, 0);
-				} else {
-					error = EBUSY;
-				}
-			}
-			if (error && try == MUTEX_LOCK)
-				error = mutex_lock_kernel(mp, tsp, msp);
+		if ((error = _ceil_mylist_add(mp)) != 0) {
+			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
+			return (error);
 		}
+		if (myprio < ceil)
+			_ceil_prio_inherit(ceil);
+	}
 
-		if (error) {
-			if (mtype & PTHREAD_PRIO_INHERIT) {
-				switch (error) {
-				case EOWNERDEAD:
-				case ENOTRECOVERABLE:
-					if (mtype & PTHREAD_MUTEX_ROBUST_NP)
-						break;
-					if (error == EOWNERDEAD) {
-						/*
-						 * We own the mutex; unlock it.
-						 * It becomes ENOTRECOVERABLE.
-						 * All waiters are waked up.
-						 */
-						mp->mutex_owner = 0;
-						mp->mutex_ownerpid = 0;
-						DTRACE_PROBE2(plockstat,
-						    mutex__release, mp, 0);
-						mp->mutex_lockw = LOCKCLEAR;
-						(void) ___lwp_mutex_unlock(mp);
-					}
-					/* FALLTHROUGH */
-				case EDEADLK:
-					if (try == MUTEX_LOCK)
-						stall();
-					error = EBUSY;
-					break;
-				}
-			}
-			if ((mtype & PTHREAD_PRIO_PROTECT) &&
-			    error != EOWNERDEAD) {
-				(void) _ceil_mylist_del(mp);
-				if (myprio < ceil)
-					_ceil_prio_waive();
-			}
-		}
-	} else if (mtype & USYNC_PROCESS) {
+	if ((mtype & (USYNC_PROCESS | LOCK_ROBUST))
+	    == (USYNC_PROCESS | LOCK_ROBUST))
+		register_lock(mp);
+
+	if (mtype & LOCK_PRIO_INHERIT) {
+		/* go straight to the kernel */
+		if (try == MUTEX_TRY)
+			error = mutex_trylock_kernel(mp);
+		else	/* MUTEX_LOCK */
+			error = mutex_lock_kernel(mp, tsp, msp);
 		/*
-		 * This is a process shared mutex.  Protect against
-		 * forkall() while setting mp->mutex_ownerpid.
+		 * The kernel never sets or clears the lock byte
+		 * for LOCK_PRIO_INHERIT mutexes.
+		 * Set it here for consistency.
 		 */
-		enter_critical(self);
-		if (set_lock_byte(&mp->mutex_lockw) == 0) {
-			mp->mutex_owner = (uintptr_t)self;
-			mp->mutex_ownerpid = udp->pid;
-			exit_critical(self);
-			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
-		} else {
-			/* try a little harder */
-			exit_critical(self);
-			error = mutex_trylock_process(mp);
+		switch (error) {
+		case 0:
+			mp->mutex_lockw = LOCKSET;
+			break;
+		case EOWNERDEAD:
+		case ELOCKUNMAPPED:
+			mp->mutex_lockw = LOCKSET;
+			/* FALLTHROUGH */
+		case ENOTRECOVERABLE:
+			ASSERT(mtype & LOCK_ROBUST);
+			break;
+		case EDEADLK:
+			if (try == MUTEX_LOCK)
+				stall();
+			error = EBUSY;
+			break;
 		}
-		if (error && try == MUTEX_LOCK)
+	} else if (mtype & USYNC_PROCESS) {
+		error = mutex_trylock_process(mp);
+		if (error == EBUSY && try == MUTEX_LOCK)
 			error = mutex_lock_kernel(mp, tsp, msp);
 	} else  {	/* USYNC_THREAD */
-		/* try once */
-		if (set_lock_byte(&mp->mutex_lockw) == 0) {
-			mp->mutex_owner = (uintptr_t)self;
-			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
-		} else {
-			/* try a little harder if we don't own the mutex */
-			error = EBUSY;
-			if (MUTEX_OWNER(mp) != self)
-				error = mutex_trylock_adaptive(mp);
-			if (error && try == MUTEX_LOCK)		/* go park */
-				error = mutex_lock_queue(self, msp, mp, tsp);
-		}
+		error = mutex_trylock_adaptive(mp);
+		if (error == EBUSY && try == MUTEX_LOCK)
+			error = mutex_lock_queue(self, msp, mp, tsp);
 	}
 
 	switch (error) {
+	case 0:
 	case EOWNERDEAD:
 	case ELOCKUNMAPPED:
-		mp->mutex_owner = (uintptr_t)self;
-		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
-		/* FALLTHROUGH */
-	case 0:
+		if (mtype & LOCK_ROBUST)
+			remember_lock(mp);
 		if (msp)
 			record_begin_hold(msp);
 		break;
 	default:
+		if (mtype & LOCK_PRIO_PROTECT) {
+			(void) _ceil_mylist_del(mp);
+			if (myprio < ceil)
+				_ceil_prio_waive();
+		}
 		if (try == MUTEX_TRY) {
 			if (msp)
 				tdb_incr(msp->mutex_try_fail);
@@ -1619,6 +1833,7 @@ fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try)
 	 * zero, one, or both of the flags LOCK_RECURSIVE and
 	 * LOCK_ERRORCHECK are set, and that no other flags are set.
 	 */
+	ASSERT((mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0);
 	enter_critical(self);
 	if (set_lock_byte(&mp->mutex_lockw) == 0) {
 		mp->mutex_owner = (uintptr_t)self;
@@ -1629,23 +1844,11 @@ fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try)
 	}
 	exit_critical(self);
 
-	if ((mtype & ~USYNC_PROCESS) && shared_mutex_held(mp)) {
-		if (mtype & LOCK_RECURSIVE) {
-			if (mp->mutex_rcount == RECURSION_MAX)
-				return (EAGAIN);
-			mp->mutex_rcount++;
-			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0);
-			return (0);
-		}
-		if (try == MUTEX_LOCK) {
-			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
-			return (EDEADLK);
-		}
-		return (EBUSY);
-	}
+	if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && shared_mutex_held(mp))
+		return (mutex_recursion(mp, mtype, try));
 
-	/* try a little harder if we don't own the mutex */
-	if (!shared_mutex_held(mp) && mutex_trylock_process(mp) == 0)
+	/* try a little harder */
+	if (mutex_trylock_process(mp) == 0)
 		return (0);
 
 	if (try == MUTEX_LOCK)
@@ -1659,16 +1862,6 @@ fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try)
 }
 
 static int
-slow_lock(ulwp_t *self, mutex_t *mp, timespec_t *tsp)
-{
-	int error = 0;
-
-	if (MUTEX_OWNER(mp) == self || mutex_trylock_adaptive(mp) != 0)
-		error = mutex_lock_queue(self, NULL, mp, tsp);
-	return (error);
-}
-
-int
 mutex_lock_impl(mutex_t *mp, timespec_t *tsp)
 {
 	ulwp_t *self = curthread;
@@ -1694,21 +1887,8 @@ mutex_lock_impl(mutex_t *mp, timespec_t *tsp)
 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
 			return (0);
 		}
-		if (mtype && MUTEX_OWNER(mp) == self) {
-			/*
-			 * LOCK_RECURSIVE, LOCK_ERRORCHECK, or both.
-			 */
-			if (mtype & LOCK_RECURSIVE) {
-				if (mp->mutex_rcount == RECURSION_MAX)
-					return (EAGAIN);
-				mp->mutex_rcount++;
-				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
-				    1, 0);
-				return (0);
-			}
-			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
-			return (EDEADLK);	/* LOCK_ERRORCHECK */
-		}
+		if (mtype && MUTEX_OWNER(mp) == self)
+			return (mutex_recursion(mp, mtype, MUTEX_LOCK));
 		/*
 		 * We have reached a deadlock, probably because the
 		 * process is executing non-async-signal-safe code in
@@ -1736,30 +1916,18 @@ mutex_lock_impl(mutex_t *mp, timespec_t *tsp)
 	if ((gflags = self->ul_schedctl_called) != NULL &&
 	    (gflags->uf_trs_ted |
 	    (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
-
 		if (mtype & USYNC_PROCESS)
 			return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK));
-
 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
 			mp->mutex_owner = (uintptr_t)self;
 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
 			return (0);
 		}
-
-		if (mtype && MUTEX_OWNER(mp) == self) {
-			if (mtype & LOCK_RECURSIVE) {
-				if (mp->mutex_rcount == RECURSION_MAX)
-					return (EAGAIN);
-				mp->mutex_rcount++;
-				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
-				    1, 0);
-				return (0);
-			}
-			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
-			return (EDEADLK);	/* LOCK_ERRORCHECK */
-		}
-
-		return (slow_lock(self, mp, tsp));
+		if (mtype && MUTEX_OWNER(mp) == self)
+			return (mutex_recursion(mp, mtype, MUTEX_LOCK));
+		if (mutex_trylock_adaptive(mp) != 0)
+			return (mutex_lock_queue(self, NULL, mp, tsp));
+		return (0);
 	}
 
 	/* else do it the long way */
@@ -1808,22 +1976,6 @@ _pthread_mutex_reltimedlock_np(mutex_t *mp, const timespec_t *reltime)
 	return (error);
 }
 
-static int
-slow_trylock(mutex_t *mp, ulwp_t *self)
-{
-	if (MUTEX_OWNER(mp) == self ||
-	    mutex_trylock_adaptive(mp) != 0) {
-		uberdata_t *udp = self->ul_uberdata;
-
-		if (__td_event_report(self, TD_LOCK_TRY, udp)) {
-			self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
-			tdb_event(TD_LOCK_TRY, udp);
-		}
-		return (EBUSY);
-	}
-	return (0);
-}
-
 #pragma weak _private_mutex_trylock = __mutex_trylock
 #pragma weak mutex_trylock = __mutex_trylock
 #pragma weak _mutex_trylock = __mutex_trylock
@@ -1856,17 +2008,8 @@ __mutex_trylock(mutex_t *mp)
 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
 			return (0);
 		}
-		if (mtype && MUTEX_OWNER(mp) == self) {
-			if (mtype & LOCK_RECURSIVE) {
-				if (mp->mutex_rcount == RECURSION_MAX)
-					return (EAGAIN);
-				mp->mutex_rcount++;
-				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
-				    1, 0);
-				return (0);
-			}
-			return (EDEADLK);	/* LOCK_ERRORCHECK */
-		}
+		if (mtype && MUTEX_OWNER(mp) == self)
+			return (mutex_recursion(mp, mtype, MUTEX_TRY));
 		return (EBUSY);
 	}
 
@@ -1878,29 +2021,23 @@ __mutex_trylock(mutex_t *mp)
 	if ((gflags = self->ul_schedctl_called) != NULL &&
 	    (gflags->uf_trs_ted |
 	    (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
-
 		if (mtype & USYNC_PROCESS)
 			return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY));
-
 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
 			mp->mutex_owner = (uintptr_t)self;
 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
 			return (0);
 		}
-
-		if (mtype && MUTEX_OWNER(mp) == self) {
-			if (mtype & LOCK_RECURSIVE) {
-				if (mp->mutex_rcount == RECURSION_MAX)
-					return (EAGAIN);
-				mp->mutex_rcount++;
-				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
-				    1, 0);
-				return (0);
+		if (mtype && MUTEX_OWNER(mp) == self)
+			return (mutex_recursion(mp, mtype, MUTEX_TRY));
+		if (mutex_trylock_adaptive(mp) != 0) {
+			if (__td_event_report(self, TD_LOCK_TRY, udp)) {
+				self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
+				tdb_event(TD_LOCK_TRY, udp);
 			}
-			return (EBUSY);		/* LOCK_ERRORCHECK */
+			return (EBUSY);
 		}
-
-		return (slow_trylock(mp, self));
+		return (0);
 	}
 
 	/* else do it the long way */
@@ -1908,13 +2045,14 @@ __mutex_trylock(mutex_t *mp)
 }
 
 int
-mutex_unlock_internal(mutex_t *mp)
+mutex_unlock_internal(mutex_t *mp, int retain_robust_flags)
 {
 	ulwp_t *self = curthread;
 	uberdata_t *udp = self->ul_uberdata;
 	int mtype = mp->mutex_type;
 	tdb_mutex_stats_t *msp;
-	int error;
+	int error = 0;
+	int release_all;
 	lwpid_t lwpid;
 
 	if ((mtype & LOCK_ERRORCHECK) && !mutex_is_held(mp))
@@ -1932,50 +2070,49 @@ mutex_unlock_internal(mutex_t *mp)
 	if ((msp = MUTEX_STATS(mp, udp)) != NULL)
 		(void) record_hold_time(msp);
 
-	if (mtype &
-	    (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) {
+	if (!retain_robust_flags && !(mtype & LOCK_PRIO_INHERIT) &&
+	    (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) {
+		ASSERT(mp->mutex_type & LOCK_ROBUST);
+		mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED);
+		mp->mutex_flag |= LOCK_NOTRECOVERABLE;
+	}
+	release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0);
+
+	if (mtype & LOCK_PRIO_INHERIT) {
 		no_preempt(self);
 		mp->mutex_owner = 0;
 		mp->mutex_ownerpid = 0;
 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
-		if (mtype & PTHREAD_PRIO_INHERIT) {
-			mp->mutex_lockw = LOCKCLEAR;
-			error = ___lwp_mutex_unlock(mp);
-		} else if (mtype & USYNC_PROCESS_ROBUST) {
-			error = ___lwp_mutex_unlock(mp);
-		} else {
-			if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK)
-				(void) ___lwp_mutex_wakeup(mp);
-			error = 0;
-		}
-		if (mtype & PTHREAD_PRIO_PROTECT) {
-			if (_ceil_mylist_del(mp))
-				_ceil_prio_waive();
-		}
+		mp->mutex_lockw = LOCKCLEAR;
+		error = ___lwp_mutex_unlock(mp);
 		preempt(self);
 	} else if (mtype & USYNC_PROCESS) {
-		if (mp->mutex_lockword & WAITERMASK)
-			mutex_unlock_process(mp);
-		else {
+		if (mp->mutex_lockword & WAITERMASK) {
+			mutex_unlock_process(mp, release_all);
+		} else {
 			mp->mutex_owner = 0;
 			mp->mutex_ownerpid = 0;
 			DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
 			if (atomic_swap_32(&mp->mutex_lockword, 0) &
-			    WAITERMASK) {
+			    WAITERMASK) {  /* a waiter suddenly appeared */
 				no_preempt(self);
-				(void) ___lwp_mutex_wakeup(mp);
+				(void) ___lwp_mutex_wakeup(mp, release_all);
 				preempt(self);
 			}
 		}
-		error = 0;
 	} else {	/* USYNC_THREAD */
-		if ((lwpid = mutex_unlock_queue(mp)) != 0) {
+		if ((lwpid = mutex_unlock_queue(mp, release_all)) != 0) {
 			(void) __lwp_unpark(lwpid);
 			preempt(self);
 		}
-		error = 0;
 	}
 
+	if (mtype & LOCK_ROBUST)
+		forget_lock(mp);
+
+	if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp))
+		_ceil_prio_waive();
+
 	return (error);
 }
 
@@ -2046,7 +2183,7 @@ fast_unlock:
 						(void) __lwp_unpark(lwpid);
 					preempt(self);
 				}
-			} else if ((lwpid = mutex_unlock_queue(mp)) != 0) {
+			} else if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) {
 				(void) __lwp_unpark(lwpid);
 				preempt(self);
 			}
@@ -2082,16 +2219,16 @@ fast_unlock:
 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
 				return (0);
 			}
-			if (mp->mutex_lockword & WAITERMASK)
-				mutex_unlock_process(mp);
-			else {
+			if (mp->mutex_lockword & WAITERMASK) {
+				mutex_unlock_process(mp, 0);
+			} else {
 				mp->mutex_owner = 0;
 				mp->mutex_ownerpid = 0;
 				DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
 				if (atomic_swap_32(&mp->mutex_lockword, 0) &
 				    WAITERMASK) {
 					no_preempt(self);
-					(void) ___lwp_mutex_wakeup(mp);
+					(void) ___lwp_mutex_wakeup(mp, 0);
 					preempt(self);
 				}
 			}
@@ -2101,7 +2238,7 @@ fast_unlock:
 
 	/* else do it the long way */
 slow_unlock:
-	return (mutex_unlock_internal(mp));
+	return (mutex_unlock_internal(mp, 0));
 }
 
 /*
@@ -2176,7 +2313,7 @@ lmutex_unlock(mutex_t *mp)
 
 		if (msp)
 			(void) record_hold_time(msp);
-		if ((lwpid = mutex_unlock_queue(mp)) != 0) {
+		if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) {
 			(void) __lwp_unpark(lwpid);
 			preempt(self);
 		}
@@ -2259,39 +2396,23 @@ static int
 shared_mutex_held(mutex_t *mparg)
 {
 	/*
-	 * There is an inherent data race in the current ownership design.
-	 * The mutex_owner and mutex_ownerpid fields cannot be set or tested
-	 * atomically as a pair. The original implementation tested each
-	 * field just once. This was exposed to trivial false positives in
-	 * the case of multiple multithreaded processes with thread addresses
-	 * in common. To close the window to an acceptable level we now use a
-	 * sequence of five tests: pid-thr-pid-thr-pid. This ensures that any
-	 * single interruption will still leave one uninterrupted sequence of
-	 * pid-thr-pid tests intact.
-	 *
-	 * It is assumed that all updates are always ordered thr-pid and that
-	 * we have TSO hardware.
+	 * The 'volatile' is necessary to make sure the compiler doesn't
+	 * reorder the tests of the various components of the mutex.
+	 * They must be tested in this order:
+	 *	mutex_lockw
+	 *	mutex_owner
+	 *	mutex_ownerpid
+	 * This relies on the fact that everywhere mutex_lockw is cleared,
+	 * mutex_owner and mutex_ownerpid are cleared before mutex_lockw
+	 * is cleared, and that everywhere mutex_lockw is set, mutex_owner
+	 * and mutex_ownerpid are set after mutex_lockw is set, and that
+	 * mutex_lockw is set or cleared with a memory barrier.
 	 */
 	volatile mutex_t *mp = (volatile mutex_t *)mparg;
 	ulwp_t *self = curthread;
 	uberdata_t *udp = self->ul_uberdata;
 
-	if (mp->mutex_ownerpid != udp->pid)
-		return (0);
-
-	if (!MUTEX_OWNED(mp, self))
-		return (0);
-
-	if (mp->mutex_ownerpid != udp->pid)
-		return (0);
-
-	if (!MUTEX_OWNED(mp, self))
-		return (0);
-
-	if (mp->mutex_ownerpid != udp->pid)
-		return (0);
-
-	return (1);
+	return (MUTEX_OWNED(mp, self) && mp->mutex_ownerpid == udp->pid);
 }
 
 /*
@@ -2305,10 +2426,12 @@ shared_mutex_held(mutex_t *mparg)
 #pragma weak _mutex_held = mutex_is_held
 #pragma weak __mutex_held = mutex_is_held
 int
-mutex_is_held(mutex_t *mp)
+mutex_is_held(mutex_t *mparg)
 {
-	if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))
-		return (shared_mutex_held(mp));
+	volatile mutex_t *mp = (volatile mutex_t *)mparg;
+
+	if (mparg->mutex_type & USYNC_PROCESS)
+		return (shared_mutex_held(mparg));
 	return (MUTEX_OWNED(mp, curthread));
 }
 
@@ -2320,12 +2443,35 @@ mutex_is_held(mutex_t *mp)
 int
 __mutex_destroy(mutex_t *mp)
 {
-	mp->mutex_magic = 0;
-	mp->mutex_flag &= ~LOCK_INITED;
+	if (mp->mutex_type & USYNC_PROCESS)
+		forget_lock(mp);
+	(void) _memset(mp, 0, sizeof (*mp));
 	tdb_sync_obj_deregister(mp);
 	return (0);
 }
 
+#pragma weak mutex_consistent = __mutex_consistent
+#pragma weak _mutex_consistent = __mutex_consistent
+#pragma weak pthread_mutex_consistent_np = __mutex_consistent
+#pragma weak _pthread_mutex_consistent_np = __mutex_consistent
+int
+__mutex_consistent(mutex_t *mp)
+{
+	/*
+	 * Do this only for an inconsistent, initialized robust lock
+	 * that we hold.  For all other cases, return EINVAL.
+	 */
+	if (mutex_is_held(mp) &&
+	    (mp->mutex_type & LOCK_ROBUST) &&
+	    (mp->mutex_flag & LOCK_INITED) &&
+	    (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) {
+		mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED);
+		mp->mutex_rcount = 0;
+		return (0);
+	}
+	return (EINVAL);
+}
+
 /*
  * Spin locks are separate from ordinary mutexes,
  * but we use the same data structure for them.
@@ -2380,21 +2526,37 @@ _pthread_spin_trylock(pthread_spinlock_t *lock)
 int
 _pthread_spin_lock(pthread_spinlock_t *lock)
 {
-	volatile uint8_t *lockp =
-		(volatile uint8_t *)&((mutex_t *)lock)->mutex_lockw;
+	mutex_t *mp = (mutex_t *)lock;
+	ulwp_t *self = curthread;
+	volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw;
+	int count = 0;
+
+	ASSERT(!self->ul_critical || self->ul_bindflags);
+
+	DTRACE_PROBE1(plockstat, mutex__spin, mp);
 
-	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
 	/*
 	 * We don't care whether the owner is running on a processor.
 	 * We just spin because that's what this interface requires.
 	 */
 	for (;;) {
+		if (count < INT_MAX)
+			count++;
 		if (*lockp == 0) {	/* lock byte appears to be clear */
-			if (_pthread_spin_trylock(lock) == 0)
-				return (0);
+			no_preempt(self);
+			if (set_lock_byte(lockp) == 0)
+				break;
+			preempt(self);
 		}
 		SMT_PAUSE();
 	}
+	mp->mutex_owner = (uintptr_t)self;
+	if (mp->mutex_type == USYNC_PROCESS)
+		mp->mutex_ownerpid = self->ul_uberdata->pid;
+	preempt(self);
+	DTRACE_PROBE2(plockstat, mutex__spun, 1, count);
+	DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
+	return (0);
 }
 
 #pragma weak pthread_spin_unlock = _pthread_spin_unlock
@@ -2413,6 +2575,148 @@ _pthread_spin_unlock(pthread_spinlock_t *lock)
 	return (0);
 }
 
+#define	INITIAL_LOCKS	8	/* initialial size of ul_heldlocks.array */
+
+/*
+ * Find/allocate an entry for 'lock' in our array of held locks.
+ */
+static mutex_t **
+find_lock_entry(mutex_t *lock)
+{
+	ulwp_t *self = curthread;
+	mutex_t **remembered = NULL;
+	mutex_t **lockptr;
+	uint_t nlocks;
+
+	if ((nlocks = self->ul_heldlockcnt) != 0)
+		lockptr = self->ul_heldlocks.array;
+	else {
+		nlocks = 1;
+		lockptr = &self->ul_heldlocks.single;
+	}
+
+	for (; nlocks; nlocks--, lockptr++) {
+		if (*lockptr == lock)
+			return (lockptr);
+		if (*lockptr == NULL && remembered == NULL)
+			remembered = lockptr;
+	}
+	if (remembered != NULL) {
+		*remembered = lock;
+		return (remembered);
+	}
+
+	/*
+	 * No entry available.  Allocate more space, converting
+	 * the single entry into an array of entries if necessary.
+	 */
+	if ((nlocks = self->ul_heldlockcnt) == 0) {
+		/*
+		 * Initial allocation of the array.
+		 * Convert the single entry into an array.
+		 */
+		self->ul_heldlockcnt = nlocks = INITIAL_LOCKS;
+		lockptr = lmalloc(nlocks * sizeof (mutex_t *));
+		/*
+		 * The single entry becomes the first entry in the array.
+		 */
+		*lockptr = self->ul_heldlocks.single;
+		self->ul_heldlocks.array = lockptr;
+		/*
+		 * Return the next available entry in the array.
+		 */
+		*++lockptr = lock;
+		return (lockptr);
+	}
+	/*
+	 * Reallocate the array, double the size each time.
+	 */
+	lockptr = lmalloc(nlocks * 2 * sizeof (mutex_t *));
+	(void) _memcpy(lockptr, self->ul_heldlocks.array,
+	    nlocks * sizeof (mutex_t *));
+	lfree(self->ul_heldlocks.array, nlocks * sizeof (mutex_t *));
+	self->ul_heldlocks.array = lockptr;
+	self->ul_heldlockcnt *= 2;
+	/*
+	 * Return the next available entry in the newly allocated array.
+	 */
+	*(lockptr += nlocks) = lock;
+	return (lockptr);
+}
+
+/*
+ * Insert 'lock' into our list of held locks.
+ * Currently only used for LOCK_ROBUST mutexes.
+ */
+void
+remember_lock(mutex_t *lock)
+{
+	(void) find_lock_entry(lock);
+}
+
+/*
+ * Remove 'lock' from our list of held locks.
+ * Currently only used for LOCK_ROBUST mutexes.
+ */
+void
+forget_lock(mutex_t *lock)
+{
+	*find_lock_entry(lock) = NULL;
+}
+
+/*
+ * Free the array of held locks.
+ */
+void
+heldlock_free(ulwp_t *ulwp)
+{
+	uint_t nlocks;
+
+	if ((nlocks = ulwp->ul_heldlockcnt) != 0)
+		lfree(ulwp->ul_heldlocks.array, nlocks * sizeof (mutex_t *));
+	ulwp->ul_heldlockcnt = 0;
+	ulwp->ul_heldlocks.array = NULL;
+}
+
+/*
+ * Mark all held LOCK_ROBUST mutexes LOCK_OWNERDEAD.
+ * Called from _thrp_exit() to deal with abandoned locks.
+ */
+void
+heldlock_exit(void)
+{
+	ulwp_t *self = curthread;
+	mutex_t **lockptr;
+	uint_t nlocks;
+	mutex_t *mp;
+
+	if ((nlocks = self->ul_heldlockcnt) != 0)
+		lockptr = self->ul_heldlocks.array;
+	else {
+		nlocks = 1;
+		lockptr = &self->ul_heldlocks.single;
+	}
+
+	for (; nlocks; nlocks--, lockptr++) {
+		/*
+		 * The kernel takes care of transitioning held
+		 * LOCK_PRIO_INHERIT mutexes to LOCK_OWNERDEAD.
+		 * We avoid that case here.
+		 */
+		if ((mp = *lockptr) != NULL &&
+		    mutex_is_held(mp) &&
+		    (mp->mutex_type & (LOCK_ROBUST | LOCK_PRIO_INHERIT)) ==
+		    LOCK_ROBUST) {
+			mp->mutex_rcount = 0;
+			if (!(mp->mutex_flag & LOCK_UNMAPPED))
+				mp->mutex_flag |= LOCK_OWNERDEAD;
+			(void) mutex_unlock_internal(mp, 1);
+		}
+	}
+
+	heldlock_free(self);
+}
+
 #pragma weak cond_init = _cond_init
 /* ARGSUSED2 */
 int
@@ -2437,7 +2741,7 @@ _cond_init(cond_t *cvp, int type, void *arg)
  * The associated mutex is *not* reacquired before returning.
  * That must be done by the caller of cond_sleep_queue().
  */
-int
+static int
 cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
 {
 	ulwp_t *self = curthread;
@@ -2446,6 +2750,7 @@ cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
 	lwpid_t lwpid;
 	int signalled;
 	int error;
+	int release_all;
 
 	/*
 	 * Put ourself on the CV sleep queue, unlock the mutex, then
@@ -2460,7 +2765,12 @@ cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
 	self->ul_cvmutex = mp;
 	self->ul_cv_wake = (tsp != NULL);
 	self->ul_signalled = 0;
-	lwpid = mutex_unlock_queue(mp);
+	if (mp->mutex_flag & LOCK_OWNERDEAD) {
+		mp->mutex_flag &= ~LOCK_OWNERDEAD;
+		mp->mutex_flag |= LOCK_NOTRECOVERABLE;
+	}
+	release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0);
+	lwpid = mutex_unlock_queue(mp, release_all);
 	for (;;) {
 		set_parking_flag(self, 1);
 		queue_unlock(qp);
@@ -2549,6 +2859,7 @@ cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp,
 {
 	ulwp_t *self = curthread;
 	int error;
+	int merror;
 
 	/*
 	 * The old thread library was programmed to defer signals
@@ -2572,14 +2883,11 @@ cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp,
 	/*
 	 * Reacquire the mutex.
 	 */
-	if (set_lock_byte(&mp->mutex_lockw) == 0) {
-		mp->mutex_owner = (uintptr_t)self;
-		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
-	} else if (mutex_trylock_adaptive(mp) != 0) {
-		(void) mutex_lock_queue(self, msp, mp, NULL);
-	}
-
-	if (msp)
+	if ((merror = mutex_trylock_adaptive(mp)) == EBUSY)
+		merror = mutex_lock_queue(self, msp, mp, NULL);
+	if (merror)
+		error = merror;
+	if (msp && (merror == 0 || merror == EOWNERDEAD))
 		record_begin_hold(msp);
 
 	/*
@@ -2595,23 +2903,21 @@ cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp,
  * cond_sleep_kernel(): utility function for cond_wait_kernel().
  * See the comment ahead of cond_sleep_queue(), above.
  */
-int
+static int
 cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
 {
 	int mtype = mp->mutex_type;
 	ulwp_t *self = curthread;
 	int error;
 
-	if (mtype & PTHREAD_PRIO_PROTECT) {
-		if (_ceil_mylist_del(mp))
-			_ceil_prio_waive();
-	}
+	if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp))
+		_ceil_prio_waive();
 
 	self->ul_sp = stkptr();
 	self->ul_wchan = cvp;
 	mp->mutex_owner = 0;
 	mp->mutex_ownerpid = 0;
-	if (mtype & PTHREAD_PRIO_INHERIT)
+	if (mtype & LOCK_PRIO_INHERIT)
 		mp->mutex_lockw = LOCKCLEAR;
 	/*
 	 * ___lwp_cond_wait() returns immediately with EINTR if
@@ -2721,12 +3027,12 @@ cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
 			lock_error(mp, "recursive mutex in cond_wait",
 				cvp, NULL);
 		if (cvp->cond_type & USYNC_PROCESS) {
-			if (!(mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)))
+			if (!(mtype & USYNC_PROCESS))
 				lock_error(mp, "cond_wait", cvp,
 					"condvar process-shared, "
 					"mutex process-private");
 		} else {
-			if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))
+			if (mtype & USYNC_PROCESS)
 				lock_error(mp, "cond_wait", cvp,
 					"condvar process-private, "
 					"mutex process-shared");
@@ -2741,8 +3047,8 @@ cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
 	 */
 	rcount = mp->mutex_rcount;
 	mp->mutex_rcount = 0;
-	if ((mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST |
-	    PTHREAD_PRIO_INHERIT | PTHREAD_PRIO_PROTECT)) |
+	if ((mtype &
+	    (USYNC_PROCESS | LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT)) |
 	    (cvp->cond_type & USYNC_PROCESS))
 		error = cond_wait_kernel(cvp, mp, tsp);
 	else
@@ -3005,10 +3311,10 @@ cond_signal_internal(cond_t *cvp)
 	 * deal properly with spurious wakeups.
 	 */
 	*ulwpp = ulwp->ul_link;
+	ulwp->ul_link = NULL;
 	if (qp->qh_tail == ulwp)
 		qp->qh_tail = prev;
 	qp->qh_qlen--;
-	ulwp->ul_link = NULL;
 
 	mp = ulwp->ul_cvmutex;		/* the mutex he will acquire */
 	ulwp->ul_cvmutex = NULL;
@@ -3036,12 +3342,12 @@ cond_signal_internal(cond_t *cvp)
 }
 
 /*
- * Utility function called from cond_broadcast() and rw_queue_release()
- * to (re)allocate a big buffer to hold the lwpids of all the threads
- * to be set running after they are removed from their sleep queues.
- * Since we are holding a queue lock, we cannot call any function
- * that might acquire a lock.  mmap(), munmap() and lwp_unpark_all()
- * are simple system calls and are safe in this regard.
+ * Utility function called by mutex_wakeup_all(), cond_broadcast(),
+ * and rw_queue_release() to (re)allocate a big buffer to hold the
+ * lwpids of all the threads to be set running after they are removed
+ * from their sleep queues.  Since we are holding a queue lock, we
+ * cannot call any function that might acquire a lock.  mmap(), munmap(),
+ * lwp_unpark_all() are simple system calls and are safe in this regard.
  */
 lwpid_t *
 alloc_lwpids(lwpid_t *lwpid, int *nlwpid_ptr, int *maxlwps_ptr)
@@ -3144,10 +3450,10 @@ cond_broadcast_internal(cond_t *cvp)
 			continue;
 		}
 		*ulwpp = ulwp->ul_link;
+		ulwp->ul_link = NULL;
 		if (qp->qh_tail == ulwp)
 			qp->qh_tail = prev;
 		qp->qh_qlen--;
-		ulwp->ul_link = NULL;
 		mp = ulwp->ul_cvmutex;		/* his mutex */
 		ulwp->ul_cvmutex = NULL;
 		ASSERT(mp != NULL);
diff --git a/usr/src/lib/libc/port/threads/thr.c b/usr/src/lib/libc/port/threads/thr.c
index affa5297eb..6df0608e1c 100644
--- a/usr/src/lib/libc/port/threads/thr.c
+++ b/usr/src/lib/libc/port/threads/thr.c
@@ -117,6 +117,7 @@ uberdata_t __uberdata = {
 	NULL,			/* ulwp_replace_free */
 	NULL,			/* ulwp_replace_last */
 	NULL,			/* atforklist */
+	NULL,			/* robustlocks */
 	NULL,			/* __tdb_bootstrap */
 	{			/* tdb */
 		NULL,		/* tdb_sync_addr_hash */
@@ -798,8 +799,9 @@ _thrp_exit()
 	}
 	lmutex_unlock(&udp->link_lock);
 
-	tsd_exit();	/* deallocate thread-specific data */
-	tls_exit();	/* deallocate thread-local storage */
+	tsd_exit();		/* deallocate thread-specific data */
+	tls_exit();		/* deallocate thread-local storage */
+	heldlock_exit();	/* deal with left-over held locks */
 
 	/* block all signals to finish exiting */
 	block_all_signals(self);
@@ -1564,6 +1566,7 @@ finish_init()
 	udp->hash_mask = HASHTBLSZ - 1;
 
 	for (i = 0; i < HASHTBLSZ; i++, htp++) {
+		htp->hash_lock.mutex_flag = LOCK_INITED;
 		htp->hash_lock.mutex_magic = MUTEX_MAGIC;
 		htp->hash_cond.cond_magic = COND_MAGIC;
 	}
@@ -1610,6 +1613,7 @@ postfork1_child()
 {
 	ulwp_t *self = curthread;
 	uberdata_t *udp = self->ul_uberdata;
+	mutex_t *mp;
 	ulwp_t *next;
 	ulwp_t *ulwp;
 	int i;
@@ -1629,8 +1633,11 @@ postfork1_child()
 	if (udp->queue_head) {
 		(void) _private_memset(udp->queue_head, 0,
 			2 * QHASHSIZE * sizeof (queue_head_t));
-		for (i = 0; i < 2 * QHASHSIZE; i++)
-			udp->queue_head[i].qh_lock.mutex_magic = MUTEX_MAGIC;
+		for (i = 0; i < 2 * QHASHSIZE; i++) {
+			mp = &udp->queue_head[i].qh_lock;
+			mp->mutex_flag = LOCK_INITED;
+			mp->mutex_magic = MUTEX_MAGIC;
+		}
 	}
 
 	/*
@@ -1650,6 +1657,7 @@ postfork1_child()
 		tsd_free(ulwp);
 		tls_free(ulwp);
 		rwl_free(ulwp);
+		heldlock_free(ulwp);
 		ulwp_free(ulwp);
 	}
 	self->ul_forw = self->ul_back = udp->all_lwps = self;
diff --git a/usr/src/lib/libc/sparc/sys/_lwp_mutex_unlock.s b/usr/src/lib/libc/sparc/sys/_lwp_mutex_unlock.s
index 5cf848d7b7..fb2e0f2dce 100644
--- a/usr/src/lib/libc/sparc/sys/_lwp_mutex_unlock.s
+++ b/usr/src/lib/libc/sparc/sys/_lwp_mutex_unlock.s
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -53,7 +53,8 @@
 	  clr 	%o0				! 	return 0
 						! else (note that %o0 is still
 						!	&mutex)
-	SYSTRAP_RVAL1(lwp_mutex_wakeup)		! call kernel to wakeup waiter
+	clr	%o1				! call kernel to wakeup waiter:
+	SYSTRAP_RVAL1(lwp_mutex_wakeup)		!    lwp_mutex_wakeup(mp, 0)
 	SYSLWPERR
 2: 	RET
 	SET_SIZE(_lwp_mutex_unlock)