summaryrefslogtreecommitdiff
path: root/usr/src
diff options
context:
space:
mode:
Diffstat (limited to 'usr/src')
-rw-r--r--usr/src/head/synch.h16
-rw-r--r--usr/src/lib/libc/amd64/threads/amd64.il18
-rw-r--r--usr/src/lib/libc/i386/threads/i386.il22
-rw-r--r--usr/src/lib/libc/inc/thr_inlines.h62
-rw-r--r--usr/src/lib/libc/inc/thr_uberdata.h37
-rw-r--r--usr/src/lib/libc/port/gen/atexit.c17
-rw-r--r--usr/src/lib/libc/port/sys/lwp_rwlock.c35
-rw-r--r--usr/src/lib/libc/port/threads/assfail.c49
-rw-r--r--usr/src/lib/libc/port/threads/rwlock.c785
-rw-r--r--usr/src/lib/libc/port/threads/sigaction.c27
-rw-r--r--usr/src/lib/libc/port/threads/synch.c182
-rw-r--r--usr/src/lib/libc/sparc/Makefile2
-rw-r--r--usr/src/lib/libc/sparc/threads/sparc.il28
-rw-r--r--usr/src/lib/libc/sparcv9/threads/sparcv9.il28
-rw-r--r--usr/src/lib/libc_db/common/thread_db.c92
-rw-r--r--usr/src/uts/common/sys/synch.h16
-rw-r--r--usr/src/uts/common/syscall/lwp_sobj.c80
-rw-r--r--usr/src/uts/intel/sys/synch32.h17
-rw-r--r--usr/src/uts/sparc/sys/synch32.h17
19 files changed, 847 insertions, 683 deletions
diff --git a/usr/src/head/synch.h b/usr/src/head/synch.h
index 8d825d55a8..629570a646 100644
--- a/usr/src/head/synch.h
+++ b/usr/src/head/synch.h
@@ -2,9 +2,8 @@
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License"). You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
*
* CDDL HEADER END
*/
+
/*
- * Copyright 2003 Sun Microsystems, Inc. All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
@@ -81,12 +81,12 @@ typedef lwp_cond_t cond_t;
* Because we have to deal with C++, we cannot redefine this one as that one.
*/
typedef struct _rwlock {
- int32_t readers; /* -1 == writer else # of readers */
+ int32_t readers; /* rwstate word */
uint16_t type;
uint16_t magic;
- mutex_t mutex; /* used to indicate ownership */
- cond_t readercv; /* unused */
- cond_t writercv; /* unused */
+ mutex_t mutex; /* used with process-shared rwlocks */
+ cond_t readercv; /* used only to indicate ownership */
+ cond_t writercv; /* used only to indicate ownership */
} rwlock_t;
#ifdef __STDC__
diff --git a/usr/src/lib/libc/amd64/threads/amd64.il b/usr/src/lib/libc/amd64/threads/amd64.il
index 92b55caaad..c0b81e0c4b 100644
--- a/usr/src/lib/libc/amd64/threads/amd64.il
+++ b/usr/src/lib/libc/amd64/threads/amd64.il
@@ -46,27 +46,37 @@
xchgb %al, (%rdi)
.end
- .inline cas32, 0
+ .inline atomic_cas_32, 0
movl %esi, %eax
lock
cmpxchgl %edx, (%rdi)
.end
- .inline swap32, 0
+ .inline atomic_swap_32, 0
movl %esi, %eax
xchgl (%rdi), %eax
.end
- .inline incr32, 0
+ .inline atomic_inc_32, 0
lock
incl (%rdi)
.end
- .inline decr32, 0
+ .inline atomic_dec_32, 0
lock
decl (%rdi)
.end
+ .inline atomic_and_32, 0
+ lock
+ andl %esi, (%rdi)
+ .end
+
+ .inline atomic_or_32, 0
+ lock
+ orl %esi, (%rdi)
+ .end
+
.inline ht_pause, 0
rep / "rep nop" is equivalent to "pause"
nop
diff --git a/usr/src/lib/libc/i386/threads/i386.il b/usr/src/lib/libc/i386/threads/i386.il
index e37be525e2..9ce94d452a 100644
--- a/usr/src/lib/libc/i386/threads/i386.il
+++ b/usr/src/lib/libc/i386/threads/i386.il
@@ -48,7 +48,7 @@
xchgb %al, (%ecx)
.end
- .inline cas32, 0
+ .inline atomic_cas_32, 0
movl 0(%esp), %edx
movl 4(%esp), %eax
movl 8(%esp), %ecx
@@ -56,24 +56,38 @@
cmpxchgl %ecx, (%edx)
.end
- .inline swap32, 0
+ .inline atomic_swap_32, 0
movl 0(%esp), %ecx
movl 4(%esp), %eax
xchgl (%ecx), %eax
.end
- .inline incr32, 0
+ .inline atomic_inc_32, 0
movl 0(%esp), %eax
lock
incl (%eax)
.end
- .inline decr32, 0
+ .inline atomic_dec_32, 0
movl 0(%esp), %eax
lock
decl (%eax)
.end
+ .inline atomic_and_32, 0
+ movl 0(%esp), %ecx
+ movl 4(%esp), %eax
+ lock
+ andl %eax, (%ecx)
+ .end
+
+ .inline atomic_or_32, 0
+ movl 0(%esp), %ecx
+ movl 4(%esp), %eax
+ lock
+ orl %eax, (%ecx)
+ .end
+
.inline ht_pause, 0
rep / "rep nop" is equivalent to "pause"
nop
diff --git a/usr/src/lib/libc/inc/thr_inlines.h b/usr/src/lib/libc/inc/thr_inlines.h
index 4e2f469488..55bd645428 100644
--- a/usr/src/lib/libc/inc/thr_inlines.h
+++ b/usr/src/lib/libc/inc/thr_inlines.h
@@ -147,7 +147,7 @@ set_lock_byte(volatile uint8_t *__lockp)
}
extern __inline__ uint32_t
-swap32(volatile uint32_t *__memory, uint32_t __value)
+atomic_swap_32(volatile uint32_t *__memory, uint32_t __value)
{
#if defined(__x86)
__asm__ __volatile__(
@@ -174,7 +174,7 @@ swap32(volatile uint32_t *__memory, uint32_t __value)
}
extern __inline__ uint32_t
-cas32(volatile uint32_t *__memory, uint32_t __cmp, uint32_t __newvalue)
+atomic_cas_32(volatile uint32_t *__memory, uint32_t __cmp, uint32_t __newvalue)
{
uint32_t __oldvalue;
#if defined(__x86)
@@ -185,7 +185,7 @@ cas32(volatile uint32_t *__memory, uint32_t __cmp, uint32_t __newvalue)
#elif defined(__sparc)
__asm__ __volatile__(
"cas [%2], %3, %1"
- : "=m" (*__memory), "+r" (__oldvalue)
+ : "=m" (*__memory), "=&r" (__oldvalue)
: "r" (__memory), "r" (__cmp), "1" (__newvalue));
#else
#error "port me"
@@ -194,7 +194,7 @@ cas32(volatile uint32_t *__memory, uint32_t __cmp, uint32_t __newvalue)
}
extern __inline__ void
-incr32(volatile uint32_t *__memory)
+atomic_inc_32(volatile uint32_t *__memory)
{
#if defined(__x86)
__asm__ __volatile__(
@@ -219,7 +219,7 @@ incr32(volatile uint32_t *__memory)
}
extern __inline__ void
-decr32(volatile uint32_t *__memory)
+atomic_dec_32(volatile uint32_t *__memory)
{
#if defined(__x86)
__asm__ __volatile__(
@@ -243,6 +243,58 @@ decr32(volatile uint32_t *__memory)
#endif
}
+extern __inline__ void
+atomic_and_32(volatile uint32_t *__memory, uint32_t __bits)
+{
+#if defined(__x86)
+ __asm__ __volatile__(
+ "lock; andl %1, %0"
+ : "+m" (*__memory)
+ : "r" (__bits));
+#elif defined(__sparc)
+ uint32_t __tmp1, __tmp2;
+ __asm__ __volatile__(
+ "ld [%3], %0\n\t"
+ "1:\n\t"
+ "and %0, %4, %1\n\t"
+ "cas [%3], %0, %1\n\t"
+ "cmp %0, %1\n\t"
+ "bne,a,pn %%icc, 1b\n\t"
+ " mov %1, %0"
+ : "=&r" (__tmp1), "=&r" (__tmp2), "=m" (*__memory)
+ : "r" (__memory), "r" (__bits)
+ : "cc");
+#else
+#error "port me"
+#endif
+}
+
+extern __inline__ void
+atomic_or_32(volatile uint32_t *__memory, uint32_t __bits)
+{
+#if defined(__x86)
+ __asm__ __volatile__(
+ "lock; orl %1, %0"
+ : "+m" (*__memory)
+ : "r" (__bits));
+#elif defined(__sparc)
+ uint32_t __tmp1, __tmp2;
+ __asm__ __volatile__(
+ "ld [%3], %0\n\t"
+ "1:\n\t"
+ "or %0, %4, %1\n\t"
+ "cas [%3], %0, %1\n\t"
+ "cmp %0, %1\n\t"
+ "bne,a,pn %%icc, 1b\n\t"
+ " mov %1, %0"
+ : "=&r" (__tmp1), "=&r" (__tmp2), "=m" (*__memory)
+ : "r" (__memory), "r" (__bits)
+ : "cc");
+#else
+#error "port me"
+#endif
+}
+
#if defined(__sparc) /* only needed on sparc */
extern __inline__ ulong_t
diff --git a/usr/src/lib/libc/inc/thr_uberdata.h b/usr/src/lib/libc/inc/thr_uberdata.h
index 4b00040fcf..7dbe4d7aab 100644
--- a/usr/src/lib/libc/inc/thr_uberdata.h
+++ b/usr/src/lib/libc/inc/thr_uberdata.h
@@ -378,6 +378,8 @@ extern void queue_unlock(queue_head_t *);
extern void enqueue(queue_head_t *, struct ulwp *, void *, int);
extern struct ulwp *dequeue(queue_head_t *, void *, int *);
extern struct ulwp *queue_waiter(queue_head_t *, void *);
+extern struct ulwp *queue_unlink(queue_head_t *,
+ struct ulwp **, struct ulwp *);
extern uint8_t dequeue_self(queue_head_t *, void *);
extern void unsleep_self(void);
extern void spin_lock_set(mutex_t *);
@@ -662,21 +664,21 @@ typedef struct {
/*
- * siguaction members have 64-byte size and alignment.
- * We know that sizeof (struct sigaction) is 32 bytes for
- * both _ILP32 and _LP64, so we put the padding in the middle.
+ * siguaction members have 128-byte size and 64-byte alignment.
+ * We know that sizeof (struct sigaction) is 32 bytes for both
+ * _ILP32 and _LP64 and that sizeof (rwlock_t) is 64 bytes.
*/
typedef struct {
- mutex_t sig_lock;
- char sig_pad[64 - (sizeof (mutex_t) + sizeof (struct sigaction))];
+ rwlock_t sig_lock;
struct sigaction sig_uaction;
+ char sig_pad[128 - sizeof (rwlock_t) - sizeof (struct sigaction)];
} siguaction_t;
#ifdef _SYSCALL32
typedef struct {
- mutex_t sig_lock;
- char sig_pad[64 - (sizeof (mutex_t) + sizeof (struct sigaction32))];
+ rwlock_t sig_lock;
struct sigaction32 sig_uaction;
+ char sig_pad[128 - sizeof (rwlock_t) - sizeof (struct sigaction32)];
} siguaction32_t;
#endif /* _SYSCALL32 */
@@ -1080,6 +1082,14 @@ extern void _flush_windows(void);
#endif
extern void set_curthread(void *);
+/*
+ * Utility function used by cond_broadcast() and rw_unlock()
+ * when waking up many threads (more than MAXLWPS) all at once.
+ */
+#define MAXLWPS 128 /* max remembered lwpids before overflow */
+#define NEWLWPS 2048 /* max remembered lwpids at first overflow */
+extern lwpid_t *alloc_lwpids(lwpid_t *, int *, int *);
+
/* enter a critical section */
#define enter_critical(self) (self->ul_critical++)
@@ -1152,6 +1162,9 @@ extern void _lwp_start(void);
extern void _lwp_terminate(void);
extern void lmutex_lock(mutex_t *);
extern void lmutex_unlock(mutex_t *);
+extern void lrw_rdlock(rwlock_t *);
+extern void lrw_wrlock(rwlock_t *);
+extern void lrw_unlock(rwlock_t *);
extern void sig_mutex_lock(mutex_t *);
extern void sig_mutex_unlock(mutex_t *);
extern int sig_mutex_trylock(mutex_t *);
@@ -1361,10 +1374,12 @@ extern int _private_lwp_mutex_unlock(mutex_t *);
* inlines
*/
extern int set_lock_byte(volatile uint8_t *);
-extern uint32_t swap32(volatile uint32_t *, uint32_t);
-extern uint32_t cas32(volatile uint32_t *, uint32_t, uint32_t);
-extern void incr32(volatile uint32_t *);
-extern void decr32(volatile uint32_t *);
+extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t);
+extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t);
+extern void atomic_inc_32(volatile uint32_t *);
+extern void atomic_dec_32(volatile uint32_t *);
+extern void atomic_and_32(volatile uint32_t *, uint32_t);
+extern void atomic_or_32(volatile uint32_t *, uint32_t);
#if defined(__sparc)
extern ulong_t caller(void);
extern ulong_t getfp(void);
diff --git a/usr/src/lib/libc/port/gen/atexit.c b/usr/src/lib/libc/port/gen/atexit.c
index 2104eefea3..b4dc369c5f 100644
--- a/usr/src/lib/libc/port/gen/atexit.c
+++ b/usr/src/lib/libc/port/gen/atexit.c
@@ -2,9 +2,8 @@
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License"). You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
@@ -21,7 +20,7 @@
*/
/*
- * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
@@ -159,7 +158,7 @@ _preexec_sig_unload(Lc_addr_range_t range[], uint_t count)
{
uberdata_t *udp = curthread->ul_uberdata;
int sig;
- mutex_t *mp;
+ rwlock_t *rwlp;
struct sigaction *sap;
struct sigaction oact;
void (*handler)();
@@ -170,10 +169,10 @@ again:
handler = sap->sa_handler;
if (handler != SIG_DFL && handler != SIG_IGN &&
in_range(handler, range, count)) {
- mp = &udp->siguaction[sig].sig_lock;
- lmutex_lock(mp);
+ rwlp = &udp->siguaction[sig].sig_lock;
+ lrw_wrlock(rwlp);
if (handler != sap->sa_handler) {
- lmutex_unlock(mp);
+ lrw_unlock(rwlp);
goto again;
}
sap->sa_handler = SIG_DFL;
@@ -183,7 +182,7 @@ again:
oact.sa_handler != SIG_DFL &&
oact.sa_handler != SIG_IGN)
(void) __sigaction(sig, sap, NULL);
- lmutex_unlock(mp);
+ lrw_unlock(rwlp);
}
}
}
diff --git a/usr/src/lib/libc/port/sys/lwp_rwlock.c b/usr/src/lib/libc/port/sys/lwp_rwlock.c
index d164cc948b..c91e60c7fa 100644
--- a/usr/src/lib/libc/port/sys/lwp_rwlock.c
+++ b/usr/src/lib/libc/port/sys/lwp_rwlock.c
@@ -2,9 +2,8 @@
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License"). You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
*
* CDDL HEADER END
*/
+
/*
- * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
@@ -41,27 +41,19 @@
int
__lwp_rwlock_rdlock(rwlock_t *rwl, timespec_t *tsp)
{
- int rval;
-
- do {
- rval = syscall(SYS_lwp_rwlock_sys,
- SUBSYS_lwp_rwlock_rdlock, rwl, tsp);
- } while (rval == -1 && errno == EINTR);
-
- return (rval == -1 ? errno : 0);
+ if (syscall(SYS_lwp_rwlock_sys,
+ SUBSYS_lwp_rwlock_rdlock, rwl, tsp) == -1)
+ return (errno);
+ return (0);
}
int
__lwp_rwlock_wrlock(rwlock_t *rwl, timespec_t *tsp)
{
- int rval;
-
- do {
- rval = syscall(SYS_lwp_rwlock_sys,
- SUBSYS_lwp_rwlock_wrlock, rwl, tsp);
- } while (rval == -1 && errno == EINTR);
-
- return (rval == -1 ? errno : 0);
+ if (syscall(SYS_lwp_rwlock_sys,
+ SUBSYS_lwp_rwlock_wrlock, rwl, tsp) == -1)
+ return (errno);
+ return (0);
}
int
@@ -85,7 +77,8 @@ __lwp_rwlock_trywrlock(rwlock_t *rwl)
int
__lwp_rwlock_unlock(rwlock_t *rwl)
{
- if (syscall(SYS_lwp_rwlock_sys, SUBSYS_lwp_rwlock_unlock, rwl) == -1)
+ if (syscall(SYS_lwp_rwlock_sys,
+ SUBSYS_lwp_rwlock_unlock, rwl) == -1)
return (errno);
return (0);
}
diff --git a/usr/src/lib/libc/port/threads/assfail.c b/usr/src/lib/libc/port/threads/assfail.c
index 989a36923a..0ffd9a3219 100644
--- a/usr/src/lib/libc/port/threads/assfail.c
+++ b/usr/src/lib/libc/port/threads/assfail.c
@@ -20,7 +20,7 @@
*/
/*
- * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
@@ -248,11 +248,13 @@ rwlock_error(const rwlock_t *rp, const char *who, const char *msg)
{
/* take a snapshot of the rwlock before it changes (we hope) */
rwlock_t rcopy = *rp;
+ uint32_t rwstate;
char buf[800];
uberdata_t *udp;
ulwp_t *self;
lwpid_t lwpid;
pid_t pid;
+ int process;
/* avoid recursion deadlock */
if ((self = __curthread()) != NULL) {
@@ -272,6 +274,9 @@ rwlock_error(const rwlock_t *rp, const char *who, const char *msg)
pid = _private_getpid();
}
+ rwstate = (uint32_t)rcopy.rwlock_readers;
+ process = (rcopy.rwlock_type & USYNC_PROCESS);
+
(void) strcpy(buf,
"\n*** _THREAD_ERROR_DETECTION: lock usage error detected ***\n");
(void) strcat(buf, who);
@@ -283,43 +288,29 @@ rwlock_error(const rwlock_t *rp, const char *who, const char *msg)
ultos((uint64_t)(uintptr_t)self, 16, buf + strlen(buf));
(void) strcat(buf, " thread-id ");
ultos((uint64_t)lwpid, 10, buf + strlen(buf));
- if (rcopy.rwlock_type & USYNC_PROCESS) {
- uint32_t *rwstate = (uint32_t *)&rcopy.rwlock_readers;
-
+ if (process) {
(void) strcat(buf, " in process ");
ultos((uint64_t)pid, 10, buf + strlen(buf));
-
- if (*rwstate & URW_WRITE_LOCKED) {
- (void) strcat(buf, "\nthe lock writer owner is ");
- ultos((uint64_t)rcopy.rwlock_owner, 16,
- buf + strlen(buf));
+ }
+ if (rwstate & URW_WRITE_LOCKED) {
+ (void) strcat(buf, "\nthe writer lock owner is ");
+ ultos((uint64_t)rcopy.rwlock_owner, 16,
+ buf + strlen(buf));
+ if (process) {
(void) strcat(buf, " in process ");
ultos((uint64_t)rcopy.rwlock_ownerpid, 10,
buf + strlen(buf));
- } else if (*rwstate & URW_READERS_MASK) {
- (void) strcat(buf, "\nthe lock is owned by ");
- ultos((uint64_t)(*rwstate & URW_READERS_MASK), 10,
- buf + strlen(buf));
- (void) strcat(buf, " readers");
- } else
- (void) strcat(buf, "\nthe lock is unowned");
-
- if (*rwstate & URW_HAS_WAITERS) {
- (void) strcat(buf, "\nand appears to have waiters");
- if (*rwstate & URW_WRITE_WANTED)
- (void) strcat(buf,
- " (including at least one writer)");
}
- } else if (rcopy.rwlock_readers < 0) {
- (void) strcat(buf, "\nthe lock writer owner is ");
- ultos((uint64_t)rcopy.rwlock_mowner, 16, buf + strlen(buf));
- } else if (rcopy.rwlock_readers > 0) {
- (void) strcat(buf, "\nthe lock is owned by ");
- ultos((uint64_t)rcopy.rwlock_readers, 10, buf + strlen(buf));
- (void) strcat(buf, "readers");
+ } else if (rwstate & URW_READERS_MASK) {
+ (void) strcat(buf, "\nthe reader lock is held by ");
+ ultos((uint64_t)(rwstate & URW_READERS_MASK), 10,
+ buf + strlen(buf));
+ (void) strcat(buf, " readers");
} else {
(void) strcat(buf, "\nthe lock is unowned");
}
+ if (rwstate & URW_HAS_WAITERS)
+ (void) strcat(buf, "\nand the lock appears to have waiters");
(void) strcat(buf, "\n\n");
(void) _write(2, buf, strlen(buf));
if (udp->uberflags.uf_thread_error_detection >= 2)
diff --git a/usr/src/lib/libc/port/threads/rwlock.c b/usr/src/lib/libc/port/threads/rwlock.c
index 3e04c82601..408f74489e 100644
--- a/usr/src/lib/libc/port/threads/rwlock.c
+++ b/usr/src/lib/libc/port/threads/rwlock.c
@@ -2,9 +2,8 @@
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License"). You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
*
* CDDL HEADER END
*/
+
/*
- * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
@@ -28,7 +28,6 @@
#include "lint.h"
#include "thr_uberdata.h"
-
#include <sys/sdt.h>
#define TRY_FLAG 0x10
@@ -39,8 +38,15 @@
#define NLOCKS 4 /* initial number of readlock_t structs allocated */
+#define ASSERT_CONSISTENT_STATE(readers) \
+ ASSERT(!((readers) & URW_WRITE_LOCKED) || \
+ ((readers) & ~URW_HAS_WAITERS) == URW_WRITE_LOCKED)
+
/*
* Find/allocate an entry for rwlp in our array of rwlocks held for reading.
+ * We must be deferring signals for this to be safe.
+ * Else if we are returning an entry with ul_rdlocks == 0,
+ * it could be reassigned behind our back in a signal handler.
*/
static readlock_t *
rwl_entry(rwlock_t *rwlp)
@@ -50,6 +56,9 @@ rwl_entry(rwlock_t *rwlp)
readlock_t *readlockp;
uint_t nlocks;
+ /* we must be deferring signals */
+ ASSERT((self->ul_critical + self->ul_sigdefer) != 0);
+
if ((nlocks = self->ul_rdlocks) != 0)
readlockp = self->ul_readlock.array;
else {
@@ -131,35 +140,40 @@ rwl_free(ulwp_t *ulwp)
int
_rw_read_held(rwlock_t *rwlp)
{
- ulwp_t *self;
+ volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
+ uint32_t readers;
+ ulwp_t *self = curthread;
readlock_t *readlockp;
uint_t nlocks;
+ int rval = 0;
- /* quick answer */
- if (rwlp->rwlock_type == USYNC_PROCESS) {
- if (!((uint32_t)rwlp->rwlock_readers & URW_READERS_MASK))
- return (0);
- } else if (rwlp->rwlock_readers <= 0) {
- return (0);
- }
+ no_preempt(self);
- /*
- * The lock is held for reading by some thread.
- * Search our array of rwlocks held for reading for a match.
- */
- self = curthread;
- if ((nlocks = self->ul_rdlocks) != 0)
- readlockp = self->ul_readlock.array;
- else {
- nlocks = 1;
- readlockp = &self->ul_readlock.single;
+ readers = *rwstate;
+ ASSERT_CONSISTENT_STATE(readers);
+ if (!(readers & URW_WRITE_LOCKED) &&
+ (readers & URW_READERS_MASK) != 0) {
+ /*
+ * The lock is held for reading by some thread.
+ * Search our array of rwlocks held for reading for a match.
+ */
+ if ((nlocks = self->ul_rdlocks) != 0)
+ readlockp = self->ul_readlock.array;
+ else {
+ nlocks = 1;
+ readlockp = &self->ul_readlock.single;
+ }
+ for (; nlocks; nlocks--, readlockp++) {
+ if (readlockp->rd_rwlock == rwlp) {
+ if (readlockp->rd_count)
+ rval = 1;
+ break;
+ }
+ }
}
- for (; nlocks; nlocks--, readlockp++)
- if (readlockp->rd_rwlock == rwlp)
- return (readlockp->rd_count? 1 : 0);
-
- return (0);
+ preempt(self);
+ return (rval);
}
/*
@@ -171,16 +185,22 @@ _rw_read_held(rwlock_t *rwlp)
int
_rw_write_held(rwlock_t *rwlp)
{
+ volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
+ uint32_t readers;
ulwp_t *self = curthread;
- uberdata_t *udp = self->ul_uberdata;
+ int rval;
+
+ no_preempt(self);
- if (rwlp->rwlock_type == USYNC_PROCESS)
- return (((uint32_t)rwlp->rwlock_readers & URW_WRITE_LOCKED) &&
- (rwlp->rwlock_ownerpid == udp->pid) &&
- (rwlp->rwlock_owner == (uintptr_t)self));
+ readers = *rwstate;
+ ASSERT_CONSISTENT_STATE(readers);
+ rval = ((readers & URW_WRITE_LOCKED) &&
+ rwlp->rwlock_owner == (uintptr_t)self &&
+ (rwlp->rwlock_type == USYNC_THREAD ||
+ rwlp->rwlock_ownerpid == self->ul_uberdata->pid));
- /* USYNC_THREAD */
- return (rwlp->rwlock_readers == -1 && mutex_is_held(&rwlp->mutex));
+ preempt(self);
+ return (rval);
}
#pragma weak rwlock_init = __rwlock_init
@@ -195,19 +215,15 @@ __rwlock_init(rwlock_t *rwlp, int type, void *arg)
* Once reinitialized, we can no longer be holding a read or write lock.
* We can do nothing about other threads that are holding read locks.
*/
- if (rw_read_is_held(rwlp))
- rwl_entry(rwlp)->rd_count = 0;
+ sigoff(curthread);
+ rwl_entry(rwlp)->rd_count = 0;
+ sigon(curthread);
(void) _memset(rwlp, 0, sizeof (*rwlp));
rwlp->rwlock_type = (uint16_t)type;
rwlp->rwlock_magic = RWL_MAGIC;
- rwlp->rwlock_readers = 0;
rwlp->mutex.mutex_type = (uint8_t)type;
rwlp->mutex.mutex_flag = LOCK_INITED;
rwlp->mutex.mutex_magic = MUTEX_MAGIC;
- rwlp->readercv.cond_type = (uint16_t)type;
- rwlp->readercv.cond_magic = COND_MAGIC;
- rwlp->writercv.cond_type = (uint16_t)type;
- rwlp->writercv.cond_magic = COND_MAGIC;
return (0);
}
@@ -222,59 +238,191 @@ __rwlock_destroy(rwlock_t *rwlp)
* Once destroyed, we can no longer be holding a read or write lock.
* We can do nothing about other threads that are holding read locks.
*/
- if (rw_read_is_held(rwlp))
- rwl_entry(rwlp)->rd_count = 0;
+ sigoff(curthread);
+ rwl_entry(rwlp)->rd_count = 0;
+ sigon(curthread);
rwlp->rwlock_magic = 0;
tdb_sync_obj_deregister(rwlp);
return (0);
}
/*
- * Wake up the next thread sleeping on the rwlock queue and then
+ * Attempt to acquire a readers lock. Return true on success.
+ */
+static int
+read_lock_try(rwlock_t *rwlp, int ignore_waiters_flag)
+{
+ volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
+ uint32_t mask = ignore_waiters_flag?
+ URW_WRITE_LOCKED : (URW_HAS_WAITERS | URW_WRITE_LOCKED);
+ uint32_t readers;
+ ulwp_t *self = curthread;
+
+ no_preempt(self);
+ while (((readers = *rwstate) & mask) == 0) {
+ if (atomic_cas_32(rwstate, readers, readers + 1) == readers) {
+ preempt(self);
+ return (1);
+ }
+ }
+ preempt(self);
+ return (0);
+}
+
+/*
+ * Attempt to release a reader lock. Return true on success.
+ */
+static int
+read_unlock_try(rwlock_t *rwlp)
+{
+ volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
+ uint32_t readers;
+ ulwp_t *self = curthread;
+
+ no_preempt(self);
+ while (((readers = *rwstate) & URW_HAS_WAITERS) == 0) {
+ if (atomic_cas_32(rwstate, readers, readers - 1) == readers) {
+ preempt(self);
+ return (1);
+ }
+ }
+ preempt(self);
+ return (0);
+}
+
+/*
+ * Attempt to acquire a writer lock. Return true on success.
+ */
+static int
+write_lock_try(rwlock_t *rwlp, int ignore_waiters_flag)
+{
+ volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
+ uint32_t mask = ignore_waiters_flag?
+ (URW_WRITE_LOCKED | URW_READERS_MASK) :
+ (URW_HAS_WAITERS | URW_WRITE_LOCKED | URW_READERS_MASK);
+ ulwp_t *self = curthread;
+ uint32_t readers;
+
+ no_preempt(self);
+ while (((readers = *rwstate) & mask) == 0) {
+ if (atomic_cas_32(rwstate, readers, readers | URW_WRITE_LOCKED)
+ == readers) {
+ preempt(self);
+ return (1);
+ }
+ }
+ preempt(self);
+ return (0);
+}
+
+/*
+ * Attempt to release a writer lock. Return true on success.
+ */
+static int
+write_unlock_try(rwlock_t *rwlp)
+{
+ volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
+ uint32_t readers;
+ ulwp_t *self = curthread;
+
+ no_preempt(self);
+ while (((readers = *rwstate) & URW_HAS_WAITERS) == 0) {
+ if (atomic_cas_32(rwstate, readers, 0) == readers) {
+ preempt(self);
+ return (1);
+ }
+ }
+ preempt(self);
+ return (0);
+}
+
+/*
+ * Wake up thread(s) sleeping on the rwlock queue and then
* drop the queue lock. Return non-zero if we wake up someone.
- *
- * This is called whenever a thread releases the lock and whenever a
- * thread successfully or unsuccessfully attempts to acquire the lock.
- * (Basically, whenever the state of the queue might have changed.)
- *
- * We wake up at most one thread. If there are more threads to be
- * awakened, the next one will be waked up by the thread we wake up.
- * This ensures that queued threads will acquire the lock in priority
- * order and that queued writers will take precedence over queued
- * readers of the same priority.
+ * This is called when a thread releases a lock that appears to have waiters.
*/
static int
rw_queue_release(queue_head_t *qp, rwlock_t *rwlp)
{
+ volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
+ uint32_t readers;
+ uint32_t writers;
+ int nlwpid = 0;
+ int maxlwps = MAXLWPS;
+ ulwp_t *self;
+ ulwp_t **ulwpp;
ulwp_t *ulwp;
- int more;
+ ulwp_t *prev = NULL;
+ lwpid_t buffer[MAXLWPS];
+ lwpid_t *lwpid = buffer;
- if (rwlp->rwlock_readers >= 0 && rwlp->rwlock_mwaiters) {
- /*
- * The lock is free or at least is available to readers
- * and there are (or might be) waiters on the queue.
- */
- if (rwlp->rwlock_readers != 0 &&
- (ulwp = queue_waiter(qp, rwlp)) == NULL)
- rwlp->rwlock_mwaiters = 0;
- else if (rwlp->rwlock_readers == 0 || !ulwp->ul_writer) {
- if ((ulwp = dequeue(qp, rwlp, &more)) == NULL)
- rwlp->rwlock_mwaiters = 0;
- else {
- ulwp_t *self = curthread;
- lwpid_t lwpid = ulwp->ul_lwpid;
-
- rwlp->rwlock_mwaiters = (more? 1 : 0);
- no_preempt(self);
- queue_unlock(qp);
- (void) __lwp_unpark(lwpid);
- preempt(self);
- return (1);
- }
+ readers = *rwstate;
+ ASSERT_CONSISTENT_STATE(readers);
+ if (!(readers & URW_HAS_WAITERS)) {
+ queue_unlock(qp);
+ return (0);
+ }
+ readers &= URW_READERS_MASK;
+ writers = 0;
+
+ /*
+ * Walk the list of waiters and prepare to wake up as
+ * many readers as we encounter before encountering
+ * a writer. If the first thread on the list is a
+ * writer, stop there and wake it up.
+ *
+ * We keep track of lwpids that are to be unparked in lwpid[].
+ * __lwp_unpark_all() is called to unpark all of them after
+ * they have been removed from the sleep queue and the sleep
+ * queue lock has been dropped. If we run out of space in our
+ * on-stack buffer, we need to allocate more but we can't call
+ * lmalloc() because we are holding a queue lock when the overflow
+ * occurs and lmalloc() acquires a lock. We can't use alloca()
+ * either because the application may have allocated a small
+ * stack and we don't want to overrun the stack. So we call
+ * alloc_lwpids() to allocate a bigger buffer using the mmap()
+ * system call directly since that path acquires no locks.
+ */
+ ulwpp = &qp->qh_head;
+ while ((ulwp = *ulwpp) != NULL) {
+ if (ulwp->ul_wchan != rwlp) {
+ prev = ulwp;
+ ulwpp = &ulwp->ul_link;
+ continue;
+ }
+ if (ulwp->ul_writer) {
+ if (writers != 0 || readers != 0)
+ break;
+ /* one writer to wake */
+ writers++;
+ } else {
+ if (writers != 0)
+ break;
+ /* at least one reader to wake */
+ readers++;
+ if (nlwpid == maxlwps)
+ lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps);
}
+ (void) queue_unlink(qp, ulwpp, prev);
+ lwpid[nlwpid++] = ulwp->ul_lwpid;
}
- queue_unlock(qp);
- return (0);
+ if (ulwp == NULL)
+ atomic_and_32(rwstate, ~URW_HAS_WAITERS);
+ if (nlwpid == 0) {
+ queue_unlock(qp);
+ } else {
+ self = curthread;
+ no_preempt(self);
+ queue_unlock(qp);
+ if (nlwpid == 1)
+ (void) __lwp_unpark(lwpid[0]);
+ else
+ (void) __lwp_unpark_all(lwpid, nlwpid);
+ preempt(self);
+ }
+ if (lwpid != buffer)
+ (void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t));
+ return (nlwpid != 0);
}
/*
@@ -290,11 +438,12 @@ rw_queue_release(queue_head_t *qp, rwlock_t *rwlp)
int
shared_rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr)
{
- uint32_t *rwstate = (uint32_t *)&rwlp->readers;
- ulwp_t *self = curthread;
- uberdata_t *udp = self->ul_uberdata;
+ volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
+ mutex_t *mp = &rwlp->mutex;
+ /* LINTED set but not used */
+ uint32_t readers;
int try_flag;
- int error = 0;
+ int error;
try_flag = (rd_wr & TRY_FLAG);
rd_wr &= ~TRY_FLAG;
@@ -305,140 +454,56 @@ shared_rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr)
}
do {
- if ((error = _private_mutex_lock(&rwlp->mutex)) != 0)
+ if (try_flag && (*rwstate & URW_WRITE_LOCKED)) {
+ error = EBUSY;
+ break;
+ }
+ if ((error = _private_mutex_lock(mp)) != 0)
break;
-
if (rd_wr == READ_LOCK) {
- /*
- * We are a reader.
- */
-
- if ((*rwstate & ~URW_READERS_MASK) == 0) {
- (*rwstate)++;
- (void) _private_mutex_unlock(&rwlp->mutex);
- } else if (try_flag) {
- if (*rwstate & URW_WRITE_LOCKED) {
- error = EBUSY;
- (void) _private_mutex_unlock(
- &rwlp->mutex);
- } else {
- /*
- * We have a higher priority than any
- * queued waiters, or the waiters bit
- * may be inaccurate. Only the kernel
- * knows for sure.
- */
- rwlp->rwlock_mowner = 0;
- rwlp->rwlock_mownerpid = 0;
- error = __lwp_rwlock_tryrdlock(rwlp);
- }
- } else {
- rwlp->rwlock_mowner = 0;
- rwlp->rwlock_mownerpid = 0;
- error = __lwp_rwlock_rdlock(rwlp, tsp);
+ if (read_lock_try(rwlp, 0)) {
+ (void) _private_mutex_unlock(mp);
+ break;
}
} else {
- /*
- * We are a writer.
- */
-
- if (*rwstate == 0) {
- *rwstate = URW_WRITE_LOCKED;
- (void) _private_mutex_unlock(&rwlp->mutex);
- } else if (try_flag) {
- if (*rwstate & URW_WRITE_LOCKED) {
- error = EBUSY;
- (void) _private_mutex_unlock(
- &rwlp->mutex);
- } else {
- /*
- * The waiters bit may be inaccurate.
- * Only the kernel knows for sure.
- */
- rwlp->rwlock_mowner = 0;
- rwlp->rwlock_mownerpid = 0;
- error = __lwp_rwlock_trywrlock(rwlp);
- }
- } else {
- rwlp->rwlock_mowner = 0;
- rwlp->rwlock_mownerpid = 0;
- error = __lwp_rwlock_wrlock(rwlp, tsp);
+ if (write_lock_try(rwlp, 0)) {
+ (void) _private_mutex_unlock(mp);
+ break;
}
}
- } while (error == EAGAIN);
-
- if (error == 0) {
- if (rd_wr == WRITE_LOCK) {
- rwlp->rwlock_owner = (uintptr_t)self;
- rwlp->rwlock_ownerpid = udp->pid;
- }
- if (!try_flag) {
- DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, 1);
- }
- DTRACE_PROBE2(plockstat, rw__acquire, rwlp, rd_wr);
- } else if (!try_flag) {
- DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, 0);
- DTRACE_PROBE3(plockstat, rw__error, rwlp, rd_wr, error);
- }
- return (error);
-}
-
-/*
- * Code for unlock of process-shared (USYNC_PROCESS) rwlocks.
- *
- * Note: if the lock appears to have waiters we call __lwp_rwlock_unlock()
- * holding the mutex. This returns with the mutex still held (for us to
- * release).
- */
-int
-shared_rwlock_unlock(rwlock_t *rwlp, int *waked)
-{
- uint32_t *rwstate = (uint32_t *)&rwlp->readers;
- int error = 0;
-
- if ((error = _private_mutex_lock(&rwlp->mutex)) != 0)
- return (error);
-
- /* Reset flag used to suggest caller yields. */
- *waked = 0;
-
- /* Our right to unlock was checked in __rw_unlock(). */
- if (*rwstate & URW_WRITE_LOCKED) {
- rwlp->rwlock_owner = 0;
- rwlp->rwlock_ownerpid = 0;
- }
-
- if ((*rwstate & ~URW_READERS_MASK) == 0) {
- /* Simple multiple readers, no waiters case. */
- if (*rwstate > 0)
- (*rwstate)--;
- } else if (!(*rwstate & URW_HAS_WAITERS)) {
- /* Simple no waiters case (i.e. was write locked). */
- *rwstate = 0;
- } else {
+ atomic_or_32(rwstate, URW_HAS_WAITERS);
+ readers = *rwstate;
+ ASSERT_CONSISTENT_STATE(readers);
/*
- * We appear to have waiters so we must call into the kernel.
- * If there are waiters a full handoff will occur (rwstate
- * will be updated, and one or more threads will be awoken).
+ * The calls to __lwp_rwlock_*() below will release the mutex,
+ * so we need a dtrace probe here.
*/
- error = __lwp_rwlock_unlock(rwlp);
-
- /* Suggest caller yields. */
- *waked = 1;
- }
-
- (void) _private_mutex_unlock(&rwlp->mutex);
+ mp->mutex_owner = 0;
+ DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
+ /*
+ * The waiters bit may be inaccurate.
+ * Only the kernel knows for sure.
+ */
+ if (rd_wr == READ_LOCK) {
+ if (try_flag)
+ error = __lwp_rwlock_tryrdlock(rwlp);
+ else
+ error = __lwp_rwlock_rdlock(rwlp, tsp);
+ } else {
+ if (try_flag)
+ error = __lwp_rwlock_trywrlock(rwlp);
+ else
+ error = __lwp_rwlock_wrlock(rwlp, tsp);
+ }
+ } while (error == EAGAIN || error == EINTR);
- if (error) {
- DTRACE_PROBE3(plockstat, rw__error, rwlp, 0, error);
- } else {
- DTRACE_PROBE2(plockstat, rw__release, rwlp, READ_LOCK);
+ if (!try_flag) {
+ DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, error == 0);
}
return (error);
}
-
/*
* Common code for rdlock, timedrdlock, wrlock, timedwrlock, tryrdlock,
* and trywrlock for process-private (USYNC_THREAD) rwlocks.
@@ -446,6 +511,8 @@ shared_rwlock_unlock(rwlock_t *rwlp, int *waked)
int
rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr)
{
+ volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
+ uint32_t readers;
ulwp_t *self = curthread;
queue_head_t *qp;
ulwp_t *ulwp;
@@ -456,60 +523,29 @@ rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr)
rd_wr &= ~TRY_FLAG;
ASSERT(rd_wr == READ_LOCK || rd_wr == WRITE_LOCK);
- /*
- * Optimize for the case of having only a single thread.
- * (Most likely a traditional single-threaded application.)
- * We don't need the protection of queue_lock() in this case.
- * We need to defer signals, however (the other form of concurrency).
- */
- if (!self->ul_uberdata->uberflags.uf_mt) {
- sigoff(self);
- if (rwlp->rwlock_readers < 0 ||
- (rd_wr == WRITE_LOCK && rwlp->rwlock_readers != 0)) {
- sigon(self);
- if (try_flag)
- return (EBUSY);
- /*
- * Sombody other than ourself owns the lock. (If we
- * owned the lock, either for reading or writing, we
- * would already have returned EDEADLK in our caller.)
- * This can happen only in the child of fork1() when
- * some now-defunct thread was holding the lock when
- * the fork1() was executed by the current thread.
- * In this case, we just fall into the long way
- * to block, either forever or with a timeout.
- */
- ASSERT(MUTEX_OWNER(&rwlp->mutex) != self);
- } else {
- if (rd_wr == READ_LOCK)
- rwlp->rwlock_readers++;
- else {
- rwlp->rwlock_readers = -1;
- rwlp->rwlock_mlockw = LOCKSET;
- rwlp->rwlock_mowner = (uintptr_t)self;
- }
- sigon(self);
- DTRACE_PROBE2(plockstat, rw__acquire, rwlp, rd_wr);
- return (0);
- }
- }
-
if (!try_flag) {
DTRACE_PROBE2(plockstat, rw__block, rwlp, rd_wr);
}
- /*
- * Do it the long way.
- */
qp = queue_lock(rwlp, MX);
+retry:
while (error == 0) {
- if (rwlp->rwlock_readers < 0 ||
- (rd_wr == WRITE_LOCK && rwlp->rwlock_readers != 0))
+ if (rd_wr == READ_LOCK) {
+ if (read_lock_try(rwlp, 0))
+ goto out;
+ } else {
+ if (write_lock_try(rwlp, 0))
+ goto out;
+ }
+ atomic_or_32(rwstate, URW_HAS_WAITERS);
+ readers = *rwstate;
+ ASSERT_CONSISTENT_STATE(readers);
+ if ((readers & URW_WRITE_LOCKED) ||
+ (rd_wr == WRITE_LOCK &&
+ (readers & URW_READERS_MASK) != 0))
/* EMPTY */; /* somebody holds the lock */
- else if (!rwlp->rwlock_mwaiters)
- break; /* no queued waiters */
else if ((ulwp = queue_waiter(qp, rwlp)) == NULL) {
- rwlp->rwlock_mwaiters = 0;
+ atomic_and_32(rwstate, ~URW_HAS_WAITERS);
break; /* no queued waiters */
} else {
int our_pri = real_priority(self);
@@ -547,7 +583,6 @@ rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr)
*/
self->ul_writer = rd_wr; /* *must* be 0 or 1 */
enqueue(qp, self, rwlp, MX);
- rwlp->rwlock_mwaiters = 1;
set_parking_flag(self, 1);
queue_unlock(qp);
if ((error = __lwp_park(tsp, 0)) == EINTR)
@@ -555,29 +590,26 @@ rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr)
self->ul_writer = 0;
set_parking_flag(self, 0);
qp = queue_lock(rwlp, MX);
- if (self->ul_sleepq) /* timeout or spurious wakeup */
- rwlp->rwlock_mwaiters = dequeue_self(qp, rwlp);
+ if (self->ul_sleepq && dequeue_self(qp, rwlp) == 0)
+ atomic_and_32(rwstate, ~URW_HAS_WAITERS);
}
if (error == 0) {
- if (rd_wr == READ_LOCK)
- rwlp->rwlock_readers++;
- else {
- rwlp->rwlock_readers = -1;
- /* make it look like we acquired the embedded mutex */
- rwlp->rwlock_mlockw = LOCKSET;
- rwlp->rwlock_mowner = (uintptr_t)self;
- }
- if (!try_flag) {
- DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, 1);
+ if (rd_wr == READ_LOCK) {
+ if (!read_lock_try(rwlp, 1))
+ goto retry;
+ } else {
+ if (!write_lock_try(rwlp, 1))
+ goto retry;
}
- DTRACE_PROBE2(plockstat, rw__acquire, rwlp, rd_wr);
- } else if (!try_flag) {
- DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, 0);
- DTRACE_PROBE3(plockstat, rw__error, rwlp, rd_wr, error);
}
- (void) rw_queue_release(qp, rwlp);
+out:
+ queue_unlock(qp);
+
+ if (!try_flag) {
+ DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, error == 0);
+ }
return (error);
}
@@ -595,14 +627,19 @@ rw_rdlock_impl(rwlock_t *rwlp, timespec_t *tsp)
* If we already hold a readers lock on this rwlock,
* just increment our reference count and return.
*/
+ sigoff(self);
readlockp = rwl_entry(rwlp);
if (readlockp->rd_count != 0) {
- if (readlockp->rd_count == READ_LOCK_MAX)
- return (EAGAIN);
- readlockp->rd_count++;
- DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK);
- return (0);
+ if (readlockp->rd_count == READ_LOCK_MAX) {
+ sigon(self);
+ error = EAGAIN;
+ goto out;
+ }
+ sigon(self);
+ error = 0;
+ goto out;
}
+ sigon(self);
/*
* If we hold the writer lock, bail out.
@@ -611,18 +648,27 @@ rw_rdlock_impl(rwlock_t *rwlp, timespec_t *tsp)
if (self->ul_error_detection)
rwlock_error(rwlp, "rwlock_rdlock",
"calling thread owns the writer lock");
- return (EDEADLK);
+ error = EDEADLK;
+ goto out;
}
- if (rwlp->rwlock_type == USYNC_PROCESS) /* kernel-level */
+ if (read_lock_try(rwlp, 0))
+ error = 0;
+ else if (rwlp->rwlock_type == USYNC_PROCESS) /* kernel-level */
error = shared_rwlock_lock(rwlp, tsp, READ_LOCK);
else /* user-level */
error = rwlock_lock(rwlp, tsp, READ_LOCK);
+out:
if (error == 0) {
- readlockp->rd_count = 1;
+ sigoff(self);
+ rwl_entry(rwlp)->rd_count++;
+ sigon(self);
if (rwsp)
tdb_incr(rwsp->rw_rdlock);
+ DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK);
+ } else {
+ DTRACE_PROBE3(plockstat, rw__error, rwlp, READ_LOCK, error);
}
return (error);
@@ -691,7 +737,8 @@ rw_wrlock_impl(rwlock_t *rwlp, timespec_t *tsp)
if (self->ul_error_detection)
rwlock_error(rwlp, "rwlock_wrlock",
"calling thread owns the readers lock");
- return (EDEADLK);
+ error = EDEADLK;
+ goto out;
}
/*
@@ -701,20 +748,30 @@ rw_wrlock_impl(rwlock_t *rwlp, timespec_t *tsp)
if (self->ul_error_detection)
rwlock_error(rwlp, "rwlock_wrlock",
"calling thread owns the writer lock");
- return (EDEADLK);
+ error = EDEADLK;
+ goto out;
}
- if (rwlp->rwlock_type == USYNC_PROCESS) { /* kernel-level */
+ if (write_lock_try(rwlp, 0))
+ error = 0;
+ else if (rwlp->rwlock_type == USYNC_PROCESS) /* kernel-level */
error = shared_rwlock_lock(rwlp, tsp, WRITE_LOCK);
- } else { /* user-level */
+ else /* user-level */
error = rwlock_lock(rwlp, tsp, WRITE_LOCK);
- }
- if (error == 0 && rwsp) {
- tdb_incr(rwsp->rw_wrlock);
- rwsp->rw_wrlock_begin_hold = gethrtime();
+out:
+ if (error == 0) {
+ rwlp->rwlock_owner = (uintptr_t)self;
+ if (rwlp->rwlock_type == USYNC_PROCESS)
+ rwlp->rwlock_ownerpid = udp->pid;
+ if (rwsp) {
+ tdb_incr(rwsp->rw_wrlock);
+ rwsp->rw_wrlock_begin_hold = gethrtime();
+ }
+ DTRACE_PROBE2(plockstat, rw__acquire, rwlp, WRITE_LOCK);
+ } else {
+ DTRACE_PROBE3(plockstat, rw__error, rwlp, WRITE_LOCK, error);
}
-
return (error);
}
@@ -788,24 +845,41 @@ __rw_tryrdlock(rwlock_t *rwlp)
* If we already hold a readers lock on this rwlock,
* just increment our reference count and return.
*/
+ sigoff(self);
readlockp = rwl_entry(rwlp);
if (readlockp->rd_count != 0) {
- if (readlockp->rd_count == READ_LOCK_MAX)
- return (EAGAIN);
- readlockp->rd_count++;
- DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK);
- return (0);
+ if (readlockp->rd_count == READ_LOCK_MAX) {
+ sigon(self);
+ error = EAGAIN;
+ goto out;
+ }
+ sigon(self);
+ error = 0;
+ goto out;
}
+ sigon(self);
- if (rwlp->rwlock_type == USYNC_PROCESS) /* kernel-level */
+ if (read_lock_try(rwlp, 0))
+ error = 0;
+ else if (rwlp->rwlock_type == USYNC_PROCESS) /* kernel-level */
error = shared_rwlock_lock(rwlp, NULL, READ_LOCK_TRY);
else /* user-level */
error = rwlock_lock(rwlp, NULL, READ_LOCK_TRY);
- if (error == 0)
- readlockp->rd_count = 1;
- else if (rwsp)
- tdb_incr(rwsp->rw_rdlock_try_fail);
+out:
+ if (error == 0) {
+ sigoff(self);
+ rwl_entry(rwlp)->rd_count++;
+ sigon(self);
+ DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK);
+ } else {
+ if (rwsp)
+ tdb_incr(rwsp->rw_rdlock_try_fail);
+ if (error != EBUSY) {
+ DTRACE_PROBE3(plockstat, rw__error, rwlp, READ_LOCK,
+ error);
+ }
+ }
return (error);
}
@@ -822,21 +896,32 @@ __rw_trywrlock(rwlock_t *rwlp)
tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
int error;
- ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
+ ASSERT(!self->ul_critical || self->ul_bindflags);
if (rwsp)
tdb_incr(rwsp->rw_wrlock_try);
- if (rwlp->rwlock_type == USYNC_PROCESS) { /* kernel-level */
+ if (write_lock_try(rwlp, 0))
+ error = 0;
+ else if (rwlp->rwlock_type == USYNC_PROCESS) /* kernel-level */
error = shared_rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY);
- } else { /* user-level */
+ else /* user-level */
error = rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY);
- }
- if (rwsp) {
- if (error)
- tdb_incr(rwsp->rw_wrlock_try_fail);
- else
+
+ if (error == 0) {
+ rwlp->rwlock_owner = (uintptr_t)self;
+ if (rwlp->rwlock_type == USYNC_PROCESS)
+ rwlp->rwlock_ownerpid = udp->pid;
+ if (rwsp)
rwsp->rw_wrlock_begin_hold = gethrtime();
+ DTRACE_PROBE2(plockstat, rw__acquire, rwlp, WRITE_LOCK);
+ } else {
+ if (rwsp)
+ tdb_incr(rwsp->rw_wrlock_try_fail);
+ if (error != EBUSY) {
+ DTRACE_PROBE3(plockstat, rw__error, rwlp, WRITE_LOCK,
+ error);
+ }
}
return (error);
}
@@ -848,23 +933,26 @@ __rw_trywrlock(rwlock_t *rwlp)
int
__rw_unlock(rwlock_t *rwlp)
{
+ volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
+ uint32_t readers;
ulwp_t *self = curthread;
uberdata_t *udp = self->ul_uberdata;
tdb_rwlock_stats_t *rwsp;
- int32_t lock_count;
- int waked;
-
- /* fetch the lock count once; it may change underfoot */
- lock_count = rwlp->rwlock_readers;
- if (rwlp->rwlock_type == USYNC_PROCESS) {
- /* munge it from rwstate */
- if (lock_count & URW_WRITE_LOCKED)
- lock_count = -1;
- else
- lock_count &= URW_READERS_MASK;
+ queue_head_t *qp;
+ int rd_wr;
+ int waked = 0;
+
+ readers = *rwstate;
+ ASSERT_CONSISTENT_STATE(readers);
+ if (readers & URW_WRITE_LOCKED) {
+ rd_wr = WRITE_LOCK;
+ readers = 0;
+ } else {
+ rd_wr = READ_LOCK;
+ readers &= URW_READERS_MASK;
}
- if (lock_count < 0) {
+ if (rd_wr == WRITE_LOCK) {
/*
* Since the writer lock is held, we'd better be
* holding it, else we cannot legitimately be here.
@@ -882,12 +970,18 @@ __rw_unlock(rwlock_t *rwlp)
gethrtime() - rwsp->rw_wrlock_begin_hold;
rwsp->rw_wrlock_begin_hold = 0;
}
- } else if (lock_count > 0) {
+ rwlp->rwlock_owner = 0;
+ rwlp->rwlock_ownerpid = 0;
+ } else if (readers > 0) {
/*
* A readers lock is held; if we don't hold one, bail out.
*/
- readlock_t *readlockp = rwl_entry(rwlp);
+ readlock_t *readlockp;
+
+ sigoff(self);
+ readlockp = rwl_entry(rwlp);
if (readlockp->rd_count == 0) {
+ sigon(self);
if (self->ul_error_detection)
rwlock_error(rwlp, "rwlock_unlock",
"readers lock held, "
@@ -899,9 +993,10 @@ __rw_unlock(rwlock_t *rwlp)
* just decrement our reference count and return.
*/
if (--readlockp->rd_count != 0) {
- DTRACE_PROBE2(plockstat, rw__release, rwlp, READ_LOCK);
- return (0);
+ sigon(self);
+ goto out;
}
+ sigon(self);
} else {
/*
* This is a usage error.
@@ -912,45 +1007,27 @@ __rw_unlock(rwlock_t *rwlp)
return (EPERM);
}
- if (rwlp->rwlock_type == USYNC_PROCESS) { /* kernel-level */
- (void) shared_rwlock_unlock(rwlp, &waked);
- } else if (!udp->uberflags.uf_mt) { /* single threaded */
- /*
- * In the case of having only a single thread, we don't
- * need the protection of queue_lock() (this parallels
- * the optimization made in rwlock_lock(), above).
- * As in rwlock_lock(), we need to defer signals.
- */
- sigoff(self);
- if (rwlp->rwlock_readers > 0) {
- rwlp->rwlock_readers--;
- DTRACE_PROBE2(plockstat, rw__release, rwlp, READ_LOCK);
- } else {
- rwlp->rwlock_readers = 0;
- /* make it look like we released the embedded mutex */
- rwlp->rwlock_mowner = 0;
- rwlp->rwlock_mlockw = LOCKCLEAR;
- DTRACE_PROBE2(plockstat, rw__release, rwlp, WRITE_LOCK);
- }
- sigon(self);
- waked = 0;
- } else { /* multithreaded */
- queue_head_t *qp;
-
+ if (rd_wr == WRITE_LOCK && write_unlock_try(rwlp)) {
+ /* EMPTY */;
+ } else if (rd_wr == READ_LOCK && read_unlock_try(rwlp)) {
+ /* EMPTY */;
+ } else if (rwlp->rwlock_type == USYNC_PROCESS) {
+ (void) _private_mutex_lock(&rwlp->mutex);
+ (void) __lwp_rwlock_unlock(rwlp);
+ (void) _private_mutex_unlock(&rwlp->mutex);
+ waked = 1;
+ } else {
qp = queue_lock(rwlp, MX);
- if (rwlp->rwlock_readers > 0) {
- rwlp->rwlock_readers--;
- DTRACE_PROBE2(plockstat, rw__release, rwlp, READ_LOCK);
- } else {
- rwlp->rwlock_readers = 0;
- /* make it look like we released the embedded mutex */
- rwlp->rwlock_mowner = 0;
- rwlp->rwlock_mlockw = LOCKCLEAR;
- DTRACE_PROBE2(plockstat, rw__release, rwlp, WRITE_LOCK);
- }
+ if (rd_wr == READ_LOCK)
+ atomic_dec_32(rwstate);
+ else
+ atomic_and_32(rwstate, ~URW_WRITE_LOCKED);
waked = rw_queue_release(qp, rwlp);
}
+out:
+ DTRACE_PROBE2(plockstat, rw__release, rwlp, rd_wr);
+
/*
* Yield to the thread we just waked up, just in case we might
* be about to grab the rwlock again immediately upon return.
diff --git a/usr/src/lib/libc/port/threads/sigaction.c b/usr/src/lib/libc/port/threads/sigaction.c
index 529530f2af..3d8f747100 100644
--- a/usr/src/lib/libc/port/threads/sigaction.c
+++ b/usr/src/lib/libc/port/threads/sigaction.c
@@ -83,6 +83,9 @@ call_user_handler(int sig, siginfo_t *sip, ucontext_t *ucp)
* while holding the sig's sig_lock for the least possible time.
* We must acquire the sig's sig_lock because some thread running
* in sigaction() might be establishing a new signal handler.
+ * The code in sigaction() acquires the writer lock; here
+ * we acquire the readers lock to ehance concurrency in the
+ * face of heavy signal traffic, such as generated by java.
*
* Locking exceptions:
* No locking for a child of vfork().
@@ -103,12 +106,12 @@ call_user_handler(int sig, siginfo_t *sip, ucontext_t *ucp)
/* we wish this assignment could be atomic */
(void) _private_memcpy(&uact, (void *)sap, sizeof (uact));
} else {
- mutex_t *mp = &udp->siguaction[sig].sig_lock;
- lmutex_lock(mp);
+ rwlock_t *rwlp = &udp->siguaction[sig].sig_lock;
+ lrw_rdlock(rwlp);
(void) _private_memcpy(&uact, (void *)sap, sizeof (uact));
if (sig == SIGCANCEL && (sap->sa_flags & SA_RESETHAND))
sap->sa_sigaction = SIG_DFL;
- lmutex_unlock(mp);
+ lrw_unlock(rwlp);
}
/*
@@ -324,7 +327,7 @@ _libc_sigaction(int sig, const struct sigaction *nact, struct sigaction *oact)
}
if (!self->ul_vfork)
- lmutex_lock(&udp->siguaction[sig].sig_lock);
+ lrw_wrlock(&udp->siguaction[sig].sig_lock);
oaction = udp->siguaction[sig].sig_uaction;
@@ -401,7 +404,7 @@ _libc_sigaction(int sig, const struct sigaction *nact, struct sigaction *oact)
}
if (!self->ul_vfork)
- lmutex_unlock(&udp->siguaction[sig].sig_lock);
+ lrw_unlock(&udp->siguaction[sig].sig_lock);
return (rv);
}
@@ -633,7 +636,7 @@ _sigprocmask(int how, const sigset_t *set, sigset_t *oset)
/*
* Called at library initialization to set up signal handling.
- * All we really do is initialize the sig_lock mutexes.
+ * All we really do is initialize the sig_lock rwlocks.
* All signal handlers are either SIG_DFL or SIG_IGN on exec().
* However, if any signal handlers were established on alternate
* link maps before the primary link map has been initialized,
@@ -645,10 +648,14 @@ signal_init()
uberdata_t *udp = curthread->ul_uberdata;
struct sigaction *sap;
struct sigaction act;
+ rwlock_t *rwlp;
int sig;
for (sig = 0; sig < NSIG; sig++) {
- udp->siguaction[sig].sig_lock.mutex_magic = MUTEX_MAGIC;
+ rwlp = &udp->siguaction[sig].sig_lock;
+ rwlp->rwlock_magic = RWL_MAGIC;
+ rwlp->mutex.mutex_flag = LOCK_INITED;
+ rwlp->mutex.mutex_magic = MUTEX_MAGIC;
sap = &udp->siguaction[sig].sig_uaction;
if (sap->sa_sigaction != SIG_DFL &&
sap->sa_sigaction != SIG_IGN &&
@@ -694,13 +701,13 @@ void
setup_cancelsig(int sig)
{
uberdata_t *udp = curthread->ul_uberdata;
- mutex_t *mp = &udp->siguaction[sig].sig_lock;
+ rwlock_t *rwlp = &udp->siguaction[sig].sig_lock;
struct sigaction act;
ASSERT(sig == SIGCANCEL || sig == SIGAIOCANCEL);
- lmutex_lock(mp);
+ lrw_rdlock(rwlp);
act = udp->siguaction[sig].sig_uaction;
- lmutex_unlock(mp);
+ lrw_unlock(rwlp);
if (act.sa_sigaction == SIG_DFL ||
act.sa_sigaction == SIG_IGN)
act.sa_flags = SA_SIGINFO;
diff --git a/usr/src/lib/libc/port/threads/synch.c b/usr/src/lib/libc/port/threads/synch.c
index 9c6e918620..ec9fabc226 100644
--- a/usr/src/lib/libc/port/threads/synch.c
+++ b/usr/src/lib/libc/port/threads/synch.c
@@ -20,7 +20,7 @@
*/
/*
- * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
@@ -292,7 +292,7 @@ spin_lock_clear(mutex_t *mp)
ulwp_t *self = curthread;
mp->mutex_owner = 0;
- if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) {
+ if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK) {
(void) ___lwp_mutex_wakeup(mp);
if (self->ul_spin_lock_wakeup != UINT_MAX)
self->ul_spin_lock_wakeup++;
@@ -569,18 +569,10 @@ queue_slot(queue_head_t *qp, void *wchan, int *more, ulwp_t **prevp)
}
ulwp_t *
-dequeue(queue_head_t *qp, void *wchan, int *more)
+queue_unlink(queue_head_t *qp, ulwp_t **ulwpp, ulwp_t *prev)
{
- ulwp_t **ulwpp;
ulwp_t *ulwp;
- ulwp_t *prev;
-
- if ((ulwpp = queue_slot(qp, wchan, more, &prev)) == NULL)
- return (NULL);
- /*
- * Dequeue the waiter.
- */
ulwp = *ulwpp;
*ulwpp = ulwp->ul_link;
ulwp->ul_link = NULL;
@@ -593,6 +585,17 @@ dequeue(queue_head_t *qp, void *wchan, int *more)
return (ulwp);
}
+ulwp_t *
+dequeue(queue_head_t *qp, void *wchan, int *more)
+{
+ ulwp_t **ulwpp;
+ ulwp_t *prev;
+
+ if ((ulwpp = queue_slot(qp, wchan, more, &prev)) == NULL)
+ return (NULL);
+ return (queue_unlink(qp, ulwpp, prev));
+}
+
/*
* Return a pointer to the highest priority thread sleeping on wchan.
*/
@@ -623,16 +626,10 @@ dequeue_self(queue_head_t *qp, void *wchan)
prev = ulwp, ulwpp = &ulwp->ul_link) {
if (ulwp == self) {
/* dequeue ourself */
- *ulwpp = self->ul_link;
- if (qp->qh_tail == self)
- qp->qh_tail = prev;
- qp->qh_qlen--;
ASSERT(self->ul_wchan == wchan);
+ (void) queue_unlink(qp, ulwpp, prev);
self->ul_cvmutex = NULL;
- self->ul_sleepq = NULL;
- self->ul_wchan = NULL;
self->ul_cv_wake = 0;
- self->ul_link = NULL;
found = 1;
break;
}
@@ -670,7 +667,6 @@ unsleep_self(void)
* to recursion. Just manipulate self->ul_critical directly.
*/
self->ul_critical++;
- self->ul_writer = 0;
while (self->ul_sleepq != NULL) {
qp = queue_lock(self->ul_wchan, self->ul_qtype);
/*
@@ -679,8 +675,10 @@ unsleep_self(void)
* If so, just loop around and try again.
* dequeue_self() clears self->ul_sleepq.
*/
- if (qp == self->ul_sleepq)
+ if (qp == self->ul_sleepq) {
(void) dequeue_self(qp, self->ul_wchan);
+ self->ul_writer = 0;
+ }
queue_unlock(qp);
}
self->ul_critical--;
@@ -969,11 +967,11 @@ mutex_trylock_adaptive(mutex_t *mp)
* programs will not suffer in any case.
*/
enter_critical(self); /* protects ul_schedctl */
- incr32(&mp->mutex_spinners);
+ atomic_inc_32(&mp->mutex_spinners);
for (count = 0; count < max; count++) {
if (*lockp == 0 && set_lock_byte(lockp) == 0) {
*ownerp = (uintptr_t)self;
- decr32(&mp->mutex_spinners);
+ atomic_dec_32(&mp->mutex_spinners);
exit_critical(self);
DTRACE_PROBE2(plockstat, mutex__spun, 1, count);
DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
@@ -1003,7 +1001,7 @@ mutex_trylock_adaptive(mutex_t *mp)
scp->sc_state != SC_ONPROC))
break;
}
- decr32(&mp->mutex_spinners);
+ atomic_dec_32(&mp->mutex_spinners);
exit_critical(self);
DTRACE_PROBE2(plockstat, mutex__spun, 0, count);
@@ -1173,8 +1171,8 @@ mutex_unlock_queue(mutex_t *mp)
if (!(*lockw & WAITERMASK)) { /* no waiter exists right now */
mp->mutex_owner = 0;
DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
- if (!(swap32(lockw, 0) & WAITERMASK)) /* still no waiters */
- return (0);
+ if (!(atomic_swap_32(lockw, 0) & WAITERMASK))
+ return (0); /* still no waiters */
no_preempt(self); /* ensure a prompt wakeup */
lwpid = mutex_wakeup(mp);
} else {
@@ -1183,7 +1181,8 @@ mutex_unlock_queue(mutex_t *mp)
spinp = (volatile uint32_t *)&mp->mutex_spinners;
mp->mutex_owner = 0;
DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
- (void) swap32(lockw, WAITER); /* clear lock, retain waiter */
+ /* clear lock, retain waiter */
+ (void) atomic_swap_32(lockw, WAITER);
/*
* We spin here fewer times than mutex_trylock_adaptive().
@@ -1249,14 +1248,14 @@ mutex_unlock_process(mutex_t *mp)
DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
if (count == 0) {
/* clear lock, test waiter */
- if (!(swap32(&mp->mutex_lockword, 0) & WAITERMASK)) {
+ if (!(atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK)) {
/* no waiters now */
preempt(self);
return;
}
} else {
/* clear lock, retain waiter */
- (void) swap32(&mp->mutex_lockword, WAITER);
+ (void) atomic_swap_32(&mp->mutex_lockword, WAITER);
lockp = (volatile uint8_t *)&mp->mutex_lockw;
while (--count >= 0) {
if (*lockp != 0) {
@@ -1945,7 +1944,7 @@ mutex_unlock_internal(mutex_t *mp)
} else if (mtype & USYNC_PROCESS_ROBUST) {
error = ___lwp_mutex_unlock(mp);
} else {
- if (swap32(&mp->mutex_lockword, 0) & WAITERMASK)
+ if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK)
(void) ___lwp_mutex_wakeup(mp);
error = 0;
}
@@ -1961,7 +1960,8 @@ mutex_unlock_internal(mutex_t *mp)
mp->mutex_owner = 0;
mp->mutex_ownerpid = 0;
DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
- if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) {
+ if (atomic_swap_32(&mp->mutex_lockword, 0) &
+ WAITERMASK) {
no_preempt(self);
(void) ___lwp_mutex_wakeup(mp);
preempt(self);
@@ -2038,7 +2038,7 @@ fast_unlock:
/* no waiter exists right now */
mp->mutex_owner = 0;
DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
- if (swap32(&mp->mutex_lockword, 0) &
+ if (atomic_swap_32(&mp->mutex_lockword, 0) &
WAITERMASK) {
/* a waiter suddenly appeared */
no_preempt(self);
@@ -2088,7 +2088,7 @@ fast_unlock:
mp->mutex_owner = 0;
mp->mutex_ownerpid = 0;
DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
- if (swap32(&mp->mutex_lockword, 0) &
+ if (atomic_swap_32(&mp->mutex_lockword, 0) &
WAITERMASK) {
no_preempt(self);
(void) ___lwp_mutex_wakeup(mp);
@@ -2408,7 +2408,7 @@ _pthread_spin_unlock(pthread_spinlock_t *lock)
mp->mutex_owner = 0;
mp->mutex_ownerpid = 0;
DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
- (void) swap32(&mp->mutex_lockword, 0);
+ (void) atomic_swap_32(&mp->mutex_lockword, 0);
preempt(self);
return (0);
}
@@ -3035,8 +3035,53 @@ cond_signal_internal(cond_t *cvp)
return (error);
}
-#define MAXLWPS 128 /* max remembered lwpids before overflow */
-#define NEWLWPS 2048 /* max remembered lwpids at first overflow */
+/*
+ * Utility function called from cond_broadcast() and rw_queue_release()
+ * to (re)allocate a big buffer to hold the lwpids of all the threads
+ * to be set running after they are removed from their sleep queues.
+ * Since we are holding a queue lock, we cannot call any function
+ * that might acquire a lock. mmap(), munmap() and lwp_unpark_all()
+ * are simple system calls and are safe in this regard.
+ */
+lwpid_t *
+alloc_lwpids(lwpid_t *lwpid, int *nlwpid_ptr, int *maxlwps_ptr)
+{
+ /*
+ * Allocate NEWLWPS ids on the first overflow.
+ * Double the allocation each time after that.
+ */
+ int nlwpid = *nlwpid_ptr;
+ int maxlwps = *maxlwps_ptr;
+ int first_allocation;
+ int newlwps;
+ void *vaddr;
+
+ ASSERT(nlwpid == maxlwps);
+
+ first_allocation = (maxlwps == MAXLWPS);
+ newlwps = first_allocation? NEWLWPS : 2 * maxlwps;
+ vaddr = _private_mmap(NULL, newlwps * sizeof (lwpid_t),
+ PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0);
+
+ if (vaddr == MAP_FAILED) {
+ /*
+ * Let's hope this never happens.
+ * If it does, then we have a terrible
+ * thundering herd on our hands.
+ */
+ (void) __lwp_unpark_all(lwpid, nlwpid);
+ *nlwpid_ptr = 0;
+ } else {
+ (void) _memcpy(vaddr, lwpid, maxlwps * sizeof (lwpid_t));
+ if (!first_allocation)
+ (void) _private_munmap(lwpid,
+ maxlwps * sizeof (lwpid_t));
+ lwpid = vaddr;
+ *maxlwps_ptr = newlwps;
+ }
+
+ return (lwpid);
+}
#pragma weak pthread_cond_broadcast = cond_broadcast_internal
#pragma weak _pthread_cond_broadcast = cond_broadcast_internal
@@ -3051,16 +3096,15 @@ cond_broadcast_internal(cond_t *cvp)
int error = 0;
queue_head_t *qp;
mutex_t *mp;
- queue_head_t *mqp;
mutex_t *mp_cache = NULL;
- queue_head_t *mqp_cache = NULL;
+ queue_head_t *mqp = NULL;
ulwp_t **ulwpp;
ulwp_t *ulwp;
ulwp_t *prev = NULL;
- lwpid_t buffer[MAXLWPS];
- lwpid_t *lwpid = buffer;
int nlwpid = 0;
int maxlwps = MAXLWPS;
+ lwpid_t buffer[MAXLWPS];
+ lwpid_t *lwpid = buffer;
if (csp)
tdb_incr(csp->cond_broadcast);
@@ -3085,89 +3129,61 @@ cond_broadcast_internal(cond_t *cvp)
* on-stack buffer, we need to allocate more but we can't call
* lmalloc() because we are holding a queue lock when the overflow
* occurs and lmalloc() acquires a lock. We can't use alloca()
- * either because the application may have allocated a small stack
- * and we don't want to overrun the stack. So we use the mmap()
+ * either because the application may have allocated a small
+ * stack and we don't want to overrun the stack. So we call
+ * alloc_lwpids() to allocate a bigger buffer using the mmap()
* system call directly since that path acquires no locks.
*/
qp = queue_lock(cvp, CV);
cvp->cond_waiters_user = 0;
ulwpp = &qp->qh_head;
while ((ulwp = *ulwpp) != NULL) {
-
if (ulwp->ul_wchan != cvp) {
prev = ulwp;
ulwpp = &ulwp->ul_link;
continue;
}
-
*ulwpp = ulwp->ul_link;
if (qp->qh_tail == ulwp)
qp->qh_tail = prev;
qp->qh_qlen--;
ulwp->ul_link = NULL;
-
mp = ulwp->ul_cvmutex; /* his mutex */
ulwp->ul_cvmutex = NULL;
ASSERT(mp != NULL);
-
if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) {
ulwp->ul_sleepq = NULL;
ulwp->ul_wchan = NULL;
ulwp->ul_cv_wake = 0;
- if (nlwpid == maxlwps) {
- /*
- * Allocate NEWLWPS ids on the first overflow.
- * Double the allocation each time after that.
- */
- int newlwps = (lwpid == buffer)? NEWLWPS :
- 2 * maxlwps;
- void *vaddr = _private_mmap(NULL,
- newlwps * sizeof (lwpid_t),
- PROT_READ|PROT_WRITE,
- MAP_PRIVATE|MAP_ANON, -1, (off_t)0);
- if (vaddr == MAP_FAILED) {
- /*
- * Let's hope this never happens.
- * If it does, then we have a terrible
- * thundering herd on our hands.
- */
- (void) __lwp_unpark_all(lwpid, nlwpid);
- nlwpid = 0;
- } else {
- (void) _memcpy(vaddr, lwpid,
- maxlwps * sizeof (lwpid_t));
- if (lwpid != buffer)
- (void) _private_munmap(lwpid,
- maxlwps * sizeof (lwpid_t));
- lwpid = vaddr;
- maxlwps = newlwps;
- }
- }
+ if (nlwpid == maxlwps)
+ lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps);
lwpid[nlwpid++] = ulwp->ul_lwpid;
} else {
if (mp != mp_cache) {
- if (mqp_cache != NULL)
- queue_unlock(mqp_cache);
- mqp_cache = queue_lock(mp, MX);
mp_cache = mp;
+ if (mqp != NULL)
+ queue_unlock(mqp);
+ mqp = queue_lock(mp, MX);
}
- mqp = mqp_cache;
enqueue(mqp, ulwp, mp, MX);
mp->mutex_waiters = 1;
}
}
- if (mqp_cache != NULL)
- queue_unlock(mqp_cache);
- queue_unlock(qp);
- if (nlwpid) {
+ if (mqp != NULL)
+ queue_unlock(mqp);
+ if (nlwpid == 0) {
+ queue_unlock(qp);
+ } else {
+ no_preempt(self);
+ queue_unlock(qp);
if (nlwpid == 1)
(void) __lwp_unpark(lwpid[0]);
else
(void) __lwp_unpark_all(lwpid, nlwpid);
+ preempt(self);
}
if (lwpid != buffer)
(void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t));
-
return (error);
}
diff --git a/usr/src/lib/libc/sparc/Makefile b/usr/src/lib/libc/sparc/Makefile
index 61472f35b6..63e8e88ea4 100644
--- a/usr/src/lib/libc/sparc/Makefile
+++ b/usr/src/lib/libc/sparc/Makefile
@@ -1111,7 +1111,7 @@ TIL= \
$(TIL:%=pics/%) := CFLAGS += $(LIBCBASE)/threads/sparc.il
# special kludge for inlines with 'cas':
-pics/synch.o pics/lwp.o := sparc_CFLAGS += -_gcc=-Wa,-xarch=v8plus
+pics/rwlock.o pics/synch.o pics/lwp.o := sparc_CFLAGS += -_gcc=-Wa,-xarch=v8plus
# Files in port/fp subdirectory that need base.il inline template
IL= \
diff --git a/usr/src/lib/libc/sparc/threads/sparc.il b/usr/src/lib/libc/sparc/threads/sparc.il
index c3f81e69b3..57d351ae6d 100644
--- a/usr/src/lib/libc/sparc/threads/sparc.il
+++ b/usr/src/lib/libc/sparc/threads/sparc.il
@@ -55,12 +55,12 @@
* 32-bit applications that use %g5 as an invariant register.
*/
- .inline cas32, 0
+ .inline atomic_cas_32, 0
.word 0xd5e21009 ! cas [%o0], %o1, %o2
mov %o2, %o0
.end
- .inline swap32, 0
+ .inline atomic_swap_32, 0
ld [%o0], %o2
1:
mov %o1, %o3
@@ -71,7 +71,7 @@
mov %o3, %o0
.end
- .inline incr32, 0
+ .inline atomic_inc_32, 0
ld [%o0], %o2
1:
add %o2, 1, %o3
@@ -81,7 +81,7 @@
mov %o3, %o2
.end
- .inline decr32, 0
+ .inline atomic_dec_32, 0
ld [%o0], %o2
1:
sub %o2, 1, %o3
@@ -91,6 +91,26 @@
mov %o3, %o2
.end
+ .inline atomic_and_32, 0
+ ld [%o0], %o2
+1:
+ and %o2, %o1, %o3
+ .word 0xd7e2100a ! cas [%o0], %o2, %o3
+ cmp %o2, %o3
+ bne,a 1b
+ mov %o3, %o2
+ .end
+
+ .inline atomic_or_32, 0
+ ld [%o0], %o2
+1:
+ or %o2, %o1, %o3
+ .word 0xd7e2100a ! cas [%o0], %o2, %o3
+ cmp %o2, %o3
+ bne,a 1b
+ mov %o3, %o2
+ .end
+
.inline caller, 0
mov %i7, %o0
.end
diff --git a/usr/src/lib/libc/sparcv9/threads/sparcv9.il b/usr/src/lib/libc/sparcv9/threads/sparcv9.il
index f84623f93c..4f1d543f6c 100644
--- a/usr/src/lib/libc/sparcv9/threads/sparcv9.il
+++ b/usr/src/lib/libc/sparcv9/threads/sparcv9.il
@@ -52,12 +52,12 @@
membar #LoadLoad
.end
- .inline cas32, 0
+ .inline atomic_cas_32, 0
cas [%o0], %o1, %o2
mov %o2, %o0
.end
- .inline swap32, 0
+ .inline atomic_swap_32, 0
ld [%o0], %o2
1:
mov %o1, %o3
@@ -68,7 +68,7 @@
mov %o3, %o0
.end
- .inline incr32, 0
+ .inline atomic_inc_32, 0
ld [%o0], %o2
1:
add %o2, 1, %o3
@@ -78,7 +78,7 @@
mov %o3, %o2
.end
- .inline decr32, 0
+ .inline atomic_dec_32, 0
ld [%o0], %o2
1:
sub %o2, 1, %o3
@@ -88,6 +88,26 @@
mov %o3, %o2
.end
+ .inline atomic_and_32, 0
+ ld [%o0], %o2
+1:
+ and %o2, %o1, %o3
+ cas [%o0], %o2, %o3
+ cmp %o2, %o3
+ bne,a,pn %icc, 1b
+ mov %o3, %o2
+ .end
+
+ .inline atomic_or_32, 0
+ ld [%o0], %o2
+1:
+ or %o2, %o1, %o3
+ cas [%o0], %o2, %o3
+ cmp %o2, %o3
+ bne,a,pn %icc, 1b
+ mov %o3, %o2
+ .end
+
.inline caller, 0
mov %i7, %o0
.end
diff --git a/usr/src/lib/libc_db/common/thread_db.c b/usr/src/lib/libc_db/common/thread_db.c
index 7759b68a20..9c10183772 100644
--- a/usr/src/lib/libc_db/common/thread_db.c
+++ b/usr/src/lib/libc_db/common/thread_db.c
@@ -2,9 +2,8 @@
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License"). You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
*
* CDDL HEADER END
*/
+
/*
- * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
@@ -2556,42 +2556,35 @@ sync_get_info_common(const td_synchandle_t *sh_p, struct ps_prochandle *ph_p,
si_p->si_data = (psaddr_t)generic_so.semaphore.count;
break;
case RWL_MAGIC:
+ {
+ uint32_t rwstate;
+
if (trunc && ps_pdread(ph_p, sh_p->sh_unique,
&generic_so.rwlock, sizeof (generic_so.rwlock)) != PS_OK)
return (TD_DBERR);
si_p->si_type = TD_SYNC_RWLOCK;
si_p->si_shared_type = generic_so.rwlock.rwlock_type;
si_p->si_size = sizeof (generic_so.rwlock);
- if (generic_so.rwlock.rwlock_type == USYNC_PROCESS) {
- uint32_t *rwstate =
- (uint32_t *)&si_p->si_state.nreaders;
-
- if (*rwstate & URW_WRITE_LOCKED) {
- si_p->si_state.nreaders = -1;
- si_p->si_is_wlock = 1;
- si_p->si_owner.th_ta_p = sh_p->sh_ta_p;
- si_p->si_owner.th_unique =
- generic_so.rwlock.rwlock_owner;
- } else if (*rwstate & URW_READERS_MASK)
- si_p->si_state.nreaders =
- *rwstate & URW_READERS_MASK;
- else
- si_p->si_state.nreaders = 0;
- si_p->si_has_waiters = (*rwstate & URW_HAS_WAITERS);
+
+ rwstate = (uint32_t)generic_so.rwlock.rwlock_readers;
+ if (rwstate & URW_WRITE_LOCKED) {
+ si_p->si_state.nreaders = -1;
+ si_p->si_is_wlock = 1;
+ si_p->si_owner.th_ta_p = sh_p->sh_ta_p;
+ si_p->si_owner.th_unique =
+ generic_so.rwlock.rwlock_owner;
+ if (si_p->si_shared_type & USYNC_PROCESS)
+ si_p->si_ownerpid =
+ generic_so.rwlock.rwlock_ownerpid;
} else {
- si_p->si_state.nreaders = generic_so.rwlock.readers;
- si_p->si_has_waiters =
- generic_so.rwlock.rwlock_mwaiters;
- if (si_p->si_state.nreaders == -1) {
- si_p->si_is_wlock = 1;
- si_p->si_owner.th_ta_p = sh_p->sh_ta_p;
- si_p->si_owner.th_unique =
- generic_so.rwlock.rwlock_mowner;
- }
+ si_p->si_state.nreaders = (rwstate & URW_READERS_MASK);
}
+ si_p->si_has_waiters = ((rwstate & URW_HAS_WAITERS) != 0);
+
/* this is useless but the old interface provided it */
si_p->si_data = (psaddr_t)generic_so.rwlock.readers;
break;
+ }
default:
return (TD_BADSH);
}
@@ -2823,36 +2816,18 @@ __td_sync_get_stats(const td_synchandle_t *sh_p, td_syncstats_t *ss_p)
}
case TDB_RWLOCK:
{
- psaddr_t cond_addr;
- tdb_sync_stats_t cond_stats;
td_rwlock_stats_t *rwsp = &ss_p->ss_un.rwlock;
ss_p->ss_info.si_type = TD_SYNC_RWLOCK;
ss_p->ss_info.si_size = sizeof (rwlock_t);
rwsp->rw_rdlock =
sync_stats.un.rwlock.rw_rdlock;
- cond_addr = (psaddr_t)&((rwlock_t *)sh_p->sh_unique)->readercv;
- if (read_sync_stats(ta_p, hashaddr, cond_addr, &cond_stats)
- == TD_OK) {
- rwsp->rw_rdlock_sleep =
- cond_stats.un.cond.cond_wait;
- rwsp->rw_rdlock_sleep_time =
- cond_stats.un.cond.cond_wait_sleep_time;
- }
rwsp->rw_rdlock_try =
sync_stats.un.rwlock.rw_rdlock_try;
rwsp->rw_rdlock_try_fail =
sync_stats.un.rwlock.rw_rdlock_try_fail;
rwsp->rw_wrlock =
sync_stats.un.rwlock.rw_wrlock;
- cond_addr = (psaddr_t)&((rwlock_t *)sh_p->sh_unique)->writercv;
- if (read_sync_stats(ta_p, hashaddr, cond_addr, &cond_stats)
- == TD_OK) {
- rwsp->rw_wrlock_sleep =
- cond_stats.un.cond.cond_wait;
- rwsp->rw_wrlock_sleep_time =
- cond_stats.un.cond.cond_wait_sleep_time;
- }
rwsp->rw_wrlock_hold_time =
sync_stats.un.rwlock.rw_wrlock_hold_time;
rwsp->rw_wrlock_try =
@@ -2900,8 +2875,8 @@ out:
* Change the state of a synchronization variable.
* 1) mutex lock state set to value
* 2) semaphore's count set to value
- * 3) writer's lock set to value
- * 4) reader's lock number of readers set to value
+ * 3) writer's lock set by value < 0
+ * 4) reader's lock number of readers set to value >= 0
* Currently unused by dbx.
*/
#pragma weak td_sync_setstate = __td_sync_setstate
@@ -2912,6 +2887,7 @@ __td_sync_setstate(const td_synchandle_t *sh_p, long lvalue)
int trunc = 0;
td_err_e return_val;
td_so_un_t generic_so;
+ uint32_t *rwstate;
int value = (int)lvalue;
if ((ph_p = ph_lock_sh(sh_p, &return_val)) == NULL)
@@ -2975,18 +2951,12 @@ __td_sync_setstate(const td_synchandle_t *sh_p, long lvalue)
return_val = TD_DBERR;
break;
}
- if (generic_so.rwlock.rwlock_type == USYNC_PROCESS) {
- uint32_t *rwstate =
- (uint32_t *)&generic_so.rwlock.readers;
- if (value < 0)
- *rwstate = URW_WRITE_LOCKED;
- else if (value > 0)
- *rwstate = (value & URW_READERS_MASK);
- else
- *rwstate = 0;
- } else
- generic_so.rwlock.readers = value;
-
+ rwstate = (uint32_t *)&generic_so.rwlock.readers;
+ *rwstate &= URW_HAS_WAITERS;
+ if (value < 0)
+ *rwstate |= URW_WRITE_LOCKED;
+ else
+ *rwstate |= (value & URW_READERS_MASK);
if (ps_pdwrite(ph_p, sh_p->sh_unique, &generic_so.rwlock,
sizeof (generic_so.rwlock)) != PS_OK)
return_val = TD_DBERR;
diff --git a/usr/src/uts/common/sys/synch.h b/usr/src/uts/common/sys/synch.h
index 8f52d720cb..9b428e6040 100644
--- a/usr/src/uts/common/sys/synch.h
+++ b/usr/src/uts/common/sys/synch.h
@@ -2,9 +2,8 @@
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License"). You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
*
* CDDL HEADER END
*/
+
/*
- * Copyright 2003 Sun Microsystems, Inc. All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
@@ -116,12 +116,12 @@ typedef struct _lwp_sema {
* for rwlock_t in head/sync.h that we cannot change.
*/
typedef struct _lwp_rwlock {
- int32_t readers; /* -1 == writer else # of readers */
+ int32_t readers; /* rwstate word */
uint16_t type;
uint16_t magic;
- lwp_mutex_t mutex; /* used to indicate ownership */
- lwp_cond_t readercv; /* unused */
- lwp_cond_t writercv; /* unused */
+ lwp_mutex_t mutex; /* used with process-shared rwlocks */
+ lwp_cond_t readercv; /* used only to indicate ownership */
+ lwp_cond_t writercv; /* used only to indicate ownership */
} lwp_rwlock_t;
#endif /* _ASM */
diff --git a/usr/src/uts/common/syscall/lwp_sobj.c b/usr/src/uts/common/syscall/lwp_sobj.c
index 5b255912a0..65d9236a32 100644
--- a/usr/src/uts/common/syscall/lwp_sobj.c
+++ b/usr/src/uts/common/syscall/lwp_sobj.c
@@ -2,9 +2,8 @@
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License"). You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
@@ -19,15 +18,15 @@
*
* CDDL HEADER END
*/
-/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
-/* All Rights Reserved */
-
/*
- * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
+/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
+/* All Rights Reserved */
+
#pragma ident "%Z%%M% %I% %E% SMI"
#include <sys/param.h>
@@ -2187,7 +2186,7 @@ out:
*
* If the first thread is a reader we scan the queue releasing all readers
* until we hit a writer or the end of the queue. If the first thread is a
- * writer we still need to check for another writer (i.e. URW_WRITE_WANTED).
+ * writer we still need to check for another writer.
*/
void
lwp_rwlock_release(lwpchan_t *lwpchan, lwp_rwlock_t *rw)
@@ -2218,9 +2217,7 @@ lwp_rwlock_release(lwpchan_t *lwpchan, lwp_rwlock_t *rw)
/* tpp already set for next thread. */
continue;
} else {
- rwstate |=
- (URW_WRITE_WANTED|URW_HAS_WAITERS);
-
+ rwstate |= URW_HAS_WAITERS;
/* We need look no further. */
break;
}
@@ -2236,8 +2233,11 @@ lwp_rwlock_release(lwpchan_t *lwpchan, lwp_rwlock_t *rw)
/* tpp already set for next thread. */
continue;
- } else
+ } else {
rwstate |= URW_HAS_WAITERS;
+ /* We need look no further. */
+ break;
+ }
}
}
tpp = &tp->t_link;
@@ -2398,54 +2398,42 @@ lwp_rwlock_lock(lwp_rwlock_t *rw, timespec_t *tsp, int rd_wr)
/*
* Fetch the current rwlock state.
*
- * The possibility of spurious wake-ups or killed waiters means that
- * rwstate's URW_HAS_WAITERS and URW_WRITE_WANTED bits may indicate
- * false positives. We only fix these if they are important to us.
+ * The possibility of spurious wake-ups or killed waiters means
+ * rwstate's URW_HAS_WAITERS bit may indicate false positives.
+ * We only fix these if they are important to us.
*
* Although various error states can be observed here (e.g. the lock
* is not held, but there are waiters) we assume these are applicaton
* errors and so we take no corrective action.
*/
fuword32_noerr(&rw->rwlock_readers, &rwstate);
-
/*
- * If the lock is uncontended we can acquire it here. These tests
- * should have already been done at user-level, we just need to be
- * sure.
+ * We cannot legitimately get here from user-level
+ * without URW_HAS_WAITERS being set.
+ * Set it now to guard against user-level error.
*/
- if (rd_wr == READ_LOCK) {
- if ((rwstate & ~URW_READERS_MASK) == 0) {
- rwstate++;
- acquired = 1;
- }
- } else if (rwstate == 0) {
- rwstate = URW_WRITE_LOCKED;
- acquired = 1;
- }
+ rwstate |= URW_HAS_WAITERS;
/*
- * We can only try harder if the lock isn't held by a writer.
+ * We can try only if the lock isn't held by a writer.
*/
- if (!acquired && !(rwstate & URW_WRITE_LOCKED)) {
+ if (!(rwstate & URW_WRITE_LOCKED)) {
tp = lwp_queue_waiter(&lwpchan);
if (tp == NULL) {
/*
* Hmmm, rwstate indicates waiters but there are
* none queued. This could just be the result of a
- * spurious wakeup, so let's fix it.
- */
- rwstate &= URW_READERS_MASK;
-
- /*
- * We now have another chance to acquire the lock
- * uncontended, but this is the last chance for a
- * writer to acquire the lock without blocking.
+ * spurious wakeup, so let's ignore it.
+ *
+ * We now have a chance to acquire the lock
+ * uncontended, but this is the last chance for
+ * a writer to acquire the lock without blocking.
*/
if (rd_wr == READ_LOCK) {
rwstate++;
acquired = 1;
- } else if (rwstate == 0) {
- rwstate = URW_WRITE_LOCKED;
+ } else if ((rwstate & URW_READERS_MASK) == 0) {
+ rwstate |= URW_WRITE_LOCKED;
acquired = 1;
}
} else if (rd_wr == READ_LOCK) {
@@ -2475,7 +2463,7 @@ lwp_rwlock_lock(lwp_rwlock_t *rw, timespec_t *tsp, int rd_wr)
if (acquired || try_flag || time_error) {
/*
- * We're not going to block this time!
+ * We're not going to block this time.
*/
suword32_noerr(&rw->rwlock_readers, rwstate);
lwpchan_unlock(&lwpchan, LWPCHAN_CVPOOL);
@@ -2510,16 +2498,16 @@ lwp_rwlock_lock(lwp_rwlock_t *rw, timespec_t *tsp, int rd_wr)
* We're about to block, so indicate what kind of waiter we are.
*/
t->t_writer = 0;
- rwstate |= URW_HAS_WAITERS;
- if (rd_wr == WRITE_LOCK) {
+ if (rd_wr == WRITE_LOCK)
t->t_writer = TRW_WANT_WRITE;
- rwstate |= URW_WRITE_WANTED;
- }
suword32_noerr(&rw->rwlock_readers, rwstate);
/*
* Unlock the rwlock's mutex (pagefaults are possible here).
*/
+ suword32_noerr((uint32_t *)&mp->mutex_owner, 0);
+ suword32_noerr((uint32_t *)&mp->mutex_owner + 1, 0);
+ suword32_noerr(&mp->mutex_ownerpid, 0);
ulock_clear(&mp->mutex_lockw);
fuword8_noerr(&mp->mutex_waiters, &mwaiters);
if (mwaiters != 0) {
@@ -2622,6 +2610,8 @@ out_drop:
lwpchan_lock(&mlwpchan, LWPCHAN_MPPOOL);
mlocked = 1;
}
+ suword32_noerr((uint32_t *)&mp->mutex_owner, 0);
+ suword32_noerr((uint32_t *)&mp->mutex_owner + 1, 0);
suword32_noerr(&mp->mutex_ownerpid, 0);
ulock_clear(&mp->mutex_lockw);
fuword8_noerr(&mp->mutex_waiters, &mwaiters);
diff --git a/usr/src/uts/intel/sys/synch32.h b/usr/src/uts/intel/sys/synch32.h
index 3966fac47d..1674444b6d 100644
--- a/usr/src/uts/intel/sys/synch32.h
+++ b/usr/src/uts/intel/sys/synch32.h
@@ -2,9 +2,8 @@
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License"). You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
*
* CDDL HEADER END
*/
+
/*
- * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
@@ -67,17 +67,12 @@ extern "C" {
#define rwlock_readers readers
#define rwlock_type type
#define rwlock_magic magic
-#define rwlock_mwaiters mutex.mutex_waiters
-#define rwlock_mlockw mutex.mutex_lockw
-#define rwlock_mowner mutex.mutex_owner
-#define rwlock_mownerpid mutex.mutex_ownerpid
#define rwlock_owner readercv.data
#define rwlock_ownerpid writercv.data
#define URW_HAS_WAITERS 0x80000000
-#define URW_WRITE_WANTED 0x40000000
-#define URW_WRITE_LOCKED 0x20000000
-#define URW_READERS_MASK 0x1fffffff
+#define URW_WRITE_LOCKED 0x40000000
+#define URW_READERS_MASK 0x3fffffff
#ifdef __cplusplus
}
diff --git a/usr/src/uts/sparc/sys/synch32.h b/usr/src/uts/sparc/sys/synch32.h
index 1d45147bda..7fe623540f 100644
--- a/usr/src/uts/sparc/sys/synch32.h
+++ b/usr/src/uts/sparc/sys/synch32.h
@@ -2,9 +2,8 @@
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License"). You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
*
* CDDL HEADER END
*/
+
/*
- * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
@@ -67,17 +67,12 @@ extern "C" {
#define rwlock_readers readers
#define rwlock_type type
#define rwlock_magic magic
-#define rwlock_mwaiters mutex.mutex_waiters
-#define rwlock_mlockw mutex.mutex_lockw
-#define rwlock_mowner mutex.mutex_owner
-#define rwlock_mownerpid mutex.mutex_ownerpid
#define rwlock_owner readercv.data
#define rwlock_ownerpid writercv.data
#define URW_HAS_WAITERS 0x80000000
-#define URW_WRITE_WANTED 0x40000000
-#define URW_WRITE_LOCKED 0x20000000
-#define URW_READERS_MASK 0x1fffffff
+#define URW_WRITE_LOCKED 0x40000000
+#define URW_READERS_MASK 0x3fffffff
#ifdef __cplusplus
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-04-27 23:22:21 +0000