Sync with the current sources from onnv-gate.

The solaris sources use a non-portable create-thread-suspended flag when
spawning the update thread; I've thrown together a pthreads portable
equivalent.  This has not had any real level of testing.

These changes include a lock around the underlying brk() call; the lack of lock
in earlier revisions of this repo may be the reason that
UMEM_OPTIONS=backend=sbrk was flaky.



git-svn-id: https://labs.omniti.com/portableumem/trunk@59 1665872d-e22b-0410-9e5d-a57ad4215e6d

13 files changed, 465 insertions, 188 deletions

diff --git a/envvar.c b/envvar.c
index 1a09a29..9db420c 100644
--- a/envvar.c
+++ b/envvar.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,8 +18,9 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 /*
@@ -115,6 +115,9 @@ static char *safe_getenv(const char *name)
 
 static arg_process_t umem_log_process;
 
+static size_t umem_size_tempval;
+static arg_process_t umem_size_process;
+
 const char *____umem_environ_msg_options = "-- UMEM_OPTIONS --";
 
 static umem_env_item_t umem_options_items[] = {
@@ -144,8 +147,27 @@ static umem_env_item_t umem_options_items[] = {
 		NULL, 0,	&umem_reap_interval
 	},
 
+	{ "size_add",		"Private",	ITEM_SPECIAL,
+		"add a size to the cache size table",
+		NULL, 0, NULL,
+		&umem_size_tempval,		&umem_size_process
+	},
+	{ "size_clear",		"Private",	ITEM_SPECIAL,
+		"clear all but the largest size from the cache size table",
+		NULL, 0, NULL,
+		&umem_size_tempval,		&umem_size_process
+	},
+	{ "size_remove",	"Private",	ITEM_SPECIAL,
+	    "remove a size from the cache size table",
+		NULL, 0, NULL,
+		&umem_size_tempval,		&umem_size_process
+	},
 #ifndef _WIN32
 #ifndef UMEM_STANDALONE
+	{ "sbrk_minalloc",	"Private",	ITEM_SIZE,
+		"The minimum allocation chunk for the sbrk(2) heap.",
+		NULL, 0, NULL,	&vmem_sbrk_minalloc
+	},
 	{ "sbrk_pagesize",	"Private",	ITEM_SIZE,
 		"The preferred page size for the sbrk(2) heap.",
 		NULL, 0, NULL,	&vmem_sbrk_pagesize
@@ -413,6 +435,49 @@ umem_log_process(const umem_env_item_t *item, const char *item_arg)
 	return (ARG_SUCCESS);
 }
 
+static int
+umem_size_process(const umem_env_item_t *item, const char *item_arg)
+{
+	const char *name = item->item_name;
+	void (*action_func)(size_t);
+
+	size_t result;
+
+	int ret;
+
+	if (strcmp(name, "size_clear") == 0) {
+		if (item_arg != NULL) {
+			log_message("%s: %s: does not take a value. ignored\n",
+			    CURRENT, name);
+			return (ARG_BAD);
+		}
+		umem_alloc_sizes_clear();
+		return (ARG_SUCCESS);
+	} else if (strcmp(name, "size_add") == 0) {
+		action_func = umem_alloc_sizes_add;
+	} else if (strcmp(name, "size_remove") == 0) {
+		action_func = umem_alloc_sizes_remove;
+	} else {
+		log_message("%s: %s: internally unrecognized\n",
+		    CURRENT, name, name, name);
+		return (ARG_BAD);
+	}
+
+	if (item_arg == NULL) {
+		log_message("%s: %s: requires a value. ignored\n",
+		    CURRENT, name);
+		return (ARG_BAD);
+	}
+
+	ret = item_size_process(item, item_arg);
+	if (ret != ARG_SUCCESS)
+		return (ret);
+
+	result = *item->item_size_target;
+	action_func(result);
+	return (ARG_SUCCESS);
+}
+
 #ifndef UMEM_STANDALONE
 static int
 umem_backend_process(const umem_env_item_t *item, const char *item_arg)
@@ -649,6 +714,7 @@ umem_setup_envvars(int invalid)
 # define dlerror()		0
 #endif
 	state = STATE_DLOPEN;
+
 	/* get a handle to the "a.out" object */
 	if ((h = dlopen(0, RTLD_FIRST | RTLD_LAZY)) != NULL) {
 		for (cur_env = umem_envvars; cur_env->env_name != NULL;
diff --git a/misc.c b/misc.c
index 3fe3e94..35861c2 100644
--- a/misc.c
+++ b/misc.c
@@ -20,7 +20,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 /*
@@ -50,10 +50,6 @@
 #include <umem_impl.h>
 #include "misc.h"
 
-#ifdef ECELERITY
-#include "util.h"
-#endif
-
 #define	UMEM_ERRFD	2	/* goes to standard error */
 #define	UMEM_MAX_ERROR_SIZE 4096 /* error messages are truncated to this */
 
@@ -80,15 +76,12 @@ static uint_t umem_error_end = 0;
 }
 
 static void
-umem_log_enter(const char *error_str, int serious)
+umem_log_enter(const char *error_str)
 {
 	int looped;
 	char c;
 
 	looped = 0;
-#ifdef ECELERITY
-	mem_printf(serious ? DCRITICAL : DINFO, "umem: %s", error_str);
-#endif
 
 	(void) mutex_lock(&umem_error_lock);
 
@@ -121,7 +114,7 @@ umem_error_enter(const char *error_str)
 		(void) write(UMEM_ERRFD, error_str, strlen(error_str));
 #endif
 
-	umem_log_enter(error_str, 1);
+	umem_log_enter(error_str);
 }
 
 int
@@ -207,7 +200,7 @@ log_message(const char *format, ...)
 		(void) write(UMEM_ERRFD, buf, strlen(buf));
 #endif
 
-	umem_log_enter(buf, 0);
+	umem_log_enter(buf);
 }
 
 #ifndef UMEM_STANDALONE
@@ -296,6 +289,7 @@ print_sym(void *pointer)
 		return (1);
 	}
 #else
+	umem_printf("?? (0x%p)", pointer);
 	return 0;
 #endif
 }
diff --git a/sol_compat.h b/sol_compat.h
index 7ea25be..0f7846f 100644
--- a/sol_compat.h
+++ b/sol_compat.h
@@ -205,6 +205,14 @@ static INLINE uint64_t umem_atomic_inc64(uint64_t *val)
 #define P2SAMEHIGHBIT(x, y)    (((x) ^ (y)) < ((x) & (y)))
 #define IS_P2ALIGNED(v, a) ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0)
 #define ISP2(x)    (((x) & ((x) - 1)) == 0)
+/*
+ * return TRUE if adding len to off would cause it to cross an align
+ * boundary.
+ * eg, P2BOUNDARY(0x1234, 0xe0, 0x100) == TRUE (0x1234 + 0xe0 == 0x1314)
+ * eg, P2BOUNDARY(0x1234, 0x50, 0x100) == FALSE (0x1234 + 0x50 == 0x1284)
+ */
+#define P2BOUNDARY(off, len, align) \
+    (((off) ^ ((off) + (len) - 1)) > (align) - 1)
 
 /* beware! umem only uses these atomic adds for incrementing by 1 */
 #define atomic_add_64(lvalptr, delta) umem_atomic_inc64(lvalptr)
diff --git a/umem.c b/umem.c
index 7da095a..d93fd79 100644
--- a/umem.c
+++ b/umem.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License.
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 /*
@@ -88,6 +87,7 @@
  *
  *	* KM_SLEEP v.s. UMEM_NOFAIL
  *
+ *	* lock ordering
  *
  * 2. Initialization
  * -----------------
@@ -362,6 +362,32 @@
  *	If a constructor callback _does_ do a UMEM_NOFAIL allocation, and
  *	the nofail callback does a non-local exit, we will leak the
  *	partially-constructed buffer.
+ *
+ *
+ * 6. Lock Ordering
+ * ----------------
+ * umem has a few more locks than kmem does, mostly in the update path.  The
+ * overall lock ordering (earlier locks must be acquired first) is:
+ *
+ *	umem_init_lock
+ *
+ *	vmem_list_lock
+ *	vmem_nosleep_lock.vmpl_mutex
+ *	vmem_t's:
+ *		vm_lock
+ *	sbrk_lock
+ *
+ *	umem_cache_lock
+ *	umem_update_lock
+ *	umem_flags_lock
+ *	umem_cache_t's:
+ *		cache_cpu[*].cc_lock
+ *		cache_depot_lock
+ *		cache_lock
+ *	umem_log_header_t's:
+ *		lh_cpu[*].clh_lock
+ *		lh_lock
+ *
  * \endcode
  */
 
@@ -413,8 +439,12 @@ size_t pagesize;
  * bytes, so that it will be 64-byte aligned.  For all multiples of 64,
  * the next kmem_cache_size greater than or equal to it must be a
  * multiple of 64.
+ *
+ * This table must be in sorted order, from smallest to highest.  The
+ * highest slot must be UMEM_MAXBUF, and every slot afterwards must be
+ * zero.
  */
-static const int umem_alloc_sizes[] = {
+static int umem_alloc_sizes[] = {
 #ifdef _LP64
 	1 * 8,
 	1 * 16,
@@ -433,17 +463,19 @@ static const int umem_alloc_sizes[] = {
 	P2ALIGN(8192 / 7, 64),
 	P2ALIGN(8192 / 6, 64),
 	P2ALIGN(8192 / 5, 64),
-	P2ALIGN(8192 / 4, 64),
+	P2ALIGN(8192 / 4, 64), 2304,
 	P2ALIGN(8192 / 3, 64),
-	P2ALIGN(8192 / 2, 64),
-	P2ALIGN(8192 / 1, 64),
+	P2ALIGN(8192 / 2, 64), 4544,
+	P2ALIGN(8192 / 1, 64), 9216,
 	4096 * 3,
-	8192 * 2,
+	UMEM_MAXBUF,				/* = 8192 * 2 */
+	/* 24 slots for user expansion */
+	0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0,
 };
 #define	NUM_ALLOC_SIZES (sizeof (umem_alloc_sizes) / sizeof (*umem_alloc_sizes))
 
-#define	UMEM_MAXBUF	16384
-
 static umem_magtype_t umem_magtype[] = {
 	{ 1,	8,	3200,	65536	},
 	{ 3,	16,	256,	32768	},
@@ -757,6 +789,8 @@ umem_remove_updates(umem_cache_t *cp)
 	 * Get it out of the active state
 	 */
 	while (cp->cache_uflags & UMU_ACTIVE) {
+		int cancel_state;
+
 		ASSERT(cp->cache_unext == NULL);
 
 		cp->cache_uflags |= UMU_NOTIFY;
@@ -768,7 +802,10 @@ umem_remove_updates(umem_cache_t *cp)
 		ASSERT(umem_update_thr != thr_self() &&
 		    umem_st_update_thr != thr_self());
 
-		(void) _cond_wait(&umem_update_cv, &umem_update_lock);
+		(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
+		    &cancel_state);
+		(void) cond_wait(&umem_update_cv, &umem_update_lock);
+		(void) pthread_setcancelstate(cancel_state, NULL);
 	}
 	/*
 	 * Get it out of the Work Requested state
@@ -1097,7 +1134,7 @@ umem_log_enter(umem_log_header_t *lhp, void *data, size_t size)
 {
 	void *logspace;
 	umem_cpu_log_header_t *clhp =
-	    &(lhp->lh_cpu[CPU(umem_cpu_mask)->cpu_number]);
+	    &lhp->lh_cpu[CPU(umem_cpu_mask)->cpu_number];
 
 	if (lhp == NULL || umem_logging == 0)
 		return (NULL);
@@ -2769,6 +2806,88 @@ umem_cache_destroy(umem_cache_t *cp)
 	vmem_free(umem_cache_arena, cp, UMEM_CACHE_SIZE(umem_max_ncpus));
 }
 
+void
+umem_alloc_sizes_clear(void)
+{
+	int i;
+
+	umem_alloc_sizes[0] = UMEM_MAXBUF;
+	for (i = 1; i < NUM_ALLOC_SIZES; i++)
+		umem_alloc_sizes[i] = 0;
+}
+
+void
+umem_alloc_sizes_add(size_t size_arg)
+{
+	int i, j;
+	size_t size = size_arg;
+
+	if (size == 0) {
+		log_message("size_add: cannot add zero-sized cache\n",
+		    size, UMEM_MAXBUF);
+		return;
+	}
+
+	if (size > UMEM_MAXBUF) {
+		log_message("size_add: %ld > %d, cannot add\n", size,
+		    UMEM_MAXBUF);
+		return;
+	}
+
+	if (umem_alloc_sizes[NUM_ALLOC_SIZES - 1] != 0) {
+		log_message("size_add: no space in alloc_table for %d\n",
+		    size);
+		return;
+	}
+
+	if (P2PHASE(size, UMEM_ALIGN) != 0) {
+		size = P2ROUNDUP(size, UMEM_ALIGN);
+		log_message("size_add: rounding %d up to %d\n", size_arg,
+		    size);
+	}
+
+	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
+		int cur = umem_alloc_sizes[i];
+		if (cur == size) {
+			log_message("size_add: %ld already in table\n",
+			    size);
+			return;
+		}
+		if (cur > size)
+			break;
+	}
+
+	for (j = NUM_ALLOC_SIZES - 1; j > i; j--)
+		umem_alloc_sizes[j] = umem_alloc_sizes[j-1];
+	umem_alloc_sizes[i] = size;
+}
+
+void
+umem_alloc_sizes_remove(size_t size)
+{
+	int i;
+
+	if (size == UMEM_MAXBUF) {
+		log_message("size_remove: cannot remove %ld\n", size);
+		return;
+	}
+
+	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
+		int cur = umem_alloc_sizes[i];
+		if (cur == size)
+			break;
+		else if (cur > size || cur == 0) {
+			log_message("size_remove: %ld not found in table\n",
+			    size);
+			return;
+		}
+	}
+
+	for (; i + 1 < NUM_ALLOC_SIZES; i++)
+		umem_alloc_sizes[i] = umem_alloc_sizes[i+1];
+	umem_alloc_sizes[i] = 0;
+}
+
 static int
 umem_cache_init(void)
 {
@@ -2861,6 +2980,10 @@ umem_cache_init(void)
 	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
 		size_t cache_size = umem_alloc_sizes[i];
 		size_t align = 0;
+
+		if (cache_size == 0)
+			break;		/* 0 terminates the list */
+
 		/*
 		 * If they allocate a multiple of the coherency granularity,
 		 * they get a coherency-granularity-aligned address.
@@ -2888,6 +3011,9 @@ umem_cache_init(void)
 	for (i = 0; i < NUM_ALLOC_SIZES; i++) {
 		size_t cache_size = umem_alloc_sizes[i];
 
+		if (cache_size == 0)
+			break;		/* 0 terminates the list */
+
 		cp = umem_alloc_caches[i];
 
 		while (size <= cache_size) {
@@ -2895,6 +3021,7 @@ umem_cache_init(void)
 			size += UMEM_ALIGN;
 		}
 	}
+	ASSERT(size - UMEM_ALIGN == UMEM_MAXBUF);
 	return (1);
 }
 
@@ -2904,7 +3031,7 @@ umem_cache_init(void)
  */
 void
 umem_startup(caddr_t start, size_t len, size_t pagesize, caddr_t minstack,
-    caddr_t maxstack) 
+	caddr_t maxstack) 
 {
 #ifdef UMEM_STANDALONE
 	int idx;
@@ -2989,9 +3116,16 @@ umem_init(void)
 			 * someone else beat us to initializing umem.  Wait
 			 * for them to complete, then return.
 			 */
-			while (umem_ready == UMEM_READY_INITING)
-				(void) _cond_wait(&umem_init_cv,
+			while (umem_ready == UMEM_READY_INITING) {
+				int cancel_state;
+
+				(void) pthread_setcancelstate(
+				    PTHREAD_CANCEL_DISABLE, &cancel_state);
+				(void) cond_wait(&umem_init_cv,
 				    &umem_init_lock);
+				(void) pthread_setcancelstate(
+				    cancel_state, NULL);
+			}
 			ASSERT(umem_ready == UMEM_READY ||
 			    umem_ready == UMEM_READY_INIT_FAILED);
 			(void) mutex_unlock(&umem_init_lock);
diff --git a/umem_base.h b/umem_base.h
index ad3cc1e..09774b5 100644
--- a/umem_base.h
+++ b/umem_base.h
@@ -120,6 +120,10 @@ extern void umem_process_updates(void);
 extern void umem_cache_applyall(void (*)(umem_cache_t *));
 extern void umem_cache_update(umem_cache_t *);
 
+extern void umem_alloc_sizes_add(size_t);
+extern void umem_alloc_sizes_clear(void);
+extern void umem_alloc_sizes_remove(size_t);
+
 /*
  * umem_fork.c: private interfaces
  */
diff --git a/umem_fail.c b/umem_fail.c
index d38dc7c..30c4b58 100644
--- a/umem_fail.c
+++ b/umem_fail.c
@@ -137,12 +137,6 @@ umem_panic(const char *format, ...)
 	if (format[strlen(format)-1] != '\n')
 		umem_error_enter("\n");
 
-#ifdef ECELERITY
-	va_start(va, format);
-	ec_debug_vprintf(DCRITICAL, DMEM, format, va);
-	va_end(va);
-#endif
-	
 	print_stacktrace();
 
 	umem_do_abort();
@@ -171,7 +165,6 @@ __umem_assert_failed(const char *assertion, const char *file, int line)
 {
 	umem_panic("Assertion failed: %s, file %s, line %d\n",
 	    assertion, file, line);
-	umem_do_abort();
 	/*NOTREACHED*/
 	return (0);
 }
diff --git a/umem_fork.c b/umem_fork.c
index 03fac89..76f7e56 100644
--- a/umem_fork.c
+++ b/umem_fork.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -19,18 +18,13 @@
  *
  * CDDL HEADER END
  */
+
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
-/*
- * Portions Copyright 2006-2008 Message Systems, Inc.
- */
-
-/* #pragma ident	"@(#)umem_fork.c	1.3	05/06/08 SMI" */
 
 #include "config.h"
-/* #include "mtlib.h" */
 #include "umem_base.h"
 #include "vmem_base.h"
 
@@ -38,7 +32,8 @@
 #include <unistd.h>
 
 /*
- * The following functions are for pre- and post-fork1(2) handling.
+ * The following functions are for pre- and post-fork1(2) handling.  See
+ * "Lock Ordering" in lib/libumem/common/umem.c for the lock ordering used.
  */
 
 static void
@@ -108,6 +103,10 @@ umem_lockup(void)
 		(void) umem_init();
 		(void) mutex_lock(&umem_init_lock);
 	}
+
+	vmem_lockup();
+	vmem_sbrk_lockup();
+
 	(void) mutex_lock(&umem_cache_lock);
 	(void) mutex_lock(&umem_update_lock);
 	(void) mutex_lock(&umem_flags_lock);
@@ -124,46 +123,30 @@ umem_lockup(void)
 
 	(void) cond_broadcast(&umem_update_cv);
 
-	vmem_sbrk_lockup();
-	vmem_lockup();
-}
-
-static void
-umem_release(void)
-{
-	umem_cache_t *cp;
-
-	vmem_release();
-	vmem_sbrk_release();
-
-	umem_release_log_header(umem_slab_log);
-	umem_release_log_header(umem_failure_log);
-	umem_release_log_header(umem_content_log);
-	umem_release_log_header(umem_transaction_log);
-
-	for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
-	    cp = cp->cache_next)
-		umem_release_cache(cp);
-	umem_release_cache(&umem_null_cache);
-
-	(void) mutex_unlock(&umem_flags_lock);
-	(void) mutex_unlock(&umem_update_lock);
-	(void) mutex_unlock(&umem_cache_lock);
-	(void) mutex_unlock(&umem_init_lock);
 }
 
 static void
-umem_release_child(void)
+umem_do_release(int as_child)
 {
 	umem_cache_t *cp;
+	int cleanup_update = 0;
 
 	/*
-	 * Clean up the update state
+	 * Clean up the update state if we are the child process and
+	 * another thread was processing updates.
 	 */
-	umem_update_thr = 0;
+	if (as_child) {
+		if (umem_update_thr != thr_self()) {
+			umem_update_thr = 0;
+			cleanup_update = 1;
+		}
+		if (umem_st_update_thr != thr_self()) {
+			umem_st_update_thr = 0;
+			cleanup_update = 1;
+		}
+	}
 
-	if (umem_st_update_thr != thr_self()) {
-		umem_st_update_thr = 0;
+	if (cleanup_update) {
 		umem_reaping = UMEM_REAP_DONE;
 
 		for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
@@ -196,7 +179,36 @@ umem_release_child(void)
 		}
 	}
 
-	umem_release();
+	umem_release_log_header(umem_slab_log);
+	umem_release_log_header(umem_failure_log);
+	umem_release_log_header(umem_content_log);
+	umem_release_log_header(umem_transaction_log);
+
+	for (cp = umem_null_cache.cache_next; cp != &umem_null_cache;
+	    cp = cp->cache_next)
+		umem_release_cache(cp);
+	umem_release_cache(&umem_null_cache);
+
+	(void) mutex_unlock(&umem_flags_lock);
+	(void) mutex_unlock(&umem_update_lock);
+	(void) mutex_unlock(&umem_cache_lock);
+
+	vmem_sbrk_release();
+	vmem_release();
+
+	(void) mutex_unlock(&umem_init_lock);
+}
+
+static void
+umem_release(void)
+{
+	umem_do_release(0);
+}
+
+static void
+umem_release_child(void)
+{
+	umem_do_release(1);
 }
 #endif
 
diff --git a/umem_update_thread.c b/umem_update_thread.c
index 85de1ed..6516061 100644
--- a/umem_update_thread.c
+++ b/umem_update_thread.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 /*
@@ -35,10 +34,18 @@
 
 #include <signal.h>
 
-/*
- * we use the _ version, since we don't want to be cancelled.
- */
-extern int _cond_timedwait(cond_t *cv, mutex_t *mutex, const timespec_t *delay);
+struct umem_suspend_signal_object {
+	/* locked by creating thread; unlocked when umem_update_thread
+	 * can proceed */
+	pthread_mutex_t mtx;
+	/* lock associated with the condition variable */
+	pthread_mutex_t cmtx;
+	/* condition variable is signalled by umem_update_thread when
+	 * it has obtained the mtx; it is then safe for the creating
+	 * thread to clean up its stack (on which this object resides) */
+	pthread_cond_t cond;
+	int flag;
+};
 
 /*ARGSUSED*/
 static THR_RETURN
@@ -46,6 +53,12 @@ THR_API umem_update_thread(void *arg)
 {
 	struct timeval now;
 	int in_update = 0;
+	struct umem_suspend_signal_object *obj = arg;
+
+	pthread_mutex_lock(&obj->mtx);
+	obj->flag = 1;
+	pthread_cond_signal(&obj->cond);
+	obj = NULL;
 
 	(void) mutex_lock(&umem_update_lock);
 
@@ -110,12 +123,16 @@ THR_API umem_update_thread(void *arg)
 		 * next update, or someone wakes us.
 		 */
 		if (umem_null_cache.cache_unext == &umem_null_cache) {
+			int cancel_state;
 			timespec_t abs_time;
 			abs_time.tv_sec = umem_update_next.tv_sec;
 			abs_time.tv_nsec = umem_update_next.tv_usec * 1000;
 
-			(void) _cond_timedwait(&umem_update_cv,
+			(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
+			    &cancel_state);
+			(void) cond_timedwait(&umem_update_cv,
 			    &umem_update_lock, &abs_time);
+			(void) pthread_setcancelstate(cancel_state, NULL);
 		}
 	}
 	/* LINTED no return statement */
@@ -127,6 +144,10 @@ umem_create_update_thread(void)
 #ifndef _WIN32
 	sigset_t sigmask, oldmask;
 #endif
+	pthread_t newthread;
+	pthread_attr_t attr;
+	struct umem_suspend_signal_object obj;
+	int cancel_state;
 
 	ASSERT(MUTEX_HELD(&umem_update_lock));
 	ASSERT(umem_update_thr == 0);
@@ -136,18 +157,65 @@ umem_create_update_thread(void)
 	 * The update thread handles no signals
 	 */
 	(void) sigfillset(&sigmask);
-	(void) thr_sigsetmask(SIG_BLOCK, &sigmask, &oldmask);
+	(void) pthread_sigmask(SIG_BLOCK, &sigmask, &oldmask);
 #endif
-	if (thr_create(NULL, 0, umem_update_thread, NULL,
-	    THR_BOUND | THR_DAEMON | THR_DETACHED, &umem_update_thr) == 0) {
+	/*
+	 * drop the umem_update_lock; we cannot hold locks acquired in
+	 * pre-fork handler while calling thr_create or thr_continue().
+	 */
+
+	(void) mutex_unlock(&umem_update_lock);
+
+	pthread_attr_init(&attr);
+	pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+
+	pthread_mutex_init(&obj.mtx, NULL);
+	pthread_mutex_init(&obj.cmtx, NULL);
+	pthread_cond_init(&obj.cond, NULL);
+	obj.flag = 0;
+	pthread_mutex_lock(&obj.mtx);
+
+	if (pthread_create(&newthread, &attr, umem_update_thread, &obj) == 0) {
 #ifndef _WIN32
-		(void) thr_sigsetmask(SIG_SETMASK, &oldmask, NULL);
+		(void) pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
 #endif
+		(void) mutex_lock(&umem_update_lock);
+		/*
+		 * due to the locking in umem_reap(), only one thread can
+		 * ever call umem_create_update_thread() at a time.  This
+		 * must be the case for this code to work.
+		 */
+
+		ASSERT(umem_update_thr == 0);
+		umem_update_thr = newthread;
+		(void) mutex_unlock(&umem_update_lock);
+
+		/* tell the thread to continue */
+		pthread_mutex_unlock(&obj.mtx);
+
+		pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state);
+		/* wait for it to be done with obj */
+		pthread_mutex_lock(&obj.cmtx);
+		do {
+			if (obj.flag) {
+				break;
+			}
+			ASSERT(pthread_cond_wait(&obj.cond, &obj.cmtx) == 0);
+		} while (1);
+		pthread_setcancelstate(cancel_state, NULL);
+		pthread_mutex_destroy(&obj.mtx);
+		pthread_mutex_destroy(&obj.cmtx);
+		pthread_cond_destroy(&obj.cond);
+
+		(void) mutex_lock(&umem_update_lock);
+
 		return (1);
+	} else { /* thr_create failed */
+		(void) thr_sigsetmask(SIG_SETMASK, &oldmask, NULL);
+		(void) mutex_lock(&umem_update_lock);
+		pthread_mutex_destroy(&obj.mtx);
+		pthread_mutex_destroy(&obj.cmtx);
+		pthread_cond_destroy(&obj.cond);
 	}
-	umem_update_thr = 0;
-#ifndef _WIN32
-	(void) thr_sigsetmask(SIG_SETMASK, &oldmask, NULL);
-#endif
 	return (0);
 }
diff --git a/vmem.c b/vmem.c
index 1b8981a..7a5d659 100644
--- a/vmem.c
+++ b/vmem.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -37,7 +36,7 @@
  *	Proceedings of the 2001 Usenix Conference.
  *	Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
  *
- * For the "Big Theory Statement", see usr/src/common/os/vmem.c
+ * For the "Big Theory Statement", see usr/src/uts/common/os/vmem.c
  *
  * 1. Overview of changes
  * ------------------------------
@@ -231,12 +230,6 @@ uint32_t vmem_mtbf;		/* mean time between failures [default: off] */
 size_t vmem_seg_size = sizeof (vmem_seg_t);
 
 /*
- * we use the _ version, since we don't want to be cancelled.
- * Actually, this is automatically taken care of by including "mtlib.h".
- */
-extern int _cond_wait(cond_t *cv, mutex_t *mutex);
-
-/*
  * Insert/delete from arena list (type 'a') or next-of-kin list (type 'k').
  */
 #define	VMEM_INSERT(vprev, vsp, type)					\
@@ -775,6 +768,8 @@ vmem_nextfit_alloc(vmem_t *vmp, size_t size, int vmflag)
 			break;
 		vsp = vsp->vs_anext;
 		if (vsp == rotor) {
+			int cancel_state;
+
 			/*
 			 * We've come full circle.  One possibility is that the
 			 * there's actually enough space, but the rotor itself
@@ -799,7 +794,10 @@ vmem_nextfit_alloc(vmem_t *vmp, size_t size, int vmflag)
 				    0, 0, NULL, NULL, vmflag & VM_UMFLAGS));
 			}
 			vmp->vm_kstat.vk_wait++;
-			(void) _cond_wait(&vmp->vm_cv, &vmp->vm_lock);
+			(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
+			    &cancel_state);
+			(void) cond_wait(&vmp->vm_cv, &vmp->vm_lock);
+			(void) pthread_setcancelstate(cancel_state, NULL);
 			vsp = rotor->vs_anext;
 		}
 	}
@@ -867,6 +865,8 @@ vmem_xalloc(vmem_t *vmp, size_t size, size_t align, size_t phase,
 
 	(void) mutex_lock(&vmp->vm_lock);
 	for (;;) {
+		int cancel_state;
+
 		if (vmp->vm_nsegfree < VMEM_MINFREE &&
 		    !vmem_populate(vmp, vmflag))
 			break;
@@ -930,7 +930,7 @@ vmem_xalloc(vmem_t *vmp, size_t size, size_t align, size_t phase,
 			start = MAX(vsp->vs_start, (uintptr_t)minaddr);
 			end = MIN(vsp->vs_end - 1, (uintptr_t)maxaddr - 1) + 1;
 			taddr = P2PHASEUP(start, align, phase);
-			if (P2CROSS(taddr, taddr + size - 1, nocross))
+			if (P2BOUNDARY(taddr, size, nocross))
 				taddr +=
 				    P2ROUNDUP(P2NPHASE(taddr, nocross), align);
 			if ((taddr - start) + size > end - start ||
@@ -986,7 +986,10 @@ vmem_xalloc(vmem_t *vmp, size_t size, size_t align, size_t phase,
 		if (vmflag & VM_NOSLEEP)
 			break;
 		vmp->vm_kstat.vk_wait++;
-		(void) _cond_wait(&vmp->vm_cv, &vmp->vm_lock);
+		(void) pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,
+		    &cancel_state);
+		(void) cond_wait(&vmp->vm_cv, &vmp->vm_lock);
+		(void) pthread_setcancelstate(cancel_state, NULL);
 	}
 	if (vbest != NULL) {
 		ASSERT(vbest->vs_type == VMEM_FREE);
@@ -994,7 +997,7 @@ vmem_xalloc(vmem_t *vmp, size_t size, size_t align, size_t phase,
 		(void) vmem_seg_alloc(vmp, vbest, addr, size);
 		(void) mutex_unlock(&vmp->vm_lock);
 		ASSERT(P2PHASE(addr, align) == phase);
-		ASSERT(!P2CROSS(addr, addr + size - 1, nocross));
+		ASSERT(!P2BOUNDARY(addr, size, nocross));
 		ASSERT(addr >= (uintptr_t)minaddr);
 		ASSERT(addr + size - 1 <= (uintptr_t)maxaddr - 1);
 		return ((void *)addr);
diff --git a/vmem_base.c b/vmem_base.c
index d43ecde..40a988b 100644
--- a/vmem_base.c
+++ b/vmem_base.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
diff --git a/vmem_base.h b/vmem_base.h
index 06c1efc..dd9482f 100644
--- a/vmem_base.h
+++ b/vmem_base.h
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License, (the "License").
+   You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -60,6 +59,7 @@ extern void vmem_reap(void);		/* vmem_populate()-safe reap */
 
 extern size_t pagesize;
 extern size_t vmem_sbrk_pagesize;
+extern size_t vmem_sbrk_minalloc;
 
 extern uint_t vmem_backend;
 #define	VMEM_BACKEND_SBRK	0x0000001
diff --git a/vmem_mmap.c b/vmem_mmap.c
index 2abb2f0..d3cab97 100644
--- a/vmem_mmap.c
+++ b/vmem_mmap.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2002 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 /*
@@ -66,11 +65,9 @@ vmem_mmap_alloc(vmem_t *src, size_t size, int vmflags)
 
 	ret = vmem_alloc(src, size, vmflags);
 #ifndef _WIN32
-	if (ret != NULL
-		&&
+	if (ret != NULL &&
 	    mmap(ret, size, ALLOC_PROT, ALLOC_FLAGS | MAP_FIXED, -1, 0) ==
-	    MAP_FAILED
-		) {
+	    MAP_FAILED) {
 		vmem_free(src, ret, size);
 		vmem_reap();
 
@@ -116,18 +113,11 @@ vmem_mmap_top_alloc(vmem_t *src, size_t size, int vmflags)
 #ifdef _WIN32
 	buf = VirtualAlloc(NULL, size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
 	if (buf == NULL) buf = MAP_FAILED;
+#elif defined(MAP_ALIGN)
+	buf = mmap((void*)CHUNKSIZE, size, FREE_PROT, FREE_FLAGS | MAP_ALIGN,
+			-1, 0);
 #else
-	buf = mmap(
-#ifdef MAP_ALIGN
-			(void *)CHUNKSIZE,
-#else
-			0,
-#endif
-			size, FREE_PROT, FREE_FLAGS
-#ifdef MAP_ALIGN
-			| MAP_ALIGN
-#endif
-			, -1, 0);
+	buf = mmap(0, size, FREE_PROT, FREE_FLAGS, -1, 0);
 #endif
 
 	if (buf != MAP_FAILED) {
@@ -170,8 +160,7 @@ vmem_mmap_arena(vmem_alloc_t **a_out, vmem_free_t **f_out)
 #endif
 	
 	if (mmap_heap == NULL) {
-		mmap_heap = vmem_init("mmap_top", 
-			CHUNKSIZE,
+		mmap_heap = vmem_init("mmap_top", CHUNKSIZE,
 		    vmem_mmap_top_alloc, vmem_free,
 		    "mmap_heap", NULL, 0, pagesize,
 		    vmem_mmap_alloc, vmem_mmap_free);
diff --git a/vmem_sbrk.c b/vmem_sbrk.c
index c06b57a..cbeff98 100644
--- a/vmem_sbrk.c
+++ b/vmem_sbrk.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 /*
@@ -70,7 +69,8 @@
 
 size_t vmem_sbrk_pagesize = 0; /* the preferred page size of the heap */
 
-#define	MIN_ALLOC	(64*1024)
+#define	VMEM_SBRK_MINALLOC	(64 * 1024)
+size_t vmem_sbrk_minalloc = VMEM_SBRK_MINALLOC; /* minimum allocation */
 
 static size_t real_pagesize;
 static vmem_t *sbrk_heap;
@@ -90,8 +90,9 @@ static sbrk_fail_t sbrk_fails = {
 };
 
 static mutex_t sbrk_faillock = DEFAULTMUTEX;
+static mutex_t sbrk_lock = DEFAULTMUTEX;
 
-/*
+ /*
  * _sbrk_grow_aligned() aligns the old break to a low_align boundry,
  * adds min_size, aligns to a high_align boundry, and calls _brk_unlocked()
  * to set the new break.  The low_aligned-aligned value is returned, and
@@ -100,48 +101,52 @@ static mutex_t sbrk_faillock = DEFAULTMUTEX;
  * Unlike sbrk(2), _sbrk_grow_aligned takes an unsigned size, and does
  * not allow shrinking the heap.
  */
-void *
+static void *
 _sbrk_grow_aligned(size_t min_size, size_t low_align, size_t high_align,
     size_t *actual_size)
 {
-  uintptr_t old_brk;
-  uintptr_t ret_brk;
-  uintptr_t high_brk;
-  uintptr_t new_brk;
-  int brk_result;
+	uintptr_t old_brk;
+	uintptr_t ret_brk;
+	uintptr_t high_brk;
+	uintptr_t new_brk;
+	int brk_result;
 
 #define ALIGNSZ   16
 #define BRKALIGN(x) (caddr_t)P2ROUNDUP((uintptr_t)(x), ALIGNSZ)
-  
-  if ((low_align & (low_align - 1)) != 0 ||
-      (high_align & (high_align - 1)) != 0) {
-    errno = EINVAL;
-    return ((void *)-1);
-  }
-  low_align = MAX(low_align, ALIGNSZ);
-  high_align = MAX(high_align, ALIGNSZ);
-
-  old_brk = (uintptr_t)BRKALIGN(sbrk(0));
-  ret_brk = P2ROUNDUP(old_brk, low_align);
-  high_brk = ret_brk + min_size;
-  new_brk = P2ROUNDUP(high_brk, high_align);
-
-  /*
-   * Check for overflow
-   */
-  if (ret_brk < old_brk || high_brk < ret_brk || new_brk < high_brk) {
-    errno = ENOMEM;
-    return ((void *)-1);
-  }
-
-  brk_result = brk((void *)new_brk);
-
-  if (brk_result != 0)
-    return ((void *)-1);
-
-  if (actual_size != NULL)
-    *actual_size = (new_brk - ret_brk);
-  return ((void *)ret_brk);
+
+	if ((low_align & (low_align - 1)) != 0 ||
+			(high_align & (high_align - 1)) != 0) {
+		errno = EINVAL;
+		return ((void *)-1);
+	}
+	low_align = MAX(low_align, ALIGNSZ);
+	high_align = MAX(high_align, ALIGNSZ);
+
+	mutex_lock(&sbrk_lock);
+
+	old_brk = (uintptr_t)BRKALIGN(sbrk(0));
+	ret_brk = P2ROUNDUP(old_brk, low_align);
+	high_brk = ret_brk + min_size;
+	new_brk = P2ROUNDUP(high_brk, high_align);
+
+	/*
+	 * Check for overflow
+	 */
+	if (ret_brk < old_brk || high_brk < ret_brk || new_brk < high_brk) {
+		mutex_unlock(&sbrk_lock);
+		errno = ENOMEM;
+		return ((void *)-1);
+	}
+
+	brk_result = brk((void *)new_brk);
+	mutex_unlock(&sbrk_lock);
+
+	if (brk_result != 0)
+		return ((void *)-1);
+
+	if (actual_size != NULL)
+		*actual_size = (new_brk - ret_brk);
+	return ((void *)ret_brk);
 }
 
 /*
@@ -210,9 +215,6 @@ vmem_sbrk_tryfail(vmem_t *src, size_t size, int vmflags)
 static void *
 vmem_sbrk_alloc(vmem_t *src, size_t size, int vmflags)
 {
-	extern void *_sbrk_grow_aligned(size_t min_size, size_t low_align,
-	    size_t high_align, size_t *actual_size);
-
 	void *ret;
 	void *buf;
 	size_t buf_size;
@@ -234,7 +236,7 @@ vmem_sbrk_alloc(vmem_t *src, size_t size, int vmflags)
 	    (ret = vmem_sbrk_tryfail(src, size, vmflags)) != NULL)
 		return (ret);
 
-	buf_size = MAX(size, MIN_ALLOC);
+	buf_size = MAX(size, vmem_sbrk_minalloc);
 
 	/*
 	 * buf_size gets overwritten with the actual allocated size
@@ -311,6 +313,11 @@ vmem_sbrk_arena(vmem_alloc_t **a_out, vmem_free_t **f_out)
 		}
 		vmem_sbrk_pagesize = heap_size;
 
+		/* validate vmem_sbrk_minalloc */
+		if (vmem_sbrk_minalloc < VMEM_SBRK_MINALLOC)
+			vmem_sbrk_minalloc = VMEM_SBRK_MINALLOC;
+		vmem_sbrk_minalloc = P2ROUNDUP(vmem_sbrk_minalloc, heap_size);
+
 		sbrk_heap = vmem_init("sbrk_top", real_pagesize,
 		    vmem_sbrk_alloc, vmem_free,
 		    "sbrk_heap", NULL, 0, real_pagesize,