OpenSolaris Launch

1 files changed, 1790 insertions, 0 deletions

diff --git a/usr/src/uts/common/os/lwp.c b/usr/src/uts/common/os/lwp.c
new file mode 100644
index 0000000000..fca3e5f898
--- /dev/null
+++ b/usr/src/uts/common/os/lwp.c
@@ -0,0 +1,1790 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+#pragma ident	"%Z%%M%	%I%	%E% SMI"
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/sysmacros.h>
+#include <sys/systm.h>
+#include <sys/thread.h>
+#include <sys/proc.h>
+#include <sys/task.h>
+#include <sys/project.h>
+#include <sys/signal.h>
+#include <sys/errno.h>
+#include <sys/vmparam.h>
+#include <sys/stack.h>
+#include <sys/procfs.h>
+#include <sys/prsystm.h>
+#include <sys/cpuvar.h>
+#include <sys/kmem.h>
+#include <sys/vtrace.h>
+#include <sys/door.h>
+#include <vm/seg_kp.h>
+#include <sys/debug.h>
+#include <sys/tnf.h>
+#include <sys/schedctl.h>
+#include <sys/poll.h>
+#include <sys/copyops.h>
+#include <sys/lwp_upimutex_impl.h>
+#include <sys/cpupart.h>
+#include <sys/lgrp.h>
+#include <sys/rctl.h>
+#include <sys/contract_impl.h>
+#include <sys/cpc_impl.h>
+#include <sys/sdt.h>
+#include <sys/cmn_err.h>
+
+void *segkp_lwp;		/* cookie for pool of segkp resources */
+
+/*
+ * Create a thread that appears to be stopped at sys_rtt.
+ */
+klwp_t *
+lwp_create(void (*proc)(), caddr_t arg, size_t len, proc_t *p,
+    int state, int pri, const k_sigset_t *smask, int cid, id_t lwpid)
+{
+	klwp_t *lwp = NULL;
+	kthread_t *t;
+	kthread_t *tx;
+	cpupart_t *oldpart = NULL;
+	size_t	stksize;
+	caddr_t lwpdata = NULL;
+	processorid_t	binding;
+	int err = 0;
+	kproject_t *oldkpj, *newkpj;
+	void *bufp = NULL;
+	klwp_t *curlwp = ttolwp(curthread);
+	lwpent_t *lep;
+	lwpdir_t *old_dir = NULL;
+	uint_t old_dirsz = 0;
+	lwpdir_t **old_hash = NULL;
+	uint_t old_hashsz = 0;
+	int i;
+	int rctlfail = 0;
+
+	mutex_enter(&p->p_lock);
+	mutex_enter(&p->p_zone->zone_nlwps_lock);
+	/*
+	 * don't enforce rctl limits on system processes
+	 */
+	if (cid != syscid) {
+		if (p->p_task->tk_nlwps >= p->p_task->tk_nlwps_ctl)
+			if (rctl_test(rc_task_lwps, p->p_task->tk_rctls, p,
+			    1, 0) & RCT_DENY)
+				rctlfail = 1;
+		if (p->p_task->tk_proj->kpj_nlwps >=
+		    p->p_task->tk_proj->kpj_nlwps_ctl)
+			if (rctl_test(rc_project_nlwps,
+			    p->p_task->tk_proj->kpj_rctls, p, 1, 0)
+			    & RCT_DENY)
+				rctlfail = 1;
+		if (p->p_zone->zone_nlwps >= p->p_zone->zone_nlwps_ctl)
+			if (rctl_test(rc_zone_nlwps, p->p_zone->zone_rctls, p,
+			    1, 0) & RCT_DENY)
+				rctlfail = 1;
+	}
+	if (rctlfail) {
+		mutex_exit(&p->p_zone->zone_nlwps_lock);
+		mutex_exit(&p->p_lock);
+		return (NULL);
+	}
+	p->p_task->tk_nlwps++;
+	p->p_task->tk_proj->kpj_nlwps++;
+	p->p_zone->zone_nlwps++;
+	mutex_exit(&p->p_zone->zone_nlwps_lock);
+	mutex_exit(&p->p_lock);
+
+	if (curlwp == NULL || (stksize = curlwp->lwp_childstksz) == 0)
+		stksize = lwp_default_stksize;
+
+	/*
+	 * Try to reclaim a <lwp,stack> from 'deathrow'
+	 */
+	if (stksize == lwp_default_stksize) {
+		if (lwp_reapcnt > 0) {
+			mutex_enter(&reaplock);
+			if ((t = lwp_deathrow) != NULL) {
+				ASSERT(t->t_swap);
+				lwp_deathrow = t->t_forw;
+				lwp_reapcnt--;
+				lwpdata = t->t_swap;
+				lwp = t->t_lwp;
+			}
+			mutex_exit(&reaplock);
+			if (t) {
+				t->t_swap = NULL;
+				lwp_stk_fini(t->t_lwp);
+				t->t_lwp = NULL;
+				t->t_forw = NULL;
+				thread_free(t);
+			}
+		}
+		if (lwpdata == NULL &&
+		    (lwpdata = (caddr_t)segkp_cache_get(segkp_lwp)) == NULL) {
+			mutex_enter(&p->p_lock);
+			mutex_enter(&p->p_zone->zone_nlwps_lock);
+			p->p_task->tk_nlwps--;
+			p->p_task->tk_proj->kpj_nlwps--;
+			p->p_zone->zone_nlwps--;
+			mutex_exit(&p->p_zone->zone_nlwps_lock);
+			mutex_exit(&p->p_lock);
+			return (NULL);
+		}
+	} else {
+		stksize = roundup(stksize, PAGESIZE);
+		if ((lwpdata = (caddr_t)segkp_get(segkp, stksize,
+		    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED))) == NULL) {
+			mutex_enter(&p->p_lock);
+			mutex_enter(&p->p_zone->zone_nlwps_lock);
+			p->p_task->tk_nlwps--;
+			p->p_task->tk_proj->kpj_nlwps--;
+			p->p_zone->zone_nlwps--;
+			mutex_exit(&p->p_zone->zone_nlwps_lock);
+			mutex_exit(&p->p_lock);
+			return (NULL);
+		}
+	}
+
+	/*
+	 * Create a thread, initializing the stack pointer
+	 */
+	t = thread_create(lwpdata, stksize, NULL, NULL, 0, p, TS_STOPPED, pri);
+
+	t->t_swap = lwpdata;	/* Start of page-able data */
+	if (lwp == NULL)
+		lwp = kmem_cache_alloc(lwp_cache, KM_SLEEP);
+	bzero(lwp, sizeof (*lwp));
+	t->t_lwp = lwp;
+
+	t->t_hold = *smask;
+	lwp->lwp_thread = t;
+	lwp->lwp_procp = p;
+	lwp->lwp_sigaltstack.ss_flags = SS_DISABLE;
+	if (curlwp != NULL && curlwp->lwp_childstksz != 0)
+		lwp->lwp_childstksz = curlwp->lwp_childstksz;
+
+	t->t_stk = lwp_stk_init(lwp, t->t_stk);
+	thread_load(t, proc, arg, len);
+
+	/*
+	 * Allocate the SIGPROF buffer if ITIMER_REALPROF is in effect.
+	 */
+	if (timerisset(&p->p_rprof_timer.it_value))
+		t->t_rprof = kmem_zalloc(sizeof (struct rprof), KM_SLEEP);
+
+	if (cid != NOCLASS)
+		(void) CL_ALLOC(&bufp, cid, KM_SLEEP);
+
+	/*
+	 * Allocate an lwp directory entry for the new lwp.
+	 */
+	lep = kmem_zalloc(sizeof (*lep), KM_SLEEP);
+
+	mutex_enter(&p->p_lock);
+grow:
+	/*
+	 * Grow the lwp (thread) directory and lwpid hash table if necessary.
+	 * A note on the growth algorithm:
+	 *	The new lwp directory size is computed as:
+	 *		new = 2 * old + 2
+	 *	Starting with an initial size of 2 (see exec_common()),
+	 *	this yields numbers that are a power of two minus 2:
+	 *		2, 6, 14, 30, 62, 126, 254, 510, 1022, ...
+	 *	The size of the lwpid hash table must be a power of two
+	 *	and must be commensurate in size with the lwp directory
+	 *	so that hash bucket chains remain short.  Therefore,
+	 *	the lwpid hash table size is computed as:
+	 *		hashsz = (dirsz + 2) / 2
+	 *	which leads to these hash table sizes corresponding to
+	 *	the above directory sizes:
+	 *		2, 4, 8, 16, 32, 64, 128, 256, 512, ...
+	 */
+	while (p->p_lwpfree == NULL) {
+		uint_t dirsz = p->p_lwpdir_sz;
+		uint_t new_dirsz;
+		uint_t new_hashsz;
+		lwpdir_t *new_dir;
+		lwpdir_t *ldp;
+		lwpdir_t **new_hash;
+
+		mutex_exit(&p->p_lock);
+
+		if (old_dir != NULL) {
+			kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
+			kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
+			old_dir = NULL;
+			old_dirsz = 0;
+			old_hash = NULL;
+			old_hashsz = 0;
+		}
+		new_dirsz = 2 * dirsz + 2;
+		new_dir = kmem_zalloc(new_dirsz * sizeof (lwpdir_t), KM_SLEEP);
+		for (ldp = new_dir, i = 1; i < new_dirsz; i++, ldp++)
+			ldp->ld_next = ldp + 1;
+		new_hashsz = (new_dirsz + 2) / 2;
+		new_hash = kmem_zalloc(new_hashsz * sizeof (lwpdir_t *),
+			KM_SLEEP);
+
+		mutex_enter(&p->p_lock);
+		if (p == curproc)
+			prbarrier(p);
+
+		if (dirsz != p->p_lwpdir_sz || p->p_lwpfree != NULL) {
+			/*
+			 * Someone else beat us to it or some lwp exited.
+			 * Set up to free our memory and take a lap.
+			 */
+			old_dir = new_dir;
+			old_dirsz = new_dirsz;
+			old_hash = new_hash;
+			old_hashsz = new_hashsz;
+		} else {
+			old_dir = p->p_lwpdir;
+			old_dirsz = p->p_lwpdir_sz;
+			old_hash = p->p_tidhash;
+			old_hashsz = p->p_tidhash_sz;
+			p->p_lwpdir = new_dir;
+			p->p_lwpfree = new_dir;
+			p->p_lwpdir_sz = new_dirsz;
+			p->p_tidhash = new_hash;
+			p->p_tidhash_sz = new_hashsz;
+			/*
+			 * We simply hash in all of the old directory entries.
+			 * This works because the old directory has no empty
+			 * slots and the new hash table starts out empty.
+			 * This reproduces the original directory ordering
+			 * (required for /proc directory semantics).
+			 */
+			for (ldp = old_dir, i = 0; i < dirsz; i++, ldp++)
+				lwp_hash_in(p, ldp->ld_entry);
+			/*
+			 * Defer freeing memory until we drop p->p_lock,
+			 */
+		}
+	}
+
+	/*
+	 * Block the process against /proc while we manipulate p->p_tlist,
+	 * unless lwp_create() was called by /proc for the PCAGENT operation.
+	 * We want to do this early enough so that we don't drop p->p_lock
+	 * until the thread is put on the p->p_tlist.
+	 */
+	if (p == curproc) {
+		prbarrier(p);
+		/*
+		 * If the current lwp has been requested to stop, do so now.
+		 * Otherwise we have a race condition between /proc attempting
+		 * to stop the process and this thread creating a new lwp
+		 * that was not seen when the /proc PCSTOP request was issued.
+		 * We rely on stop() to call prbarrier(p) before returning.
+		 */
+		while ((curthread->t_proc_flag & TP_PRSTOP) &&
+		    !ttolwp(curthread)->lwp_nostop)
+			stop(PR_REQUESTED, 0);
+
+		/*
+		 * If process is exiting, there could be a race between
+		 * the agent lwp creation and the new lwp currently being
+		 * created. So to prevent this race lwp creation is failed
+		 * if the process is exiting.
+		 */
+		if (p->p_flag & (SEXITLWPS|SKILLED)) {
+			err = 1;
+			goto error;
+		}
+
+		/*
+		 * Since we might have dropped p->p_lock, the
+		 * lwp directory free list might have changed.
+		 */
+		if (p->p_lwpfree == NULL)
+			goto grow;
+	}
+
+	kpreempt_disable();	/* can't grab cpu_lock here */
+
+	/*
+	 * Inherit processor and processor set bindings from curthread,
+	 * unless we're creating a new kernel process, in which case
+	 * clear all bindings.
+	 */
+	if (cid == syscid) {
+		t->t_bind_cpu = binding = PBIND_NONE;
+		t->t_cpupart = oldpart = &cp_default;
+		t->t_bind_pset = PS_NONE;
+	} else {
+		binding = curthread->t_bind_cpu;
+		t->t_bind_cpu = binding;
+		oldpart = t->t_cpupart;
+		t->t_cpupart = curthread->t_cpupart;
+		t->t_bind_pset = curthread->t_bind_pset;
+	}
+
+	/*
+	 * thread_create() initializes this thread's home lgroup to the root.
+	 * Choose a more suitable lgroup, since this thread is associated
+	 * with an lwp.
+	 */
+	ASSERT(oldpart != NULL);
+	if (binding != PBIND_NONE && t->t_affinitycnt == 0) {
+		t->t_bound_cpu = cpu[binding];
+		if (t->t_lpl != t->t_bound_cpu->cpu_lpl)
+			lgrp_move_thread(t, t->t_bound_cpu->cpu_lpl, 1);
+	} else {
+		lgrp_move_thread(t, lgrp_choose(t, t->t_cpupart), 1);
+	}
+
+	kpreempt_enable();
+
+	/*
+	 * make sure lpl points to our own partition
+	 */
+	ASSERT(t->t_lpl >= t->t_cpupart->cp_lgrploads);
+	ASSERT(t->t_lpl < t->t_cpupart->cp_lgrploads +
+	    t->t_cpupart->cp_nlgrploads);
+
+	/*
+	 * If we're creating a new process, then inherit the project from our
+	 * parent. If we're only creating an additional lwp then use the
+	 * project pointer of the target process.
+	 */
+	if (p->p_task == NULL)
+		newkpj = ttoproj(curthread);
+	else
+		newkpj = p->p_task->tk_proj;
+
+	/*
+	 * It is safe to point the thread to the new project without holding it
+	 * since we're holding the target process' p_lock here and therefore
+	 * we're guaranteed that it will not move to another project.
+	 */
+	oldkpj = ttoproj(t);
+	if (newkpj != oldkpj) {
+		t->t_proj = newkpj;
+		(void) project_hold(newkpj);
+		project_rele(oldkpj);
+	}
+
+	if (cid != NOCLASS) {
+		/*
+		 * If the lwp is being created in the current process
+		 * and matches the current thread's scheduling class,
+		 * we should propagate the current thread's scheduling
+		 * parameters by calling CL_FORK.  Otherwise just use
+		 * the defaults by calling CL_ENTERCLASS.
+		 */
+		if (p != curproc || curthread->t_cid != cid) {
+			err = CL_ENTERCLASS(t, cid, NULL, NULL, bufp);
+			t->t_pri = pri;	/* CL_ENTERCLASS may have changed it */
+		} else {
+			t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
+			err = CL_FORK(curthread, t, bufp);
+			t->t_cid = cid;
+		}
+		if (err)
+			goto error;
+		else
+			bufp = NULL;
+	}
+
+	/*
+	 * If we were given an lwpid then use it, else allocate one.
+	 */
+	if (lwpid != 0)
+		t->t_tid = lwpid;
+	else {
+		/*
+		 * lwp/thread id 0 is never valid; reserved for special checks.
+		 * lwp/thread id 1 is reserved for the main thread.
+		 * Start again at 2 when INT_MAX has been reached
+		 * (id_t is a signed 32-bit integer).
+		 */
+		id_t prev_id = p->p_lwpid;	/* last allocated tid */
+
+		do {			/* avoid lwpid duplication */
+			if (p->p_lwpid == INT_MAX) {
+				p->p_flag |= SLWPWRAP;
+				p->p_lwpid = 1;
+			}
+			if ((t->t_tid = ++p->p_lwpid) == prev_id) {
+				/*
+				 * All lwpids are allocated; fail the request.
+				 */
+				err = 1;
+				goto error;
+			}
+			/*
+			 * We only need to worry about colliding with an id
+			 * that's already in use if this process has
+			 * cycled through all available lwp ids.
+			 */
+			if ((p->p_flag & SLWPWRAP) == 0)
+				break;
+		} while (lwp_hash_lookup(p, t->t_tid) != NULL);
+	}
+	p->p_lwpcnt++;
+	t->t_waitfor = -1;
+
+	/*
+	 * Turn microstate accounting on for thread if on for process.
+	 */
+	if (p->p_flag & SMSACCT)
+		t->t_proc_flag |= TP_MSACCT;
+
+	/*
+	 * If the process has watchpoints, mark the new thread as such.
+	 */
+	if (pr_watch_active(p))
+		watch_enable(t);
+
+	/*
+	 * The lwp is being created in the stopped state.
+	 * We set all the necessary flags to indicate that fact here.
+	 * We omit the TS_CREATE flag from t_schedflag so that the lwp
+	 * cannot be set running until the caller is finished with it,
+	 * even if lwp_continue() is called on it after we drop p->p_lock.
+	 * When the caller is finished with the newly-created lwp,
+	 * the caller must call lwp_create_done() to allow the lwp
+	 * to be set running.  If the TP_HOLDLWP is left set, the
+	 * lwp will suspend itself after reaching system call exit.
+	 */
+	init_mstate(t, LMS_STOPPED);
+	t->t_proc_flag |= TP_HOLDLWP;
+	t->t_schedflag |= (TS_ALLSTART & ~(TS_CSTART | TS_CREATE));
+	t->t_whystop = PR_SUSPENDED;
+	t->t_whatstop = SUSPEND_NORMAL;
+	t->t_sig_check = 1;	/* ensure that TP_HOLDLWP is honored */
+
+	/*
+	 * Set system call processing flags in case tracing or profiling
+	 * is set.  The first system call will evaluate these and turn
+	 * them off if they aren't needed.
+	 */
+	t->t_pre_sys = 1;
+	t->t_post_sys = 1;
+
+	/*
+	 * Insert the new thread into the list of all threads.
+	 */
+	if ((tx = p->p_tlist) == NULL) {
+		t->t_back = t;
+		t->t_forw = t;
+		p->p_tlist = t;
+	} else {
+		t->t_forw = tx;
+		t->t_back = tx->t_back;
+		tx->t_back->t_forw = t;
+		tx->t_back = t;
+	}
+
+	/*
+	 * Insert the new lwp into an lwp directory slot position
+	 * and into the lwpid hash table.
+	 */
+	lep->le_thread = t;
+	lep->le_lwpid = t->t_tid;
+	lep->le_start = t->t_start;
+	lwp_hash_in(p, lep);
+
+	if (state == TS_RUN) {
+		/*
+		 * We set the new lwp running immediately.
+		 */
+		t->t_proc_flag &= ~TP_HOLDLWP;
+		lwp_create_done(t);
+	}
+
+error:
+	if (err) {
+		/*
+		 * We have failed to create an lwp, so decrement the number
+		 * of lwps in the task and let the lgroup load averages know
+		 * that this thread isn't going to show up.
+		 */
+		kpreempt_disable();
+		lgrp_move_thread(t, NULL, 1);
+		kpreempt_enable();
+
+		ASSERT(MUTEX_HELD(&p->p_lock));
+		mutex_enter(&p->p_zone->zone_nlwps_lock);
+		p->p_task->tk_nlwps--;
+		p->p_task->tk_proj->kpj_nlwps--;
+		p->p_zone->zone_nlwps--;
+		mutex_exit(&p->p_zone->zone_nlwps_lock);
+		if (cid != NOCLASS && bufp != NULL)
+			CL_FREE(cid, bufp);
+
+		mutex_exit(&p->p_lock);
+		t->t_state = TS_FREE;
+		thread_rele(t);
+
+		/*
+		 * We need to remove t from the list of all threads
+		 * because thread_exit()/lwp_exit() isn't called on t.
+		 */
+		mutex_enter(&pidlock);
+		ASSERT(t != t->t_next);		/* t0 never exits */
+		t->t_next->t_prev = t->t_prev;
+		t->t_prev->t_next = t->t_next;
+		mutex_exit(&pidlock);
+
+		thread_free(t);
+		kmem_free(lep, sizeof (*lep));
+		lwp = NULL;
+	} else {
+		mutex_exit(&p->p_lock);
+	}
+
+	if (old_dir != NULL) {
+		kmem_free(old_dir, old_dirsz * sizeof (*old_dir));
+		kmem_free(old_hash, old_hashsz * sizeof (*old_hash));
+	}
+
+	DTRACE_PROC1(lwp__create, kthread_t *, t);
+	return (lwp);
+}
+
+/*
+ * lwp_create_done() is called by the caller of lwp_create() to set the
+ * newly-created lwp running after the caller has finished manipulating it.
+ */
+void
+lwp_create_done(kthread_t *t)
+{
+	proc_t *p = ttoproc(t);
+
+	ASSERT(MUTEX_HELD(&p->p_lock));
+
+	/*
+	 * We set the TS_CREATE and TS_CSTART flags and call setrun_locked().
+	 * (The absence of the TS_CREATE flag prevents the lwp from running
+	 * until we are finished with it, even if lwp_continue() is called on
+	 * it by some other lwp in the process or elsewhere in the kernel.)
+	 */
+	thread_lock(t);
+	ASSERT(t->t_state == TS_STOPPED && !(t->t_schedflag & TS_CREATE));
+	/*
+	 * If TS_CSTART is set, lwp_continue(t) has been called and
+	 * has already incremented p_lwprcnt; avoid doing this twice.
+	 */
+	if (!(t->t_schedflag & TS_CSTART))
+		p->p_lwprcnt++;
+	t->t_schedflag |= (TS_CSTART | TS_CREATE);
+	setrun_locked(t);
+	thread_unlock(t);
+}
+
+/*
+ * Copy an LWP's active templates, and clear the latest contracts.
+ */
+void
+lwp_ctmpl_copy(klwp_t *dst, klwp_t *src)
+{
+	int i;
+
+	for (i = 0; i < ct_ntypes; i++) {
+		dst->lwp_ct_active[i] = ctmpl_dup(src->lwp_ct_active[i]);
+		dst->lwp_ct_latest[i] = NULL;
+	}
+}
+
+/*
+ * Clear an LWP's contract template state.
+ */
+void
+lwp_ctmpl_clear(klwp_t *lwp)
+{
+	ct_template_t *tmpl;
+	int i;
+
+	for (i = 0; i < ct_ntypes; i++) {
+		if ((tmpl = lwp->lwp_ct_active[i]) != NULL) {
+			ctmpl_free(tmpl);
+			lwp->lwp_ct_active[i] = NULL;
+		}
+
+		if (lwp->lwp_ct_latest[i] != NULL) {
+			contract_rele(lwp->lwp_ct_latest[i]);
+			lwp->lwp_ct_latest[i] = NULL;
+		}
+	}
+}
+
+/*
+ * Individual lwp exit.
+ * If this is the last lwp, exit the whole process.
+ */
+void
+lwp_exit(void)
+{
+	kthread_t *t = curthread;
+	klwp_t *lwp = ttolwp(t);
+	proc_t *p = ttoproc(t);
+
+	ASSERT(MUTEX_HELD(&p->p_lock));
+
+	mutex_exit(&p->p_lock);
+
+#if defined(__sparc)
+	/*
+	 * Ensure that the user stack is fully abandoned..
+	 */
+	trash_user_windows();
+#endif
+
+	tsd_exit();			/* free thread specific data */
+
+	kcpc_passivate();		/* Clean up performance counter state */
+
+	pollcleanup();
+
+	if (t->t_door)
+		door_slam();
+
+	if (t->t_schedctl != NULL)
+		schedctl_lwp_cleanup(t);
+
+	if (t->t_upimutex != NULL)
+		upimutex_cleanup();
+
+	mutex_enter(&p->p_lock);
+	lwp_cleanup();
+
+	/*
+	 * When this process is dumping core, its lwps are held here
+	 * until the core dump is finished. Then exitlwps() is called
+	 * again to release these lwps so that they can finish exiting.
+	 */
+	if (p->p_flag & SCOREDUMP)
+		stop(PR_SUSPENDED, SUSPEND_NORMAL);
+
+	/*
+	 * Block the process against /proc now that we have really acquired
+	 * p->p_lock (to decrement p_lwpcnt and manipulate p_tlist at least).
+	 */
+	prbarrier(p);
+
+	/*
+	 * Call proc_exit() if this is the last non-daemon lwp in the process.
+	 */
+	if (!(t->t_proc_flag & TP_DAEMON) &&
+	    p->p_lwpcnt == p->p_lwpdaemon + 1) {
+		int why, what;
+
+		if (p->p_flag & SEXITLWPS) {
+			why = CLD_KILLED;
+			what = SIGKILL;
+		} else {
+			why = CLD_EXITED;
+			what = 0;
+		}
+		mutex_exit(&p->p_lock);
+		if (proc_exit(why, what) == 0) {
+			/* Restarting init. */
+			return;
+		}
+
+		/*
+		 * proc_exit() returns a non-zero value when some other
+		 * lwp got there first.  We just have to continue in
+		 * lwp_exit().
+		 */
+		mutex_enter(&p->p_lock);
+		ASSERT(curproc->p_flag & SEXITLWPS);
+		prbarrier(p);
+	}
+
+	DTRACE_PROC(lwp__exit);
+
+	/*
+	 * If the lwp is a detached lwp or if the process is exiting,
+	 * remove (lwp_hash_out()) the lwp from the lwp directory.
+	 * Otherwise null out the lwp's le_thread pointer in the lwp
+	 * directory so that other threads will see it as a zombie lwp.
+	 */
+	prlwpexit(t);		/* notify /proc */
+	if (!(t->t_proc_flag & TP_TWAIT) || (p->p_flag & SEXITLWPS))
+		lwp_hash_out(p, t->t_tid);
+	else {
+		ASSERT(!(t->t_proc_flag & TP_DAEMON));
+		p->p_lwpdir[t->t_dslot].ld_entry->le_thread = NULL;
+		p->p_zombcnt++;
+		cv_broadcast(&p->p_lwpexit);
+	}
+	if (t->t_proc_flag & TP_DAEMON) {
+		p->p_lwpdaemon--;
+		t->t_proc_flag &= ~TP_DAEMON;
+	}
+	t->t_proc_flag &= ~TP_TWAIT;
+
+	/*
+	 * Maintain accurate lwp count for task.max-lwps resource control.
+	 */
+	mutex_enter(&p->p_zone->zone_nlwps_lock);
+	p->p_task->tk_nlwps--;
+	p->p_task->tk_proj->kpj_nlwps--;
+	p->p_zone->zone_nlwps--;
+	mutex_exit(&p->p_zone->zone_nlwps_lock);
+
+	CL_EXIT(t);		/* tell the scheduler that t is exiting */
+	ASSERT(p->p_lwpcnt != 0);
+	p->p_lwpcnt--;
+
+	/*
+	 * If all remaining non-daemon lwps are waiting in lwp_wait(),
+	 * wake them up so someone can return EDEADLK.
+	 * (See the block comment preceeding lwp_wait().)
+	 */
+	if (p->p_lwpcnt == p->p_lwpdaemon + (p->p_lwpwait - p->p_lwpdwait))
+		cv_broadcast(&p->p_lwpexit);
+
+	t->t_proc_flag |= TP_LWPEXIT;
+	term_mstate(t);
+#ifndef NPROBE
+	/* Kernel probe */
+	if (t->t_tnf_tpdp)
+		tnf_thread_exit();
+#endif /* NPROBE */
+
+	t->t_forw->t_back = t->t_back;
+	t->t_back->t_forw = t->t_forw;
+	if (t == p->p_tlist)
+		p->p_tlist = t->t_forw;
+
+	/*
+	 * Clean up the signal state.
+	 */
+	if (t->t_sigqueue != NULL)
+		sigdelq(p, t, 0);
+	if (lwp->lwp_curinfo != NULL) {
+		siginfofree(lwp->lwp_curinfo);
+		lwp->lwp_curinfo = NULL;
+	}
+
+	thread_rele(t);
+
+	/*
+	 * Terminated lwps are associated with process zero and are put onto
+	 * death-row by resume().  Avoid preemption after resetting t->t_procp.
+	 */
+	t->t_preempt++;
+	t->t_procp = &p0;
+
+	/*
+	 * Notify the HAT about the change of address space
+	 */
+	hat_thread_exit(t);
+	/*
+	 * When this is the last running lwp in this process and some lwp is
+	 * waiting for this condition to become true, or this thread was being
+	 * suspended, then the waiting lwp is awakened.
+	 *
+	 * Also, if the process is exiting, we may have a thread waiting in
+	 * exitlwps() that needs to be notified.
+	 */
+	if (--p->p_lwprcnt == 0 || (t->t_proc_flag & TP_HOLDLWP) ||
+	    (p->p_flag & SEXITLWPS))
+		cv_broadcast(&p->p_holdlwps);
+
+	/*
+	 * Need to drop p_lock so we can reacquire pidlock.
+	 */
+	mutex_exit(&p->p_lock);
+	mutex_enter(&pidlock);
+
+	ASSERT(t != t->t_next);		/* t0 never exits */
+	t->t_next->t_prev = t->t_prev;
+	t->t_prev->t_next = t->t_next;
+	cv_broadcast(&t->t_joincv);	/* wake up anyone in thread_join */
+	mutex_exit(&pidlock);
+
+	lwp_pcb_exit();
+
+	if (t->t_ctx != NULL)
+		exitctx(t);
+
+	t->t_state = TS_ZOMB;
+	swtch_from_zombie();
+	/* never returns */
+}
+
+
+/*
+ * Cleanup function for an exiting lwp.
+ * Called both from lwp_exit() and from proc_exit().
+ * p->p_lock is repeatedly released and grabbed in this function.
+ */
+void
+lwp_cleanup(void)
+{
+	kthread_t *t = curthread;
+	proc_t *p = ttoproc(t);
+
+	ASSERT(MUTEX_HELD(&p->p_lock));
+
+	/* untimeout any lwp-bound realtime timers */
+	if (p->p_itimer != NULL)
+		timer_lwpexit();
+
+	/*
+	 * If this is the /proc agent lwp that is exiting, readjust p_lwpid
+	 * so it appears that the agent never existed, and clear p_agenttp.
+	 */
+	if (t == p->p_agenttp) {
+		ASSERT(t->t_tid == p->p_lwpid);
+		p->p_lwpid--;
+		p->p_agenttp = NULL;
+	}
+
+	/*
+	 * Do lgroup bookkeeping to account for thread exiting.
+	 */
+	kpreempt_disable();
+	lgrp_move_thread(t, NULL, 1);
+	kpreempt_enable();
+
+	lwp_ctmpl_clear(ttolwp(t));
+}
+
+int
+lwp_suspend(kthread_t *t)
+{
+	int tid;
+	proc_t *p = ttoproc(t);
+
+	ASSERT(MUTEX_HELD(&p->p_lock));
+
+	/*
+	 * Set the thread's TP_HOLDLWP flag so it will stop in holdlwp().
+	 * If an lwp is stopping itself, there is no need to wait.
+	 */
+	t->t_proc_flag |= TP_HOLDLWP;
+	if (t == curthread) {
+		t->t_sig_check = 1;
+	} else {
+		/*
+		 * Make sure the lwp stops promptly.
+		 */
+		thread_lock(t);
+		t->t_sig_check = 1;
+		/*
+		 * XXX Should use virtual stop like /proc does instead of
+		 * XXX waking the thread to get it to stop.
+		 */
+		if (t->t_state == TS_SLEEP && (t->t_flag & T_WAKEABLE))
+			setrun_locked(t);
+		else if (t->t_state == TS_ONPROC && t->t_cpu != CPU)
+			poke_cpu(t->t_cpu->cpu_id);
+		tid = t->t_tid;	 /* remember thread ID */
+		/*
+		 * Wait for lwp to stop
+		 */
+		while (!SUSPENDED(t)) {
+			/*
+			 * Drop the thread lock before waiting and reacquire it
+			 * afterwards, so the thread can change its t_state
+			 * field.
+			 */
+			thread_unlock(t);
+
+			/*
+			 * Check if aborted by exitlwps().
+			 */
+			if (p->p_flag & SEXITLWPS)
+				lwp_exit();
+
+			/*
+			 * Cooperate with jobcontrol signals and /proc stopping
+			 * by calling cv_wait_sig() to wait for the target
+			 * lwp to stop.  Just using cv_wait() can lead to
+			 * deadlock because, if some other lwp has stopped
+			 * by either of these mechanisms, then p_lwprcnt will
+			 * never become zero if we do a cv_wait().
+			 */
+			if (!cv_wait_sig(&p->p_holdlwps, &p->p_lock))
+				return (EINTR);
+
+			/*
+			 * Check to see if thread died while we were
+			 * waiting for it to suspend.
+			 */
+			if (idtot(p, tid) == NULL)
+				return (ESRCH);
+
+			thread_lock(t);
+			/*
+			 * If TP_HOLDLWP flag goes away, lwp_continue() must
+			 * have been called while we were waiting, so cancel
+			 * the suspend.
+			 */
+			if ((t->t_proc_flag & TP_HOLDLWP) == 0) {
+				thread_unlock(t);
+				return (0);
+			}
+		}
+		thread_unlock(t);
+	}
+	return (0);
+}
+
+/*
+ * continue a lwp that's been stopped by lwp_suspend().
+ */
+void
+lwp_continue(kthread_t *t)
+{
+	proc_t *p = ttoproc(t);
+	int was_suspended = t->t_proc_flag & TP_HOLDLWP;
+
+	ASSERT(MUTEX_HELD(&p->p_lock));
+
+	t->t_proc_flag &= ~TP_HOLDLWP;
+	thread_lock(t);
+	if (SUSPENDED(t) &&
+	    !(p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH))) {
+		p->p_lwprcnt++;
+		t->t_schedflag |= TS_CSTART;
+		setrun_locked(t);
+	}
+	thread_unlock(t);
+	/*
+	 * Wakeup anyone waiting for this thread to be suspended
+	 */
+	if (was_suspended)
+		cv_broadcast(&p->p_holdlwps);
+}
+
+/*
+ * ********************************
+ *  Miscellaneous lwp routines	  *
+ * ********************************
+ */
+/*
+ * When a process is undergoing a forkall(), its p_flag is set to SHOLDFORK.
+ * This will cause the process's lwps to stop at a hold point.  A hold
+ * point is where a kernel thread has a flat stack.  This is at the
+ * return from a system call and at the return from a user level trap.
+ *
+ * When a process is undergoing a fork1() or vfork(), its p_flag is set to
+ * SHOLDFORK1.  This will cause the process's lwps to stop at a modified
+ * hold point.  The lwps in the process are not being cloned, so they
+ * are held at the usual hold points and also within issig_forreal().
+ * This has the side-effect that their system calls do not return
+ * showing EINTR.
+ *
+ * An lwp can also be held.  This is identified by the TP_HOLDLWP flag on
+ * the thread.  The TP_HOLDLWP flag is set in lwp_suspend(), where the active
+ * lwp is waiting for the target lwp to be stopped.
+ */
+void
+holdlwp(void)
+{
+	proc_t *p = curproc;
+	kthread_t *t = curthread;
+
+	mutex_enter(&p->p_lock);
+	/*
+	 * Don't terminate immediately if the process is dumping core.
+	 * Once the process has dumped core, all lwps are terminated.
+	 */
+	if (!(p->p_flag & SCOREDUMP)) {
+		if ((p->p_flag & SEXITLWPS) || (t->t_proc_flag & TP_EXITLWP))
+			lwp_exit();
+	}
+	if (!(ISHOLD(p)) && !(p->p_flag & (SHOLDFORK1 | SHOLDWATCH))) {
+		mutex_exit(&p->p_lock);
+		return;
+	}
+	/*
+	 * stop() decrements p->p_lwprcnt and cv_signal()s &p->p_holdlwps
+	 * when p->p_lwprcnt becomes zero.
+	 */
+	stop(PR_SUSPENDED, SUSPEND_NORMAL);
+	if (p->p_flag & SEXITLWPS)
+		lwp_exit();
+	mutex_exit(&p->p_lock);
+}
+
+/*
+ * Have all lwps within the process hold at a point where they are
+ * cloneable (SHOLDFORK) or just safe w.r.t. fork1 (SHOLDFORK1).
+ */
+int
+holdlwps(int holdflag)
+{
+	proc_t *p = curproc;
+
+	ASSERT(holdflag == SHOLDFORK || holdflag == SHOLDFORK1);
+	mutex_enter(&p->p_lock);
+	schedctl_finish_sigblock(curthread);
+again:
+	while (p->p_flag & (SEXITLWPS | SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
+		/*
+		 * If another lwp is doing a forkall() or proc_exit(), bail out.
+		 */
+		if (p->p_flag & (SEXITLWPS | SHOLDFORK)) {
+			mutex_exit(&p->p_lock);
+			return (0);
+		}
+		/*
+		 * Another lwp is doing a fork1() or is undergoing
+		 * watchpoint activity.  We hold here for it to complete.
+		 */
+		stop(PR_SUSPENDED, SUSPEND_NORMAL);
+	}
+	p->p_flag |= holdflag;
+	pokelwps(p);
+	--p->p_lwprcnt;
+	/*
+	 * Wait for the process to become quiescent (p->p_lwprcnt == 0).
+	 */
+	while (p->p_lwprcnt > 0) {
+		/*
+		 * Check if aborted by exitlwps().
+		 * Also check if SHOLDWATCH is set; it takes precedence.
+		 */
+		if (p->p_flag & (SEXITLWPS | SHOLDWATCH)) {
+			p->p_lwprcnt++;
+			p->p_flag &= ~holdflag;
+			cv_broadcast(&p->p_holdlwps);
+			goto again;
+		}
+		/*
+		 * Cooperate with jobcontrol signals and /proc stopping.
+		 * If some other lwp has stopped by either of these
+		 * mechanisms, then p_lwprcnt will never become zero
+		 * and the process will appear deadlocked unless we
+		 * stop here in sympathy with the other lwp before
+		 * doing the cv_wait() below.
+		 *
+		 * If the other lwp stops after we do the cv_wait(), it
+		 * will wake us up to loop around and do the sympathy stop.
+		 *
+		 * Since stop() drops p->p_lock, we must start from
+		 * the top again on returning from stop().
+		 */
+		if (p->p_stopsig | (curthread->t_proc_flag & TP_PRSTOP)) {
+			int whystop = p->p_stopsig? PR_JOBCONTROL :
+			    PR_REQUESTED;
+			p->p_lwprcnt++;
+			p->p_flag &= ~holdflag;
+			stop(whystop, p->p_stopsig);
+			goto again;
+		}
+		cv_wait(&p->p_holdlwps, &p->p_lock);
+	}
+	p->p_lwprcnt++;
+	p->p_flag &= ~holdflag;
+	mutex_exit(&p->p_lock);
+	return (1);
+}
+
+/*
+ * See comments for holdwatch(), below.
+ */
+static int
+holdcheck(int clearflags)
+{
+	proc_t *p = curproc;
+
+	/*
+	 * If we are trying to exit, that takes precedence over anything else.
+	 */
+	if (p->p_flag & SEXITLWPS) {
+		p->p_lwprcnt++;
+		p->p_flag &= ~clearflags;
+		lwp_exit();
+	}
+
+	/*
+	 * If another thread is calling fork1(), stop the current thread so the
+	 * other can complete.
+	 */
+	if (p->p_flag & SHOLDFORK1) {
+		p->p_lwprcnt++;
+		stop(PR_SUSPENDED, SUSPEND_NORMAL);
+		if (p->p_flag & SEXITLWPS) {
+			p->p_flag &= ~clearflags;
+			lwp_exit();
+		}
+		return (-1);
+	}
+
+	/*
+	 * If another thread is calling fork(), then indicate we are doing
+	 * watchpoint activity.  This will cause holdlwps() above to stop the
+	 * forking thread, at which point we can continue with watchpoint
+	 * activity.
+	 */
+	if (p->p_flag & SHOLDFORK) {
+		p->p_lwprcnt++;
+		while (p->p_flag & SHOLDFORK) {
+			p->p_flag |= SHOLDWATCH;
+			cv_broadcast(&p->p_holdlwps);
+			cv_wait(&p->p_holdlwps, &p->p_lock);
+			p->p_flag &= ~SHOLDWATCH;
+		}
+		return (-1);
+	}
+
+	return (0);
+}
+
+/*
+ * Stop all lwps within the process, holding themselves in the kernel while the
+ * active lwp undergoes watchpoint activity.  This is more complicated than
+ * expected because stop() relies on calling holdwatch() in order to copyin data
+ * from the user's address space.  A double barrier is used to prevent an
+ * infinite loop.
+ *
+ * 	o The first thread into holdwatch() is the 'master' thread and does
+ *        the following:
+ *
+ *              - Sets SHOLDWATCH on the current process
+ *              - Sets TP_WATCHSTOP on the current thread
+ *              - Waits for all threads to be either stopped or have
+ *                TP_WATCHSTOP set.
+ *              - Sets the SWATCHOK flag on the process
+ *              - Unsets TP_WATCHSTOP
+ *              - Waits for the other threads to completely stop
+ *              - Unsets SWATCHOK
+ *
+ * 	o If SHOLDWATCH is already set when we enter this function, then another
+ *        thread is already trying to stop this thread.  This 'slave' thread
+ *        does the following:
+ *
+ *              - Sets TP_WATCHSTOP on the current thread
+ *              - Waits for SWATCHOK flag to be set
+ *              - Calls stop()
+ *
+ * 	o If SWATCHOK is set on the process, then this function immediately
+ *        returns, as we must have been called via stop().
+ *
+ * In addition, there are other flags that take precedence over SHOLDWATCH:
+ *
+ * 	o If SEXITLWPS is set, exit immediately.
+ *
+ * 	o If SHOLDFORK1 is set, wait for fork1() to complete.
+ *
+ * 	o If SHOLDFORK is set, then watchpoint activity takes precedence In this
+ *        case, set SHOLDWATCH, signalling the forking thread to stop first.
+ *
+ * 	o If the process is being stopped via /proc (TP_PRSTOP is set), then we
+ *        stop the current thread.
+ *
+ * Returns 0 if all threads have been quiesced.  Returns non-zero if not all
+ * threads were stopped, or the list of watched pages has changed.
+ */
+int
+holdwatch(void)
+{
+	proc_t *p = curproc;
+	kthread_t *t = curthread;
+	int ret = 0;
+
+	mutex_enter(&p->p_lock);
+
+	p->p_lwprcnt--;
+
+	/*
+	 * Check for bail-out conditions as outlined above.
+	 */
+	if (holdcheck(0) != 0) {
+		mutex_exit(&p->p_lock);
+		return (-1);
+	}
+
+	if (!(p->p_flag & SHOLDWATCH)) {
+		/*
+		 * We are the master watchpoint thread.  Set SHOLDWATCH and poke
+		 * the other threads.
+		 */
+		p->p_flag |= SHOLDWATCH;
+		pokelwps(p);
+
+		/*
+		 * Wait for all threads to be stopped or have TP_WATCHSTOP set.
+		 */
+		while (pr_allstopped(p, 1) > 0) {
+			if (holdcheck(SHOLDWATCH) != 0) {
+				p->p_flag &= ~SHOLDWATCH;
+				mutex_exit(&p->p_lock);
+				return (-1);
+			}
+
+			cv_wait(&p->p_holdlwps, &p->p_lock);
+		}
+
+		/*
+		 * All threads are now stopped or in the process of stopping.
+		 * Set SWATCHOK and let them stop completely.
+		 */
+		p->p_flag |= SWATCHOK;
+		t->t_proc_flag &= ~TP_WATCHSTOP;
+		cv_broadcast(&p->p_holdlwps);
+
+		while (pr_allstopped(p, 0) > 0) {
+			/*
+			 * At first glance, it may appear that we don't need a
+			 * call to holdcheck() here.  But if the process gets a
+			 * SIGKILL signal, one of our stopped threads may have
+			 * been awakened and is waiting in exitlwps(), which
+			 * takes precedence over watchpoints.
+			 */
+			if (holdcheck(SHOLDWATCH | SWATCHOK) != 0) {
+				p->p_flag &= ~(SHOLDWATCH | SWATCHOK);
+				mutex_exit(&p->p_lock);
+				return (-1);
+			}
+
+			cv_wait(&p->p_holdlwps, &p->p_lock);
+		}
+
+		/*
+		 * All threads are now completely stopped.
+		 */
+		p->p_flag &= ~SWATCHOK;
+		p->p_flag &= ~SHOLDWATCH;
+		p->p_lwprcnt++;
+
+	} else if (!(p->p_flag & SWATCHOK)) {
+
+		/*
+		 * SHOLDWATCH is set, so another thread is trying to do
+		 * watchpoint activity.  Indicate this thread is stopping, and
+		 * wait for the OK from the master thread.
+		 */
+		t->t_proc_flag |= TP_WATCHSTOP;
+		cv_broadcast(&p->p_holdlwps);
+
+		while (!(p->p_flag & SWATCHOK)) {
+			if (holdcheck(0) != 0) {
+				t->t_proc_flag &= ~TP_WATCHSTOP;
+				mutex_exit(&p->p_lock);
+				return (-1);
+			}
+
+			cv_wait(&p->p_holdlwps, &p->p_lock);
+		}
+
+		/*
+		 * Once the master thread has given the OK, this thread can
+		 * actually call stop().
+		 */
+		t->t_proc_flag &= ~TP_WATCHSTOP;
+		p->p_lwprcnt++;
+
+		stop(PR_SUSPENDED, SUSPEND_NORMAL);
+
+		/*
+		 * It's not OK to do watchpoint activity, notify caller to
+		 * retry.
+		 */
+		ret = -1;
+
+	} else {
+
+		/*
+		 * The only way we can hit the case where SHOLDWATCH is set and
+		 * SWATCHOK is set is if we are triggering this from within a
+		 * stop() call.  Assert that this is the case.
+		 */
+
+		ASSERT(t->t_proc_flag & TP_STOPPING);
+		p->p_lwprcnt++;
+	}
+
+	mutex_exit(&p->p_lock);
+
+	return (ret);
+}
+
+/*
+ * force all interruptible lwps to trap into the kernel.
+ */
+void
+pokelwps(proc_t *p)
+{
+	kthread_t *t;
+
+	ASSERT(MUTEX_HELD(&p->p_lock));
+
+	t = p->p_tlist;
+	do {
+		if (t == curthread)
+			continue;
+		thread_lock(t);
+		aston(t);	/* make thread trap or do post_syscall */
+		if (t->t_state == TS_SLEEP) {
+			if (t->t_flag & T_WAKEABLE)
+				setrun_locked(t);
+		} else if (t->t_state == TS_STOPPED) {
+			/*
+			 * Ensure that proc_exit() is not blocked by lwps
+			 * that were stopped via jobcontrol or /proc.
+			 */
+			if (p->p_flag & SEXITLWPS) {
+				p->p_stopsig = 0;
+				t->t_schedflag |= (TS_XSTART | TS_PSTART);
+				setrun_locked(t);
+			}
+			/*
+			 * If we are holding lwps for a forkall(),
+			 * force lwps that have been suspended via
+			 * lwp_suspend() and are suspended inside
+			 * of a system call to proceed to their
+			 * holdlwp() points where they are clonable.
+			 */
+			if ((p->p_flag & SHOLDFORK) && SUSPENDED(t)) {
+				if ((t->t_schedflag & TS_CSTART) == 0) {
+					p->p_lwprcnt++;
+					t->t_schedflag |= TS_CSTART;
+					setrun_locked(t);
+				}
+			}
+		} else if (t->t_state == TS_ONPROC) {
+			if (t->t_cpu != CPU)
+				poke_cpu(t->t_cpu->cpu_id);
+		}
+		thread_unlock(t);
+	} while ((t = t->t_forw) != p->p_tlist);
+}
+
+/*
+ * undo the effects of holdlwps() or holdwatch().
+ */
+void
+continuelwps(proc_t *p)
+{
+	kthread_t *t;
+
+	/*
+	 * If this flag is set, then the original holdwatch() didn't actually
+	 * stop the process.  See comments for holdwatch().
+	 */
+	if (p->p_flag & SWATCHOK) {
+		ASSERT(curthread->t_proc_flag & TP_STOPPING);
+		return;
+	}
+
+	ASSERT(MUTEX_HELD(&p->p_lock));
+	ASSERT((p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) == 0);
+
+	t = p->p_tlist;
+	do {
+		thread_lock(t);		/* SUSPENDED looks at t_schedflag */
+		if (SUSPENDED(t) && !(t->t_proc_flag & TP_HOLDLWP)) {
+			p->p_lwprcnt++;
+			t->t_schedflag |= TS_CSTART;
+			setrun_locked(t);
+		}
+		thread_unlock(t);
+	} while ((t = t->t_forw) != p->p_tlist);
+}
+
+/*
+ * Force all other LWPs in the current process other than the caller to exit,
+ * and then cv_wait() on p_holdlwps for them to exit.  The exitlwps() function
+ * is typically used in these situations:
+ *
+ *   (a) prior to an exec() system call
+ *   (b) prior to dumping a core file
+ *   (c) prior to a uadmin() shutdown
+ *
+ * If the 'coredump' flag is set, other LWPs are quiesced but not destroyed.
+ * Multiple threads in the process can call this function at one time by
+ * triggering execs or core dumps simultaneously, so the SEXITLWPS bit is used
+ * to declare one particular thread the winner who gets to kill the others.
+ * If a thread wins the exitlwps() dance, zero is returned; otherwise an
+ * appropriate errno value is returned to caller for its system call to return.
+ */
+int
+exitlwps(int coredump)
+{
+	proc_t *p = curproc;
+	int heldcnt;
+
+	if (curthread->t_door)
+		door_slam();
+	if (p->p_door_list)
+		door_revoke_all();
+	if (curthread->t_schedctl != NULL)
+		schedctl_lwp_cleanup(curthread);
+
+	/*
+	 * Ensure that before starting to wait for other lwps to exit,
+	 * cleanup all upimutexes held by curthread. Otherwise, some other
+	 * lwp could be waiting (uninterruptibly) for a upimutex held by
+	 * curthread, and the call to pokelwps() below would deadlock.
+	 * Even if a blocked upimutex_lock is made interruptible,
+	 * curthread's upimutexes need to be unlocked: do it here.
+	 */
+	if (curthread->t_upimutex != NULL)
+		upimutex_cleanup();
+
+	/*
+	 * Grab p_lock in order to check and set SEXITLWPS to declare a winner.
+	 * We must also block any further /proc access from this point forward.
+	 */
+	mutex_enter(&p->p_lock);
+	prbarrier(p);
+
+	if (p->p_flag & SEXITLWPS) {
+		mutex_exit(&p->p_lock);
+		aston(curthread);	/* force a trip through post_syscall */
+		return (set_errno(EINTR));
+	}
+
+	p->p_flag |= SEXITLWPS;
+	if (coredump)		/* tell other lwps to stop, not exit */
+		p->p_flag |= SCOREDUMP;
+
+	/*
+	 * Give precedence to exitlwps() if a holdlwps() is
+	 * in progress. The lwp doing the holdlwps() operation
+	 * is aborted when it is awakened.
+	 */
+	while (p->p_flag & (SHOLDFORK | SHOLDFORK1 | SHOLDWATCH)) {
+		cv_broadcast(&p->p_holdlwps);
+		cv_wait(&p->p_holdlwps, &p->p_lock);
+	}
+	p->p_flag |= SHOLDFORK;
+	pokelwps(p);
+
+	/*
+	 * Wait for process to become quiescent.
+	 */
+	--p->p_lwprcnt;
+	while (p->p_lwprcnt > 0)
+		cv_wait(&p->p_holdlwps, &p->p_lock);
+	p->p_lwprcnt++;
+	ASSERT(p->p_lwprcnt == 1);
+
+	/*
+	 * The SCOREDUMP flag puts the process into a quiescent
+	 * state.  The process's lwps remain attached to this
+	 * process until exitlwps() is called again without the
+	 * 'coredump' flag set, then the lwps are terminated
+	 * and the process can exit.
+	 */
+	if (coredump) {
+		p->p_flag &= ~(SCOREDUMP | SHOLDFORK | SEXITLWPS);
+		goto out;
+	}
+
+	/*
+	 * Determine if there are any lwps left dangling in
+	 * the stopped state.  This happens when exitlwps()
+	 * aborts a holdlwps() operation.
+	 */
+	p->p_flag &= ~SHOLDFORK;
+	if ((heldcnt = p->p_lwpcnt) > 1) {
+		kthread_t *t;
+		for (t = curthread->t_forw; --heldcnt > 0; t = t->t_forw) {
+			t->t_proc_flag &= ~TP_TWAIT;
+			lwp_continue(t);
+		}
+	}
+
+	/*
+	 * Wait for all other lwps to exit.
+	 */
+	--p->p_lwprcnt;
+	while (p->p_lwpcnt > 1)
+		cv_wait(&p->p_holdlwps, &p->p_lock);
+	++p->p_lwprcnt;
+	ASSERT(p->p_lwpcnt == 1 && p->p_lwprcnt == 1);
+
+	p->p_flag &= ~SEXITLWPS;
+	curthread->t_proc_flag &= ~TP_TWAIT;
+
+out:
+	if (!coredump && p->p_zombcnt) {	/* cleanup the zombie lwps */
+		lwpdir_t *ldp;
+		lwpent_t *lep;
+		int i;
+
+		prbarrier(p);
+		for (ldp = p->p_lwpdir, i = 0; i < p->p_lwpdir_sz; i++, ldp++) {
+			lep = ldp->ld_entry;
+			if (lep != NULL && lep->le_thread != curthread) {
+				ASSERT(lep->le_thread == NULL);
+				p->p_zombcnt--;
+				lwp_hash_out(p, lep->le_lwpid);
+			}
+		}
+		ASSERT(p->p_zombcnt == 0);
+	}
+
+	/*
+	 * If some other LWP in the process wanted us to suspend ourself,
+	 * then we will not do it.  The other LWP is now terminated and
+	 * no one will ever continue us again if we suspend ourself.
+	 */
+	curthread->t_proc_flag &= ~TP_HOLDLWP;
+	p->p_flag &= ~(SHOLDFORK | SHOLDFORK1 | SHOLDWATCH | SLWPWRAP);
+	mutex_exit(&p->p_lock);
+	return (0);
+}
+
+/*
+ * duplicate a lwp.
+ */
+klwp_t *
+forklwp(klwp_t *lwp, proc_t *cp, id_t lwpid)
+{
+	klwp_t *clwp;
+	void *tregs, *tfpu;
+	kthread_t *t = lwptot(lwp);
+	kthread_t *ct;
+	proc_t *p = lwptoproc(lwp);
+	int cid;
+	void *bufp;
+	int val;
+
+	ASSERT(p == curproc);
+	ASSERT(t == curthread || (SUSPENDED(t) && lwp->lwp_asleep == 0));
+
+#if defined(__sparc)
+	if (t == curthread)
+		(void) flush_user_windows_to_stack(NULL);
+#endif
+
+	if (t == curthread)
+		/* copy args out of registers first */
+		(void) save_syscall_args();
+	clwp = lwp_create(cp->p_lwpcnt == 0 ? lwp_rtt_initial : lwp_rtt,
+	    NULL, 0, cp, TS_STOPPED, t->t_pri, &t->t_hold, NOCLASS, lwpid);
+	if (clwp == NULL)
+		return (NULL);
+
+	/*
+	 * most of the parent's lwp can be copied to its duplicate,
+	 * except for the fields that are unique to each lwp, like
+	 * lwp_thread, lwp_procp, lwp_regs, and lwp_ap.
+	 */
+	ct = clwp->lwp_thread;
+	tregs = clwp->lwp_regs;
+	tfpu = clwp->lwp_fpu;
+
+	/* copy parent lwp to child lwp */
+	*clwp = *lwp;
+
+	/* fix up child's lwp */
+
+	clwp->lwp_pcb.pcb_flags = 0;
+#if defined(__sparc)
+	clwp->lwp_pcb.pcb_step = STEP_NONE;
+#endif
+	clwp->lwp_cursig = 0;
+	clwp->lwp_extsig = 0;
+	clwp->lwp_curinfo = (struct sigqueue *)0;
+	clwp->lwp_thread = ct;
+	ct->t_sysnum = t->t_sysnum;
+	clwp->lwp_regs = tregs;
+	clwp->lwp_fpu = tfpu;
+	clwp->lwp_ap = clwp->lwp_arg;
+	clwp->lwp_procp = cp;
+	bzero(clwp->lwp_timer, sizeof (clwp->lwp_timer));
+	init_mstate(ct, LMS_STOPPED);
+	bzero(&clwp->lwp_ru, sizeof (clwp->lwp_ru));
+	clwp->lwp_lastfault = 0;
+	clwp->lwp_lastfaddr = 0;
+
+	/* copy parent's struct regs to child. */
+	lwp_forkregs(lwp, clwp);
+
+	/*
+	 * fork device context, if any.
+	 *
+	 * Someday we could do the work to support the possibility of
+	 * forkctx() or lwp_createctx() failing.  Currently, this would
+	 * only be needed on x86 for the occasional process using a
+	 * private LDT.
+	 */
+	if (t->t_ctx)
+		forkctx(t, ct);
+
+	/* fix door state in the child */
+	if (t->t_door)
+		door_fork(t, ct);
+
+	/* copy current contract templates, clear latest contracts */
+	lwp_ctmpl_copy(clwp, lwp);
+
+	mutex_enter(&cp->p_lock);
+	/* lwp_create() set the TP_HOLDLWP flag */
+	if (!(t->t_proc_flag & TP_HOLDLWP))
+		ct->t_proc_flag &= ~TP_HOLDLWP;
+	if (cp->p_flag & SMSACCT)
+		ct->t_proc_flag |= TP_MSACCT;
+	mutex_exit(&cp->p_lock);
+
+retry:
+	cid = t->t_cid;
+
+	val = CL_ALLOC(&bufp, cid, KM_SLEEP);
+	ASSERT(val == 0);
+
+	mutex_enter(&p->p_lock);
+	if (cid != t->t_cid) {
+		/*
+		 * Someone just changed this thread's scheduling class,
+		 * so try pre-allocating the buffer again.  Hopefully we
+		 * don't hit this often.
+		 */
+		mutex_exit(&p->p_lock);
+		CL_FREE(cid, bufp);
+		goto retry;
+	}
+
+	ct->t_unpark = t->t_unpark;
+	ct->t_clfuncs = t->t_clfuncs;
+	CL_FORK(t, ct, bufp);
+	ct->t_cid = t->t_cid;	/* after data allocated so prgetpsinfo works */
+	mutex_exit(&p->p_lock);
+
+	return (clwp);
+}
+
+/*
+ * Add a new lwp entry to the lwp directory and to the lwpid hash table.
+ */
+void
+lwp_hash_in(proc_t *p, lwpent_t *lep)
+{
+	lwpdir_t **ldpp;
+	lwpdir_t *ldp;
+	kthread_t *t;
+
+	/*
+	 * Allocate a directory element from the free list.
+	 * Code elsewhere guarantees a free slot.
+	 */
+	ldp = p->p_lwpfree;
+	p->p_lwpfree = ldp->ld_next;
+	ASSERT(ldp->ld_entry == NULL);
+	ldp->ld_entry = lep;
+
+	/*
+	 * Insert it into the lwpid hash table.
+	 */
+	ldpp = &p->p_tidhash[TIDHASH(p, lep->le_lwpid)];
+	ldp->ld_next = *ldpp;
+	*ldpp = ldp;
+
+	/*
+	 * Set the active thread's directory slot entry.
+	 */
+	if ((t = lep->le_thread) != NULL) {
+		ASSERT(lep->le_lwpid == t->t_tid);
+		t->t_dslot = (int)(ldp - p->p_lwpdir);
+	}
+}
+
+/*
+ * Remove an lwp from the lwpid hash table and free its directory entry.
+ * This is done when a detached lwp exits in lwp_exit() or
+ * when a non-detached lwp is waited for in lwp_wait() or
+ * when a zombie lwp is detached in lwp_detach().
+ */
+void
+lwp_hash_out(proc_t *p, id_t lwpid)
+{
+	lwpdir_t **ldpp;
+	lwpdir_t *ldp;
+	lwpent_t *lep;
+
+	for (ldpp = &p->p_tidhash[TIDHASH(p, lwpid)];
+	    (ldp = *ldpp) != NULL; ldpp = &ldp->ld_next) {
+		lep = ldp->ld_entry;
+		if (lep->le_lwpid == lwpid) {
+			prlwpfree(p, lep);	/* /proc deals with le_trace */
+			*ldpp = ldp->ld_next;
+			ldp->ld_entry = NULL;
+			ldp->ld_next = p->p_lwpfree;
+			p->p_lwpfree = ldp;
+			kmem_free(lep, sizeof (*lep));
+			break;
+		}
+	}
+}
+
+/*
+ * Lookup an lwp in the lwpid hash table by lwpid.
+ */
+lwpdir_t *
+lwp_hash_lookup(proc_t *p, id_t lwpid)
+{
+	lwpdir_t *ldp;
+
+	/*
+	 * The process may be exiting, after p_tidhash has been set to NULL in
+	 * proc_exit() but before prfee() has been called.  Return failure in
+	 * this case.
+	 */
+	if (p->p_tidhash == NULL)
+		return (NULL);
+
+	for (ldp = p->p_tidhash[TIDHASH(p, lwpid)];
+	    ldp != NULL; ldp = ldp->ld_next) {
+		if (ldp->ld_entry->le_lwpid == lwpid)
+			return (ldp);
+	}
+
+	return (NULL);
+}
+
+/*
+ * Update the indicated LWP usage statistic for the current LWP.
+ */
+void
+lwp_stat_update(lwp_stat_id_t lwp_stat_id, long inc)
+{
+	klwp_t *lwp = ttolwp(curthread);
+
+	if (lwp == NULL)
+		return;
+
+	switch (lwp_stat_id) {
+	case LWP_STAT_INBLK:
+		lwp->lwp_ru.inblock += inc;
+		break;
+	case LWP_STAT_OUBLK:
+		lwp->lwp_ru.oublock += inc;
+		break;
+	case LWP_STAT_MSGRCV:
+		lwp->lwp_ru.msgrcv += inc;
+		break;
+	case LWP_STAT_MSGSND:
+		lwp->lwp_ru.msgsnd += inc;
+		break;
+	default:
+		panic("lwp_stat_update: invalid lwp_stat_id 0x%x", lwp_stat_id);
+	}
+}