6580293 fork1 leaks active door lwps

4 files changed, 116 insertions, 30 deletions

diff --git a/usr/src/lib/libc/amd64/sys/door.s b/usr/src/lib/libc/amd64/sys/door.s
index e1df3c8db4..7ab1a747e6 100644
--- a/usr/src/lib/libc/amd64/sys/door.s
+++ b/usr/src/lib/libc/amd64/sys/door.s
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -100,7 +99,7 @@
 door_restart:
 	movq	$DOOR_RETURN, %r9	/* subcode */
 	SYSTRAP_RVAL1(door)
-	jb	2f			/* errno is set */
+	jb	3f			/* errno is set */
 	/*
 	 * On return, we're serving a door_call.  Our stack looks like this:
 	 *
@@ -127,16 +126,19 @@ door_restart:
 	movq	DOOR_DESC_SIZE(%rsp), %r8
 	movq	DOOR_PC(%rsp), %rax
 	call	*%rax
+2:
 	/* Exit the thread if we return here */
 	movq	$0, %rdi
 	call	_thr_terminate
 	/* NOTREACHED */
-2:
+3:
 	/*
 	 * Error during door_return call.  Repark the thread in the kernel if
 	 * the error code is EINTR (or ERESTART) and this lwp is still part
 	 * of the same process.
 	 */
+	cmpl	$EEXIST, %eax		/* exit if EEXIST is returned */
+	je	2b
 	cmpl	$ERESTART, %eax		/* ERESTART is same as EINTR */
 	jne	3f
 	movl	$EINTR, %eax
diff --git a/usr/src/lib/libc/i386/sys/door.s b/usr/src/lib/libc/i386/sys/door.s
index b2a42587d6..c36a5f3232 100644
--- a/usr/src/lib/libc/i386/sys/door.s
+++ b/usr/src/lib/libc/i386/sys/door.s
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -148,7 +147,7 @@
 
 door_restart:
 	SYSTRAP_RVAL1(door)
-	jb	2f			/* errno is set */
+	jb	3f			/* errno is set */
 	/*
 	 * On return, we're serving a door_call.  Our stack looks like this:
 	 *
@@ -177,11 +176,13 @@ door_restart:
 	/* Call the door server function now */
 	movl	DOOR_PC(%esp), %eax
 	call	*%eax
+
+2:
 	/* Exit the thread if we return here */
 	pushl	$0
 	call	_thr_terminate
 	/* NOTREACHED */
-2:
+3:
 	/*
 	 * Error during door_return call.  Repark the thread in the kernel if
 	 * the error code is EINTR (or ERESTART) and this lwp is still part
@@ -191,12 +192,14 @@ door_restart:
 	 * corrupted by a partial door call, so we refresh the system call
 	 * arguments.
 	 */
+	cmpl	$EEXIST, %eax		/* exit this thread if EEXIST */
+	je	2b
 	cmpl	$ERESTART, %eax		/* ERESTART is same as EINTR */
-	jne	3f
+	jne	4f
 	movl	$EINTR, %eax
-3:
+4:
 	cmpl	$EINTR, %eax		/* interrupted while waiting? */
-	jne	4f			/* if not, return the error */
+	jne	5f			/* if not, return the error */
 	_prologue_
 	call	_private_getpid		/* get current process id */
 	movl	_daref_(door_create_pid), %edx
@@ -204,7 +207,7 @@ door_restart:
 	_epilogue_
 	cmpl	%eax, %edx		/* same process? */
 	movl	$EINTR, %eax	/* if no, return EINTR (child of forkall) */
-	jne	4f
+	jne	5f
 
 	movl	$0, 4(%esp)		/* clear arguments and restart */
 	movl	$0, 8(%esp)
@@ -213,7 +216,7 @@ door_restart:
 	movl	%edi, 20(%esp)		/* refresh ssize */
 	movl	$DOOR_RETURN, 24(%esp)	/* refresh syscall subcode */
 	jmp	door_restart
-4:
+5:
 	/* Something bad happened during the door_return */
 	addl	$28, %esp
 	popl	%esi
diff --git a/usr/src/lib/libc/sparc/sys/door.s b/usr/src/lib/libc/sparc/sys/door.s
index 3b1abf187e..89e7452f6d 100644
--- a/usr/src/lib/libc/sparc/sys/door.s
+++ b/usr/src/lib/libc/sparc/sys/door.s
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -100,7 +99,7 @@
 door_restart:
 	mov	DOOR_RETURN, %o5	/* subcode */
 	SYSTRAP_RVAL1(door)
-	bcs,pn	%icc, 2f			/* errno is set */
+	bcs,pn	%icc, 3f			/* errno is set */
 	ld	[%sp + DOOR_SERVERS], %g1	/* (delay) load nservers */
 	/*
 	 * On return, we're serving a door_call.  Our stack looks like this:
@@ -139,20 +138,27 @@ door_restart:
 	jmpl	%g1, %o7
 	ldn	[%sp + DOOR_DESC_SIZE], %o4
 
+2:
 	/* Exit the thread if we return here */
 	call	_thr_terminate
 	mov	%g0, %o0
 	/* NOTREACHED */
-2:
+
+3:
 	/*
 	 * Error during door_return call.  Repark the thread in the kernel if
 	 * the error code is EINTR (or ERESTART) and this lwp is still part
-	 * of the same process.
+	 * of the same process. If error is EEXIST then we don't need
+	 * this server thread.
 	 */
+	cmp	%o0, EEXIST
+	be	2b
+	nop
 	cmp	%o0, ERESTART		/* ERESTART is same as EINTR */
-	be,a	3f
+	be,a	4f
 	mov	EINTR, %o0
-3:
+
+4:
 	cmp	%o0, EINTR		/* interrupted while waiting? */
 	bne	__cerror		/* if not, return the error */
 	nop
diff --git a/usr/src/uts/common/fs/doorfs/door_sys.c b/usr/src/uts/common/fs/doorfs/door_sys.c
index ec9d650d99..f0f8a2b4cb 100644
--- a/usr/src/uts/common/fs/doorfs/door_sys.c
+++ b/usr/src/uts/common/fs/doorfs/door_sys.c
@@ -90,6 +90,12 @@ size_t	door_max_upcall_reply = 1024 * 1024;
 uint_t door_max_desc = 1024;
 
 /*
+ * The maximum time door server thread waits before exiting if there
+ * is no request from client. This is expressed in seconds.
+ */
+int door_srv_timeout = 5;
+
+/*
  * Definition of a door handle, used by other kernel subsystems when
  * calling door functions.  This is really a file structure but we
  * want to hide that fact.
@@ -341,7 +347,7 @@ static int door_overflow(kthread_t *, caddr_t, size_t, door_desc_t *, uint_t);
 static int door_args(kthread_t *, int);
 static int door_results(kthread_t *, caddr_t, size_t, door_desc_t *, uint_t);
 static int door_copy(struct as *, caddr_t, caddr_t, uint_t);
-static void	door_server_exit(proc_t *, kthread_t *);
+static int	door_server_exit(proc_t *, kthread_t *);
 static void	door_release_server(door_node_t *, kthread_t *);
 static kthread_t	*door_get_server(door_node_t *);
 static door_node_t	*door_lookup(int, file_t **);
@@ -1322,6 +1328,13 @@ door_return(caddr_t data_ptr, size_t data_size,
 	door_node_t	*dp;
 	door_server_t	*st;		/* curthread door_data */
 	door_client_t	*ct;		/* caller door_data */
+	clock_t timeleft = 0;
+	timeout_id_t id = NULL;
+
+	if (door_srv_timeout > 0)
+		timeleft = door_srv_timeout * (hz/100) * 100;
+	else
+		timeleft = -1;
 
 	st = door_my_server(1);
 
@@ -1387,6 +1400,13 @@ out:
 	/* Put ourselves on the available server thread list */
 	door_release_server(st->d_pool, curthread);
 
+	if (timeleft > 0) {
+		id = realtime_timeout((void (*)(void *))setrun,
+		    curthread, timeleft);
+	} else {
+		id = NULL;
+	}
+
 	/*
 	 * Make sure the caller is still waiting to be resumed
 	 */
@@ -1415,6 +1435,8 @@ out:
 			 */
 			thread_onproc(caller, cp);
 			disp_lock_exit_high(tlp);
+
+			/* when server returns results to client */
 			shuttle_resume(caller, &door_knob);
 		} else {
 			/* May have been setrun or in stop state */
@@ -1422,9 +1444,16 @@ out:
 			shuttle_swtch(&door_knob);
 		}
 	} else {
+		/* no client */
 		shuttle_swtch(&door_knob);
 	}
 
+
+	if (id != NULL) {
+		timeleft = untimeout(id);
+		id = NULL;
+	}
+
 	/*
 	 * We've sprung to life. Determine if we are part of a door
 	 * invocation, or just interrupted
@@ -1457,24 +1486,52 @@ out:
 				ct = DOOR_CLIENT(caller->t_door);
 			else
 				ct = NULL;
+
+			/*
+			 * Recalculate timeout since there was a
+			 * door invocation.
+			 */
+			if (door_srv_timeout > 0) {
+				timeleft = door_srv_timeout *
+				    (hz/100) * 100;
+			} else {
+				timeleft = -1;
+			}
+
 			goto out;
 		}
 		mutex_exit(&door_knob);
 		return (0);
 	} else {
+		int empty;
+
 		/*
 		 * We are not involved in a door_invocation.
 		 * Check for /proc related activity...
 		 */
 		st->d_caller = NULL;
-		door_server_exit(curproc, curthread);
+		empty = door_server_exit(curproc, curthread);
 		mutex_exit(&door_knob);
+
+		/*
+		 * We return EEXIST, which terminates this thread,
+		 * if there was no door invocation for past
+		 * door_srv_timeout seconds. But if all the
+		 * servers threads are busy, then this server
+		 * thread should not exit.
+		 */
+		if (door_srv_timeout > 0 && timeleft <= 0 &&
+		    empty == 0) {
+			return (set_errno(EEXIST));
+		}
+
 		if (ISSIG(curthread, FORREAL) ||
 		    lwp->lwp_sysabort || MUSTRETURN(curproc, curthread)) {
 			lwp->lwp_asleep = 0;
 			lwp->lwp_sysabort = 0;
 			return (set_errno(EINTR));
 		}
+
 		/* Go back and wait for another request */
 		lwp->lwp_asleep = 0;
 		mutex_enter(&door_knob);
@@ -1864,12 +1921,13 @@ door_release_server(door_node_t *dp, kthread_t *t)
 /*
  * Remove a server thread from the pool if present.
  */
-static void
+static int
 door_server_exit(proc_t *p, kthread_t *t)
 {
 	door_pool_t *pool;
 	kthread_t **next;
 	door_server_t *st = DOOR_SERVER(t->t_door);
+	int empty = 1;	/* assume door server list is empty */
 
 	ASSERT(MUTEX_HELD(&door_knob));
 	if (st->d_pool != NULL) {
@@ -1883,10 +1941,27 @@ door_server_exit(proc_t *p, kthread_t *t)
 	while (*next != NULL) {
 		if (*next == t) {
 			*next = DOOR_SERVER(t->t_door)->d_servers;
-			return;
+			if (empty && *next != NULL) {
+				/* door server list is not empty */
+				empty = 0;
+			}
+
+			return (empty);
 		}
 		next = &(DOOR_SERVER((*next)->t_door)->d_servers);
+
+		/* door server list is not empty */
+		empty = 0;
 	}
+
+	/*
+	 * If empty is set to 1, it means that there are no available
+	 * door server threads hence caller should not let the last
+	 * server thread go away. If empty is 0, then it means that
+	 * there is at least one available door server thread in the
+	 * pool.
+	 */
+	return (empty);
 }
 
 /*