6474978 hung rpc calls cause automounter to hang

6475460 The automounter is file descriptor challenged

2 files changed, 24 insertions, 14 deletions

diff --git a/usr/src/cmd/fs.d/autofs/autod_main.c b/usr/src/cmd/fs.d/autofs/autod_main.c
index 3b64fcc902..5aa5ff13f5 100644
--- a/usr/src/cmd/fs.d/autofs/autod_main.c
+++ b/usr/src/cmd/fs.d/autofs/autod_main.c
@@ -90,13 +90,6 @@ extern int _autofssys(int, void *);
 
 #define	CTIME_BUF_LEN 26
 
-/*
- * XXX - this limit was imposed due to resource problems - even though
- * we can and do try and set the rlimit to be able to handle more threads,
- * fopen() doesn't allow more than 256 fp's.
- */
-#define	MAXTHREADS 64
-
 #define	RESOURCE_FACTOR 8
 #ifdef DEBUG
 #define	AUTOFS_DOOR	"/var/run/autofs_door"
@@ -210,6 +203,19 @@ main(argc, argv)
 	(void) ns_setup(NULL, NULL);
 
 	/*
+	 * we're using doors and its thread management now so we need to
+	 * make sure we have more than the default of 256 file descriptors
+	 * available.
+	 */
+	rlset.rlim_cur = RLIM_INFINITY;
+	rlset.rlim_max = RLIM_INFINITY;
+	if (setrlimit(RLIMIT_NOFILE, &rlset) == -1)
+		syslog(LOG_ERR, "setrlimit failed for %s: %s", AUTOMOUNTD,
+		    strerror(errno));
+
+	(void) enable_extended_FILE_stdio(-1, -1);
+
+	/*
 	 * establish our lock on the lock file and write our pid to it.
 	 * exit if some other process holds the lock, or if there's any
 	 * error in writing/locking the file.
diff --git a/usr/src/lib/libnsl/rpc/clnt_vc.c b/usr/src/lib/libnsl/rpc/clnt_vc.c
index 6e72e821a7..42d5246ebe 100644
--- a/usr/src/lib/libnsl/rpc/clnt_vc.c
+++ b/usr/src/lib/libnsl/rpc/clnt_vc.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -21,7 +20,7 @@
  */
 
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
@@ -250,6 +249,12 @@ _clnt_vc_create_timed(int fd, struct netbuf *svcaddr, rpcprog_t prog,
 	}
 	ct->ct_addr.buf = NULL;
 
+	/*
+	 * The only use of vctbl_lock is for serializing the creation of
+	 * vctbl. Once created the lock needs to be released so we don't
+	 * hold it across the set_up_connection() call and end up with a
+	 * bunch of threads stuck waiting for the mutex.
+	 */
 	sig_mutex_lock(&vctbl_lock);
 
 	if ((vctbl == NULL) && ((vctbl = rpc_fd_init()) == NULL)) {
@@ -260,6 +265,8 @@ _clnt_vc_create_timed(int fd, struct netbuf *svcaddr, rpcprog_t prog,
 		goto err;
 	}
 
+	sig_mutex_unlock(&vctbl_lock);
+
 	ct->ct_io_mode = RPC_CL_BLOCKING;
 	ct->ct_blocking_mode = RPC_CL_BLOCKING_FLUSH;
 
@@ -276,16 +283,13 @@ _clnt_vc_create_timed(int fd, struct netbuf *svcaddr, rpcprog_t prog,
 		rpc_createerr.cf_stat = RPC_SYSTEMERROR;
 		rpc_createerr.cf_error.re_terrno = errno;
 		rpc_createerr.cf_error.re_errno = 0;
-		sig_mutex_unlock(&vctbl_lock);
 		goto err;
 	}
 	ct->ct_is_blocking = flag & O_NONBLOCK ? FALSE : TRUE;
 
 	if (set_up_connection(fd, svcaddr, ct, tp) == FALSE) {
-		sig_mutex_unlock(&vctbl_lock);
 		goto err;
 	}
-	sig_mutex_unlock(&vctbl_lock);
 
 	/*
 	 * Set up other members of private data struct