6477912 /etc/hosts name lookup sporadically fail on sparks based builds

6479036 nscd/nss_nisplus.so.1 core dumps when processing a getservent request
6481313 nscd drops core during DNS resolution
6481603 Installing SUNWCrnet had complains about nscd due to missing SUNWgss/SUNWgssc

5 files changed, 131 insertions, 79 deletions

diff --git a/usr/src/cmd/nscd/Makefile b/usr/src/cmd/nscd/Makefile
index c29806a5c2..7947b35380 100644
--- a/usr/src/cmd/nscd/Makefile
+++ b/usr/src/cmd/nscd/Makefile
@@ -74,7 +74,7 @@ LINTFLAGS +=	-erroff=E_NAME_DEF_NOT_USED2
 # GPROF_FLAG=	-xpg
 # DEBUG_FLAG=	-g
 
-PROGLIBS=	$(LDLIBS) -lresolv -lnsl -lsocket -lumem -lscf -lavl -lgss
+PROGLIBS=	$(LDLIBS) -lresolv -lnsl -lsocket -lumem -lscf -lavl
 NISPROGLIBS=	$(LDLIBS) -lnsl
 
 # install macros and rule
diff --git a/usr/src/cmd/nscd/nscd_switch.c b/usr/src/cmd/nscd/nscd_switch.c
index 9306915cf2..d55fbb27e5 100644
--- a/usr/src/cmd/nscd/nscd_switch.c
+++ b/usr/src/cmd/nscd/nscd_switch.c
@@ -437,6 +437,65 @@ try_local(
 }
 
 static nscd_rc_t
+get_gss_func(void **func_p)
+{
+	char		*me = "get_gss_func";
+	static void	*handle = NULL;
+	static mutex_t	func_lock = DEFAULTMUTEX;
+	void		*sym;
+	char		*func_name = "gss_inquire_cred";
+	static void	*func = NULL;
+
+	if (handle != NULL && func_p != NULL && func != NULL) {
+		(void) memcpy(func_p, &func, sizeof (void *));
+		return (NSCD_SUCCESS);
+	}
+
+	(void) mutex_lock(&func_lock);
+
+	/* close the handle if requested */
+	if (func_p == NULL) {
+		if (handle != NULL) {
+			(void) dlclose(handle);
+			func = NULL;
+		}
+		(void) mutex_unlock(&func_lock);
+		return (NSCD_SUCCESS);
+	}
+
+	if (handle != NULL && func != NULL) {
+		(void) memcpy(func_p, &func, sizeof (void *));
+		(void) mutex_unlock(&func_lock);
+		return (NSCD_SUCCESS);
+	}
+
+	if (handle == NULL) {
+		handle = dlopen("libgss.so.1", RTLD_LAZY);
+		if (handle == NULL) {
+			_NSCD_LOG(NSCD_LOG_SWITCH_ENGINE,
+				NSCD_LOG_LEVEL_ERROR)
+			(me, "unable to dlopen libgss.so.1\n");
+			(void) mutex_unlock(&func_lock);
+			return (NSCD_CFG_DLOPEN_ERROR);
+		}
+	}
+
+	if ((sym = dlsym(handle, func_name)) == NULL) {
+
+		_NSCD_LOG(NSCD_LOG_SWITCH_ENGINE, NSCD_LOG_LEVEL_ERROR)
+		(me, "unable to find symbol %s\n", func_name);
+		(void) mutex_unlock(&func_lock);
+		return (NSCD_CFG_DLSYM_ERROR);
+	} else {
+		(void) memcpy(func_p, &sym, sizeof (void *));
+		(void) memcpy(&func, &sym, sizeof (void *));
+	}
+
+	(void) mutex_unlock(&func_lock);
+	return (NSCD_SUCCESS);
+}
+
+static nscd_rc_t
 get_dns_funcs(int dnsi, void **func_p)
 {
 	char		*me = "get_dns_funcs";
@@ -523,7 +582,6 @@ set_fallback_flag(char *srcname, nss_status_t rc)
 {
 	char	*me = "set_fallback_flag";
 	if (strcmp(srcname, "ldap") == 0 && rc == NSS_NOTFOUND) {
-
 		_NSCD_LOG(NSCD_LOG_SWITCH_ENGINE, NSCD_LOG_LEVEL_DEBUG)
 		(me, "NSS_NOTFOUND (ldap): fallback to main nscd "
 		"may be needed\n");
@@ -1234,16 +1292,21 @@ nss_psearch(void *buffer, size_t length)
 	 * nscd get a chance to process the lookup
 	 */
 	if (swret.fallback == 1 && status == NSS_NOTFOUND) {
-		OM_uint32 stat;
+		OM_uint32	(*func)();
+		OM_uint32	stat;
+		nscd_rc_t	rc;
 
-		if (gss_inquire_cred(&stat, GSS_C_NO_CREDENTIAL,
-			NULL, NULL, NULL, NULL) != GSS_S_COMPLETE) {
+		rc = get_gss_func((void **)&func);
+		if (rc == NSCD_SUCCESS) {
+			if (func(&stat, GSS_C_NO_CREDENTIAL,
+				NULL, NULL, NULL, NULL) != GSS_S_COMPLETE) {
 
-			_NSCD_LOG(NSCD_LOG_SWITCH_ENGINE,
-				NSCD_LOG_LEVEL_DEBUG)
+				_NSCD_LOG(NSCD_LOG_SWITCH_ENGINE,
+					NSCD_LOG_LEVEL_DEBUG)
 			(me, "NSS_ALTRETRY: fallback to main nscd needed\n");
 
-			status = NSS_ALTRETRY;
+				status = NSS_ALTRETRY;
+			}
 		}
 	}
 
@@ -1362,15 +1425,20 @@ nss_psetent(void *buffer, size_t length, pid_t pid)
 	 * setent call
 	 */
 	if (_whoami == NSCD_CHILD) {
-		OM_uint32 stat;
+		OM_uint32	(*func)();
+		OM_uint32	stat;
+		nscd_rc_t	rc;
 
-		if (gss_inquire_cred(&stat, GSS_C_NO_CREDENTIAL,
-			NULL, NULL, NULL, NULL) != GSS_S_COMPLETE) {
+		rc = get_gss_func((void **)&func);
+		if (rc == NSCD_SUCCESS) {
+			if (func(&stat, GSS_C_NO_CREDENTIAL,
+				NULL, NULL, NULL, NULL) != GSS_S_COMPLETE) {
 
-			_NSCD_LOG(NSCD_LOG_SWITCH_ENGINE,
-				NSCD_LOG_LEVEL_DEBUG)
+				_NSCD_LOG(NSCD_LOG_SWITCH_ENGINE,
+					NSCD_LOG_LEVEL_DEBUG)
 			(me, "NSS_TRYLOCAL: fallback to caller process\n");
-			NSCD_RETURN_STATUS(pbuf, NSS_TRYLOCAL, 0);
+				NSCD_RETURN_STATUS(pbuf, NSS_TRYLOCAL, 0);
+			}
 		}
 	}
 
diff --git a/usr/src/lib/nsswitch/dns/common/dns_common.c b/usr/src/lib/nsswitch/dns/common/dns_common.c
index 0ed02f1d23..d7254f6202 100644
--- a/usr/src/lib/nsswitch/dns/common/dns_common.c
+++ b/usr/src/lib/nsswitch/dns/common/dns_common.c
@@ -287,28 +287,6 @@ _nss_dns_constr(dns_backend_op_t ops[], int n_ops)
  *	nsswitch lookup format.
  */
 
-struct tsd_priv {
-	struct __res_state *statp;	/* dns state block */
-	union msg {
-		uchar_t	buf[NS_MAXMSG];	/* max legal DNS answer size */
-		HEADER	h;
-	} resbuf;
-	char aliases[NS_MAXMSG];	/* set of aliases */
-};
-
-static void ghttlcleanup(void *ptr)
-{
-	struct tsd_priv	*priv = (struct tsd_priv *)ptr;
-
-	if (priv) {
-		if (priv->statp != NULL) {
-			res_nclose(priv->statp);
-			free((void *)priv->statp);
-		}
-		free(ptr);
-	}
-}
-
 nss_status_t
 _nss_dns_gethost_withttl(void *buffer, size_t bufsize, int ipnode)
 {
@@ -321,10 +299,12 @@ _nss_dns_gethost_withttl(void *buffer, size_t bufsize, int ipnode)
 	size_t		bsize, blen;
 	char		*bptr;
 	/* resolver query variables */
-	static mutex_t		keylock;
-	static thread_key_t	key;
-	static int		once_per_keyname = 0;
-	struct tsd_priv		*tsd = NULL;
+	struct __res_state stat, *statp;	/* dns state block */
+	union msg {
+		uchar_t	buf[NS_MAXMSG];		/* max legal DNS answer size */
+		HEADER	h;
+	} resbuf;
+	char aliases[NS_MAXMSG];		/* set of aliases */
 	const char	*name;
 	int		qtype;
 	/* answer parsing variables */
@@ -348,29 +328,12 @@ _nss_dns_gethost_withttl(void *buffer, size_t bufsize, int ipnode)
 	char		*ap, *apc;
 	int		hlen, alen, iplen, len;
 
-	if (!once_per_keyname) {
-		(void) mutex_lock(&keylock);
-		if (!once_per_keyname) {
-			(void) thr_keycreate(&key, ghttlcleanup);
-			once_per_keyname++;
-		}
-		(void) mutex_unlock(&keylock);
-	}
-	(void) thr_getspecific(key, (void **)&tsd);
-	if (tsd == NULL) {
-		tsd = (struct tsd_priv *)calloc(1, sizeof (struct tsd_priv));
-		(void) thr_setspecific(key, (void *)tsd);
-		(void) thr_getspecific(key, (void **)&tsd);
-		tsd->statp = (struct __res_state *)
-				calloc(1, sizeof (struct __res_state));
-		if (tsd->statp == NULL)
-			return (NSS_ERROR);
-		if (res_ninit(tsd->statp) == -1) {
-			free(tsd->statp);
-			return (NSS_ERROR);
-		}
-	}
-	ap = apc = (char *)tsd->aliases;
+	statp = &stat;
+	(void) memset(statp, '\0', sizeof (struct __res_state));
+	if (res_ninit(statp) == -1)
+		return (NSS_ERROR);
+
+	ap = apc = (char *)aliases;
 	alen = 0;
 	ttl = (nssuint_t)0xFFFFFFF;		/* start w/max, find smaller */
 
@@ -380,13 +343,16 @@ _nss_dns_gethost_withttl(void *buffer, size_t bufsize, int ipnode)
 	blen = 0;
 	sret = nss_packed_getkey(buffer, bufsize, &dbname, &dbop, &arg);
 	if (sret != NSS_SUCCESS) {
+		res_nclose(statp);
 		return (NSS_ERROR);
 	}
 
 	if (ipnode) {
 		/* initially only handle the simple cases */
-		if (arg.key.ipnode.flags != 0)
+		if (arg.key.ipnode.flags != 0) {
+			res_nclose(statp);
 			return (NSS_ERROR);
+		}
 		name = arg.key.ipnode.name;
 		if (arg.key.ipnode.af_family == AF_INET6)
 			qtype = T_AAAA;
@@ -396,42 +362,50 @@ _nss_dns_gethost_withttl(void *buffer, size_t bufsize, int ipnode)
 		name = arg.key.name;
 		qtype = T_A;
 	}
-	ret = res_nsearch(tsd->statp, name, C_IN, qtype,
-				tsd->resbuf.buf, NS_MAXMSG);
+	ret = res_nsearch(statp, name, C_IN, qtype, resbuf.buf, NS_MAXMSG);
 	if (ret == -1) {
-		if (tsd->statp->res_h_errno == HOST_NOT_FOUND) {
+		if (statp->res_h_errno == HOST_NOT_FOUND) {
 			pbuf->p_herrno = HOST_NOT_FOUND;
 			pbuf->p_status = NSS_NOTFOUND;
 			pbuf->data_len = 0;
+			res_nclose(statp);
 			return (NSS_NOTFOUND);
 		}
 		/* else lookup error - handle in general code */
+		res_nclose(statp);
 		return (NSS_ERROR);
 	}
 
-	cp = tsd->resbuf.buf;
-	hp = (HEADER *)&tsd->resbuf.h;
+	cp = resbuf.buf;
+	hp = (HEADER *)&resbuf.h;
 	bom = cp;
 	eom = cp + ret;
 
 	ancount = ntohs(hp->ancount);
 	qdcount = ntohs(hp->qdcount);
 	cp += HFIXEDSZ;
-	if (qdcount != 1)
+	if (qdcount != 1) {
+		res_nclose(statp);
 		return (NSS_ERROR);
+	}
 	n = dn_expand(bom, eom, cp, host, MAXHOSTNAMELEN);
 	if (n < 0) {
+		res_nclose(statp);
 		return (NSS_ERROR);
 	} else
 		hlen = strlen(host);
 	cp += n + QFIXEDSZ;
-	if (cp > eom)
+	if (cp > eom) {
+		res_nclose(statp);
 		return (NSS_ERROR);
+	}
 	while (ancount-- > 0 && cp < eom && blen < bsize) {
 		n = dn_expand(bom, eom, cp, ans, MAXHOSTNAMELEN);
 		if (n > 0) {
-			if (strncasecmp(host, ans, hlen) != 0)
+			if (strncasecmp(host, ans, hlen) != 0) {
+				res_nclose(statp);
 				return (NSS_ERROR);	/* spoof? */
+			}
 		}
 		cp += n;
 		/* bounds check */
@@ -485,8 +459,10 @@ _nss_dns_gethost_withttl(void *buffer, size_t bufsize, int ipnode)
 		}
 		af = (type == T_A ? AF_INET : AF_INET6);
 		np = inet_ntop(af, (void *)cp, nbuf, INET6_ADDRSTRLEN);
-		if (np == NULL)
+		if (np == NULL) {
+			res_nclose(statp);
 			return (NSS_ERROR);
+		}
 		cp += n;
 		/* append IP host aliases to results */
 		iplen = strlen(np);
@@ -526,5 +502,6 @@ _nss_dns_gethost_withttl(void *buffer, size_t bufsize, int ipnode)
 	pbuf->data_len = blen;
 	pttl = (nssuint_t *)((void *)((char *)pbuf + pbuf->ext_off));
 	*pttl = ttl;
+	res_nclose(statp);
 	return (NSS_SUCCESS);
 }
diff --git a/usr/src/lib/nsswitch/files/common/gethostent.c b/usr/src/lib/nsswitch/files/common/gethostent.c
index 37914ed4fa..f7cd99af44 100644
--- a/usr/src/lib/nsswitch/files/common/gethostent.c
+++ b/usr/src/lib/nsswitch/files/common/gethostent.c
@@ -511,6 +511,7 @@ __nss_files_XY_hostbyname(be, args, filter, type)
 							ap, alen);
 						buffer += alen;
 						buflen -= alen;
+						args->returnlen += alen;
 						alias_end += alen;
 					}
 
@@ -541,12 +542,11 @@ __nss_files_XY_hostbyname(be, args, filter, type)
 				buffer += cplen;
 				*buffer = '\0';
 				buflen -= cplen;
+				args->returnlen += cplen;
 				if (alias_end == NULL)
 					alias_end = buffer;
 			}
 
-			args->returnlen += linelen;
-
 			/*
 			 * If this is the first one, save the canonical
 			 * name for future matches and continue.
diff --git a/usr/src/lib/nsswitch/nisplus/common/getservent.c b/usr/src/lib/nsswitch/nisplus/common/getservent.c
index b3138f8c1a..390da4c5c7 100644
--- a/usr/src/lib/nsswitch/nisplus/common/getservent.c
+++ b/usr/src/lib/nsswitch/nisplus/common/getservent.c
@@ -87,12 +87,19 @@ nis_object2str(nobj, obj, be, argp)
 	char			*buffer, *linep, *limit;
 	char			*cname, *port, *proto, *endnum;
 	int			buflen, cnamelen, portlen, protolen;
-	const char		*protokey;
-	int			protokeylen, stat;
+	const char		*protokey = NULL;
+	int			protokeylen = 0, stat;
 	struct	entry_col	*ecol;
 
-	protokey = argp->key.serv.proto;
-	protokeylen = (protokey) ? strlen(protokey) : 0;
+	/*
+	 * For getent request, we don't want to set protokey
+	 * and protokeylen, since argp->key.serv.proto won't
+	 * be initialized.
+	 */
+	if (be->table_path == NULL) {
+		protokey = argp->key.serv.proto;
+		protokeylen = (protokey) ? strlen(protokey) : 0;
+	}
 
 	for (; nobj > 0; obj++, nobj--) {
 		if (obj->zo_data.zo_type != NIS_ENTRY_OBJ ||