7 files changed, 252 insertions, 71 deletions

diff --git a/usr/src/lib/libcmdutils/Makefile.com b/usr/src/lib/libcmdutils/Makefile.com
index 7961146e1a..2f98fe5f25 100644
--- a/usr/src/lib/libcmdutils/Makefile.com
+++ b/usr/src/lib/libcmdutils/Makefile.com
@@ -20,11 +20,12 @@
 #
 #
 # Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013 RackTop Systems.
 #
 
 LIBRARY=	libcmdutils.a
 VERS=		.1
-CMD_OBJS=	avltree.o sysattrs.o writefile.o process_xattrs.o
+CMD_OBJS=	avltree.o sysattrs.o writefile.o process_xattrs.o uid.o gid.o
 COM_OBJS=	list.o
 OBJECTS=	$(CMD_OBJS) $(COM_OBJS)
 
diff --git a/usr/src/lib/libcmdutils/common/gid.c b/usr/src/lib/libcmdutils/common/gid.c
new file mode 100644
index 0000000000..63456810ea
--- /dev/null
+++ b/usr/src/lib/libcmdutils/common/gid.c
@@ -0,0 +1,111 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
+/*	  All Rights Reserved  	*/
+
+/*
+ * Copyright (c) 2013 RackTop Systems.
+ */
+
+#include <errno.h>
+#include <sys/types.h>
+#include <stdio.h>
+#include <userdefs.h>
+#include <grp.h>
+#include <libcmdutils.h>
+
+static int findunusedgid(gid_t start, gid_t stop, gid_t *ret);
+static boolean_t isreservedgid(gid_t gid);
+
+/*
+ * Find the highest unused uid. If the highest unused gid is "stop",
+ * then attempt to find a hole in the range. Returns 0 on success.
+ */
+int
+findnextgid(gid_t start, gid_t stop, gid_t *ret)
+{
+	gid_t gid = start;
+	struct group *grp;
+	boolean_t overflow = B_FALSE;
+
+	setgrent();
+	for (grp = getgrent(); grp != NULL; grp = getgrent()) {
+		if (isreservedgid(grp->gr_gid))		/* Skip reserved IDs */
+			continue;
+		if (grp->gr_gid >= gid) {
+			if (grp->gr_gid == stop) {	/* Overflow check */
+				overflow = B_TRUE;
+				break;
+			}
+			gid = grp->gr_gid + 1;
+		}
+	}
+	if (grp == NULL && errno != 0) {
+		endgrent();
+		return (-1);
+	}
+	endgrent();
+	if (overflow == B_TRUE)				/* Find a hole */
+		return (findunusedgid(start, stop, ret));
+	while (isreservedgid(gid) && gid < stop)	/* Skip reserved IDs */
+		gid++;
+	*ret = gid;
+	return (0);
+}
+
+/*
+ * Check to see whether the gid is a reserved gid
+ * -- nobody, noaccess or nogroup
+ */
+static boolean_t
+isreservedgid(gid_t gid)
+{
+	return (gid == 60001 || gid == 60002 || gid == 65534);
+}
+
+/*
+ * findunusedgid() attempts to return the next valid usable id between the
+ * supplied upper and lower limits. Returns 0 on success.
+ */
+static int
+findunusedgid(gid_t start, gid_t stop, gid_t *ret)
+{
+	gid_t gid;
+
+	for (gid = start; gid <= stop; gid++) {
+		if (isreservedgid(gid))
+			continue;
+		if (getgrgid(gid) == NULL) {
+			if (errno != 0)
+				return (-1);
+			break;
+		}
+	}
+	if (gid > stop)
+		return (-1);
+	*ret = gid;
+	return (0);
+}
diff --git a/usr/src/lib/libcmdutils/common/mapfile-vers b/usr/src/lib/libcmdutils/common/mapfile-vers
index 74f59e8fdb..e4c5940c31 100644
--- a/usr/src/lib/libcmdutils/common/mapfile-vers
+++ b/usr/src/lib/libcmdutils/common/mapfile-vers
@@ -20,6 +20,7 @@
 #
 #
 # Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013 RackTop Systems.
 #
 
 #
@@ -42,6 +43,8 @@ SYMBOL_VERSION SUNWprivate_1.1 {
     global:
 	add_tnode;
 	destroy_tree;
+	findnextgid;
+	findnextuid;
 	list_create;
 	list_destroy;
 	list_head;
diff --git a/usr/src/lib/libcmdutils/common/uid.c b/usr/src/lib/libcmdutils/common/uid.c
new file mode 100644
index 0000000000..a5ca6c45bd
--- /dev/null
+++ b/usr/src/lib/libcmdutils/common/uid.c
@@ -0,0 +1,112 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 1997-2001 by Sun Microsystems, Inc.
+ * All rights reserved.
+ */
+
+/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
+/*	  All Rights Reserved  	*/
+
+/*
+ * Copyright (c) 2013 RackTop Systems.
+ */
+
+#include <errno.h>
+#include <sys/types.h>
+#include <stdio.h>
+#include <userdefs.h>
+#include <pwd.h>
+#include <libcmdutils.h>
+
+static int findunuseduid(uid_t start, uid_t stop, uid_t *ret);
+static boolean_t isreserveduid(uid_t uid);
+
+/*
+ * Find the highest unused uid. If the highest unused uid is "stop",
+ * then attempt to find a hole in the range. Returns 0 on success.
+ */
+int
+findnextuid(uid_t start, uid_t stop, uid_t *ret)
+{
+	uid_t uid = start;
+	struct passwd *pwd;
+	boolean_t overflow = B_FALSE;
+
+	setpwent();
+	for (pwd = getpwent(); pwd != NULL; pwd = getpwent()) {
+		if (isreserveduid(pwd->pw_uid))		/* Skip reserved IDs */
+			continue;
+		if (pwd->pw_uid >= uid) {
+			if (pwd->pw_uid == stop) {	/* Overflow check */
+				overflow = B_TRUE;
+				break;
+			}
+			uid = pwd->pw_uid + 1;
+		}
+	}
+	if (pwd == NULL && errno != 0) {
+		endpwent();
+		return (-1);
+	}
+	endpwent();
+	if (overflow == B_TRUE)				/* Find a hole */
+		return (findunuseduid(start, stop, ret));
+	while (isreserveduid(uid) && uid < stop)	/* Skip reserved IDs */
+		uid++;
+	*ret = uid;
+	return (0);
+}
+
+/*
+ * Check to see whether the uid is a reserved uid
+ * -- nobody, noaccess or nobody4
+ */
+static boolean_t
+isreserveduid(uid_t uid)
+{
+	return (uid == 60001 || uid == 60002 || uid == 65534);
+}
+
+/*
+ * findunuseduid() attempts to return the next valid usable id between the
+ * supplied upper and lower limits. Returns 0 on success.
+ */
+static int
+findunuseduid(uid_t start, uid_t stop, uid_t *ret)
+{
+	uid_t uid;
+
+	for (uid = start; uid <= stop; uid++) {
+		if (isreserveduid(uid))
+			continue;
+		if (getpwuid(uid) == NULL) {
+			if (errno != 0)
+				return (-1);
+			break;
+		}
+	}
+	if (uid > stop)
+		return (-1);
+	*ret = uid;
+	return (0);
+}
diff --git a/usr/src/lib/libcmdutils/libcmdutils.h b/usr/src/lib/libcmdutils/libcmdutils.h
index d1c3a0b193..c315e0fbef 100644
--- a/usr/src/lib/libcmdutils/libcmdutils.h
+++ b/usr/src/lib/libcmdutils/libcmdutils.h
@@ -22,6 +22,9 @@
  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright (c) 2013 RackTop Systems.
+ */
 
 /*
  * Declarations for the functions in libcmdutils.
@@ -30,8 +33,6 @@
 #ifndef	_LIBCMDUTILS_H
 #define	_LIBCMDUTILS_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /*
  * This is a private header file.  Applications should not directly include
  * this file.
@@ -125,6 +126,20 @@ extern int add_tnode(avl_tree_t **, dev_t, ino_t);
  */
 extern void destroy_tree(avl_tree_t *);
 
+
+
+		/* user/group id helpers */
+
+/*
+ * Used to get the next available user id in given range.
+ */
+extern int findnextuid(uid_t, uid_t, uid_t *);
+
+/*
+ * Used to get the next available group id in given range.
+ */
+extern int findnextgid(gid_t, gid_t, gid_t *);
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/usr/src/lib/smbsrv/libsmb/Makefile.com b/usr/src/lib/smbsrv/libsmb/Makefile.com
index 4b86caeb95..dbf479e106 100644
--- a/usr/src/lib/smbsrv/libsmb/Makefile.com
+++ b/usr/src/lib/smbsrv/libsmb/Makefile.com
@@ -73,6 +73,8 @@ include ../../Makefile.lib
 
 INCS += -I$(SRC)/common/smbsrv
 
+LINTCHECKFLAGS += -erroff=E_INCONS_ARG_DECL2
+
 LDLIBS +=	$(MACH_LDLIBS)
 LDLIBS +=	-lscf -lmd -luuid -lnsl -lpkcs11 -lsec -lsocket -lresolv
 LDLIBS +=	-lidmap -lreparse -lnvpair -lcmdutils -lavl -lc
diff --git a/usr/src/lib/smbsrv/libsmb/common/smb_lgrp.c b/usr/src/lib/smbsrv/libsmb/common/smb_lgrp.c
index 406d7c5db0..3cab135341 100644
--- a/usr/src/lib/smbsrv/libsmb/common/smb_lgrp.c
+++ b/usr/src/lib/smbsrv/libsmb/common/smb_lgrp.c
@@ -22,6 +22,7 @@
 /*
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013 RackTop Systems.
  */
 
 #include <stdlib.h>
@@ -38,6 +39,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/param.h>
+#include <libcmdutils.h>
 
 /*
  * Local domain SID (aka machine SID) is not stored in the domain table
@@ -123,7 +125,7 @@
 #define	SMB_LGRP_PGRP_GRPBUFSIZ	5120
 #define	SMB_LGRP_PGRP_GROUP	"/etc/group"
 #define	SMB_LGRP_PGRP_MAXGLEN	9	/* max length of group name */
-#define	SMB_LGRP_PGRP_DEFRID	99	/* max reserved id */
+#define	SMB_LGRP_PGRP_DEFRID	1000	/* lowest cifs created gid */
 
 #define	SMB_LGRP_PGRP_NOTUNIQUE	0
 #define	SMB_LGRP_PGRP_RESERVED	1
@@ -2545,70 +2547,6 @@ smb_lgrp_pgrp_valid_gname(char *group)
 }
 
 /*
- * smb_lgrp_pgrp_valid_gid
- *
- * Check to see that the gid is not a reserved gid
- * -- nobody (60001), noaccess (60002) or nogroup (65534)
- */
-static int
-smb_lgrp_pgrp_valid_gid(gid_t gid)
-{
-	return (gid != 60001 && gid != 60002 && gid != 65534);
-}
-
-/*
- * smb_lgrp_pgrp_findnextgid(void)
- *
- * This method finds the next valid GID.
- * It sorts the used GIDs in decreasing order to return MAXUSED + 1.
- * It then adds one to obtain the next valid GID.
- * On failure, -1 is returned. On success, a valid GID is returned.
- */
-static int
-smb_lgrp_pgrp_findnextgid(void)
-{
-	FILE *fptr;
-	gid_t last, next;
-	int gid;
-
-	if ((fptr = popen("exec sh -c "
-	    "\"getent group|cut -f3 -d:|sort -nr|uniq \" 2>/dev/null",
-	    "r")) == NULL)
-		return (-1);
-
-	if (fscanf(fptr, "%u\n", &next) == EOF) {
-		(void) pclose(fptr);
-		return (SMB_LGRP_PGRP_DEFRID + 1);
-	}
-
-	last = MAXUID;
-	gid = -1;
-	do {
-		if (!smb_lgrp_pgrp_valid_gid(next))
-			continue;
-
-		if (next <= SMB_LGRP_PGRP_DEFRID) {
-			if (last != SMB_LGRP_PGRP_DEFRID + 1)
-				gid = SMB_LGRP_PGRP_DEFRID + 1;
-			break;
-		}
-
-		if ((gid = next + 1) != last) {
-			while (!smb_lgrp_pgrp_valid_gid((gid_t)gid))
-				gid++;
-			if (gid > 0 && gid < last)
-				break;
-		}
-
-		gid = -1;
-		last = next;
-	} while (fscanf(fptr, "%u\n", &next) != EOF);
-
-	(void) pclose(fptr);
-	return (gid);
-}
-
-/*
  * smb_lgrp_pgrp_add
  *
  * Create a posix group with the given name.
@@ -2619,7 +2557,7 @@ smb_lgrp_pgrp_add(char *group)
 {
 	FILE *etcgrp;
 	FILE *etctmp;
-	int o_mask, gret;
+	int o_mask;
 	int newdone = 0;
 	struct stat sb;
 	char buf[SMB_LGRP_PGRP_GRPBUFSIZ];
@@ -2630,9 +2568,8 @@ smb_lgrp_pgrp_add(char *group)
 	if ((rc == SMB_LGRP_PGRP_INVALID) || (rc == SMB_LGRP_PGRP_NOTUNIQUE))
 		return (-1);
 
-	if ((gret = smb_lgrp_pgrp_findnextgid()) < 0)
+	if ((findnextgid(SMB_LGRP_PGRP_DEFRID, MAXUID, &gid)) != 0)
 		return (-1);
-	gid = gret;
 
 	if ((etcgrp = fopen(SMB_LGRP_PGRP_GROUP, "r")) == NULL)
 		return (-1);