[illumos-gate merge]

commit 7484d76e78bc19298de9589214be103d65cf3989
    12263 libc: NULL pointer errors (sparc)
commit b36afad7ffe84071c2c6831936cc1c524bd1ca90
    12216 Clean up libc-tests smatch
commit 69c811ab73b7ce531454837ae68c4343e8724e0b
    9965 Want support for O_DIRECTORY
commit 2fe8bc68ec8e8e8e05997b3ac2f081bfdded45ab
    12275 check_rtime(1onbld): BUNDEF_OBJ and BUNUSED_OBJ
commit 5fe3b0929d8a195dbfa4196d72a3f9b15b745171
    12084 idm_conn_event_handler mishandles CA_DROP action

	Conflicts:
	usr/src/test/os-tests/tests/Makefile

1 files changed, 30 insertions, 4 deletions

diff --git a/usr/src/uts/common/fs/vnode.c b/usr/src/uts/common/fs/vnode.c
index e16dbbbc8d..9e0b071999 100644
--- a/usr/src/uts/common/fs/vnode.c
+++ b/usr/src/uts/common/fs/vnode.c
@@ -972,6 +972,7 @@ vn_openat(
 	int estale_retry = 0;
 	struct shrlock shr;
 	struct shr_locowner shr_own;
+	boolean_t create;
 
 	mode = 0;
 	accessflags = 0;
@@ -991,8 +992,31 @@ vn_openat(
 	if (filemode & FAPPEND)
 		accessflags |= V_APPEND;
 
+	/*
+	 * We need to handle the case of FCREAT | FDIRECTORY and the case of
+	 * FEXCL. If all three are specified, then we always fail because we
+	 * cannot create a directory through this interface and FEXCL says we
+	 * need to fail the request if we can't create it. If, however, only
+	 * FCREAT | FDIRECTORY are specified, then we can treat this as the case
+	 * of opening a file that already exists. If it exists, we can do
+	 * something and if not, we fail. Effectively FCREAT | FDIRECTORY is
+	 * treated as FDIRECTORY.
+	 */
+	if ((filemode & (FCREAT | FDIRECTORY | FEXCL)) ==
+	    (FCREAT | FDIRECTORY | FEXCL)) {
+		return (EINVAL);
+	}
+
+	if ((filemode & (FCREAT | FDIRECTORY)) == (FCREAT | FDIRECTORY)) {
+		create = B_FALSE;
+	} else if ((filemode & FCREAT) != 0) {
+		create = B_TRUE;
+	} else {
+		create = B_FALSE;
+	}
+
 top:
-	if (filemode & FCREAT) {
+	if (create) {
 		enum vcexcl excl;
 
 		/*
@@ -1089,11 +1113,13 @@ top:
 		 */
 		if (error = VOP_ACCESS(vp, mode, accessflags, CRED(), NULL))
 			goto out;
+
 		/*
-		 * Require FSEARCH to return a directory.
-		 * Require FEXEC to return a regular file.
+		 * Require FSEARCH and FDIRECTORY to return a directory. Require
+		 * FEXEC to return a regular file.
 		 */
-		if ((filemode & FSEARCH) && vp->v_type != VDIR) {
+		if ((filemode & (FSEARCH|FDIRECTORY)) != 0 &&
+		    vp->v_type != VDIR) {
 			error = ENOTDIR;
 			goto out;
 		}