OS-8277 lx can use O_DIRECTORY directly (#358)

Reviewed by: Dan McDonald <danmcd@joyent.com>
Reviewed by: Mike Zeller <mike.zeller@joyent.com>
Approved by: Dan McDonald <danmcd@joyent.com>

1 files changed, 9 insertions, 72 deletions

diff --git a/usr/src/uts/common/brand/lx/syscall/lx_open.c b/usr/src/uts/common/brand/lx/syscall/lx_open.c
index 76672aece1..137e6b786c 100644
--- a/usr/src/uts/common/brand/lx/syscall/lx_open.c
+++ b/usr/src/uts/common/brand/lx/syscall/lx_open.c
@@ -54,17 +54,17 @@ ltos_open_flags(int input)
 {
 	int flags;
 
-	if (input & LX_O_PATH) {
+	if (input & LX_O_PATH)
 		input &= (LX_O_DIRECTORY | LX_O_NOFOLLOW | LX_O_CLOEXEC);
-	}
 
-	/* This depends on the Linux ACCMODE flags being the same as SunOS. */
+	/*
+	 * The illumos O_ACCMODE also includes O_SEARCH|O_EXEC
+	 * so this has the effect of stripping those here.
+	 */
 	flags = (input & LX_O_ACCMODE);
 
-	if (input & LX_O_CREAT) {
+	if (input & LX_O_CREAT)
 		flags |= O_CREAT;
-	}
-
 	if (input & LX_O_EXCL)
 		flags |= O_EXCL;
 	if (input & LX_O_NOCTTY)
@@ -83,6 +83,8 @@ ltos_open_flags(int input)
 		flags |= O_NOFOLLOW;
 	if (input & LX_O_CLOEXEC)
 		flags |= O_CLOEXEC;
+	if (input & LX_O_DIRECTORY)
+		flags |= O_DIRECTORY;
 
 	/*
 	 * Linux uses the LX_O_DIRECT flag to do raw, synchronous I/O to the
@@ -169,78 +171,13 @@ lx_openat(int atfd, char *path, int fmode, int cmode)
 
 	flags = ltos_open_flags(fmode);
 
-	/*
-	 * We use the FSEARCH flag to make sure this is a directory. We have to
-	 * explicitly add 1 to emulate the FREAD/FWRITE mapping of the OPENMODE
-	 * macro since it won't get set via OPENMODE when FSEARCH is used.
-	 */
-	if (fmode & LX_O_DIRECTORY) {
-		flags |= FSEARCH;
-		flags++;
-	}
-
 	if (flags & O_CREAT)
 		mode = (mode_t)cmode;
 
 	ttolwp(curthread)->lwp_errno = 0;
 	fd = openat(atfd, path, flags, mode);
 	if (ttolwp(curthread)->lwp_errno != 0) {
-		if ((fmode & LX_O_DIRECTORY) &&
-		    ttolwp(curthread)->lwp_errno != ENOTDIR) {
-			/*
-			 * We got an error trying to open a file as a directory.
-			 * We need to determine if we should return the original
-			 * error or ENOTDIR.
-			 */
-			vnode_t *startvp;
-			vnode_t *vp;
-			int oerror, error = 0;
-
-			oerror = ttolwp(curthread)->lwp_errno;
-
-			if (atfd == AT_FDCWD) {
-				/* regular open */
-				startvp = NULL;
-			} else {
-				char startchar;
-
-				if (copyin(path, &startchar, sizeof (char)))
-					return (set_errno(oerror));
-
-				/* if startchar is / then startfd is ignored */
-				if (startchar == '/') {
-					startvp = NULL;
-				} else {
-					file_t *startfp;
-
-					if ((startfp = getf(atfd)) == NULL)
-						return (set_errno(oerror));
-					startvp = startfp->f_vnode;
-					VN_HOLD(startvp);
-					releasef(atfd);
-				}
-			}
-
-			if (lookupnameat(path, UIO_USERSPACE,
-			    (fmode & LX_O_NOFOLLOW) ?  NO_FOLLOW : FOLLOW,
-			    NULLVPP, &vp, startvp) != 0) {
-				if (startvp != NULL)
-					VN_RELE(startvp);
-				return (set_errno(oerror));
-			}
-
-			if (startvp != NULL)
-				VN_RELE(startvp);
-
-			if (vp->v_type != VDIR)
-				error = ENOTDIR;
-
-			VN_RELE(vp);
-			if (error != 0)
-				return (set_errno(ENOTDIR));
-
-			(void) set_errno(oerror);
-		} else if ((fmode & LX_O_NOFOLLOW) && (fmode & LX_O_PATH) &&
+		if ((fmode & LX_O_NOFOLLOW) && (fmode & LX_O_PATH) &&
 		    ttolwp(curthread)->lwp_errno == ELOOP) {
 			/*
 			 * On Linux, if O_NOFOLLOW and O_PATH are set together