OS-8278 lx improve O_PATH|O_NOFOLLOW emulation for symlinks

8 files changed, 67 insertions, 18 deletions

diff --git a/usr/src/cmd/ptools/pfiles/pfiles.c b/usr/src/cmd/ptools/pfiles/pfiles.c
index 474650faa4..8d6cacff2e 100644
--- a/usr/src/cmd/ptools/pfiles/pfiles.c
+++ b/usr/src/cmd/ptools/pfiles/pfiles.c
@@ -25,7 +25,7 @@
  */
 /*
  * Copyright (c) 2017 Joyent, Inc.  All Rights reserved.
- * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
+ * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #include <stdio.h>
@@ -44,6 +44,7 @@
 #include <sys/mkdev.h>
 #include <sys/stropts.h>
 #include <sys/timod.h>
+#include <sys/file.h>
 #include <sys/un.h>
 #include <libproc.h>
 #include <netinet/in.h>
@@ -364,7 +365,8 @@ dofcntl(struct ps_prochandle *Pr, const prfdinfo_t *info, int mandatory,
 
 #define	ALL_O_FLAGS	O_ACCMODE | O_NDELAY | O_NONBLOCK | O_APPEND | \
 			O_SYNC | O_DSYNC | O_RSYNC | O_XATTR | \
-			O_CREAT | O_TRUNC | O_EXCL | O_NOCTTY | O_LARGEFILE
+			O_CREAT | O_TRUNC | O_EXCL | O_NOCTTY | O_LARGEFILE | \
+			__FLXPATH
 
 static void
 show_fileflags(int flags)
@@ -417,6 +419,8 @@ show_fileflags(int flags)
 		(void) strcat(str, "|O_LARGEFILE");
 	if (flags & O_XATTR)
 		(void) strcat(str, "|O_XATTR");
+	if (flags & __FLXPATH)
+		(void) strcat(str, "|__FLXPATH");
 	if (flags & ~(ALL_O_FLAGS))
 		(void) sprintf(str + strlen(str), "|0x%x",
 		    flags & ~(ALL_O_FLAGS));
diff --git a/usr/src/cmd/sgs/libconv/common/corenote.c b/usr/src/cmd/sgs/libconv/common/corenote.c
index 10900b0d27..73913c5eaf 100644
--- a/usr/src/cmd/sgs/libconv/common/corenote.c
+++ b/usr/src/cmd/sgs/libconv/common/corenote.c
@@ -27,6 +27,7 @@
  * Copyright 2012 DEY Storage Systems, Inc.  All rights reserved.
  * Copyright (c) 2018 Joyent, Inc.
  * Copyright 2020 Oxide Computer Company
+ * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
  */
 
 /*
@@ -2480,6 +2481,7 @@ conv_cnote_fileflags(uint32_t fileflags, Conv_fmt_flags_t fmt_flags,
 		{ 0x2000,	MSG_PR_O_LARGEFILE },
 		{ 0x20000,	MSG_PR_O_NOFOLLOW },
 		{ 0x40000,	MSG_PR_O_NOLINKS },
+		{ 0x80000000,	MSG_PR___FLXPATH },
 		{ 0, 0 },
 	};
 
diff --git a/usr/src/cmd/sgs/libconv/common/corenote.msg b/usr/src/cmd/sgs/libconv/common/corenote.msg
index 7c75c5735e..d8ecedaff5 100644
--- a/usr/src/cmd/sgs/libconv/common/corenote.msg
+++ b/usr/src/cmd/sgs/libconv/common/corenote.msg
@@ -26,6 +26,7 @@
 # Copyright 2012 DEY Storage Systems, Inc.  All rights reserved.
 # Copyright (c) 2018 Joyent, Inc.
 # Copyright 2020 Oxide Computer Company
+# Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
 #
 
 @ MSG_NT_PRSTATUS		"[ NT_PRSTATUS ]"
@@ -1116,6 +1117,7 @@
 @ MSG_PR_O_XATTR			"O_XATTR"
 @ MSG_PR_O_NOFOLLOW			"O_NOFOLLOW"
 @ MSG_PR_O_NOLINKS			"O_NOLINKS"
+@ MSG_PR___FLXPATH			"__FLXPATH"
 
 @ MSG_S_IFIFO				"S_IFIFO"
 @ MSG_S_IFCHR				"S_IFCHR"
diff --git a/usr/src/cmd/truss/codes.c b/usr/src/cmd/truss/codes.c
index fd6e02bad6..95b15ca787 100644
--- a/usr/src/cmd/truss/codes.c
+++ b/usr/src/cmd/truss/codes.c
@@ -25,6 +25,7 @@
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  * Copyright 2020 Joyent, Inc.
  * Copyright (c) 2014, OmniTI Computer Consulting, Inc. All rights reserved.
+ * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
@@ -1966,7 +1967,7 @@ pathconfname(int code)
 #define	ALL_O_FLAGS \
 	(O_NDELAY|O_APPEND|O_SYNC|O_DSYNC|O_NONBLOCK|O_CREAT|O_TRUNC\
 	|O_EXCL|O_NOCTTY|O_LARGEFILE|O_RSYNC|O_XATTR|O_NOFOLLOW|O_NOLINKS\
-	|O_CLOEXEC|O_DIRECTORY|O_DIRECT|FXATTRDIROPEN)
+	|O_CLOEXEC|O_DIRECTORY|O_DIRECT|FXATTRDIROPEN|__FLXPATH)
 
 const char *
 openarg(private_t *pri, int arg)
@@ -2032,6 +2033,8 @@ openarg(private_t *pri, int arg)
 		(void) strlcat(str, "|O_DIRECT", sizeof (pri->code_buf));
 	if (arg & FXATTRDIROPEN)
 		(void) strlcat(str, "|FXATTRDIROPEN", sizeof (pri->code_buf));
+	if (arg & __FLXPATH)
+		(void) strlcat(str, "|__FLXPATH", sizeof (pri->code_buf));
 
 	return ((const char *)str);
 }
diff --git a/usr/src/uts/common/brand/lx/syscall/lx_open.c b/usr/src/uts/common/brand/lx/syscall/lx_open.c
index 137e6b786c..5d5e4397ce 100644
--- a/usr/src/uts/common/brand/lx/syscall/lx_open.c
+++ b/usr/src/uts/common/brand/lx/syscall/lx_open.c
@@ -23,7 +23,7 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  * Copyright 2018 Joyent, Inc.
- * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
+ * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
  */
 
 #include <sys/systm.h>
@@ -171,13 +171,19 @@ lx_openat(int atfd, char *path, int fmode, int cmode)
 
 	flags = ltos_open_flags(fmode);
 
+	if ((fmode & (LX_O_NOFOLLOW|LX_O_PATH|__FLXPATH)) ==
+	    (LX_O_NOFOLLOW|LX_O_PATH|__FLXPATH)) {
+		flags |= __FLXPATH;
+	}
+
 	if (flags & O_CREAT)
 		mode = (mode_t)cmode;
 
 	ttolwp(curthread)->lwp_errno = 0;
 	fd = openat(atfd, path, flags, mode);
 	if (ttolwp(curthread)->lwp_errno != 0) {
-		if ((fmode & LX_O_NOFOLLOW) && (fmode & LX_O_PATH) &&
+		if ((fmode & (LX_O_NOFOLLOW|LX_O_PATH|__FLXPATH)) ==
+		    (LX_O_NOFOLLOW|LX_O_PATH) &&
 		    ttolwp(curthread)->lwp_errno == ELOOP) {
 			/*
 			 * On Linux, if O_NOFOLLOW and O_PATH are set together
@@ -189,17 +195,14 @@ lx_openat(int atfd, char *path, int fmode, int cmode)
 			 * fstatat(2), linkat(2), and readlinkat(2) alongside
 			 * an empty pathname.
 			 *
-			 * We do not have a way to return such a file
-			 * descriptor in illumos so open it without NO_FOLLOW
-			 * and allow the postprocess to emulate O_PATH by
-			 * removing the read and write flags.
-			 * This is enough to keep recent systemd happy
-			 * although any attempt to use the fd for the above
-			 * listed calls without a pathname will fail or modify
-			 * the symlink target.
+			 * illumos has a private interface flag that causes
+			 * openat() to return a file descriptor attached to
+			 * the symlink's vnode. This, in conjunction with the
+			 * other adjustments made in lx_open_postprocess()
+			 * for O_PATH, is enough to satisfy systemd and
+			 * other parts of Linux.
 			 */
-			return (lx_openat(atfd, path, fmode & ~LX_O_NOFOLLOW,
-			    cmode));
+			return (lx_openat(atfd, path, fmode|__FLXPATH, cmode));
 		}
 
 		if (ttolwp(curthread)->lwp_errno == EINTR)
diff --git a/usr/src/uts/common/fs/vnode.c b/usr/src/uts/common/fs/vnode.c
index 3babdbecdb..e1359f47de 100644
--- a/usr/src/uts/common/fs/vnode.c
+++ b/usr/src/uts/common/fs/vnode.c
@@ -24,6 +24,7 @@
  * Copyright 2020 Joyent, Inc.
  * Copyright 2016 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
+ * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
  */
 
 /*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T	*/
@@ -1165,7 +1166,20 @@ top:
 	 * Do remaining checks for FNOFOLLOW and FNOLINKS.
 	 */
 	if ((filemode & FNOFOLLOW) && vp->v_type == VLNK) {
-		error = ELOOP;
+		/*
+		 * The __FLXPATH flag is a private interface for use by the lx
+		 * brand in order to emulate open(O_NOFOLLOW|O_PATH) which,
+		 * when a symbolic link is encountered, returns a file
+		 * descriptor which references it.
+		 * See uts/common/brand/lx/syscall/lx_open.c
+		 *
+		 * When this flag is set, VOP_OPEN() is not called (for a
+		 * symlink, most filesystems will return ENOSYS anyway)
+		 * and the link's vnode is returned to be linked to the
+		 * file descriptor.
+		 */
+		if ((filemode & __FLXPATH) == 0)
+			error = ELOOP;
 		goto out;
 	}
 	if (filemode & FNOLINKS) {
diff --git a/usr/src/uts/common/os/fio.c b/usr/src/uts/common/os/fio.c
index ec89cb0657..300725c0ba 100644
--- a/usr/src/uts/common/os/fio.c
+++ b/usr/src/uts/common/os/fio.c
@@ -22,6 +22,7 @@
 /*
  * Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2017, Joyent Inc.
+ * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
@@ -958,7 +959,22 @@ closef(file_t *fp)
 
 	vp = fp->f_vnode;
 
-	error = VOP_CLOSE(vp, flag, count, offset, fp->f_cred, NULL);
+	/*
+	 * The __FLXPATH flag is a private interface for use by the lx
+	 * brand in order to emulate open(O_NOFOLLOW|O_PATH) which,
+	 * when a symbolic link is encountered, returns a file
+	 * descriptor which references it.
+	 * See uts/common/brand/lx/syscall/lx_open.c
+	 *
+	 * When this flag is set, VOP_OPEN() will not have been called when
+	 * this file descriptor was opened, and VOP_CLOSE() should not be
+	 * called here (for a symlink, most filesystems would return ENOSYS
+	 * anyway)
+	 */
+	if (fp->f_flag2 & (__FLXPATH >> 16))
+		error = 0;
+	else
+		error = VOP_CLOSE(vp, flag, count, offset, fp->f_cred, NULL);
 
 	if (count > 1) {
 		mutex_exit(&fp->f_tlock);
@@ -1118,7 +1134,7 @@ falloc(vnode_t *vp, int flag, file_t **fpp, int *fdp)
 	mutex_enter(&fp->f_tlock);
 	fp->f_count = 1;
 	fp->f_flag = (ushort_t)flag;
-	fp->f_flag2 = (flag & (FSEARCH|FEXEC)) >> 16;
+	fp->f_flag2 = (flag & (FSEARCH|FEXEC|__FLXPATH)) >> 16;
 	fp->f_vnode = vp;
 	fp->f_offset = 0;
 	fp->f_audit_data = 0;
diff --git a/usr/src/uts/common/sys/file.h b/usr/src/uts/common/sys/file.h
index c7d67a5d47..be96a68d46 100644
--- a/usr/src/uts/common/sys/file.h
+++ b/usr/src/uts/common/sys/file.h
@@ -28,6 +28,7 @@
 
 /* Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved. */
 /* Copyright 2020 Joyent, Inc. */
+/* Copyright 2021 OmniOS Community Edition (OmniOSce) Association. */
 
 #ifndef _SYS_FILE_H
 #define	_SYS_FILE_H
@@ -120,6 +121,10 @@ typedef struct fpollinfo {
 #define	FCLOEXEC	0x800000	/* O_CLOEXEC = 0x800000 */
 #define	FDIRECTORY	0x1000000	/* O_DIRECTORY = 0x1000000 */
 #define	FDIRECT		0x2000000	/* O_DIRECT = 0x2000000 */
+/*
+ * Private interface for lx O_PATH|O_NOFOLLOW emulation for symlinks.
+ */
+#define	__FLXPATH	0x80000000
 
 #if defined(_KERNEL) || defined(_FAKE_KERNEL)