OS-6018 NFSv3 interaction with Linux statd fails

Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Approved by: Patrick Mooney <patrick.mooney@joyent.com>

10 files changed, 151 insertions, 17 deletions

diff --git a/usr/src/uts/common/brand/lx/os/lx_brand.c b/usr/src/uts/common/brand/lx/os/lx_brand.c
index 4dfa0bc69e..b6ecc856e7 100644
--- a/usr/src/uts/common/brand/lx/os/lx_brand.c
+++ b/usr/src/uts/common/brand/lx/os/lx_brand.c
@@ -210,6 +210,7 @@ extern void lx_socket_init();
 extern void lx_socket_fini();
 
 extern int lx_start_nfs_lockd();
+extern void lx_upcall_statd();
 
 lx_systrace_f *lx_systrace_entry_ptr;
 lx_systrace_f *lx_systrace_return_ptr;
@@ -305,7 +306,8 @@ struct brand_ops lx_brops = {
 	NULL,
 #endif
 	B_FALSE,			/* b_intp_parse_arg */
-	lx_clearbrand			/* b_clearbrand */
+	lx_clearbrand,			/* b_clearbrand */
+	lx_upcall_statd			/* b_rpc_statd */
 };
 
 struct brand_mach_ops lx_mops = {
diff --git a/usr/src/uts/common/brand/lx/os/lx_start_lockd.c b/usr/src/uts/common/brand/lx/os/lx_lockd.c
index 727f715cfb..234e815d3f 100644
--- a/usr/src/uts/common/brand/lx/os/lx_start_lockd.c
+++ b/usr/src/uts/common/brand/lx/os/lx_lockd.c
@@ -14,8 +14,18 @@
  */
 
 /*
- * Start an NFS lockd (lx_lockd) process inside the zone. This uses the same
- * technique as used in our lx cgroupfs to launch a release agent process.
+ * lx_start_nfs_lockd() starts an NFS lockd (lx_lockd) process inside the zone.
+ * This uses the same technique as used in our lx cgroupfs to launch a release
+ * agent process. This is called implicitly when an NFS mount syscall occurs
+ * within the zone. See the user-level lx_lockd source for the "big theory"
+ * comment behind this.
+ *
+ * lx_upcall_statd() is a brand hook that interposes on the rpc.statd RPC
+ * handling so that we can interface to a Linux rpc.statd that must run
+ * when NFSv3 locking is in use. The rpc.statd handles server or client reboots
+ * and interacts with the lockd to reclaim locks after the server reboots. The
+ * rcp.statd also informs the server when we reboot, so the server can release
+ * the locks we held.
  */
 
 #include <sys/types.h>
@@ -35,9 +45,19 @@
 #include <sys/brand.h>
 #include <sys/lx_brand.h>
 #include <sys/pathname.h>
+#include <rpcsvc/nlm_prot.h>
+#include <rpcsvc/sm_inter.h>
+#include <klm/nlm_impl.h>
 
 #define	LX_LOCKD_PATH	"/native/usr/lib/brand/lx/lx_lockd"
 
+/* Linux lockd RPC called by statd when it detects an NFS server reboot */
+#define	LX_NLMPROC_NSM_NOTIFY	16
+
+/* From uts/common/klm/nlm_impl.c */
+extern void nlm_netbuf_to_netobj(struct netbuf *, int *, netobj *);
+extern void nlm_nsm_clnt_init(CLIENT *, struct nlm_nsm *);
+
 /*
  * Check if the current lockd is still running.
  */
@@ -228,3 +248,86 @@ lx_start_nfs_lockd()
 	 */
 	return (newproc(lx_run_lockd, NULL, cid, minclsyspri - 1, NULL, -1));
 }
+
+void
+lx_upcall_statd(int op, struct nlm_globals *g, struct nlm_host *host)
+{
+	struct nlm_nsm *nsm;
+	struct mon args;
+	struct mon_id *mip = &args.mon_id;
+	int family;
+	netobj obj;
+	enum clnt_stat stat;
+
+	/*
+	 * For Linux rpc.statd monitor registration, the Linux NSMPROC_MON and
+	 * NSMPROC_UNMON RPC upcalls correspond almost directly to the native
+	 * SM_MON and SM_UNMON RPC upcalls. The key differences with the native
+	 * registration is that in our nlm_host_monitor function we make two
+	 * RPC calls:
+	 *    - the first RPC (nsmaddrproc1_reg_1) uses our private 'nsm_addr'
+	 *	RPC protocol to register the lockd RPC information that statd
+	 *	should call when it detects that the remote server rebooted
+	 *    - the second RPC (sm_mon_1) tells statd the information about the
+	 *	remote server to be monitored
+	 * For Linux, there is only a single RPC from the kernel to the local
+	 * statd. This RPC is equivalent to our sm_mon_1 code, but it uses the
+	 * Linux-private NLMPROC_NSM_NOTIFY lockd procedure in the 'my_proc'
+	 * RPC parameter. This corresponds to our private 'nsm_addr' code, and
+	 * tells statd which lockd RPC to call when it detects a server reboot.
+	 *
+	 * Because our sm_mon_1 RPC is so similar to the Linux RPC, we can use
+	 * that directly and simply set the expected value in the 'my_proc'
+	 * argument.
+	 *
+	 * Within the kernel lockd RPC handling, the nlm_prog_3_dtable dispatch
+	 * table has an entry for each lockd RPC function. Thus, this table also
+	 * contains an entry for the Linux NLMPROC_NSM_NOTIFY procedure. That
+	 * procedure number is unused by the native lockd code, so there is no
+	 * conflict with dispatching that procedure. The implementation of the
+	 * procedure corresponds to the native, private NLM_SM_NOTIFY1
+	 * procedure which is called by the native rpc.statd.
+	 *
+	 * The Linux RPC call to "unmonitor" a host expects the same arguments
+	 * as we pass to monitor, so that is also handled here by this same
+	 * brand hook.
+	 */
+	nlm_netbuf_to_netobj(&host->nh_addr, &family, &obj);
+	nsm = &g->nlm_nsm;
+
+	bzero(&args, sizeof (args));
+
+	mip->mon_name = host->nh_name;
+	mip->my_id.my_name = uts_nodename();
+	mip->my_id.my_prog = NLM_PROG;
+	mip->my_id.my_vers = NLM_SM;
+	mip->my_id.my_proc = LX_NLMPROC_NSM_NOTIFY;
+	if (op == SM_MON) {
+		bcopy(&host->nh_sysid, args.priv, sizeof (uint16_t));
+	}
+
+	sema_p(&nsm->ns_sem);
+	nlm_nsm_clnt_init(nsm->ns_handle, nsm);
+	if (op == SM_MON) {
+		struct sm_stat_res mres;
+
+		bzero(&mres, sizeof (mres));
+		stat = sm_mon_1(&args, &mres, nsm->ns_handle);
+	} else {
+		struct sm_stat ures;
+
+		ASSERT(op == SM_UNMON);
+		bzero(&ures, sizeof (ures));
+		stat = sm_unmon_1(mip, &ures, nsm->ns_handle);
+	}
+	sema_v(&nsm->ns_sem);
+
+	if (stat != RPC_SUCCESS) {
+		NLM_WARN("Failed to contact local statd, stat=%d", stat);
+		if (op == SM_MON) {
+			mutex_enter(&g->lock);
+			host->nh_flags &= ~NLM_NH_MONITORED;
+			mutex_exit(&g->lock);
+		}
+	}
+}
diff --git a/usr/src/uts/common/brand/sn1/sn1_brand.c b/usr/src/uts/common/brand/sn1/sn1_brand.c
index 5726beffa2..92c0aec1e0 100644
--- a/usr/src/uts/common/brand/sn1/sn1_brand.c
+++ b/usr/src/uts/common/brand/sn1/sn1_brand.c
@@ -103,7 +103,8 @@ struct brand_ops sn1_brops = {
 	NULL,				/* b_setid_clear */
 	NULL,				/* b_pagefault */
 	B_TRUE,				/* b_intp_parse_arg */
-	NULL				/* b_clearbrand */
+	NULL,				/* b_clearbrand */
+	NULL				/* b_rpc_statd */
 };
 
 #ifdef	sparc
diff --git a/usr/src/uts/common/brand/solaris10/s10_brand.c b/usr/src/uts/common/brand/solaris10/s10_brand.c
index 6fb2ffd80d..c02fcdaef6 100644
--- a/usr/src/uts/common/brand/solaris10/s10_brand.c
+++ b/usr/src/uts/common/brand/solaris10/s10_brand.c
@@ -108,7 +108,8 @@ struct brand_ops s10_brops = {
 	NULL,				/* b_setid_clear */
 	NULL,				/* b_pagefault */
 	B_TRUE,				/* b_intp_parse_arg */
-	NULL				/* b_clearbrand */
+	NULL,				/* b_clearbrand */
+	NULL				/* b_rpc_statd */
 };
 
 #ifdef	sparc
diff --git a/usr/src/uts/common/klm/mapfile-mod b/usr/src/uts/common/klm/mapfile-mod
index 0debe6d986..b7789d81fd 100644
--- a/usr/src/uts/common/klm/mapfile-mod
+++ b/usr/src/uts/common/klm/mapfile-mod
@@ -11,6 +11,7 @@
 
 #
 # Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+# Copyright 2017 Joyent, Inc.
 #
 
 
@@ -49,6 +50,11 @@ SYMBOL_SCOPE {
 		nlm_frlock;
 		nlm_register_lock_locally;
 		nlm_shrlock;
+# These four functions are available for use within a branded zone.
+		nlm_nsm_clnt_init;
+		nlm_netbuf_to_netobj;
+		sm_mon_1;
+		sm_unmon_1;
 
 	local:
 		*;
diff --git a/usr/src/uts/common/klm/nlm_dispatch.c b/usr/src/uts/common/klm/nlm_dispatch.c
index a0ca2a56c4..8fa9940eae 100644
--- a/usr/src/uts/common/klm/nlm_dispatch.c
+++ b/usr/src/uts/common/klm/nlm_dispatch.c
@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2017 Joyent, Inc.  All rights reserved.
  */
 
 /*
@@ -412,13 +413,13 @@ nlm_prog_3_dtable[] = {
 	0,
 	0 },
 
-	{ /* 16: not used */
-	NLM_SVC_FUNC(0),
-	(xdrproc_t)0,
-	(xdrproc_t)0,
+	{ /* 16: Linux NLMPROC_NSM_NOTIFY (same handling as NLM_SM_NOTIFY1) */
+	NLM_SVC_FUNC(nlm_sm_notify1_2_svc),
+	(xdrproc_t)xdr_nlm_sm_status,
+	(xdrproc_t)xdr_void,
 	NULL,
 	0,
-	0 },
+	NLM_DISP_NOREMOTE },
 
 	{ /* 17: NLM_SM_NOTIFY1 */
 	NLM_SVC_FUNC(nlm_sm_notify1_2_svc),
diff --git a/usr/src/uts/common/klm/nlm_impl.c b/usr/src/uts/common/klm/nlm_impl.c
index 1e9033a17c..f283828013 100644
--- a/usr/src/uts/common/klm/nlm_impl.c
+++ b/usr/src/uts/common/klm/nlm_impl.c
@@ -28,6 +28,7 @@
 /*
  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright 2017 Joyent, Inc.  All rights reserved.
  */
 
 /*
@@ -57,6 +58,7 @@
 #include <sys/queue.h>
 #include <sys/bitmap.h>
 #include <sys/sdt.h>
+#include <sys/brand.h>
 #include <netinet/in.h>
 
 #include <rpc/rpc.h>
@@ -202,6 +204,12 @@ static struct nlm_knc nlm_netconfigs[] = { /* (g) */
 };
 
 /*
+ * NLM functions which can be called by a brand hook.
+ */
+void nlm_netbuf_to_netobj(struct netbuf *, int *, netobj *);
+void nlm_nsm_clnt_init(CLIENT *, struct nlm_nsm *);
+
+/*
  * NLM misc. function
  */
 static void nlm_copy_netbuf(struct netbuf *, struct netbuf *);
@@ -210,8 +218,6 @@ static void nlm_kmem_reclaim(void *);
 static void nlm_pool_shutdown(void);
 static void nlm_suspend_zone(struct nlm_globals *);
 static void nlm_resume_zone(struct nlm_globals *);
-static void nlm_nsm_clnt_init(CLIENT *, struct nlm_nsm *);
-static void nlm_netbuf_to_netobj(struct netbuf *, int *, netobj *);
 
 /*
  * NLM thread functions
@@ -1839,6 +1845,12 @@ nlm_host_unmonitor(struct nlm_globals *g, struct nlm_host *host)
 		return;
 
 	host->nh_flags &= ~NLM_NH_MONITORED;
+
+	if (ZONE_IS_BRANDED(curzone) && ZBROP(curzone)->b_rpc_statd != NULL) {
+		ZBROP(curzone)->b_rpc_statd(SM_UNMON, g, host);
+		return;
+	}
+
 	stat = nlm_nsm_unmon(&g->nlm_nsm, host->nh_name);
 	if (stat != RPC_SUCCESS) {
 		NLM_WARN("NLM: Failed to contact statd, stat=%d\n", stat);
@@ -1877,6 +1889,11 @@ nlm_host_monitor(struct nlm_globals *g, struct nlm_host *host, int state)
 	host->nh_flags |= NLM_NH_MONITORED;
 	mutex_exit(&host->nh_lock);
 
+	if (ZONE_IS_BRANDED(curzone) && ZBROP(curzone)->b_rpc_statd != NULL) {
+		ZBROP(curzone)->b_rpc_statd(SM_MON, g, host);
+		return;
+	}
+
 	/*
 	 * Before we begin monitoring the host register the network address
 	 * associated with this hostname.
@@ -2781,14 +2798,14 @@ nlm_cprresume(void)
 	rw_exit(&lm_lck);
 }
 
-static void
+void
 nlm_nsm_clnt_init(CLIENT *clnt, struct nlm_nsm *nsm)
 {
 	(void) clnt_tli_kinit(clnt, &nsm->ns_knc, &nsm->ns_addr, 0,
 	    NLM_RPC_RETRIES, kcred);
 }
 
-static void
+void
 nlm_netbuf_to_netobj(struct netbuf *addr, int *family, netobj *obj)
 {
 	/* LINTED pointer alignment */
diff --git a/usr/src/uts/common/sys/brand.h b/usr/src/uts/common/sys/brand.h
index cf22a0f509..e7d4fb894e 100644
--- a/usr/src/uts/common/sys/brand.h
+++ b/usr/src/uts/common/sys/brand.h
@@ -150,6 +150,7 @@ struct execa;
  * b_pagefault - Trap pagefault events
  * b_intp_parse_arg - Controls interpreter argument handling (allow 1 or all)
  * b_clearbrand - Perform any actions necessary when clearing the brand.
+ * b_rpc_statd - Upcall to rpc.statd running within the zone
  */
 struct brand_ops {
 	void	(*b_init_brand_data)(zone_t *, kmutex_t *);
@@ -200,6 +201,7 @@ struct brand_ops {
 	    enum seg_rw);
 	boolean_t b_intp_parse_arg;
 	void	(*b_clearbrand)(proc_t *, boolean_t);
+	void	(*b_rpc_statd)(int, void *, void *);
 };
 
 /*
diff --git a/usr/src/uts/intel/Makefile.files b/usr/src/uts/intel/Makefile.files
index b45d1a1539..a1bca02510 100644
--- a/usr/src/uts/intel/Makefile.files
+++ b/usr/src/uts/intel/Makefile.files
@@ -334,7 +334,7 @@ LX_BRAND_OBJS  =		\
 	lx_mkdir.o		\
 	lx_modify_ldt.o		\
 	lx_mount.o		\
-	lx_start_lockd.o	\
+	lx_lockd.o		\
 	lx_open.o		\
 	lx_personality.o	\
 	lx_pgrp.o		\
diff --git a/usr/src/uts/intel/lx_brand/Makefile b/usr/src/uts/intel/lx_brand/Makefile
index b0a4e96412..819c22566c 100644
--- a/usr/src/uts/intel/lx_brand/Makefile
+++ b/usr/src/uts/intel/lx_brand/Makefile
@@ -21,7 +21,7 @@
 #
 # Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
-# Copyright 2016 Joyent, Inc.
+# Copyright 2017 Joyent, Inc.
 #
 #	This makefile drives the production of the kernel component of
 #	the lx brand
@@ -68,7 +68,8 @@ AS_INC_PATH	+= -I$(UTSBASE)/i86pc/genassym/$(OBJS_DIR)
 #
 CFLAGS		+= $(CCVERBOSE)
 
-LDFLAGS		+= -dy -Nexec/elfexec -Nfs/fifofs -Nfs/sockfs -Ndrv/ip -Nfs/zfs
+LDFLAGS		+= -dy -Nexec/elfexec -Nfs/fifofs -Nfs/sockfs -Ndrv/ip \
+		    -Nfs/zfs -Nmisc/klmmod
 
 #
 # For now, disable these lint checks; maintainers should endeavor