[illumos-gate merge]

commit e984c70bc7d741cd0663924c95c15fed9f645565
    8629 nvme: rework command abortion
commit 4b324362d54967a3ccfad8a9b113978ea1b38feb
    8628 nvme: use a semaphore to guard submission queue
commit 3c4c5929f4f5a266089758e0d018685b379767ad
    8643 Fix misuse of _KMEMUSER in i86pc

	Conflcits:
	usr/src/uts/common/io/nvme/nvme.c
	usr/src/uts/common/sys/cpuvar.h

7 files changed, 65 insertions, 29 deletions

diff --git a/usr/src/uts/common/io/nvme/nvme.c b/usr/src/uts/common/io/nvme/nvme.c
index 9e2127618d..42a320dff0 100644
--- a/usr/src/uts/common/io/nvme/nvme.c
+++ b/usr/src/uts/common/io/nvme/nvme.c
@@ -164,14 +164,15 @@
  * Each queue pair has its own nq_mutex, which must be held when accessing the
  * associated queue registers or the shared state of the queue pair. Callers of
  * nvme_unqueue_cmd() must make sure that nq_mutex is held, while
- * nvme_submit_cmd() and nvme_retrieve_cmd() take care of this themselves.
+ * nvme_submit_{admin,io}_cmd() and nvme_retrieve_cmd() take care of this
+ * themselves.
  *
  * Each command also has its own nc_mutex, which is associated with the
  * condition variable nc_cv. It is only used on admin commands which are run
- * synchronously. In that case it must be held across calls to nvme_submit_cmd()
- * and nvme_wait_cmd(), which is taken care of by nvme_admin_cmd(). It must also
- * be held whenever the completion state of the command is changed or while a
- * admin command timeout is handled.
+ * synchronously. In that case it must be held across calls to
+ * nvme_submit_{admin,io}_cmd() and nvme_wait_cmd(), which is taken care of by
+ * nvme_admin_cmd(). It must also be held whenever the completion state of the
+ * command is changed or while a admin command timeout is handled.
  *
  * If both nc_mutex and nq_mutex must be held, nc_mutex must be acquired first.
  * More than one nc_mutex may only be held when aborting commands. In this case,
@@ -284,7 +285,9 @@ static void nvme_free_cmd(nvme_cmd_t *);
 static nvme_cmd_t *nvme_create_nvm_cmd(nvme_namespace_t *, uint8_t,
     bd_xfer_t *);
 static void nvme_admin_cmd(nvme_cmd_t *, int);
-static void nvme_submit_cmd(nvme_qpair_t *, nvme_cmd_t *);
+static void nvme_submit_admin_cmd(nvme_qpair_t *, nvme_cmd_t *);
+static int nvme_submit_io_cmd(nvme_qpair_t *, nvme_cmd_t *);
+static void nvme_submit_cmd_common(nvme_qpair_t *, nvme_cmd_t *);
 static nvme_cmd_t *nvme_unqueue_cmd(nvme_t *, nvme_qpair_t *, int);
 static nvme_cmd_t *nvme_retrieve_cmd(nvme_t *, nvme_qpair_t *);
 static void nvme_wait_cmd(nvme_cmd_t *, uint_t);
@@ -867,11 +870,27 @@ nvme_free_cmd(nvme_cmd_t *cmd)
 }
 
 static void
-nvme_submit_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd)
+nvme_submit_admin_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd)
+{
+	sema_p(&qp->nq_sema);
+	nvme_submit_cmd_common(qp, cmd);
+}
+
+static int
+nvme_submit_io_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd)
+{
+	if (sema_tryp(&qp->nq_sema) == 0)
+		return (EAGAIN);
+
+	nvme_submit_cmd_common(qp, cmd);
+	return (0);
+}
+
+static void
+nvme_submit_cmd_common(nvme_qpair_t *qp, nvme_cmd_t *cmd)
 {
 	nvme_reg_sqtdbl_t tail = { 0 };
 
-	sema_p(&qp->nq_sema);
 	mutex_enter(&qp->nq_mutex);
 	cmd->nc_completed = B_FALSE;
 
@@ -1435,7 +1454,7 @@ nvme_async_event_task(void *arg)
 
 	/* Clear CQE and re-submit the async request. */
 	bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t));
-	nvme_submit_cmd(nvme->n_adminq, cmd);
+	nvme_submit_admin_cmd(nvme->n_adminq, cmd);
 
 	switch (event.b.ae_type) {
 	case NVME_ASYNC_TYPE_ERROR:
@@ -1549,7 +1568,7 @@ static void
 nvme_admin_cmd(nvme_cmd_t *cmd, int sec)
 {
 	mutex_enter(&cmd->nc_mutex);
-	nvme_submit_cmd(cmd->nc_nvme->n_adminq, cmd);
+	nvme_submit_admin_cmd(cmd->nc_nvme->n_adminq, cmd);
 	nvme_wait_cmd(cmd, sec);
 	mutex_exit(&cmd->nc_mutex);
 }
@@ -1563,7 +1582,7 @@ nvme_async_event(nvme_t *nvme)
 	cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT;
 	cmd->nc_callback = nvme_async_event_task;
 
-	nvme_submit_cmd(nvme->n_adminq, cmd);
+	nvme_submit_admin_cmd(nvme->n_adminq, cmd);
 }
 
 static int
@@ -3254,9 +3273,10 @@ static int
 nvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc)
 {
 	nvme_t *nvme = ns->ns_nvme;
-	nvme_cmd_t *cmd, *ret;
+	nvme_cmd_t *cmd;
 	nvme_qpair_t *ioq;
 	boolean_t poll;
+	int ret;
 
 	if (nvme->n_dead)
 		return (EIO);
@@ -3276,15 +3296,18 @@ nvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc)
 	 */
 	poll = (xfer->x_flags & BD_XFER_POLL) != 0;
 
-	nvme_submit_cmd(ioq, cmd);
+	ret = nvme_submit_io_cmd(ioq, cmd);
+
+	if (ret != 0)
+		return (ret);
 
 	if (!poll)
 		return (0);
 
 	do {
-		ret = nvme_retrieve_cmd(nvme, ioq);
-		if (ret != NULL)
-			nvme_bd_xfer_done(ret);
+		cmd = nvme_retrieve_cmd(nvme, ioq);
+		if (cmd != NULL)
+			nvme_bd_xfer_done(cmd);
 		else
 			drv_usecwait(10);
 	} while (ioq->nq_active_cmds != 0);
diff --git a/usr/src/uts/common/io/nvme/nvme_var.h b/usr/src/uts/common/io/nvme/nvme_var.h
index c988991120..dca8c57e7c 100644
--- a/usr/src/uts/common/io/nvme/nvme_var.h
+++ b/usr/src/uts/common/io/nvme/nvme_var.h
@@ -12,6 +12,7 @@
 /*
  * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
  * Copyright 2016 The MathWorks, Inc. All rights reserved.
+ * Copyright 2017 Joyent, Inc.
  */
 
 #ifndef _NVME_VAR_H
diff --git a/usr/src/uts/common/sys/cpuvar.h b/usr/src/uts/common/sys/cpuvar.h
index eccb12f05f..efd684735d 100644
--- a/usr/src/uts/common/sys/cpuvar.h
+++ b/usr/src/uts/common/sys/cpuvar.h
@@ -296,7 +296,7 @@ extern cpu_core_t cpu_core[];
  */
 #define	CPU_PSEUDO_RANDOM() (CPU->cpu_rotor++)
 
-#if defined(_KERNEL) || defined(_KMEMUSER)
+#if defined(_KERNEL) || defined(_KMEMUSER) || defined(_BOOT)
 
 #define	INTR_STACK_SIZE	MAX(DEFAULTSTKSZ, PAGESIZE)
 
@@ -387,9 +387,8 @@ extern cpu_core_t cpu_core[];
 #define	CPU_DISP_DONTSTEAL	0x01	/* CPU undergoing context swtch */
 #define	CPU_DISP_HALTED		0x02	/* CPU halted waiting for interrupt */
 
-#endif /* _KERNEL || _KMEMUSER */
-
-#if (defined(_KERNEL) || defined(_KMEMUSER))
+/* Note: inside ifdef: _KERNEL || _KMEMUSER || _BOOT */
+#if defined(_MACHDEP)
 
 /*
  * Macros for manipulating sets of CPUs as a bitmap.  Note that this
@@ -508,6 +507,17 @@ extern	void	cpuset_zero(cpuset_t *);
 
 extern cpuset_t cpu_seqid_inuse;
 
+#endif	/* _MACHDEP */
+#endif /* _KERNEL || _KMEMUSER || _BOOT */
+
+#define	CPU_CPR_OFFLINE		0x0
+#define	CPU_CPR_ONLINE		0x1
+#define	CPU_CPR_IS_OFFLINE(cpu)	(((cpu)->cpu_cpr_flags & CPU_CPR_ONLINE) == 0)
+#define	CPU_CPR_IS_ONLINE(cpu)	((cpu)->cpu_cpr_flags & CPU_CPR_ONLINE)
+#define	CPU_SET_CPR_FLAGS(cpu, flag)	((cpu)->cpu_cpr_flags |= flag)
+
+#if defined(_KERNEL) || defined(_KMEMUSER)
+
 extern struct cpu	*cpu[];		/* indexed by CPU number */
 extern struct cpu	**cpu_seq;	/* indexed by sequential CPU id */
 extern cpu_t		*cpu_list;	/* list of CPUs */
diff --git a/usr/src/uts/common/sys/param.h b/usr/src/uts/common/sys/param.h
index 0d5e0b24b9..66bd91f76f 100644
--- a/usr/src/uts/common/sys/param.h
+++ b/usr/src/uts/common/sys/param.h
@@ -156,7 +156,9 @@ extern "C" {
  * Fundamental constants of the implementation--cannot be changed easily.
  */
 
+#if !defined(_ASM)
 #define	NBPW	sizeof (int)	/* number of bytes in an integer */
+#endif	/* _ASM */
 
 #define	CMASK	022		/* default mask for file creation */
 #define	CDLIMIT	(1L<<11)	/* default max write address */
@@ -329,7 +331,7 @@ extern void drv_usecwait(clock_t);
  * an architecture. Must be included after definition of DEV_BSIZE.
  */
 
-#if (defined(_KERNEL) || defined(_KMEMUSER))
+#if defined(_KERNEL) || defined(_KMEMUSER) || defined(_BOOT)
 
 #if defined(_MACHDEP)
 #include <sys/machparam.h>
@@ -476,7 +478,7 @@ extern const int _clsize;
 }
 #endif
 
-#else	/* (defined(_KERNEL) || defined(_KMEMUSER)) */
+#else	/* defined(_KERNEL) || defined(_KMEMUSER) || defined(_BOOT) */
 
 /*
  * The following are assorted machine dependent values which can be
@@ -508,6 +510,6 @@ extern long _sysconf(int);	/* System Private interface to sysconf() */
 }
 #endif
 
-#endif	/* (defined(_KERNEL) || defined(_KMEMUSER)) &&  defined(_MACHDEP) */
+#endif	/* defined(_KERNEL) || defined(_KMEMUSER) || defined(_BOOT) */
 
 #endif	/* _SYS_PARAM_H */
diff --git a/usr/src/uts/common/sys/stddef.h b/usr/src/uts/common/sys/stddef.h
index a0bd1f83a1..640f85df7f 100644
--- a/usr/src/uts/common/sys/stddef.h
+++ b/usr/src/uts/common/sys/stddef.h
@@ -36,18 +36,17 @@ extern "C" {
 #endif
 #endif /* !offsetof */
 
-#if !defined(container_of)
 
 /*
  * We must not expose container_of() to userland, but we want it
- * to be available for early boot and for the kernel.
+ * to be available for kernel and boot programs.
  */
-#if ((defined(_KERNEL) || defined(_FAKE_KERNEL)) && !defined(_KMEMUSER)) || \
-	(defined(_BOOT) && defined(_KMEMUSER))
+#if defined(_KERNEL) || defined(_BOOT)
+#if !defined(container_of)
 #define	container_of(m, s, name)			\
 	(void *)((uintptr_t)(m) - (uintptr_t)offsetof(s, name))
 #endif
-#endif /* !container_of */
+#endif
 
 #ifdef __cplusplus
 }
diff --git a/usr/src/uts/common/xen/os/hypercall.c b/usr/src/uts/common/xen/os/hypercall.c
index 308c7246db..a906f101ea 100644
--- a/usr/src/uts/common/xen/os/hypercall.c
+++ b/usr/src/uts/common/xen/os/hypercall.c
@@ -37,6 +37,7 @@
  */
 
 #include <sys/types.h>
+#include <sys/errno.h>
 #ifndef __xpv
 #include <sys/xpv_support.h>
 #else
diff --git a/usr/src/uts/i86pc/Makefile.rules b/usr/src/uts/i86pc/Makefile.rules
index a3bf823c69..01e0e0d31e 100644
--- a/usr/src/uts/i86pc/Makefile.rules
+++ b/usr/src/uts/i86pc/Makefile.rules
@@ -222,7 +222,7 @@ DBOOT_OBJS_DIR	= dboot/$(OBJS_DIR)
 DBOOT_MACH_32	= -D_BOOT_TARGET_i386
 DBOOT_MACH_64	= -D_BOOT_TARGET_amd64
 DBOOT_DEFS	= -D_BOOT $(DBOOT_MACH_$(CLASS))
-DBOOT_DEFS	+= -D_MACHDEP -D_KMEMUSER -U_KERNEL -D_I32LPx
+DBOOT_DEFS	+= -D_MACHDEP -U_KERNEL -D_I32LPx
 DBOOT_FLAGS	= $(CCVERBOSE) $(C99MODE) $(CERRWARN) $(CCNOAUTOINLINE)
 
 DBOOT_CC_INCL	= -I$(SRC)/common -I$(SRC)/common/util $(INCLUDE_PATH)