6423041 PSARC 2006/242 'mdi interfaces to support pHCI driver locking' and related fixes

12 files changed, 1633 insertions, 1236 deletions

diff --git a/usr/src/cmd/mdb/common/modules/mpxio/mpxio_dcmds.c b/usr/src/cmd/mdb/common/modules/mpxio/mpxio_dcmds.c
index 12f1f5bbb4..0284cbe6af 100644
--- a/usr/src/cmd/mdb/common/modules/mpxio/mpxio_dcmds.c
+++ b/usr/src/cmd/mdb/common/modules/mpxio/mpxio_dcmds.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -266,29 +265,30 @@ mdivhci(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
 	mdb_printf("----------------- mdi_vhci @ %#lr ----------\n", addr);
 
 	dump_string((uintptr_t)value.vh_class, "Class name (vh_class)");
-	dump_state_str("Load Balance (vh_lb)", value.vh_lb, client_lb_str);
-	mdb_printf("vh_client_count: %19d\n", value.vh_client_count);
-
-	mdb_printf("vh_client_table: %19l#r::print struct client_hash\n",
-	    value.vh_client_table);
+	mdb_printf("vh_refcnt: %19d\n", value.vh_refcnt);
 	mdb_printf("vh_dip: %28l#r::print struct dev_info\n", value.vh_dip);
 	mdb_printf("vh_next: %27l#r::print struct mdi_vhci\n", value.vh_next);
 	mdb_printf("vh_prev: %27l#r::print struct mdi_vhci\n", value.vh_prev);
+	dump_state_str("Load Balance (vh_lb)", value.vh_lb, client_lb_str);
 	mdb_printf("vh_ops: %28l#r::print struct mdi_vhci_ops\n",
 	    value.vh_ops);
+
+	dump_mutex(value.vh_phci_mutex, "phci mutex (vh_phci_mutex):");
+	mdb_printf("vh_phci_count: %21d\n", value.vh_phci_count);
 	mdb_printf("\nvh_phci_head: %22l#r::print struct mdi_phci\n",
 	    value.vh_phci_head);
 	mdb_printf("vh_phci_tail: %22l#r::print struct mdi_phci\n",
 	    value.vh_phci_tail);
 
-	mdb_printf("vh_phci_count: %21d\n", value.vh_phci_count);
+	dump_mutex(value.vh_phci_mutex, "client mutex (vh_client_mutex):");
+	mdb_printf("vh_client_count: %19d\n", value.vh_client_count);
+	mdb_printf("vh_client_table: %19l#r::print struct client_hash\n",
+	    value.vh_client_table);
+
 	mdb_printf("List of pHCIs:\n");
 	mdb_pwalk("mdiphci_list", (mdb_walk_cb_t)mpxio_walk_cb,
 			mdiphci_cb_str, (uintptr_t)value.vh_phci_head);
-
 	mdb_printf("\n");
-	mdb_printf("vh_flags UNUSED: %19d\n", value.vh_flags);
-
 	return (DCMD_OK);
 }
 
diff --git a/usr/src/uts/common/io/ib/ibnex/ibnex.c b/usr/src/uts/common/io/ib/ibnex/ibnex.c
index be0cdf833b..a14ba9f6e3 100644
--- a/usr/src/uts/common/io/ib/ibnex/ibnex.c
+++ b/usr/src/uts/common/io/ib/ibnex/ibnex.c
@@ -1104,8 +1104,21 @@ ibnex_bus_config(dev_info_t *parent, uint_t flag,
 	dev_info_t		*cdip, *pdip = NULL;
 	ibnex_node_data_t	*node_data;
 	ibnex_port_node_t	*port_node;
+	int			use_mdi_devi_locking = 0;
 
-	ndi_devi_enter(parent, &circ);
+
+	/* Set use_mdi_devi_locking appropriately */
+	if ((mdi_component_is_phci(parent, NULL) == MDI_SUCCESS) &&
+	    ((op != BUS_CONFIG_ONE) || (op == BUS_CONFIG_ONE &&
+	    strncmp((char *)devname, IBNEX_IBPORT_CNAME, 6) != 0))) {
+		IBTF_DPRINTF_L4("ibnex", "\tbus_config: using mdi_devi_enter");
+		use_mdi_devi_locking = 1;
+	}
+
+	if (use_mdi_devi_locking)
+		mdi_devi_enter(parent, &circ);
+	else
+		ndi_devi_enter(parent, &circ);
 
 	switch (op) {
 	case BUS_CONFIG_ONE:
@@ -1118,7 +1131,10 @@ ibnex_bus_config(dev_info_t *parent, uint_t flag,
 
 		if (caddr == NULL || (strlen(caddr) == 0)) {
 			kmem_free(device_name, len);
-			ndi_devi_exit(parent, circ);
+			if (use_mdi_devi_locking)
+				mdi_devi_exit(parent, circ);
+			else
+				ndi_devi_exit(parent, circ);
 			return (NDI_FAILURE);
 		}
 
@@ -1137,7 +1153,11 @@ ibnex_bus_config(dev_info_t *parent, uint_t flag,
 		if (cdip == NULL) {
 			/* Node is not present */
 			if (strncmp(cname, IBNEX_IOC_CNAME, 3) == 0) {
-				ndi_devi_exit(parent, circ);
+				if (use_mdi_devi_locking)
+					mdi_devi_exit(parent, circ);
+				else
+					ndi_devi_exit(parent, circ);
+
 				ret = ibnex_ioc_bus_config_one(&parent, flag,
 				    op, devname, child, &need_bus_config);
 				if (!need_bus_config) {
@@ -1145,7 +1165,10 @@ ibnex_bus_config(dev_info_t *parent, uint_t flag,
 					return (ret);
 				}
 
-				ndi_devi_enter(parent, &circ);
+				if (use_mdi_devi_locking)
+					mdi_devi_enter(parent, &circ);
+				else
+					ndi_devi_enter(parent, &circ);
 			} else if ((strncmp(cname,
 			    IBNEX_IBPORT_CNAME, 6) == 0) &&
 			    (parent != ibnex.ibnex_dip)) { /* parent is HCA */
@@ -1168,7 +1191,11 @@ ibnex_bus_config(dev_info_t *parent, uint_t flag,
 				ret = IBNEX_SUCCESS;
 				ibnex_pseudo_initnodes();
 				if (parent == ibnex.ibnex_dip) {
-					ndi_devi_exit(parent, circ);
+					if (use_mdi_devi_locking)
+						mdi_devi_exit(parent, circ);
+					else
+						ndi_devi_exit(parent, circ);
+
 					mutex_enter(&ibnex.ibnex_mutex);
 					ret = ibnex_pseudo_mdi_config_one(
 					    flag, devname, child, cname,
@@ -1226,6 +1253,15 @@ ibnex_bus_config(dev_info_t *parent, uint_t flag,
 			    ", parent %p", parent);
 
 		/*
+		 * No locks to be held while calling mdi_vhci_bus_config()
+		 * ibnex_config_all_children() holds appropriate locks.
+		 */
+		if (use_mdi_devi_locking)
+			mdi_devi_exit(parent, circ);
+		else
+			ndi_devi_exit(parent, circ);
+
+		/*
 		 * Drive CONFIG requests for IB Nexus parent through
 		 * MDI. This is needed to load the HCA drivers in x86 SRP
 		 * boot case.
@@ -1234,12 +1270,16 @@ ibnex_bus_config(dev_info_t *parent, uint_t flag,
 		 * ibdm and configure all children.
 		 */
 		if (parent == ibnex.ibnex_dip) {
-			ndi_devi_exit(parent, circ);
 			ret = mdi_vhci_bus_config(parent,
 			    flag, op, devname, child, NULL);
 			return (ret);
 		} else {
 			ibnex_config_all_children(parent);
+
+			if (use_mdi_devi_locking)
+				mdi_devi_enter(parent, &circ);
+			else
+				ndi_devi_enter(parent, &circ);
 		}
 		break;
 	default:
@@ -1247,7 +1287,12 @@ ibnex_bus_config(dev_info_t *parent, uint_t flag,
 		ret = IBNEX_FAILURE;
 		break;
 	}
-	ndi_devi_exit(parent, circ);
+
+	if (use_mdi_devi_locking)
+		mdi_devi_exit(parent, circ);
+	else
+		ndi_devi_exit(parent, circ);
+
 	if (ret == IBNEX_SUCCESS) {
 		if (op == BUS_CONFIG_OBP_ARGS)
 			op = BUS_CONFIG_ONE;
@@ -1386,13 +1431,17 @@ ibnex_config_all_children(dev_info_t *parent)
 	ibdm_ioc_info_t		*ioc_list, *ioc;
 	ibdm_hca_list_t		*hca_list;
 	ib_guid_t		hca_guid;
+	int			circ;
 
 	IBTF_DPRINTF_L4("ibnex", "\tconfig_all_children: Begin");
 
 	/*
 	 * Enumerate children of this HCA, port nodes,
-	 * VPPA & HCA_SVC nodes
+	 * VPPA & HCA_SVC nodes. Use ndi_devi_enter() for
+	 * locking. IB Nexus is enumerating the children
+	 * of HCA, not MPXIO clients.
 	 */
+	ndi_devi_enter(parent, &circ);
 	hca_guid = ibtl_ibnex_hcadip2guid(parent);
 	wait_time = ibdm_ibnex_get_waittime(
 		hca_guid, &ibnex_port_settling_time);
@@ -1410,6 +1459,13 @@ ibnex_config_all_children(dev_info_t *parent)
 		    parent, &hca_list->hl_port_attr[ii]);
 	}
 	ibdm_ibnex_free_hca_list(hca_list);
+	ndi_devi_exit(parent, circ);
+
+	/*
+	 * Use mdi_devi_enter() for locking. IB Nexus is
+	 * enumerating MPXIO clients.
+	 */
+	mdi_devi_enter(parent, &circ);
 
 	ibnex_pseudo_initnodes();
 
@@ -1438,6 +1494,7 @@ ibnex_config_all_children(dev_info_t *parent)
 
 	mutex_exit(&ibnex.ibnex_mutex);
 	ibdm_ibnex_free_ioc_list(ioc);
+	mdi_devi_exit(parent, circ);
 
 	IBTF_DPRINTF_L4("ibnex", "\tconfig_all_children: End");
 }
@@ -4053,7 +4110,7 @@ ibnex_ioc_bus_config_one(dev_info_t **pdipp, uint_t flag,
 		if (ret == MDI_SUCCESS)
 			*need_bus_config = 0;
 	} else {
-		ndi_devi_enter(pdip, &circ);
+		mdi_devi_enter(pdip, &circ);
 		if (strstr((char *)devname, ":port=") != NULL) {
 			ret = ibnex_config_root_iocnode(pdip, devname);
 			ASSERT(ibnex.ibnex_dip == NULL);
@@ -4061,7 +4118,7 @@ ibnex_ioc_bus_config_one(dev_info_t **pdipp, uint_t flag,
 		} else {
 			ret = ibnex_config_ioc_node(devname, pdip);
 		}
-		ndi_devi_exit(pdip, circ);
+		mdi_devi_exit(pdip, circ);
 	}
 	return (ret);
 }
diff --git a/usr/src/uts/common/io/tphci.c b/usr/src/uts/common/io/tphci.c
index 5c577ebdeb..1aa9c4ebf1 100644
--- a/usr/src/uts/common/io/tphci.c
+++ b/usr/src/uts/common/io/tphci.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -40,7 +39,6 @@
 #include <sys/sunddi.h>
 #include <sys/sunndi.h>
 #include <sys/sunmdi.h>
-#include <sys/mdi_impldefs.h>
 #include <sys/disp.h>
 
 /* cb_ops entry points */
@@ -484,18 +482,18 @@ tphci_bus_config(dev_info_t *parent, uint_t flags,
     ddi_bus_config_op_t op, void *arg, dev_info_t **childp)
 {
 	_NOTE(ARGUNUSED(flags))
-	char *cname, *paddr, *guid, *devnm;
-	mdi_pathinfo_t *pip;
-	int len;
+	char		*cname, *paddr, *guid, *devnm;
+	mdi_pathinfo_t	*pip;
+	int		len, circ, rval;
 
 	switch (op) {
 	case BUS_CONFIG_ONE:
 		break;
 	case BUS_CONFIG_DRIVER:	/* no direct children to configure */
 	case BUS_CONFIG_ALL:
-		return (DDI_SUCCESS);
+		return (NDI_SUCCESS);
 	default:
-		return (DDI_FAILURE);
+		return (NDI_FAILURE);
 	}
 
 	/* only implement BUS_CONFIG_ONE */
@@ -507,75 +505,102 @@ tphci_bus_config(dev_info_t *parent, uint_t flags,
 		cmn_err(CE_NOTE, "tphci_bus_config -- invalid device %s",
 		    (char *)arg);
 		kmem_free(devnm, len);
-		return (DDI_FAILURE);
+		return (NDI_FAILURE);
 	}
 
-	if (mdi_pi_alloc(parent, cname, guid, paddr, 0, &pip)
-	    != MDI_SUCCESS) {
+	mdi_devi_enter(parent, &circ);
+	rval = mdi_pi_alloc(parent, cname, guid, paddr, 0, &pip);
+	kmem_free(devnm, len);
+	if (rval != MDI_SUCCESS) {
 		cmn_err(CE_NOTE, "tphci_bus_config -- mdi_pi_alloc failed");
-		kmem_free(devnm, len);
-		return (DDI_FAILURE);
+		mdi_devi_exit(parent, circ);
+		return (NDI_FAILURE);
 	}
-	kmem_free(devnm, len);
 
-	if (mdi_pi_online(pip, 0) != MDI_SUCCESS) {
+	/*
+	 * Hold the path and exit the pHCI while calling mdi_pi_online
+	 * to avoid deadlock with power management of pHCI.
+	 */
+	mdi_hold_path(pip);
+	mdi_devi_exit_phci(parent, circ);
+	rval = mdi_pi_online(pip, 0);
+	mdi_devi_enter_phci(parent, &circ);
+	mdi_rele_path(pip);
+
+	if (rval != MDI_SUCCESS) {
 		cmn_err(CE_NOTE, "tphci_bus_config -- mdi_pi_online failed");
 		(void) mdi_pi_free(pip, 0);
-		return (DDI_FAILURE);
+		mdi_devi_exit(parent, circ);
+		return (NDI_FAILURE);
 	}
 
 	if (childp) {
-		*childp = MDI_PI(pip)->pi_client->ct_dip;
+		*childp = mdi_pi_get_client(pip);
 		ndi_hold_devi(*childp);
 	}
-	return (DDI_SUCCESS);
+	mdi_devi_exit(parent, circ);
+
+	return (NDI_SUCCESS);
 }
 
 static int
 tphci_bus_unconfig(dev_info_t *parent, uint_t flags,
     ddi_bus_config_op_t op, void *arg)
 {
-	int rval, circ;
-	mdi_pathinfo_t *pip;
-	mdi_phci_t *ph;
-	char *devnm, *cname, *caddr;
+	int		rval = MDI_SUCCESS;
+	int		circ;
+	mdi_pathinfo_t	*pip, *next;
+	char		*devnm, *cname, *caddr;
 
 	switch (op) {
 	case BUS_UNCONFIG_ONE:
 		devnm = (char *)arg;
 		i_ddi_parse_name(devnm, &cname, &caddr, NULL);
 		if (strcmp(cname, "tclient") != 0)
-			return (DDI_SUCCESS);	/* no such device */
+			return (NDI_SUCCESS);	/* no such device */
+
+		mdi_devi_enter(parent, &circ);
 		pip = mdi_pi_find(parent, NULL, caddr);
-		if (pip == NULL)
-			return (DDI_SUCCESS);
-		rval = mdi_pi_offline(pip, NDI_DEVI_REMOVE);
+		if (pip) {
+			mdi_hold_path(pip);
+			mdi_devi_exit_phci(parent, circ);
+			rval = mdi_pi_offline(pip, NDI_DEVI_REMOVE);
+			mdi_devi_enter_phci(parent, &circ);
+			mdi_rele_path(pip);
+
+			if (rval == MDI_SUCCESS)
+				(void) mdi_pi_free(pip, 0);
+		}
+		mdi_devi_exit(parent, circ);
 		return (rval == MDI_SUCCESS ? NDI_SUCCESS : NDI_FAILURE);
 
 	case BUS_UNCONFIG_ALL:
 		if (flags & NDI_AUTODETACH)
-			return (DDI_FAILURE);
+			return (NDI_FAILURE);
 
-		ph = DEVI(parent)->devi_mdi_xhci;
-		ASSERT(ph != NULL);
+		mdi_devi_enter(parent, &circ);
+		next = mdi_get_next_client_path(parent, NULL);
+		while ((pip = next) != NULL) {
+			next = mdi_get_next_client_path(parent, pip);
 
-		rval = MDI_SUCCESS;
-		ndi_devi_enter(parent, &circ);
-		pip = (mdi_pathinfo_t *)ph->ph_path_head;
-		while (pip) {
+			mdi_hold_path(pip);
+			mdi_devi_exit_phci(parent, circ);
 			rval = mdi_pi_offline(pip, NDI_DEVI_REMOVE);
-			if (rval != MDI_SUCCESS) {
+			mdi_devi_enter_phci(parent, &circ);
+			mdi_rele_path(pip);
+
+			if (rval != MDI_SUCCESS)
 				break;
-			}
-			pip = (mdi_pathinfo_t *)ph->ph_path_head;
+			(void) mdi_pi_free(pip, 0);
 		}
-		ndi_devi_exit(parent, circ);
+		mdi_devi_exit(parent, circ);
 		return (rval == MDI_SUCCESS ? NDI_SUCCESS : NDI_FAILURE);
 
 	case BUS_UNCONFIG_DRIVER:	/* nothing to do */
-		return (DDI_SUCCESS);
+		return (NDI_SUCCESS);
+
 	default:
-		return (DDI_FAILURE);
+		return (NDI_FAILURE);
 	}
 	/*NOTREACHED*/
 }
diff --git a/usr/src/uts/common/io/tvhci.c b/usr/src/uts/common/io/tvhci.c
index e5fa627530..8c7034399d 100644
--- a/usr/src/uts/common/io/tvhci.c
+++ b/usr/src/uts/common/io/tvhci.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -542,163 +541,46 @@ tvh_get_phci_devname(char *cname, char *guid,
 	return (pname);
 }
 
-static int
-tvh_enum_by_phci(dev_info_t *vdip, char *devnm, int flags)
+/*
+ * Return a pointer to the guid part of the devnm.
+ * devnm format is "nodename@busaddr", busaddr format is "gGUID".
+ */
+static char *
+tvhci_devnm_to_guid(char *devnm)
 {
-	mdi_phci_t *ph;
-	mdi_vhci_t *vh = (mdi_vhci_t *)DEVI(vdip)->devi_mdi_xhci;
-	dev_info_t *pdip, *cdip;
-	char *cname, *caddr, *guid, *pname;
-	int rval = DDI_FAILURE;
-
-	(void) i_ddi_parse_name(devnm, &cname, &caddr, NULL);
-	if (cname == NULL || caddr == NULL || caddr[0] != 'g')
-		return (rval);
-
-	guid = caddr + 1;
-	pname = kmem_alloc(MAXNAMELEN, KM_SLEEP);
-
-	/* mutex_enter(&mdi_mutex); XXX need lock access */
-	ph = vh->vh_phci_head;
-	while (ph) {
-		pdip = ph->ph_dip;
-		/* mutex_exit(&mdi_mutex); */
-		(void) tvh_get_phci_devname(cname, guid, pdip, pname,
-		    MAXNAMELEN);
-		if (ndi_devi_config_one(pdip, pname, &cdip, flags)
-		    == DDI_SUCCESS) {
-			ndi_rele_devi(cdip);
-			rval = DDI_SUCCESS;
-		}
-		/* mutex_enter(&mdi_mutex); */
-		ph = ph->ph_next;
-	}
-	/* mutex_exit(&mdi_mutex); */
-
-	*(caddr - 1) = '@';	/* undo damage from i_ddi_parse_name() */
-	kmem_free(pname, MAXNAMELEN);
-	return (rval);
-}
+	char *cp = devnm;
 
-static int
-tvh_remove_by_phci(dev_info_t *vdip, char *devnm, int flags)
-{
-	int rval = DDI_SUCCESS;
-	mdi_phci_t *ph;
-	mdi_vhci_t *vh = (mdi_vhci_t *)DEVI(vdip)->devi_mdi_xhci;
-	dev_info_t *pdip;
-	char *cname, *caddr, *guid, *pname;
-
-	(void) i_ddi_parse_name(devnm, &cname, &caddr, NULL);
-	if (cname == NULL || caddr == NULL || caddr[0] != 'g')
-		return (rval);	/* devnm can't exist */
-
-	guid = caddr + 1;
-	pname = kmem_alloc(MAXNAMELEN, KM_SLEEP);
-
-	/* mutex_enter(&mdi_mutex); XXX need lock access */
-	ph = vh->vh_phci_head;
-	while (ph) {
-		pdip = ph->ph_dip;
-		/* mutex_exit(&mdi_mutex); */
-		(void) tvh_get_phci_devname(cname, guid, pdip, pname,
-		    MAXNAMELEN);
-		rval = ndi_devi_unconfig_one(pdip, pname, NULL, flags);
-		/* mutex_enter(&mdi_mutex); */
-		if (rval != NDI_SUCCESS)
-			break;
-		ph = ph->ph_next;
-	}
-	/* mutex_exit(&mdi_mutex); */
+	if (devnm == NULL)
+		return (NULL);
 
-	kmem_free(pname, MAXNAMELEN);
-	return (rval);
+	while (*cp != '\0' && *cp != '@')
+		cp++;
+	if (*cp == '@' && *(cp + 1) == 'g')
+		return (cp + 2);
+	return (NULL);
 }
 
 static int
-tvhci_bus_config(dev_info_t *parent, uint_t flags,
-    ddi_bus_config_op_t op, void *arg, dev_info_t **childp)
+tvhci_bus_config(dev_info_t *pdip, uint_t flags, ddi_bus_config_op_t op,
+    void *arg, dev_info_t **child)
 {
-	char *devnm;
-	dev_info_t *cdip;
-	int circ;
-
-	switch (op) {
-	case BUS_CONFIG_ONE:
-		break;
-	case BUS_CONFIG_ALL:
-		/* XXX call into phci's here? */
-	case BUS_CONFIG_DRIVER:
-		return (ndi_busop_bus_config(parent, flags, op, arg, childp,
-		    0));
-	default:
-		return (DDI_FAILURE);
-	}
-
-	devnm = (char *)arg;
-	ndi_devi_enter(parent, &circ);
-	cdip = ndi_devi_findchild(parent, devnm);
-	ndi_devi_exit(parent, circ);
-	if (cdip == NULL) {
-		/* call into registered phci's */
-		(void) tvh_enum_by_phci(parent, devnm, flags);
-	}
-
-	return (ndi_busop_bus_config(parent, flags, op, arg, childp, 0));
+	char *guid;
+
+	if (op == BUS_CONFIG_ONE || op == BUS_UNCONFIG_ONE)
+		guid = tvhci_devnm_to_guid((char *)arg);
+	else
+		guid = NULL;
+
+	if (mdi_vhci_bus_config(pdip, flags, op, arg, child, guid)
+	    == MDI_SUCCESS)
+		return (NDI_SUCCESS);
+	else
+		return (NDI_FAILURE);
 }
 
 static int
 tvhci_bus_unconfig(dev_info_t *parent, uint_t flags,
     ddi_bus_config_op_t op, void *arg)
 {
-	char *devnm, *slashname;
-	int rval, circ;
-	dev_info_t *cdip, *ndip;
-
-	/*
-	 * If we are not removing device nodes, pathinfo can be
-	 * left as is. So no need to disturb the phci's.
-	 */
-	if ((flags & NDI_DEVI_REMOVE) == 0) {
-		return (ndi_busop_bus_unconfig(parent, flags, op, arg));
-	}
-
-	switch (op) {
-	case BUS_UNCONFIG_ONE:
-		devnm = (char *)arg;
-		ndi_devi_enter(parent, &circ);
-		if (ndi_devi_findchild(parent, devnm) == NULL) {
-			ndi_devi_exit(parent, circ);
-			return (DDI_SUCCESS);
-		}
-		ndi_devi_exit(parent, circ);
-		return (tvh_remove_by_phci(parent, devnm, flags));
-
-	case BUS_UNCONFIG_ALL:
-		/* this functionality is for developers only */
-		rval = DDI_SUCCESS;
-		slashname = kmem_alloc(MAXNAMELEN, KM_SLEEP);
-		devnm = slashname + 1;
-		ndi_devi_enter(parent, &circ);
-		cdip = ddi_get_child(parent);
-		while (cdip != NULL) {
-			ndip = ddi_get_next_sibling(cdip);
-			(void) ddi_deviname(cdip, slashname);
-			ndi_devi_exit(parent, circ);
-			rval = tvh_remove_by_phci(parent, devnm, flags);
-			if (rval != DDI_SUCCESS) {
-				break;
-			}
-			ndi_devi_enter(parent, &circ);
-			cdip = ndip;
-		}
-		ndi_devi_exit(parent, circ);
-		return (rval);
-
-	case BUS_UNCONFIG_DRIVER:
-		/* unconfig driver never comes with NDI_DEVI_REMOVE */
-	default:
-		return (DDI_FAILURE);
-	}
-	/*NOTREACHED*/
+	return (ndi_busop_bus_unconfig(parent, flags, op, arg));
 }
diff --git a/usr/src/uts/common/os/devcfg.c b/usr/src/uts/common/os/devcfg.c
index 997d4ebb71..d6877da9d2 100644
--- a/usr/src/uts/common/os/devcfg.c
+++ b/usr/src/uts/common/os/devcfg.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -175,6 +174,8 @@ static int
 ndi_devi_config_obp_args(dev_info_t *parent, char *devnm,
     dev_info_t **childp, int flags);
 static void i_link_vhci_node(dev_info_t *);
+static void ndi_devi_exit_and_wait(dev_info_t *dip,
+    int circular, clock_t end_time);
 
 /*
  * dev_info cache and node management
@@ -976,6 +977,7 @@ attach_node(dev_info_t *dip)
 {
 	int rv;
 
+	ASSERT(DEVI_BUSY_OWNED(ddi_get_parent(dip)));
 	ASSERT(i_ddi_node_state(dip) == DS_PROBED);
 
 	NDI_CONFIG_DEBUG((CE_CONT, "attach_node: 0x%p(%s%d)\n",
@@ -1011,18 +1013,15 @@ attach_node(dev_info_t *dip)
 		    TASKQ_DEFAULTPRI, 0);
 
 	mutex_enter(&(DEVI(dip)->devi_lock));
-	DEVI_SET_ATTACHING(dip);
 	DEVI_SET_NEED_RESET(dip);
 	mutex_exit(&(DEVI(dip)->devi_lock));
 
 	rv = devi_attach(dip, DDI_ATTACH);
 
-	mutex_enter(&(DEVI(dip)->devi_lock));
-	if (rv != DDI_SUCCESS)
+	if (rv != DDI_SUCCESS) {
+		mutex_enter(&(DEVI(dip)->devi_lock));
 		DEVI_CLR_NEED_RESET(dip);
-	DEVI_CLR_ATTACHING(dip);
 
-	if (rv != DDI_SUCCESS) {
 		/* ensure that devids are unregistered */
 		if (DEVI(dip)->devi_flags & DEVI_REGISTERED_DEVID) {
 			DEVI(dip)->devi_flags &= ~DEVI_REGISTERED_DEVID;
@@ -1049,8 +1048,7 @@ attach_node(dev_info_t *dip)
 		NDI_CONFIG_DEBUG((CE_CONT, "attach_node: 0x%p(%s%d) failed\n",
 		    (void *)dip, ddi_driver_name(dip), ddi_get_instance(dip)));
 		return (DDI_FAILURE);
-	} else
-		mutex_exit(&DEVI(dip)->devi_lock);
+	}
 
 	/* successful attach, return with driver held */
 
@@ -1067,6 +1065,7 @@ detach_node(dev_info_t *dip, uint_t flag)
 	struct devnames	*dnp;
 	int		rv;
 
+	ASSERT(DEVI_BUSY_OWNED(ddi_get_parent(dip)));
 	ASSERT(i_ddi_node_state(dip) == DS_ATTACHED);
 
 	/* check references */
@@ -1076,6 +1075,18 @@ detach_node(dev_info_t *dip, uint_t flag)
 	NDI_CONFIG_DEBUG((CE_CONT, "detach_node: 0x%p(%s%d)\n",
 	    (void *)dip, ddi_driver_name(dip), ddi_get_instance(dip)));
 
+	/*
+	 * NOTE: If we are processing a pHCI node then the calling code
+	 * must detect this and ndi_devi_enter() in (vHCI, parent(pHCI))
+	 * order unless pHCI and vHCI are siblings.  Code paths leading
+	 * here that must ensure this ordering include:
+	 * unconfig_immediate_children(), devi_unconfig_one(),
+	 * ndi_devi_unconfig_one(), ndi_devi_offline().
+	 */
+	ASSERT(!MDI_PHCI(dip) ||
+	    (ddi_get_parent(mdi_devi_get_vdip(dip)) == ddi_get_parent(dip)) ||
+	    DEVI_BUSY_OWNED(mdi_devi_get_vdip(dip)));
+
 	/* Offline the device node with the mpxio framework. */
 	if (mdi_devi_offline(dip, flag) != NDI_SUCCESS) {
 		return (DDI_FAILURE);
@@ -1086,11 +1097,6 @@ detach_node(dev_info_t *dip, uint_t flag)
 		ddi_taskq_wait(DEVI(dip)->devi_taskq);
 
 	rv = devi_detach(dip, DDI_DETACH);
-	if (rv == DDI_SUCCESS) {
-		mutex_enter(&(DEVI(dip)->devi_lock));
-		DEVI_CLR_NEED_RESET(dip);
-		mutex_exit(&(DEVI(dip)->devi_lock));
-	}
 
 	if (rv != DDI_SUCCESS) {
 		NDI_CONFIG_DEBUG((CE_CONT,
@@ -1099,6 +1105,10 @@ detach_node(dev_info_t *dip, uint_t flag)
 		return (DDI_FAILURE);
 	}
 
+	mutex_enter(&(DEVI(dip)->devi_lock));
+	DEVI_CLR_NEED_RESET(dip);
+	mutex_exit(&(DEVI(dip)->devi_lock));
+
 	/* destroy the taskq */
 	if (DEVI(dip)->devi_taskq) {
 		ddi_taskq_destroy(DEVI(dip)->devi_taskq);
@@ -1292,8 +1302,18 @@ i_ndi_config_node(dev_info_t *dip, ddi_node_state_t state, uint_t flag)
 			break;
 		case DS_PROBED:
 			atomic_add_long(&devinfo_attach_detach, 1);
+
+			mutex_enter(&(DEVI(dip)->devi_lock));
+			DEVI_SET_ATTACHING(dip);
+			mutex_exit(&(DEVI(dip)->devi_lock));
+
 			if ((rv = attach_node(dip)) == DDI_SUCCESS)
 				i_ddi_set_node_state(dip, DS_ATTACHED);
+
+			mutex_enter(&(DEVI(dip)->devi_lock));
+			DEVI_CLR_ATTACHING(dip);
+			mutex_exit(&(DEVI(dip)->devi_lock));
+
 			atomic_add_long(&devinfo_attach_detach, -1);
 			break;
 		case DS_ATTACHED:
@@ -1427,14 +1447,14 @@ ddi_uninitchild(dev_info_t *dip)
 static int
 i_ddi_attachchild(dev_info_t *dip)
 {
-	int ret, circ;
-	dev_info_t *parent = ddi_get_parent(dip);
-	ASSERT(parent);
+	dev_info_t	*parent = ddi_get_parent(dip);
+	int		ret;
+
+	ASSERT(parent && DEVI_BUSY_OWNED(parent));
 
 	if ((i_ddi_node_state(dip) < DS_BOUND) || DEVI_IS_DEVICE_OFFLINE(dip))
 		return (DDI_FAILURE);
 
-	ndi_devi_enter(parent, &circ);
 	ret = i_ndi_config_node(dip, DS_READY, 0);
 	if (ret == NDI_SUCCESS) {
 		ret = DDI_SUCCESS;
@@ -1446,7 +1466,6 @@ i_ddi_attachchild(dev_info_t *dip)
 		(void) i_ndi_unconfig_node(dip, DS_INITIALIZED, 0);
 		ret = DDI_FAILURE;
 	}
-	ndi_devi_exit(parent, circ);
 
 	return (ret);
 }
@@ -1461,19 +1480,17 @@ i_ddi_attachchild(dev_info_t *dip)
 static int
 i_ddi_detachchild(dev_info_t *dip, uint_t flags)
 {
-	int ret, circ;
-	dev_info_t *parent = ddi_get_parent(dip);
-	ASSERT(parent);
+	dev_info_t	*parent = ddi_get_parent(dip);
+	int		ret;
+
+	ASSERT(parent && DEVI_BUSY_OWNED(parent));
 
-	ndi_devi_enter(parent, &circ);
 	ret = i_ndi_unconfig_node(dip, DS_PROBED, flags);
 	if (ret != DDI_SUCCESS)
 		(void) i_ndi_config_node(dip, DS_READY, 0);
 	else
 		/* allow pm_pre_probe to reestablish pm state */
 		(void) i_ndi_unconfig_node(dip, DS_INITIALIZED, 0);
-	ndi_devi_exit(parent, circ);
-
 	return (ret);
 }
 
@@ -1597,6 +1614,10 @@ ndi_devi_enter(dev_info_t *dip, int *circular)
 	struct dev_info *devi = DEVI(dip);
 	ASSERT(dip != NULL);
 
+	/* for vHCI, enforce (vHCI, pHCI) ndi_deve_enter() order */
+	ASSERT(!MDI_VHCI(dip) || (mdi_devi_pdip_entered(dip) == 0) ||
+	    DEVI_BUSY_OWNED(dip));
+
 	mutex_enter(&devi->devi_lock);
 	if (devi->devi_busy_thread == curthread) {
 		devi->devi_circular++;
@@ -1620,7 +1641,8 @@ ndi_devi_enter(dev_info_t *dip, int *circular)
 void
 ndi_devi_exit(dev_info_t *dip, int circular)
 {
-	struct dev_info *devi = DEVI(dip);
+	struct dev_info	*devi = DEVI(dip);
+	struct dev_info	*vdevi;
 	ASSERT(dip != NULL);
 
 	if (panicstr)
@@ -1636,6 +1658,53 @@ ndi_devi_exit(dev_info_t *dip, int circular)
 		cv_broadcast(&(devi->devi_cv));
 	}
 	mutex_exit(&(devi->devi_lock));
+
+	/*
+	 * For pHCI exit we issue a broadcast to vHCI for ndi_devi_config_one()
+	 * doing cv_wait on vHCI.
+	 */
+	if (MDI_PHCI(dip)) {
+		vdevi = DEVI(mdi_devi_get_vdip(dip));
+		if (vdevi) {
+			mutex_enter(&(vdevi->devi_lock));
+			if (vdevi->devi_flags & DEVI_PHCI_SIGNALS_VHCI) {
+				vdevi->devi_flags &= ~DEVI_PHCI_SIGNALS_VHCI;
+				cv_broadcast(&(vdevi->devi_cv));
+			}
+			mutex_exit(&(vdevi->devi_lock));
+		}
+	}
+}
+
+/*
+ * Release ndi_devi_enter and wait for possibility of new children, avoiding
+ * possibility of missing broadcast before getting to cv_timedwait().
+ */
+static void
+ndi_devi_exit_and_wait(dev_info_t *dip, int circular, clock_t end_time)
+{
+	struct dev_info	*devi = DEVI(dip);
+	ASSERT(dip != NULL);
+
+	if (panicstr)
+		return;
+
+	/*
+	 * We are called to wait for of a new child, and new child can
+	 * only be added if circular is zero.
+	 */
+	ASSERT(circular == 0);
+
+	/* like ndi_devi_exit with circular of zero */
+	mutex_enter(&(devi->devi_lock));
+	devi->devi_flags &= ~DEVI_BUSY;
+	ASSERT(devi->devi_busy_thread == curthread);
+	devi->devi_busy_thread = NULL;
+	cv_broadcast(&(devi->devi_cv));
+
+	/* now wait for new children while still holding devi_lock */
+	(void) cv_timedwait(&devi->devi_cv, &(devi->devi_lock), end_time);
+	mutex_exit(&(devi->devi_lock));
 }
 
 /*
@@ -4296,6 +4365,10 @@ init_bound_node_ev(dev_info_t *pdip, dev_info_t *dip, int flags)
 static int
 devi_attach_node(dev_info_t *dip, uint_t flags)
 {
+	dev_info_t *pdip = ddi_get_parent(dip);
+
+	ASSERT(pdip && DEVI_BUSY_OWNED(pdip));
+
 	mutex_enter(&(DEVI(dip)->devi_lock));
 	if (flags & NDI_DEVI_ONLINE) {
 		if (!i_ddi_devi_attached(dip))
@@ -4346,43 +4419,13 @@ devi_attach_node(dev_info_t *dip, uint_t flags)
 	return (NDI_SUCCESS);
 }
 
-/*
- * Configure all children of a nexus, assuming all spec children have
- * been made.
- */
-static int
-devi_attach_children(dev_info_t *pdip, uint_t flags, major_t major)
-{
-	dev_info_t *dip;
-
-	ASSERT(DEVI(pdip)->devi_flags & DEVI_MADE_CHILDREN);
-
-	dip = ddi_get_child(pdip);
-	while (dip) {
-		/*
-		 * NOTE: devi_attach_node() may remove the dip
-		 */
-		dev_info_t *next = ddi_get_next_sibling(dip);
-
-		/*
-		 * Configure all nexus nodes or leaf nodes with
-		 * matching driver major
-		 */
-		if ((major == (major_t)-1) ||
-		    (major == ddi_driver_major(dip)) ||
-		    ((flags & NDI_CONFIG) && (is_leaf_node(dip) == 0)))
-			(void) devi_attach_node(dip, flags);
-		dip = next;
-	}
-
-	return (NDI_SUCCESS);
-}
-
 /* internal function to config immediate children */
 static int
 config_immediate_children(dev_info_t *pdip, uint_t flags, major_t major)
 {
-	int circ;
+	dev_info_t	*child, *next;
+	int		circ;
+
 	ASSERT(i_ddi_devi_attached(pdip));
 
 	if (!NEXUS_DRV(ddi_get_driver(pdip)))
@@ -4402,7 +4445,22 @@ config_immediate_children(dev_info_t *pdip, uint_t flags, major_t major)
 	}
 	(void) i_ndi_make_spec_children(pdip, flags);
 	i_ndi_init_hw_children(pdip, flags);
-	(void) devi_attach_children(pdip, flags, major);
+
+	child = ddi_get_child(pdip);
+	while (child) {
+		/* NOTE: devi_attach_node() may remove the dip */
+		next = ddi_get_next_sibling(child);
+
+		/*
+		 * Configure all nexus nodes or leaf nodes with
+		 * matching driver major
+		 */
+		if ((major == (major_t)-1) ||
+		    (major == ddi_driver_major(child)) ||
+		    ((flags & NDI_CONFIG) && (is_leaf_node(child) == 0)))
+			(void) devi_attach_node(child, flags);
+		child = next;
+	}
 
 	ndi_devi_exit(pdip, circ);
 
@@ -4502,115 +4560,164 @@ ndi_devi_config_driver(dev_info_t *dip, int flags, major_t major)
 }
 
 /*
- * called by nexus drivers to configure/unconfigure its children
+ * Called by nexus drivers to configure its children.
  */
 static int
-devi_config_one(dev_info_t *pdip, char *devnm, dev_info_t **dipp,
+devi_config_one(dev_info_t *pdip, char *devnm, dev_info_t **cdipp,
     uint_t flags, clock_t timeout)
 {
-	int circ, probed, rv;
-	dev_info_t *dip = NULL;
-	char *name, *addr, *drivername = NULL;
-	clock_t end_time;	/* 60 sec */
+	dev_info_t	*vdip = NULL;
+	char		*drivername = NULL;
+	char		*name, *addr;
+	int		v_circ, p_circ;
+	clock_t		end_time;	/* 60 sec */
+	int		probed;
+	dev_info_t	*cdip;
+	mdi_pathinfo_t	*cpip;
+
+	*cdipp = NULL;
 
 	if (!NEXUS_DRV(ddi_get_driver(pdip)))
 		return (NDI_FAILURE);
 
-	if (MDI_PHCI(pdip)) {
-		/* Call mdi_ to configure the child */
-		rv = mdi_devi_config_one(pdip, devnm, dipp, flags, timeout);
-		if (rv == MDI_SUCCESS)
-			return (NDI_SUCCESS);
-
-		/*
-		 * Normally, we should return failure here.
-		 *
-		 * Leadville implemented an unfortunate fallback mechanism.
-		 * If a target is non-standard and scsi_vhci doesn't know
-		 * how to do failover, then the node is enumerated under
-		 * phci. Leadville specifies NDI_MDI_FALLBACK flag to
-		 * maintain the old behavior.
-		 */
-		if ((flags & NDI_MDI_FALLBACK) == 0)
-			return (NDI_FAILURE);
-	}
-
 	/* split name into "name@addr" parts */
 	i_ddi_parse_name(devnm, &name, &addr, NULL);
 
-	if (flags & NDI_PROMNAME) {
-		/*
-		 * We may have a genericname on a system that creates
-		 * drivername nodes (from .conf files).  Find the drivername
-		 * by nodeid. If we can't find a node with devnm as the
-		 * node name then we search by drivername.  This allows an
-		 * implementation to supply a genericly named boot path (disk)
-		 * and locate drivename nodes (sd).
-		 */
+	/*
+	 * If the nexus is a pHCI and we are not processing a pHCI from
+	 * mdi bus_config code then we need to know the vHCI.
+	 */
+	if (MDI_PHCI(pdip))
+		vdip = mdi_devi_get_vdip(pdip);
+
+	/*
+	 * We may have a genericname on a system that creates drivername
+	 * nodes (from .conf files).  Find the drivername by nodeid. If we
+	 * can't find a node with devnm as the node name then we search by
+	 * drivername.  This allows an implementation to supply a genericly
+	 * named boot path (disk) and locate drivename nodes (sd).
+	 */
+	if (flags & NDI_PROMNAME)
 		drivername = child_path_to_driver(pdip, name, addr);
-	}
 
-	if (timeout > 0) {
+	/*
+	 * Determine end_time: This routine should *not* be called with a
+	 * constant non-zero timeout argument, the caller should be adjusting
+	 * the timeout argument relative to when it *started* its asynchronous
+	 * enumeration.
+	 */
+	if (timeout > 0)
 		end_time = ddi_get_lbolt() + timeout;
-	}
 
-	ndi_devi_enter(pdip, &circ);
-
-reprobe:
-	probed = (DEVI(pdip)->devi_flags & DEVI_MADE_CHILDREN);
-	(void) i_ndi_make_spec_children(pdip, flags);
 	for (;;) {
-		dip = find_child_by_name(pdip, name, addr);
 		/*
-		 * Search for a node bound to the drivername driver with
-		 * the specified "@addr".
+		 * For pHCI, enter (vHCI, pHCI) and search for pathinfo/client
+		 * child - break out of for(;;) loop if child found.
+		 * NOTE: Lock order for ndi_devi_enter is (vHCI, pHCI).
 		 */
-		if (dip == NULL && drivername)
-			dip = find_child_by_driver(pdip, drivername, addr);
+		if (vdip) {
+			/* use mdi_devi_enter ordering */
+			ndi_devi_enter(vdip, &v_circ);
+			ndi_devi_enter(pdip, &p_circ);
+			cpip = mdi_pi_find(pdip, NULL, addr);
+			cdip = mdi_pi_get_client(cpip);
+			if (cdip)
+				break;
+		} else
+			ndi_devi_enter(pdip, &p_circ);
 
-		if (dip || timeout <= 0 || ddi_get_lbolt() >= end_time)
+		/*
+		 * When not a  vHCI or not all pHCI devices are required to
+		 * enumerated under the vHCI (NDI_MDI_FALLBACK) search for
+		 * devinfo child.
+		 */
+		if ((vdip == NULL) || (flags & NDI_MDI_FALLBACK)) {
+			/* determine if .conf nodes already built */
+			probed = (DEVI(pdip)->devi_flags & DEVI_MADE_CHILDREN);
+
+			/*
+			 * Search for child by name, if not found then search
+			 * for a node bound to the drivername driver with the
+			 * specified "@addr". Break out of for(;;) loop if
+			 * child found.
+			 */
+again:			(void) i_ndi_make_spec_children(pdip, flags);
+			cdip = find_child_by_name(pdip, name, addr);
+			if ((cdip == NULL) && drivername)
+				cdip = find_child_by_driver(pdip,
+				    drivername, addr);
+			if (cdip)
+				break;
+
+			/*
+			 * determine if we should reenumerate .conf nodes
+			 * and look for child again.
+			 */
+			if (probed &&
+			    i_ddi_io_initialized() &&
+			    (flags & NDI_CONFIG_REPROBE) &&
+			    ((timeout <= 0) || (ddi_get_lbolt() >= end_time))) {
+				probed = 0;
+				mutex_enter(&DEVI(pdip)->devi_lock);
+				DEVI(pdip)->devi_flags &= ~DEVI_MADE_CHILDREN;
+				mutex_exit(&DEVI(pdip)->devi_lock);
+				goto again;
+			}
+		}
+
+		/* break out of for(;;) if time expired */
+		if ((timeout <= 0) || (ddi_get_lbolt() >= end_time))
 			break;
 
 		/*
-		 * Wait up to end_time for asynchronous enumeration
+		 * Child not found, exit and wait for asynchronous enumeration
+		 * to add child (or timeout). The addition of a new child (vhci
+		 * or phci) requires the asynchronous enumeration thread to
+		 * ndi_devi_enter/ndi_devi_exit. This exit will signal devi_cv
+		 * and cause us to return from ndi_devi_exit_and_wait, after
+		 * which we loop and search for the requested child again.
 		 */
-		ndi_devi_exit(pdip, circ);
 		NDI_DEBUG(flags, (CE_CONT,
 		    "%s%d: waiting for child %s@%s, timeout %ld",
 		    ddi_driver_name(pdip), ddi_get_instance(pdip),
 		    name, addr, timeout));
+		if (vdip) {
+			/*
+			 * Mark vHCI for pHCI ndi_devi_exit broadcast.
+			 */
+			mutex_enter(&DEVI(vdip)->devi_lock);
+			DEVI(vdip)->devi_flags |=
+			    DEVI_PHCI_SIGNALS_VHCI;
+			mutex_exit(&DEVI(vdip)->devi_lock);
+			ndi_devi_exit(pdip, p_circ);
 
-		mutex_enter(&DEVI(pdip)->devi_lock);
-		(void) cv_timedwait(&DEVI(pdip)->devi_cv,
-		    &DEVI(pdip)->devi_lock, end_time);
-		mutex_exit(&DEVI(pdip)->devi_lock);
-		ndi_devi_enter(pdip, &circ);
-		(void) i_ndi_make_spec_children(pdip, flags);
-	}
-
-	if ((dip == NULL) && probed && (flags & NDI_CONFIG_REPROBE) &&
-	    i_ddi_io_initialized()) {
-		/*
-		 * reenumerate .conf nodes and probe again
-		 */
-		mutex_enter(&DEVI(pdip)->devi_lock);
-		DEVI(pdip)->devi_flags &= ~DEVI_MADE_CHILDREN;
-		mutex_exit(&DEVI(pdip)->devi_lock);
-		goto reprobe;
+			/*
+			 * NB: There is a small race window from above
+			 * ndi_devi_exit() of pdip to cv_wait() in
+			 * ndi_devi_exit_and_wait() which can result in
+			 * not immediately finding a new pHCI child
+			 * of a pHCI that uses NDI_MDI_FAILBACK.
+			 */
+			ndi_devi_exit_and_wait(vdip, v_circ, end_time);
+		} else {
+			ndi_devi_exit_and_wait(pdip, p_circ, end_time);
+		}
 	}
 
-	if (addr[0] != '\0')
+	/* done with paddr, fixup i_ddi_parse_name '@'->'\0' change */
+	if (addr && *addr != '\0')
 		*(addr - 1) = '@';
 
-	if (dip == NULL || devi_attach_node(dip, flags) != NDI_SUCCESS) {
-		ndi_devi_exit(pdip, circ);
-		return (NDI_FAILURE);
+	/* attach and hold the child, returning pointer to child */
+	if (cdip && (devi_attach_node(cdip, flags) == NDI_SUCCESS)) {
+		ndi_hold_devi(cdip);
+		*cdipp = cdip;
 	}
 
-	*dipp = dip;
-	ndi_hold_devi(dip);
-	ndi_devi_exit(pdip, circ);
-	return (NDI_SUCCESS);
+	ndi_devi_exit(pdip, p_circ);
+	if (vdip)
+		ndi_devi_exit(vdip, v_circ);
+	return (*cdipp ? NDI_SUCCESS : NDI_FAILURE);
 }
 
 /*
@@ -4716,8 +4823,10 @@ devi_detach_node(dev_info_t *dip, uint_t flags)
 	int ret = NDI_SUCCESS;
 	ddi_eventcookie_t cookie;
 
+	ASSERT(pdip && DEVI_BUSY_OWNED(pdip));
+
 	if (flags & NDI_POST_EVENT) {
-		if (pdip && i_ddi_devi_attached(pdip)) {
+		if (i_ddi_devi_attached(pdip)) {
 			if (ddi_get_eventcookie(dip, DDI_DEVI_REMOVE_EVENT,
 			    &cookie) == NDI_SUCCESS)
 				(void) ndi_post_event(dip, dip, cookie, NULL);
@@ -4788,15 +4897,51 @@ unconfig_immediate_children(
 	int flags,
 	major_t major)
 {
-	int rv = NDI_SUCCESS, circ;
+	int rv = NDI_SUCCESS;
+	int circ, vcirc;
 	dev_info_t *child;
+	dev_info_t *vdip = NULL;
+	dev_info_t *next;
 
 	ASSERT(dipp == NULL || *dipp == NULL);
 
+	/*
+	 * Scan forward to see if we will be processing a pHCI child. If we
+	 * have a child that is a pHCI and vHCI and pHCI are not siblings then
+	 * enter vHCI before parent(pHCI) to prevent deadlock with mpxio
+	 * Client power management operations.
+	 */
 	ndi_devi_enter(dip, &circ);
+	for (child = ddi_get_child(dip); child;
+	    child = ddi_get_next_sibling(child)) {
+		/* skip same nodes we skip below */
+		if (((major != (major_t)-1) &&
+		    (major != ddi_driver_major(child))) ||
+		    ((flags & NDI_AUTODETACH) && !is_leaf_node(child)))
+			continue;
+
+		if (MDI_PHCI(child)) {
+			vdip = mdi_devi_get_vdip(child);
+			/*
+			 * If vHCI and vHCI is not a sibling of pHCI
+			 * then enter in (vHCI, parent(pHCI)) order.
+			 */
+			if (vdip && (ddi_get_parent(vdip) != dip)) {
+				ndi_devi_exit(dip, circ);
+
+				/* use mdi_devi_enter ordering */
+				ndi_devi_enter(vdip, &vcirc);
+				ndi_devi_enter(dip, &circ);
+				break;
+			} else
+				vdip = NULL;
+		}
+	}
+
 	child = ddi_get_child(dip);
 	while (child) {
-		dev_info_t *next = ddi_get_next_sibling(child);
+		next = ddi_get_next_sibling(child);
+
 		if ((major != (major_t)-1) &&
 		    (major != ddi_driver_major(child))) {
 			child = next;
@@ -4822,7 +4967,11 @@ unconfig_immediate_children(
 		 */
 		child = next;
 	}
+
 	ndi_devi_exit(dip, circ);
+	if (vdip)
+		ndi_devi_exit(vdip, vcirc);
+
 	return (rv);
 }
 
@@ -4977,19 +5126,44 @@ e_ddi_devi_unconfig(dev_info_t *dip, dev_info_t **dipp, int flags)
 static int
 devi_unconfig_one(dev_info_t *pdip, char *devnm, int flags)
 {
-	int rv, circ;
-	dev_info_t *child;
+	int		rv, circ;
+	dev_info_t	*child;
+	dev_info_t	*vdip = NULL;
+	int		v_circ;
 
 	ndi_devi_enter(pdip, &circ);
 	child = ndi_devi_findchild(pdip, devnm);
-	if (child == NULL) {
+
+	/*
+	 * If child is pHCI and vHCI and pHCI are not siblings then enter vHCI
+	 * before parent(pHCI) to avoid deadlock with mpxio Client power
+	 * management operations.
+	 */
+	if (child && MDI_PHCI(child)) {
+		vdip = mdi_devi_get_vdip(child);
+		if (vdip && (ddi_get_parent(vdip) != pdip)) {
+			ndi_devi_exit(pdip, circ);
+
+			/* use mdi_devi_enter ordering */
+			ndi_devi_enter(vdip, &v_circ);
+			ndi_devi_enter(pdip, &circ);
+			child = ndi_devi_findchild(pdip, devnm);
+		} else
+			vdip = NULL;
+	}
+
+	if (child) {
+		rv = devi_detach_node(child, flags);
+	} else {
 		NDI_CONFIG_DEBUG((CE_CONT,
 		    "devi_unconfig_one: %s not found\n", devnm));
-		ndi_devi_exit(pdip, circ);
-		return (NDI_SUCCESS);
+		rv = NDI_SUCCESS;
 	}
-	rv = devi_detach_node(child, flags);
+
 	ndi_devi_exit(pdip, circ);
+	if (vdip)
+		ndi_devi_exit(pdip, v_circ);
+
 	return (rv);
 }
 
@@ -5000,10 +5174,12 @@ ndi_devi_unconfig_one(
 	dev_info_t **dipp,
 	int flags)
 {
-	int (*f)();
-	int circ, rv;
-	int pm_cookie;
-	dev_info_t *child;
+	int		(*f)();
+	int		circ, rv;
+	int		pm_cookie;
+	dev_info_t	*child;
+	dev_info_t	*vdip = NULL;
+	int		v_circ;
 	struct brevq_node *brevq = NULL;
 
 	ASSERT(i_ddi_devi_attached(pdip));
@@ -5021,12 +5197,30 @@ ndi_devi_unconfig_one(
 
 	ndi_devi_enter(pdip, &circ);
 	child = ndi_devi_findchild(pdip, devnm);
+
+	/*
+	 * If child is pHCI and vHCI and pHCI are not siblings then enter vHCI
+	 * before parent(pHCI) to avoid deadlock with mpxio Client power
+	 * management operations.
+	 */
+	if (child && MDI_PHCI(child)) {
+		vdip = mdi_devi_get_vdip(child);
+		if (vdip && (ddi_get_parent(vdip) != pdip)) {
+			ndi_devi_exit(pdip, circ);
+
+			/* use mdi_devi_enter ordering */
+			ndi_devi_enter(vdip, &v_circ);
+			ndi_devi_enter(pdip, &circ);
+			child = ndi_devi_findchild(pdip, devnm);
+		} else
+			vdip = NULL;
+	}
+
 	if (child == NULL) {
 		NDI_CONFIG_DEBUG((CE_CONT, "ndi_devi_unconfig_one: %s"
 		    " not found\n", devnm));
-		ndi_devi_exit(pdip, circ);
-		pm_post_unconfig(pdip, pm_cookie, devnm);
-		return (NDI_SUCCESS);
+		rv = NDI_SUCCESS;
+		goto out;
 	}
 
 	/*
@@ -5062,6 +5256,9 @@ ndi_devi_unconfig_one(
 
 out:
 	ndi_devi_exit(pdip, circ);
+	if (vdip)
+		ndi_devi_exit(pdip, v_circ);
+
 	pm_post_unconfig(pdip, pm_cookie, devnm);
 
 	return (rv);
@@ -5157,7 +5354,7 @@ ndi_devi_online(dev_info_t *dip, uint_t flags)
 	/* merge .conf properties */
 	(void) i_ndi_make_spec_children(pdip, flags);
 
-	flags |= (NDI_DEVI_ONLINE | NDI_CONFIG);
+	flags |= (NDI_DEVI_ONLINE | NDI_CONFIG | NDI_MTC_OFF);
 
 	if (flags & NDI_NO_EVENT) {
 		/*
@@ -5241,14 +5438,30 @@ ndi_devi_online_async(dev_info_t *dip, uint_t flags)
 int
 ndi_devi_offline(dev_info_t *dip, uint_t flags)
 {
-	int circ, rval = 0;
-	dev_info_t *pdip = ddi_get_parent(dip);
+	int		circ, rval = 0;
+	dev_info_t	*pdip = ddi_get_parent(dip);
+	dev_info_t	*vdip = NULL;
+	int		v_circ;
 	struct brevq_node *brevq = NULL;
 
 	ASSERT(pdip);
 
 	flags |= NDI_DEVI_OFFLINE;
+
+	/*
+	 * If child is pHCI and vHCI and pHCI are not siblings then enter vHCI
+	 * before parent(pHCI) to avoid deadlock with mpxio Client power
+	 * management operations.
+	 */
+	if (MDI_PHCI(dip)) {
+		vdip = mdi_devi_get_vdip(dip);
+		if (vdip && (ddi_get_parent(vdip) != pdip))
+			ndi_devi_enter(vdip, &v_circ);
+		else
+			vdip = NULL;
+	}
 	ndi_devi_enter(pdip, &circ);
+
 	if (i_ddi_node_state(dip) == DS_READY) {
 		/*
 		 * If dip is in DS_READY state, there may be cached dv_nodes
@@ -5258,7 +5471,10 @@ ndi_devi_offline(dev_info_t *dip, uint_t flags)
 		 */
 		char *devname = kmem_alloc(MAXNAMELEN + 1, KM_SLEEP);
 		(void) ddi_deviname(dip, devname);
+
 		ndi_devi_exit(pdip, circ);
+		if (vdip)
+			ndi_devi_exit(vdip, v_circ);
 
 		/*
 		 * If we own parent lock, this is part of a branch
@@ -5274,6 +5490,8 @@ ndi_devi_offline(dev_info_t *dip, uint_t flags)
 		if (rval)
 			return (NDI_FAILURE);
 
+		if (vdip)
+			ndi_devi_enter(vdip, &v_circ);
 		ndi_devi_enter(pdip, &circ);
 	}
 
@@ -5288,6 +5506,8 @@ ndi_devi_offline(dev_info_t *dip, uint_t flags)
 	}
 
 	ndi_devi_exit(pdip, circ);
+	if (vdip)
+		ndi_devi_exit(vdip, v_circ);
 
 	return (rval);
 }
@@ -5697,23 +5917,28 @@ ddi_rele_driver(major_t major)
 int
 i_ddi_attach_node_hierarchy(dev_info_t *dip)
 {
-	dev_info_t *parent;
-
-	if (i_ddi_devi_attached(dip))
-		return (DDI_SUCCESS);
+	dev_info_t	*parent;
+	int		ret, circ;
 
 	/*
-	 * Attach parent dip
+	 * Recurse up until attached parent is found.
 	 */
+	if (i_ddi_devi_attached(dip))
+		return (DDI_SUCCESS);
 	parent = ddi_get_parent(dip);
 	if (i_ddi_attach_node_hierarchy(parent) != DDI_SUCCESS)
 		return (DDI_FAILURE);
 
 	/*
-	 * Expand .conf nodes under this parent
+	 * Come top-down, expanding .conf nodes under this parent
+	 * and driving attach.
 	 */
+	ndi_devi_enter(parent, &circ);
 	(void) i_ndi_make_spec_children(parent, 0);
-	return (i_ddi_attachchild(dip));
+	ret = i_ddi_attachchild(dip);
+	ndi_devi_exit(parent, circ);
+
+	return (ret);
 }
 
 /* keep this function static */
@@ -6201,8 +6426,22 @@ mt_config_thread(void *arg)
 	gethrestime(&end_time);
 	hdl->total_time += time_diff_in_msec(start_time, end_time);
 #endif /* DEBUG */
-	if (rv != NDI_SUCCESS)
+
+	if ((rv != NDI_SUCCESS) && (hdl->mtc_error == 0)) {
 		hdl->mtc_error = rv;
+#ifdef	DEBUG
+		if ((ddidebug & DDI_DEBUG) && (major != (major_t)-1)) {
+			char	*path = kmem_alloc(MAXPATHLEN, KM_SLEEP);
+
+			(void) ddi_pathname(dip, path);
+			cmn_err(CE_NOTE, "mt_config_thread: "
+			    "op %d.%d.%x at %s failed %d",
+			    hdl->mtc_op, major, flags, path, rv);
+			kmem_free(path, MAXPATHLEN);
+		}
+#endif	/* DEBUG */
+	}
+
 	if (hdl->mtc_fdip && *hdl->mtc_fdip == NULL) {
 		*hdl->mtc_fdip = rdip;
 		rdip = NULL;
@@ -6327,7 +6566,7 @@ mt_config_children(struct mt_config_handle *hdl)
 	/* go through the list of held children */
 	for (mcd = mcd_head; mcd; mcd = mcd_head) {
 		mcd_head = mcd->mtc_next;
-		if (mtc_off)
+		if (mtc_off || (mcd->mtc_flags & NDI_MTC_OFF))
 			mt_config_thread(mcd);
 		else
 			(void) thread_create(NULL, 0, mt_config_thread, mcd,
@@ -6401,7 +6640,7 @@ mt_config_driver(struct mt_config_handle *hdl)
 	/* go through the list of held children */
 	for (mcd = mcd_head; mcd; mcd = mcd_head) {
 		mcd_head = mcd->mtc_next;
-		if (mtc_off)
+		if (mtc_off || (mcd->mtc_flags & NDI_MTC_OFF))
 			mt_config_thread(mcd);
 		else
 			(void) thread_create(NULL, 0, mt_config_thread, mcd,
diff --git a/usr/src/uts/common/os/sunmdi.c b/usr/src/uts/common/os/sunmdi.c
index df174f049a..746af5223d 100644
--- a/usr/src/uts/common/os/sunmdi.c
+++ b/usr/src/uts/common/os/sunmdi.c
@@ -30,8 +30,10 @@
  *
  * Default locking order:
  *
- * _NOTE(LOCK_ORDER(mdi_mutex, mdi_phci::ph_mutex))
- * _NOTE(LOCK_ORDER(mdi_mutex, mdi_client::ct_mutex))
+ * _NOTE(LOCK_ORDER(mdi_mutex, mdi_vhci:vh_phci_mutex);
+ * _NOTE(LOCK_ORDER(mdi_mutex, mdi_vhci:vh_client_mutex);
+ * _NOTE(LOCK_ORDER(mdi_vhci:vh_phci_mutex, mdi_phci::ph_mutex);
+ * _NOTE(LOCK_ORDER(mdi_vhci:vh_client_mutex, mdi_client::ct_mutex);
  * _NOTE(LOCK_ORDER(mdi_phci::ph_mutex mdi_pathinfo::pi_mutex))
  * _NOTE(LOCK_ORDER(mdi_phci::ph_mutex mdi_client::ct_mutex))
  * _NOTE(LOCK_ORDER(mdi_client::ct_mutex mdi_pathinfo::pi_mutex))
@@ -70,6 +72,7 @@
 #ifdef	DEBUG
 #include <sys/debug.h>
 int	mdi_debug = 1;
+int	mdi_debug_logonly = 0;
 #define	MDI_DEBUG(level, stmnt) \
 	    if (mdi_debug >= (level)) i_mdi_log stmnt
 static void i_mdi_log(int, dev_info_t *, const char *fmt, ...);
@@ -82,7 +85,7 @@ extern int	modrootloaded;
 
 /*
  * Global mutex:
- * Protects vHCI list and structure members, pHCI and Client lists.
+ * Protects vHCI list and structure members.
  */
 kmutex_t	mdi_mutex;
 
@@ -201,7 +204,6 @@ static int 		i_mdi_lba_lb(mdi_client_t *ct,
 static void		i_mdi_pm_hold_client(mdi_client_t *, int);
 static void		i_mdi_pm_rele_client(mdi_client_t *, int);
 static void		i_mdi_pm_reset_client(mdi_client_t *);
-static void		i_mdi_pm_hold_all_phci(mdi_client_t *);
 static int		i_mdi_power_all_phci(mdi_client_t *);
 static void		i_mdi_log_sysevent(dev_info_t *, char *, char *);
 
@@ -216,8 +218,6 @@ static mdi_vhci_t	*i_mdi_vhci_class2vhci(char *);
 static mdi_vhci_t	*i_devi_get_vhci(dev_info_t *);
 static mdi_phci_t	*i_devi_get_phci(dev_info_t *);
 static void		i_mdi_phci_lock(mdi_phci_t *, mdi_pathinfo_t *);
-static void		i_mdi_phci_get_client_lock(mdi_phci_t *,
-			    mdi_client_t *);
 static void		i_mdi_phci_unlock(mdi_phci_t *);
 static mdi_pathinfo_t	*i_mdi_pi_alloc(mdi_phci_t *, char *, mdi_client_t *);
 static void		i_mdi_phci_add_path(mdi_phci_t *, mdi_pathinfo_t *);
@@ -326,7 +326,7 @@ i_mdi_init()
  *		MDI_COMPONENT_PHCI
  *		MDI_COMPONENT_CLIENT
  * XXX This doesn't work under multi-level MPxIO and should be
- *	removed when clients migrate mdi_is_*() interfaces.
+ *	removed when clients migrate mdi_component_is_*() interfaces.
  */
 int
 mdi_get_component_type(dev_info_t *dip)
@@ -349,7 +349,6 @@ mdi_get_component_type(dev_info_t *dip)
  *		MDI_SUCCESS
  *		MDI_FAILURE
  */
-
 /*ARGSUSED*/
 int
 mdi_vhci_register(char *class, dev_info_t *vdip, mdi_vhci_ops_t *vops,
@@ -358,6 +357,7 @@ mdi_vhci_register(char *class, dev_info_t *vdip, mdi_vhci_ops_t *vops,
 	mdi_vhci_t		*vh = NULL;
 
 	ASSERT(vops->vo_revision == MDI_VHCI_OPS_REV);
+	ASSERT(DEVI_BUSY_OWNED(ddi_get_parent(vdip)));
 
 	i_mdi_init();
 
@@ -409,6 +409,9 @@ mdi_vhci_register(char *class, dev_info_t *vdip, mdi_vhci_ops_t *vops,
 			ddi_prop_free(load_balance);
 		}
 
+		mutex_init(&vh->vh_phci_mutex, NULL, MUTEX_DEFAULT, NULL);
+		mutex_init(&vh->vh_client_mutex, NULL, MUTEX_DEFAULT, NULL);
+
 		/*
 		 * Store the vHCI ops vectors
 		 */
@@ -449,24 +452,24 @@ mdi_vhci_register(char *class, dev_info_t *vdip, mdi_vhci_ops_t *vops,
  *		MDI_SUCCESS
  *		MDI_FAILURE
  */
-
 /*ARGSUSED*/
 int
 mdi_vhci_unregister(dev_info_t *vdip, int flags)
 {
 	mdi_vhci_t	*found, *vh, *prev = NULL;
 
+	ASSERT(DEVI_BUSY_OWNED(ddi_get_parent(vdip)));
+
 	/*
 	 * Check for invalid VHCI
 	 */
 	if ((vh = i_devi_get_vhci(vdip)) == NULL)
 		return (MDI_FAILURE);
 
-	mutex_enter(&mdi_mutex);
-
 	/*
 	 * Scan the list of registered vHCIs for a match
 	 */
+	mutex_enter(&mdi_mutex);
 	for (found = mdi_vhci_head; found != NULL; found = found->vh_next) {
 		if (found == vh)
 			break;
@@ -483,7 +486,15 @@ mdi_vhci_unregister(dev_info_t *vdip, int flags)
 	 * should have been unregistered, before a vHCI can be
 	 * unregistered.
 	 */
-	if (vh->vh_phci_count || vh->vh_client_count || vh->vh_refcnt) {
+	MDI_VHCI_PHCI_LOCK(vh);
+	if (vh->vh_refcnt || vh->vh_phci_count || vh->vh_client_count) {
+		MDI_VHCI_PHCI_UNLOCK(vh);
+		mutex_exit(&mdi_mutex);
+		return (MDI_FAILURE);
+	}
+	MDI_VHCI_PHCI_UNLOCK(vh);
+
+	if (destroy_vhci_cache(vh) != MDI_SUCCESS) {
 		mutex_exit(&mdi_mutex);
 		return (MDI_FAILURE);
 	}
@@ -499,29 +510,17 @@ mdi_vhci_unregister(dev_info_t *vdip, int flags)
 	if (vh == mdi_vhci_tail) {
 		mdi_vhci_tail = prev;
 	}
-
 	mdi_vhci_count--;
 	mutex_exit(&mdi_mutex);
 
-	if (destroy_vhci_cache(vh) != MDI_SUCCESS) {
-		/* add vhci to the global list */
-		mutex_enter(&mdi_mutex);
-		if (mdi_vhci_head == NULL)
-			mdi_vhci_head = vh;
-		else
-			mdi_vhci_tail->vh_next = vh;
-		mdi_vhci_tail = vh;
-		mdi_vhci_count++;
-		mutex_exit(&mdi_mutex);
-		return (MDI_FAILURE);
-	}
-
 	vh->vh_ops = NULL;
 	DEVI(vdip)->devi_mdi_component &= ~MDI_COMPONENT_VHCI;
 	DEVI(vdip)->devi_mdi_xhci = NULL;
 	kmem_free(vh->vh_class, strlen(vh->vh_class)+1);
 	kmem_free(vh->vh_client_table,
 	    mdi_client_table_size * sizeof (struct client_hash));
+	mutex_destroy(&vh->vh_phci_mutex);
+	mutex_destroy(&vh->vh_client_mutex);
 
 	kmem_free(vh, sizeof (mdi_vhci_t));
 	return (MDI_SUCCESS);
@@ -579,7 +578,6 @@ i_devi_get_vhci(dev_info_t *vdip)
  *		MDI_SUCCESS
  *		MDI_FAILURE
  */
-
 /*ARGSUSED*/
 int
 mdi_phci_register(char *class, dev_info_t *pdip, int flags)
@@ -589,6 +587,16 @@ mdi_phci_register(char *class, dev_info_t *pdip, int flags)
 	char			*data;
 	char			*pathname;
 
+	/*
+	 * Some subsystems, like fcp, perform pHCI registration from a
+	 * different thread than the one doing the pHCI attach(9E) - the
+	 * driver attach code is waiting for this other thread to complete.
+	 * This means we can only ASSERT DEVI_BUSY_CHANGING of parent
+	 * (indicating that some thread has done an ndi_devi_enter of parent)
+	 * not DEVI_BUSY_OWNED (which would indicate that we did the enter).
+	 */
+	ASSERT(DEVI_BUSY_CHANGING(ddi_get_parent(pdip)));
+
 	pathname = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
 	(void) ddi_pathname(pdip, pathname);
 
@@ -630,15 +638,16 @@ mdi_phci_register(char *class, dev_info_t *pdip, int flags)
 	ph->ph_unstable = 0;
 	ph->ph_vprivate = 0;
 	cv_init(&ph->ph_unstable_cv, NULL, CV_DRIVER, NULL);
-	cv_init(&ph->ph_powerchange_cv, NULL, CV_DRIVER, NULL);
 
+	MDI_PHCI_LOCK(ph);
 	MDI_PHCI_SET_POWER_UP(ph);
+	MDI_PHCI_UNLOCK(ph);
 	DEVI(pdip)->devi_mdi_component |= MDI_COMPONENT_PHCI;
 	DEVI(pdip)->devi_mdi_xhci = (caddr_t)ph;
 
 	vhcache_phci_add(vh->vh_config, ph);
 
-	mutex_enter(&mdi_mutex);
+	MDI_VHCI_PHCI_LOCK(vh);
 	if (vh->vh_phci_head == NULL) {
 		vh->vh_phci_head = ph;
 	}
@@ -647,7 +656,8 @@ mdi_phci_register(char *class, dev_info_t *pdip, int flags)
 	}
 	vh->vh_phci_tail = ph;
 	vh->vh_phci_count++;
-	mutex_exit(&mdi_mutex);
+	MDI_VHCI_PHCI_UNLOCK(vh);
+
 	i_mdi_log_sysevent(pdip, class, ESC_DDI_INITIATOR_REGISTER);
 	return (MDI_SUCCESS);
 }
@@ -662,7 +672,6 @@ mdi_phci_register(char *class, dev_info_t *pdip, int flags)
  *		MDI_SUCCESS
  *		MDI_FAILURE
  */
-
 /*ARGSUSED*/
 int
 mdi_phci_unregister(dev_info_t *pdip, int flags)
@@ -672,6 +681,8 @@ mdi_phci_unregister(dev_info_t *pdip, int flags)
 	mdi_phci_t		*tmp;
 	mdi_phci_t		*prev = NULL;
 
+	ASSERT(DEVI_BUSY_CHANGING(ddi_get_parent(pdip)));
+
 	ph = i_devi_get_phci(pdip);
 	if (ph == NULL) {
 		MDI_DEBUG(1, (CE_WARN, pdip,
@@ -687,7 +698,7 @@ mdi_phci_unregister(dev_info_t *pdip, int flags)
 		return (MDI_FAILURE);
 	}
 
-	mutex_enter(&mdi_mutex);
+	MDI_VHCI_PHCI_LOCK(vh);
 	tmp = vh->vh_phci_head;
 	while (tmp) {
 		if (tmp == ph) {
@@ -708,14 +719,12 @@ mdi_phci_unregister(dev_info_t *pdip, int flags)
 	}
 
 	vh->vh_phci_count--;
-
-	mutex_exit(&mdi_mutex);
+	MDI_VHCI_PHCI_UNLOCK(vh);
 
 	i_mdi_log_sysevent(pdip, ph->ph_vhci->vh_class,
 	    ESC_DDI_INITIATOR_UNREGISTER);
 	vhcache_phci_remove(vh->vh_config, ph);
 	cv_destroy(&ph->ph_unstable_cv);
-	cv_destroy(&ph->ph_powerchange_cv);
 	mutex_destroy(&ph->ph_mutex);
 	kmem_free(ph, sizeof (mdi_phci_t));
 	DEVI(pdip)->devi_mdi_component &= ~MDI_COMPONENT_PHCI;
@@ -738,11 +747,163 @@ i_devi_get_phci(dev_info_t *pdip)
 }
 
 /*
+ * Single thread mdi entry into devinfo node for modifying its children.
+ * If necessary we perform an ndi_devi_enter of the vHCI before doing
+ * an ndi_devi_enter of 'dip'.  We maintain circular in two parts: one
+ * for the vHCI and one for the pHCI.
+ */
+void
+mdi_devi_enter(dev_info_t *phci_dip, int *circular)
+{
+	dev_info_t	*vdip;
+	int		vcircular, pcircular;
+
+	/* Verify calling context */
+	ASSERT(MDI_PHCI(phci_dip));
+	vdip = mdi_devi_get_vdip(phci_dip);
+	ASSERT(vdip);			/* A pHCI always has a vHCI */
+
+	/*
+	 * If pHCI is detaching then the framework has already entered the
+	 * vHCI on a threads that went down the code path leading to
+	 * detach_node().  This framework enter of the vHCI during pHCI
+	 * detach is done to avoid deadlock with vHCI power management
+	 * operations which enter the vHCI and the enter down the path
+	 * to the pHCI. If pHCI is detaching then we piggyback this calls
+	 * enter of the vHCI on frameworks vHCI enter that has already
+	 * occurred - this is OK because we know that the framework thread
+	 * doing detach is waiting for our completion.
+	 *
+	 * We should DEVI_IS_DETACHING under an enter of the parent to avoid
+	 * race with detach - but we can't do that because the framework has
+	 * already entered the parent, so we have some complexity instead.
+	 */
+	for (;;) {
+		if (ndi_devi_tryenter(vdip, &vcircular)) {
+			ASSERT(vcircular != -1);
+			if (DEVI_IS_DETACHING(phci_dip)) {
+				ndi_devi_exit(vdip, vcircular);
+				vcircular = -1;
+			}
+			break;
+		} else if (DEVI_IS_DETACHING(phci_dip)) {
+			vcircular = -1;
+			break;
+		} else {
+			delay(1);
+		}
+	}
+
+	ndi_devi_enter(phci_dip, &pcircular);
+	*circular = (vcircular << 16) | (pcircular & 0xFFFF);
+}
+
+/*
+ * Release mdi_devi_enter or successful mdi_devi_tryenter.
+ */
+void
+mdi_devi_exit(dev_info_t *phci_dip, int circular)
+{
+	dev_info_t	*vdip;
+	int		vcircular, pcircular;
+
+	/* Verify calling context */
+	ASSERT(MDI_PHCI(phci_dip));
+	vdip = mdi_devi_get_vdip(phci_dip);
+	ASSERT(vdip);			/* A pHCI always has a vHCI */
+
+	/* extract two circular recursion values from single int */
+	pcircular = (short)(circular & 0xFFFF);
+	vcircular = (short)((circular >> 16) & 0xFFFF);
+
+	ndi_devi_exit(phci_dip, pcircular);
+	if (vcircular != -1)
+		ndi_devi_exit(vdip, vcircular);
+}
+
+/*
+ * The functions mdi_devi_exit_phci() and mdi_devi_enter_phci() are used
+ * around a pHCI drivers calls to mdi_pi_online/offline, after holding
+ * the pathinfo node via mdi_hold_path/mdi_rele_path, to avoid deadlock
+ * with vHCI power management code during path online/offline.  Each
+ * mdi_devi_exit_phci must have a matching mdi_devi_enter_phci, and both must
+ * occur within the scope of an active mdi_devi_enter that establishes the
+ * circular value.
+ */
+void
+mdi_devi_exit_phci(dev_info_t *phci_dip, int circular)
+{
+	int		pcircular;
+
+	/* Verify calling context */
+	ASSERT(MDI_PHCI(phci_dip));
+
+	pcircular = (short)(circular & 0xFFFF);
+	ndi_devi_exit(phci_dip, pcircular);
+}
+
+void
+mdi_devi_enter_phci(dev_info_t *phci_dip, int *circular)
+{
+	int		pcircular;
+
+	/* Verify calling context */
+	ASSERT(MDI_PHCI(phci_dip));
+
+	ndi_devi_enter(phci_dip, &pcircular);
+
+	/* verify matching mdi_devi_exit_phci/mdi_devi_enter_phci use */
+	ASSERT(pcircular == ((short)(*circular & 0xFFFF)));
+}
+
+/*
+ * mdi_devi_get_vdip():
+ *		given a pHCI dip return vHCI dip
+ */
+dev_info_t *
+mdi_devi_get_vdip(dev_info_t *pdip)
+{
+	mdi_phci_t	*ph;
+
+	ph = i_devi_get_phci(pdip);
+	if (ph && ph->ph_vhci)
+		return (ph->ph_vhci->vh_dip);
+	return (NULL);
+}
+
+/*
+ * mdi_devi_pdip_entered():
+ *		Return 1 if we are vHCI and have done an ndi_devi_enter
+ *		of a pHCI
+ */
+int
+mdi_devi_pdip_entered(dev_info_t *vdip)
+{
+	mdi_vhci_t	*vh;
+	mdi_phci_t	*ph;
+
+	vh = i_devi_get_vhci(vdip);
+	if (vh == NULL)
+		return (0);
+
+	MDI_VHCI_PHCI_LOCK(vh);
+	ph = vh->vh_phci_head;
+	while (ph) {
+		if (ph->ph_dip && DEVI_BUSY_OWNED(ph->ph_dip)) {
+			MDI_VHCI_PHCI_UNLOCK(vh);
+			return (1);
+		}
+		ph = ph->ph_next;
+	}
+	MDI_VHCI_PHCI_UNLOCK(vh);
+	return (0);
+}
+
+/*
  * mdi_phci_path2devinfo():
  * 		Utility function to search for a valid phci device given
  *		the devfs pathname.
  */
-
 dev_info_t *
 mdi_phci_path2devinfo(dev_info_t *vdip, caddr_t pathname)
 {
@@ -762,7 +923,7 @@ mdi_phci_path2devinfo(dev_info_t *vdip, caddr_t pathname)
 	}
 
 	temp_pathname = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
-	mutex_enter(&mdi_mutex);
+	MDI_VHCI_PHCI_LOCK(vh);
 	ph = vh->vh_phci_head;
 	while (ph != NULL) {
 		pdip = ph->ph_dip;
@@ -777,7 +938,7 @@ mdi_phci_path2devinfo(dev_info_t *vdip, caddr_t pathname)
 	if (ph == NULL) {
 		pdip = NULL;
 	}
-	mutex_exit(&mdi_mutex);
+	MDI_VHCI_PHCI_UNLOCK(vh);
 	kmem_free(temp_pathname, MAXPATHLEN);
 	return (pdip);
 }
@@ -834,37 +995,6 @@ i_mdi_phci_lock(mdi_phci_t *ph, mdi_pathinfo_t *pip)
 }
 
 /*
- * i_mdi_phci_get_client_lock():
- *		Lock a pHCI device
- * Return Values:
- *		None
- * Note:
- *		The default locking order is:
- *		_NOTE(LOCK_ORDER(mdi_phci::ph_mutex mdi_client::ct_mutex))
- *		But there are number of situations where locks need to be
- *		grabbed in reverse order.  This routine implements try and lock
- *		mechanism depending on the requested parameter option.
- */
-static void
-i_mdi_phci_get_client_lock(mdi_phci_t *ph, mdi_client_t *ct)
-{
-	if (ct) {
-		/* Reverse locking is requested. */
-		while (MDI_PHCI_TRYLOCK(ph) == 0) {
-			/*
-			 * tryenter failed. Try to grab again
-			 * after a small delay
-			 */
-			MDI_CLIENT_UNLOCK(ct);
-			delay(1);
-			MDI_CLIENT_LOCK(ct);
-		}
-	} else {
-		MDI_PHCI_LOCK(ph);
-	}
-}
-
-/*
  * i_mdi_phci_unlock():
  *		Unlock the pHCI component
  */
@@ -888,7 +1018,7 @@ i_mdi_devinfo_create(mdi_vhci_t *vh, char *name, char *guid,
 {
 	dev_info_t *cdip = NULL;
 
-	ASSERT(MUTEX_HELD(&mdi_mutex));
+	ASSERT(MDI_VHCI_CLIENT_LOCKED(vh));
 
 	/* Verify for duplicate entry */
 	cdip = i_mdi_devinfo_find(vh, name, guid);
@@ -936,7 +1066,6 @@ fail:
  * Return Values:
  *		Handle to a dev_info node or NULL
  */
-
 static dev_info_t *
 i_mdi_devinfo_find(mdi_vhci_t *vh, caddr_t name, char *guid)
 {
@@ -979,12 +1108,13 @@ static int
 i_mdi_devinfo_remove(dev_info_t *vdip, dev_info_t *cdip, int flags)
 {
 	int	rv = MDI_SUCCESS;
+
 	if (i_mdi_is_child_present(vdip, cdip) == MDI_SUCCESS ||
 	    (flags & MDI_CLIENT_FLAGS_DEV_NOT_SUPPORTED)) {
 		rv = ndi_devi_offline(cdip, NDI_DEVI_REMOVE);
 		if (rv != NDI_SUCCESS) {
 			MDI_DEBUG(1, (CE_NOTE, NULL, "!i_mdi_devinfo_remove:"
-			    " failed. cdip = %p\n", cdip));
+			    " failed. cdip = %p\n", (void *)cdip));
 		}
 		/*
 		 * Convert to MDI error code
@@ -1012,6 +1142,7 @@ static mdi_client_t *
 i_devi_get_client(dev_info_t *cdip)
 {
 	mdi_client_t	*ct = NULL;
+
 	if (MDI_CLIENT(cdip)) {
 		ct = (mdi_client_t *)DEVI(cdip)->devi_mdi_client;
 	}
@@ -1022,7 +1153,6 @@ i_devi_get_client(dev_info_t *cdip)
  * i_mdi_is_child_present():
  *		Search for the presence of client device dev_info node
  */
-
 static int
 i_mdi_is_child_present(dev_info_t *vdip, dev_info_t *cdip)
 {
@@ -1056,7 +1186,6 @@ i_mdi_is_child_present(dev_info_t *vdip, dev_info_t *cdip)
  *		grabbed in reverse order.  This routine implements try and lock
  *		mechanism depending on the requested parameter option.
  */
-
 static void
 i_mdi_client_lock(mdi_client_t *ct, mdi_pathinfo_t *pip)
 {
@@ -1084,7 +1213,6 @@ i_mdi_client_lock(mdi_client_t *ct, mdi_pathinfo_t *pip)
  * i_mdi_client_unlock():
  *		Unlock a client component
  */
-
 static void
 i_mdi_client_unlock(mdi_client_t *ct)
 {
@@ -1094,7 +1222,7 @@ i_mdi_client_unlock(mdi_client_t *ct)
 /*
  * i_mdi_client_alloc():
  * 		Allocate and initialize a client structure.  Caller should
- *		hold the global mdi_mutex.
+ *		hold the vhci client lock.
  * Return Values:
  *		Handle to a client component
  */
@@ -1104,7 +1232,7 @@ i_mdi_client_alloc(mdi_vhci_t *vh, char *name, char *lguid)
 {
 	mdi_client_t	*ct;
 
-	ASSERT(MUTEX_HELD(&mdi_mutex));
+	ASSERT(MDI_VHCI_CLIENT_LOCKED(vh));
 
 	/*
 	 * Allocate and initialize a component structure.
@@ -1123,9 +1251,11 @@ i_mdi_client_alloc(mdi_vhci_t *vh, char *name, char *lguid)
 	ct->ct_vprivate = NULL;
 	ct->ct_flags = 0;
 	ct->ct_state = MDI_CLIENT_STATE_FAILED;
+	MDI_CLIENT_LOCK(ct);
 	MDI_CLIENT_SET_OFFLINE(ct);
 	MDI_CLIENT_SET_DETACH(ct);
 	MDI_CLIENT_SET_POWER_UP(ct);
+	MDI_CLIENT_UNLOCK(ct);
 	ct->ct_failover_flags = 0;
 	ct->ct_failover_status = 0;
 	cv_init(&ct->ct_failover_cv, NULL, CV_DRIVER, NULL);
@@ -1150,16 +1280,16 @@ i_mdi_client_alloc(mdi_vhci_t *vh, char *name, char *lguid)
 /*
  * i_mdi_client_enlist_table():
  *		Attach the client device to the client hash table. Caller
- *		should hold the mdi_mutex
+ *		should hold the vhci client lock.
  */
-
 static void
 i_mdi_client_enlist_table(mdi_vhci_t *vh, mdi_client_t *ct)
 {
 	int 			index;
 	struct client_hash	*head;
 
-	ASSERT(MUTEX_HELD(&mdi_mutex));
+	ASSERT(MDI_VHCI_CLIENT_LOCKED(vh));
+
 	index = i_mdi_get_hash_key(ct->ct_guid);
 	head = &vh->vh_client_table[index];
 	ct->ct_hnext = (mdi_client_t *)head->ct_hash_head;
@@ -1171,9 +1301,8 @@ i_mdi_client_enlist_table(mdi_vhci_t *vh, mdi_client_t *ct)
 /*
  * i_mdi_client_delist_table():
  *		Attach the client device to the client hash table.
- *		Caller should hold the mdi_mutex
+ *		Caller should hold the vhci client lock.
  */
-
 static void
 i_mdi_client_delist_table(mdi_vhci_t *vh, mdi_client_t *ct)
 {
@@ -1183,7 +1312,8 @@ i_mdi_client_delist_table(mdi_vhci_t *vh, mdi_client_t *ct)
 	mdi_client_t		*next;
 	mdi_client_t		*last;
 
-	ASSERT(MUTEX_HELD(&mdi_mutex));
+	ASSERT(MDI_VHCI_CLIENT_LOCKED(vh));
+
 	guid = ct->ct_guid;
 	index = i_mdi_get_hash_key(guid);
 	head = &vh->vh_client_table[index];
@@ -1223,7 +1353,8 @@ i_mdi_client_free(mdi_vhci_t *vh, mdi_client_t *ct)
 	dev_info_t	*cdip;
 	dev_info_t	*vdip;
 
-	ASSERT(MUTEX_HELD(&mdi_mutex));
+	ASSERT(MDI_VHCI_CLIENT_LOCKED(vh));
+
 	vdip = vh->vh_dip;
 	cdip = ct->ct_dip;
 
@@ -1254,9 +1385,9 @@ i_mdi_client_free(mdi_vhci_t *vh, mdi_client_t *ct)
 	kmem_free(ct, sizeof (*ct));
 
 	if (cdip != NULL) {
-		mutex_exit(&mdi_mutex);
+		MDI_VHCI_CLIENT_UNLOCK(vh);
 		(void) i_mdi_devinfo_remove(vdip, cdip, flags);
-		mutex_enter(&mdi_mutex);
+		MDI_VHCI_CLIENT_LOCK(vh);
 	}
 	return (rv);
 }
@@ -1264,7 +1395,7 @@ i_mdi_client_free(mdi_vhci_t *vh, mdi_client_t *ct)
 /*
  * i_mdi_client_find():
  * 		Find the client structure corresponding to a given guid
- *		Caller should hold the mdi_mutex
+ *		Caller should hold the vhci client lock.
  */
 static mdi_client_t *
 i_mdi_client_find(mdi_vhci_t *vh, char *cname, char *guid)
@@ -1273,7 +1404,8 @@ i_mdi_client_find(mdi_vhci_t *vh, char *cname, char *guid)
 	struct client_hash	*head;
 	mdi_client_t		*ct;
 
-	ASSERT(MUTEX_HELD(&mdi_mutex));
+	ASSERT(MDI_VHCI_CLIENT_LOCKED(vh));
+
 	index = i_mdi_get_hash_key(guid);
 	head = &vh->vh_client_table[index];
 
@@ -1288,8 +1420,6 @@ i_mdi_client_find(mdi_vhci_t *vh, char *cname, char *guid)
 	return (ct);
 }
 
-
-
 /*
  * i_mdi_client_update_state():
  *		Compute and update client device state
@@ -1308,7 +1438,8 @@ static void
 i_mdi_client_update_state(mdi_client_t *ct)
 {
 	int state;
-	ASSERT(MUTEX_HELD(&ct->ct_mutex));
+
+	ASSERT(MDI_CLIENT_LOCKED(ct));
 	state = i_mdi_client_compute_state(ct, NULL);
 	MDI_CLIENT_SET_STATE(ct, state);
 }
@@ -1330,7 +1461,7 @@ i_mdi_client_compute_state(mdi_client_t *ct, mdi_phci_t *ph)
 	int		standby_count = 0;
 	mdi_pathinfo_t	*pip, *next;
 
-	ASSERT(MUTEX_HELD(&ct->ct_mutex));
+	ASSERT(MDI_CLIENT_LOCKED(ct));
 	pip = ct->ct_path_head;
 	while (pip != NULL) {
 		MDI_PI_LOCK(pip);
@@ -1340,6 +1471,7 @@ i_mdi_client_compute_state(mdi_client_t *ct, mdi_phci_t *ph)
 			pip = next;
 			continue;
 		}
+
 		if ((MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_MASK)
 				== MDI_PATHINFO_STATE_ONLINE)
 			online_count++;
@@ -1354,7 +1486,7 @@ i_mdi_client_compute_state(mdi_client_t *ct, mdi_phci_t *ph)
 		if (standby_count == 0) {
 			state = MDI_CLIENT_STATE_FAILED;
 			MDI_DEBUG(2, (CE_NOTE, NULL, "!client state: failed"
-			    " ct = %p\n", ct));
+			    " ct = %p\n", (void *)ct));
 		} else if (standby_count == 1) {
 			state = MDI_CLIENT_STATE_DEGRADED;
 		} else {
@@ -1426,7 +1558,6 @@ mdi_client_path2devinfo(dev_info_t *vdip, char *pathname)
 	return (cdip);
 }
 
-
 /*
  * mdi_client_get_path_count():
  * 		Utility function to get number of path information nodes
@@ -1653,7 +1784,7 @@ i_mdi_failover(void *arg)
 	mdi_client_t	*ct = (mdi_client_t *)arg;
 	mdi_vhci_t	*vh = ct->ct_vhci;
 
-	ASSERT(!MUTEX_HELD(&ct->ct_mutex));
+	ASSERT(!MDI_CLIENT_LOCKED(ct));
 
 	if (vh->vh_ops->vo_failover != NULL) {
 		/*
@@ -1750,12 +1881,12 @@ i_mdi_lba_lb(mdi_client_t *ct, mdi_pathinfo_t **ret_pip, struct buf *bp)
 		}
 		if (pip == NULL) {
 			MDI_DEBUG(4, (CE_NOTE, NULL,
-			    "!lba %p, no pip !!\n",
-				bp->b_blkno));
+			    "!lba %llx, no pip !!\n",
+				bp->b_lblkno));
 		} else {
 			MDI_DEBUG(4, (CE_NOTE, NULL,
-			    "!lba %p, no pip for path_index, "
-			    "pip %p\n", pip));
+			    "!lba %llx, no pip for path_index, "
+			    "pip %p\n", bp->b_lblkno, (void *)pip));
 		}
 	}
 	return (MDI_FAILURE);
@@ -1798,8 +1929,9 @@ i_mdi_lba_lb(mdi_client_t *ct, mdi_pathinfo_t **ret_pip, struct buf *bp)
  *		when the STANDBY path comes up first), during failover
  *		(to activate a STANDBY path as ONLINE).
  *
- *		The selected path in returned in a held state (ref_cnt).
- *		Caller should release the hold by calling mdi_rele_path().
+ *		The selected path is returned in a a mdi_hold_path() state
+ *		(pi_ref_cnt). Caller should release the hold by calling
+ *		mdi_rele_path().
  *
  * Return Values:
  *		MDI_SUCCESS	- Completed successfully
@@ -1851,7 +1983,7 @@ mdi_select_path(dev_info_t *cdip, struct buf *bp, int flags,
 			 * Fail this request.
 			 */
 			MDI_DEBUG(2, (CE_NOTE, cdip, "!mdi_select_path: "
-			    "client state offline ct = %p\n", ct));
+			    "client state offline ct = %p\n", (void *)ct));
 			MDI_CLIENT_UNLOCK(ct);
 			return (MDI_FAILURE);
 		}
@@ -1862,7 +1994,8 @@ mdi_select_path(dev_info_t *cdip, struct buf *bp, int flags,
 			 * caller that this device is busy.
 			 */
 			MDI_DEBUG(2, (CE_NOTE, cdip, "!mdi_select_path: "
-			    "client failover in progress ct = %p\n", ct));
+			    "client failover in progress ct = %p\n",
+			    (void *)ct));
 			MDI_CLIENT_UNLOCK(ct);
 			return (MDI_BUSY);
 		}
@@ -1873,7 +2006,8 @@ mdi_select_path(dev_info_t *cdip, struct buf *bp, int flags,
 		 * (standby) and let the probe/attach process to continue.
 		 */
 		if (MDI_CLIENT_IS_DETACHED(ct) || !i_ddi_devi_attached(cdip)) {
-			MDI_DEBUG(4, (CE_NOTE, cdip, "!Devi is onlining\n"));
+			MDI_DEBUG(4, (CE_NOTE, cdip, "!Devi is onlining "
+			    "ct = %p\n", (void *)ct));
 			MDI_CLIENT_UNLOCK(ct);
 			return (MDI_DEVI_ONLINING);
 		}
@@ -2243,31 +2377,6 @@ mdi_get_next_client_path(dev_info_t *ph_dip, mdi_pathinfo_t *pip)
 }
 
 /*
- * mdi_get_nextpath():
- *		mdi_pathinfo node walker function.  Get the next node from the
- *		client or pHCI device list.
- *
- * XXX This is wrapper function for compatibility purposes only.
- *
- *	It doesn't work under Multi-level MPxIO, where a dip
- *	is both client and phci (which link should next_path follow?).
- *	Once Leadville is modified to call mdi_get_next_phci/client_path,
- *	this interface should be removed.
- */
-void
-mdi_get_next_path(dev_info_t *dip, mdi_pathinfo_t *pip,
-    mdi_pathinfo_t **ret_pip)
-{
-	if (MDI_CLIENT(dip)) {
-		*ret_pip = mdi_get_next_phci_path(dip, pip);
-	} else if (MDI_PHCI(dip)) {
-		*ret_pip = mdi_get_next_client_path(dip, pip);
-	} else {
-		*ret_pip = NULL;
-	}
-}
-
-/*
  * mdi_hold_path():
  *		Hold the mdi_pathinfo node against unwanted unexpected free.
  * Return Values:
@@ -2305,7 +2414,6 @@ mdi_rele_path(mdi_pathinfo_t *pip)
 	}
 }
 
-
 /*
  * mdi_pi_lock():
  * 		Lock the mdi_pathinfo node.
@@ -2357,6 +2465,8 @@ mdi_pi_find(dev_info_t *pdip, char *caddr, char *paddr)
 	mdi_client_t		*ct;
 	mdi_pathinfo_t		*pip = NULL;
 
+	MDI_DEBUG(2, (CE_NOTE, pdip, "!mdi_pi_find: %s %s",
+	    caddr ? caddr : "NULL", paddr ? paddr : "NULL"));
 	if ((pdip == NULL) || (paddr == NULL)) {
 		return (NULL);
 	}
@@ -2365,7 +2475,7 @@ mdi_pi_find(dev_info_t *pdip, char *caddr, char *paddr)
 		/*
 		 * Invalid pHCI device, Nothing more to do.
 		 */
-		MDI_DEBUG(2, (CE_WARN, NULL,
+		MDI_DEBUG(2, (CE_WARN, pdip,
 		    "!mdi_pi_find: invalid phci"));
 		return (NULL);
 	}
@@ -2375,20 +2485,26 @@ mdi_pi_find(dev_info_t *pdip, char *caddr, char *paddr)
 		/*
 		 * Invalid vHCI device, Nothing more to do.
 		 */
-		MDI_DEBUG(2, (CE_WARN, NULL,
-		    "!mdi_pi_find: invalid phci"));
+		MDI_DEBUG(2, (CE_WARN, pdip,
+		    "!mdi_pi_find: invalid vhci"));
 		return (NULL);
 	}
 
 	/*
-	 * Look for client device identified by caddr (guid)
+	 * Look for pathinfo node identified by paddr.
 	 */
 	if (caddr == NULL) {
 		/*
 		 * Find a mdi_pathinfo node under pHCI list for a matching
 		 * unit address.
 		 */
-		mutex_enter(&ph->ph_mutex);
+		MDI_PHCI_LOCK(ph);
+		if (MDI_PHCI_IS_OFFLINE(ph)) {
+			MDI_DEBUG(2, (CE_WARN, pdip,
+			    "!mdi_pi_find: offline phci %p", (void *)ph));
+			MDI_PHCI_UNLOCK(ph);
+			return (NULL);
+		}
 		pip = (mdi_pathinfo_t *)ph->ph_path_head;
 
 		while (pip != NULL) {
@@ -2397,7 +2513,9 @@ mdi_pi_find(dev_info_t *pdip, char *caddr, char *paddr)
 			}
 			pip = (mdi_pathinfo_t *)MDI_PI(pip)->pi_phci_link;
 		}
-		mutex_exit(&ph->ph_mutex);
+		MDI_PHCI_UNLOCK(ph);
+		MDI_DEBUG(2, (CE_NOTE, pdip, "!mdi_pi_find: found %p",
+		    (void *)pip));
 		return (pip);
 	}
 
@@ -2412,7 +2530,7 @@ mdi_pi_find(dev_info_t *pdip, char *caddr, char *paddr)
 	/*
 	 * Find the client device corresponding to 'caddr'
 	 */
-	mutex_enter(&mdi_mutex);
+	MDI_VHCI_CLIENT_LOCK(vh);
 
 	/*
 	 * XXX - Passing NULL to the following function works as long as the
@@ -2424,8 +2542,10 @@ mdi_pi_find(dev_info_t *pdip, char *caddr, char *paddr)
 		 * Client not found, Obviously mdi_pathinfo node has not been
 		 * created yet.
 		 */
-		mutex_exit(&mdi_mutex);
-		return (pip);
+		MDI_VHCI_CLIENT_UNLOCK(vh);
+		MDI_DEBUG(2, (CE_NOTE, pdip, "!mdi_pi_find: client not "
+		    "found for caddr %s", caddr ? caddr : "NULL"));
+		return (NULL);
 	}
 
 	/*
@@ -2438,7 +2558,7 @@ mdi_pi_find(dev_info_t *pdip, char *caddr, char *paddr)
 	 * Release the global mutex as it is no more needed. Note: We always
 	 * respect the locking order while acquiring.
 	 */
-	mutex_exit(&mdi_mutex);
+	MDI_VHCI_CLIENT_UNLOCK(vh);
 
 	pip = (mdi_pathinfo_t *)ct->ct_path_head;
 	while (pip != NULL) {
@@ -2452,6 +2572,7 @@ mdi_pi_find(dev_info_t *pdip, char *caddr, char *paddr)
 		pip = (mdi_pathinfo_t *)MDI_PI(pip)->pi_client_link;
 	}
 	MDI_CLIENT_UNLOCK(ct);
+	MDI_DEBUG(2, (CE_NOTE, pdip, "!mdi_pi_find: found:: %p", (void *)pip));
 	return (pip);
 }
 
@@ -2488,6 +2609,10 @@ mdi_pi_alloc_compatible(dev_info_t *pdip, char *cname, char *caddr, char *paddr,
 	int		rv = MDI_NOMEM;
 	int		path_allocated = 0;
 
+	MDI_DEBUG(2, (CE_NOTE, pdip, "!mdi_pi_alloc_compatible: %s %s %s",
+	    cname ? cname : "NULL", caddr ? caddr : "NULL",
+	    paddr ? paddr : "NULL"));
+
 	if (pdip == NULL || cname == NULL || caddr == NULL || paddr == NULL ||
 	    ret_pip == NULL) {
 		/* Nothing more to do */
@@ -2495,12 +2620,21 @@ mdi_pi_alloc_compatible(dev_info_t *pdip, char *cname, char *caddr, char *paddr,
 	}
 
 	*ret_pip = NULL;
+
+	/* No allocations on detaching pHCI */
+	if (DEVI_IS_DETACHING(pdip)) {
+		/* Invalid pHCI device, return failure */
+		MDI_DEBUG(1, (CE_WARN, pdip,
+		    "!mdi_pi_alloc: detaching pHCI=%p", (void *)pdip));
+		return (MDI_FAILURE);
+	}
+
 	ph = i_devi_get_phci(pdip);
 	ASSERT(ph != NULL);
 	if (ph == NULL) {
 		/* Invalid pHCI device, return failure */
-		MDI_DEBUG(1, (CE_WARN, NULL,
-		    "!mdi_pi_alloc: invalid pHCI=%p", pdip));
+		MDI_DEBUG(1, (CE_WARN, pdip,
+		    "!mdi_pi_alloc: invalid pHCI=%p", (void *)pdip));
 		return (MDI_FAILURE);
 	}
 
@@ -2508,8 +2642,8 @@ mdi_pi_alloc_compatible(dev_info_t *pdip, char *cname, char *caddr, char *paddr,
 	vh = ph->ph_vhci;
 	if (vh == NULL) {
 		/* Invalid vHCI device, return failure */
-		MDI_DEBUG(1, (CE_WARN, NULL,
-		    "!mdi_pi_alloc: invalid pHCI=%p", pdip));
+		MDI_DEBUG(1, (CE_WARN, pdip,
+		    "!mdi_pi_alloc: invalid vHCI=%p", (void *)pdip));
 		MDI_PHCI_UNLOCK(ph);
 		return (MDI_FAILURE);
 	}
@@ -2519,8 +2653,8 @@ mdi_pi_alloc_compatible(dev_info_t *pdip, char *cname, char *caddr, char *paddr,
 		 * Do not allow new node creation when pHCI is in
 		 * offline/suspended states
 		 */
-		MDI_DEBUG(1, (CE_WARN, NULL,
-		    "mdi_pi_alloc: pHCI=%p is not ready", ph));
+		MDI_DEBUG(1, (CE_WARN, pdip,
+		    "mdi_pi_alloc: pHCI=%p is not ready", (void *)ph));
 		MDI_PHCI_UNLOCK(ph);
 		return (MDI_BUSY);
 	}
@@ -2528,7 +2662,7 @@ mdi_pi_alloc_compatible(dev_info_t *pdip, char *cname, char *caddr, char *paddr,
 	MDI_PHCI_UNLOCK(ph);
 
 	/* look for a matching client, create one if not found */
-	mutex_enter(&mdi_mutex);
+	MDI_VHCI_CLIENT_LOCK(vh);
 	ct = i_mdi_client_find(vh, cname, caddr);
 	if (ct == NULL) {
 		ct = i_mdi_client_alloc(vh, cname, caddr);
@@ -2551,6 +2685,7 @@ mdi_pi_alloc_compatible(dev_info_t *pdip, char *cname, char *caddr, char *paddr,
 	DEVI(cdip)->devi_mdi_component |= MDI_COMPONENT_CLIENT;
 	DEVI(cdip)->devi_mdi_client = (caddr_t)ct;
 
+	MDI_CLIENT_LOCK(ct);
 	pip = (mdi_pathinfo_t *)ct->ct_path_head;
 	while (pip != NULL) {
 		/*
@@ -2562,6 +2697,7 @@ mdi_pi_alloc_compatible(dev_info_t *pdip, char *cname, char *caddr, char *paddr,
 		}
 		pip = (mdi_pathinfo_t *)MDI_PI(pip)->pi_client_link;
 	}
+	MDI_CLIENT_UNLOCK(ct);
 
 	if (pip == NULL) {
 		/*
@@ -2578,7 +2714,7 @@ fail:
 	/*
 	 * Release the global mutex.
 	 */
-	mutex_exit(&mdi_mutex);
+	MDI_VHCI_CLIENT_UNLOCK(vh);
 
 	/*
 	 * Mark the pHCI as stable
@@ -2588,6 +2724,9 @@ fail:
 	MDI_PHCI_UNLOCK(ph);
 	*ret_pip = pip;
 
+	MDI_DEBUG(2, (CE_NOTE, pdip,
+	    "!mdi_pi_alloc_compatible: alloc %p", (void *)pip));
+
 	if (path_allocated)
 		vhcache_pi_add(vh->vh_config, MDI_PI(pip));
 
@@ -2609,7 +2748,6 @@ mdi_pi_alloc(dev_info_t *pdip, char *cname, char *caddr, char *paddr,
  * Return Values:
  *		mdi_pathinfo
  */
-
 /*ARGSUSED*/
 static mdi_pathinfo_t *
 i_mdi_pi_alloc(mdi_phci_t *ph, char *paddr, mdi_client_t *ct)
@@ -2620,6 +2758,8 @@ i_mdi_pi_alloc(mdi_phci_t *ph, char *paddr, mdi_client_t *ct)
 	int		se_flag;
 	int		kmem_flag;
 
+	ASSERT(MDI_VHCI_CLIENT_LOCKED(ph->ph_vhci));
+
 	pip = kmem_zalloc(sizeof (struct mdi_pathinfo), KM_SLEEP);
 	mutex_init(&MDI_PI(pip)->pi_mutex, NULL, MUTEX_DEFAULT, NULL);
 	MDI_PI(pip)->pi_state = MDI_PATHINFO_STATE_INIT |
@@ -2654,6 +2794,11 @@ i_mdi_pi_alloc(mdi_phci_t *ph, char *paddr, mdi_client_t *ct)
 
 	/*
 	 * Lock both dev_info nodes against changes in parallel.
+	 *
+	 * The ndi_devi_enter(Client), is atypical since the client is a leaf.
+	 * This atypical operation is done to synchronize pathinfo nodes
+	 * during devinfo snapshot (see di_register_pip) by 'pretending' that
+	 * the pathinfo nodes are children of the Client.
 	 */
 	ndi_devi_enter(ct->ct_dip, &ct_circular);
 	ndi_devi_enter(ph->ph_dip, &ph_circular);
@@ -2679,12 +2824,12 @@ i_mdi_pi_alloc(mdi_phci_t *ph, char *paddr, mdi_client_t *ct)
  * Notes:
  *		Caller should per-pHCI mutex
  */
-
 static void
 i_mdi_phci_add_path(mdi_phci_t *ph, mdi_pathinfo_t *pip)
 {
 	ASSERT(DEVI_BUSY_OWNED(ph->ph_dip));
 
+	MDI_PHCI_LOCK(ph);
 	if (ph->ph_path_head == NULL) {
 		ph->ph_path_head = pip;
 	} else {
@@ -2692,18 +2837,19 @@ i_mdi_phci_add_path(mdi_phci_t *ph, mdi_pathinfo_t *pip)
 	}
 	ph->ph_path_tail = pip;
 	ph->ph_path_count++;
+	MDI_PHCI_UNLOCK(ph);
 }
 
 /*
  * i_mdi_client_add_path():
  *		Add mdi_pathinfo node to client list
  */
-
 static void
 i_mdi_client_add_path(mdi_client_t *ct, mdi_pathinfo_t *pip)
 {
 	ASSERT(DEVI_BUSY_OWNED(ct->ct_dip));
 
+	MDI_CLIENT_LOCK(ct);
 	if (ct->ct_path_head == NULL) {
 		ct->ct_path_head = pip;
 	} else {
@@ -2711,6 +2857,7 @@ i_mdi_client_add_path(mdi_client_t *ct, mdi_pathinfo_t *pip)
 	}
 	ct->ct_path_tail = pip;
 	ct->ct_path_count++;
+	MDI_CLIENT_UNLOCK(ct);
 }
 
 /*
@@ -2722,7 +2869,6 @@ i_mdi_client_add_path(mdi_client_t *ct, mdi_pathinfo_t *pip)
  *		MDI_FAILURE
  *		MDI_BUSY
  */
-
 /*ARGSUSED*/
 int
 mdi_pi_free(mdi_pathinfo_t *pip, int flags)
@@ -2742,7 +2888,7 @@ mdi_pi_free(mdi_pathinfo_t *pip, int flags)
 		 * Invalid pHCI device, return failure
 		 */
 		MDI_DEBUG(1, (CE_WARN, NULL,
-		    "!mdi_pi_free: invalid pHCI"));
+		    "!mdi_pi_free: invalid pHCI pip=%p", (void *)pip));
 		MDI_PI_UNLOCK(pip);
 		return (MDI_FAILURE);
 	}
@@ -2752,7 +2898,7 @@ mdi_pi_free(mdi_pathinfo_t *pip, int flags)
 	if (vh == NULL) {
 		/* Invalid pHCI device, return failure */
 		MDI_DEBUG(1, (CE_WARN, NULL,
-		    "!mdi_pi_free: invalid vHCI"));
+		    "!mdi_pi_free: invalid vHCI pip=%p", (void *)pip));
 		MDI_PI_UNLOCK(pip);
 		return (MDI_FAILURE);
 	}
@@ -2764,7 +2910,7 @@ mdi_pi_free(mdi_pathinfo_t *pip, int flags)
 		 * Invalid Client device, return failure
 		 */
 		MDI_DEBUG(1, (CE_WARN, NULL,
-		    "!mdi_pi_free: invalid client"));
+		    "!mdi_pi_free: invalid client pip=%p", (void *)pip));
 		MDI_PI_UNLOCK(pip);
 		return (MDI_FAILURE);
 	}
@@ -2779,8 +2925,8 @@ mdi_pi_free(mdi_pathinfo_t *pip, int flags)
 		/*
 		 * Node is busy
 		 */
-		MDI_DEBUG(1, (CE_WARN, NULL,
-		    "!mdi_pi_free: pathinfo node is busy pip=%p", pip));
+		MDI_DEBUG(1, (CE_WARN, ct->ct_dip,
+		    "!mdi_pi_free: pathinfo node is busy pip=%p", (void *)pip));
 		MDI_PI_UNLOCK(pip);
 		return (MDI_BUSY);
 	}
@@ -2789,9 +2935,9 @@ mdi_pi_free(mdi_pathinfo_t *pip, int flags)
 		/*
 		 * Give a chance for pending I/Os to complete.
 		 */
-		MDI_DEBUG(1, (CE_NOTE, ct->ct_vhci->vh_dip, "!mdi_pi_free: "
+		MDI_DEBUG(1, (CE_NOTE, ct->ct_dip, "!mdi_pi_free: "
 		    "%d cmds still pending on path: %p\n",
-		    MDI_PI(pip)->pi_ref_cnt, pip));
+		    MDI_PI(pip)->pi_ref_cnt, (void *)pip));
 		if (cv_timedwait(&MDI_PI(pip)->pi_ref_cv,
 		    &MDI_PI(pip)->pi_mutex,
 		    ddi_get_lbolt() + drv_usectohz(60 * 1000000)) == -1) {
@@ -2799,14 +2945,14 @@ mdi_pi_free(mdi_pathinfo_t *pip, int flags)
 			 * The timeout time reached without ref_cnt being zero
 			 * being signaled.
 			 */
-			MDI_DEBUG(1, (CE_NOTE, ct->ct_vhci->vh_dip,
+			MDI_DEBUG(1, (CE_NOTE, ct->ct_dip,
 			    "!mdi_pi_free: "
 			    "Timeout reached on path %p without the cond\n",
-			    pip));
-			MDI_DEBUG(1, (CE_NOTE, ct->ct_vhci->vh_dip,
+			    (void *)pip));
+			MDI_DEBUG(1, (CE_NOTE, ct->ct_dip,
 			    "!mdi_pi_free: "
 			    "%d cmds still pending on path: %p\n",
-			    MDI_PI(pip)->pi_ref_cnt, pip));
+			    MDI_PI(pip)->pi_ref_cnt, (void *)pip));
 			MDI_PI_UNLOCK(pip);
 			return (MDI_BUSY);
 		}
@@ -2820,7 +2966,7 @@ mdi_pi_free(mdi_pathinfo_t *pip, int flags)
 
 	MDI_CLIENT_LOCK(ct);
 
-	/* Prevent further failovers till mdi_mutex is held */
+	/* Prevent further failovers till MDI_VHCI_CLIENT_LOCK is held */
 	MDI_CLIENT_SET_PATH_FREE_IN_PROGRESS(ct);
 
 	/*
@@ -2830,7 +2976,7 @@ mdi_pi_free(mdi_pathinfo_t *pip, int flags)
 		cv_wait(&ct->ct_failover_cv, &ct->ct_mutex);
 
 	MDI_CLIENT_UNLOCK(ct);
-	mutex_enter(&mdi_mutex);
+	MDI_VHCI_CLIENT_LOCK(vh);
 	MDI_CLIENT_LOCK(ct);
 	MDI_CLIENT_CLEAR_PATH_FREE_IN_PROGRESS(ct);
 
@@ -2857,12 +3003,12 @@ mdi_pi_free(mdi_pathinfo_t *pip, int flags)
 			 */
 			MDI_CLIENT_UNLOCK(ct);
 			(void) i_mdi_client_free(ct->ct_vhci, ct);
-			mutex_exit(&mdi_mutex);
+			MDI_VHCI_CLIENT_UNLOCK(vh);
 			return (rv);
 		}
 	}
 	MDI_CLIENT_UNLOCK(ct);
-	mutex_exit(&mdi_mutex);
+	MDI_VHCI_CLIENT_UNLOCK(vh);
 
 	if (rv == MDI_FAILURE)
 		vhcache_pi_add(vh->vh_config, MDI_PI(pip));
@@ -2882,11 +3028,14 @@ i_mdi_pi_free(mdi_phci_t *ph, mdi_pathinfo_t *pip, mdi_client_t *ct)
 	int	se_flag;
 	int	kmem_flag;
 
+	ASSERT(MDI_CLIENT_LOCKED(ct));
+
 	/*
 	 * remove any per-path kstats
 	 */
 	i_mdi_pi_kstat_destroy(pip);
 
+	/* See comments in i_mdi_pi_alloc() */
 	ndi_devi_enter(ct->ct_dip, &ct_circular);
 	ndi_devi_enter(ph->ph_dip, &ph_circular);
 
@@ -2925,7 +3074,6 @@ i_mdi_pi_free(mdi_phci_t *ph, mdi_pathinfo_t *pip, mdi_client_t *ct)
  * Notes:
  *		Caller should hold per-pHCI mutex
  */
-
 static void
 i_mdi_phci_remove_path(mdi_phci_t *ph, mdi_pathinfo_t *pip)
 {
@@ -2934,6 +3082,7 @@ i_mdi_phci_remove_path(mdi_phci_t *ph, mdi_pathinfo_t *pip)
 
 	ASSERT(DEVI_BUSY_OWNED(ph->ph_dip));
 
+	MDI_PHCI_LOCK(ph);
 	path = ph->ph_path_head;
 	while (path != NULL) {
 		if (path == pip) {
@@ -2961,13 +3110,13 @@ i_mdi_phci_remove_path(mdi_phci_t *ph, mdi_pathinfo_t *pip)
 	 */
 	MDI_PI(pip)->pi_phci_link = NULL;
 	MDI_PI(pip)->pi_phci = NULL;
+	MDI_PHCI_UNLOCK(ph);
 }
 
 /*
  * i_mdi_client_remove_path():
  * 		Remove a mdi_pathinfo node from client path list.
  */
-
 static void
 i_mdi_client_remove_path(mdi_client_t *ct, mdi_pathinfo_t *pip)
 {
@@ -2976,6 +3125,7 @@ i_mdi_client_remove_path(mdi_client_t *ct, mdi_pathinfo_t *pip)
 
 	ASSERT(DEVI_BUSY_OWNED(ct->ct_dip));
 
+	ASSERT(MDI_CLIENT_LOCKED(ct));
 	path = ct->ct_path_head;
 	while (path != NULL) {
 		if (path == pip) {
@@ -3034,7 +3184,7 @@ i_mdi_pi_state_change(mdi_pathinfo_t *pip, mdi_pathinfo_state_t state, int flag)
 		 */
 		MDI_PI_UNLOCK(pip);
 		MDI_DEBUG(1, (CE_WARN, NULL,
-		    "!mdi_pi_state_change: invalid phci"));
+		    "!mdi_pi_state_change: invalid phci pip=%p", (void *)pip));
 		return (MDI_FAILURE);
 	}
 
@@ -3046,7 +3196,7 @@ i_mdi_pi_state_change(mdi_pathinfo_t *pip, mdi_pathinfo_state_t state, int flag)
 		 */
 		MDI_PI_UNLOCK(pip);
 		MDI_DEBUG(1, (CE_WARN, NULL,
-		    "!mdi_pi_state_change: invalid vhci"));
+		    "!mdi_pi_state_change: invalid vhci pip=%p", (void *)pip));
 		return (MDI_FAILURE);
 	}
 
@@ -3058,7 +3208,8 @@ i_mdi_pi_state_change(mdi_pathinfo_t *pip, mdi_pathinfo_state_t state, int flag)
 		 */
 		MDI_PI_UNLOCK(pip);
 		MDI_DEBUG(1, (CE_WARN, NULL,
-		    "!mdi_pi_state_change: invalid client"));
+		    "!mdi_pi_state_change: invalid client pip=%p",
+		    (void *)pip));
 		return (MDI_FAILURE);
 	}
 
@@ -3073,9 +3224,9 @@ i_mdi_pi_state_change(mdi_pathinfo_t *pip, mdi_pathinfo_state_t state, int flag)
 		if (f != NULL) {
 			rv = (*f)(vh->vh_dip, pip, 0);
 			if (rv != MDI_SUCCESS) {
-				MDI_DEBUG(1, (CE_WARN, vh->vh_dip,
+				MDI_DEBUG(1, (CE_WARN, ct->ct_dip,
 				    "!vo_pi_init: failed vHCI=0x%p, pip=0x%p",
-				    vh, pip));
+				    (void *)vh, (void *)pip));
 				return (MDI_FAILURE);
 			}
 		}
@@ -3089,8 +3240,9 @@ i_mdi_pi_state_change(mdi_pathinfo_t *pip, mdi_pathinfo_state_t state, int flag)
 	 */
 	i_mdi_phci_lock(ph, pip);
 	if (MDI_PHCI_IS_READY(ph) == 0) {
-		MDI_DEBUG(1, (CE_WARN, NULL,
-		    "!mdi_pi_state_change: pHCI not ready, pHCI=%p", ph));
+		MDI_DEBUG(1, (CE_WARN, ct->ct_dip,
+		    "!mdi_pi_state_change: pHCI not ready, pHCI=%p",
+		    (void *)ph));
 		MDI_PI_UNLOCK(pip);
 		i_mdi_phci_unlock(ph);
 		return (MDI_BUSY);
@@ -3187,18 +3339,18 @@ i_mdi_pi_state_change(mdi_pathinfo_t *pip, mdi_pathinfo_state_t state, int flag)
 	i_mdi_client_unlock(ct);
 
 	f = vh->vh_ops->vo_pi_state_change;
-	if (f != NULL) {
+	if (f != NULL)
 		rv = (*f)(vh->vh_dip, pip, state, 0, flag);
-		if (rv == MDI_NOT_SUPPORTED) {
-			MDI_CLIENT_SET_DEV_NOT_SUPPORTED(ct);
-		}
-		if (rv != MDI_SUCCESS) {
-			MDI_DEBUG(2, (CE_WARN, vh->vh_dip,
-			    "!vo_pi_state_change: failed rv = %x", rv));
-		}
-	}
+
 	MDI_CLIENT_LOCK(ct);
 	MDI_PI_LOCK(pip);
+	if (rv == MDI_NOT_SUPPORTED) {
+		MDI_CLIENT_SET_DEV_NOT_SUPPORTED(ct);
+	}
+	if (rv != MDI_SUCCESS) {
+		MDI_DEBUG(2, (CE_WARN, ct->ct_dip,
+		    "!vo_pi_state_change: failed rv = %x", rv));
+	}
 	if (MDI_PI_IS_TRANSIENT(pip)) {
 		if (rv == MDI_SUCCESS) {
 			MDI_PI_CLEAR_TRANSIENT(pip);
@@ -3235,14 +3387,14 @@ i_mdi_pi_state_change(mdi_pathinfo_t *pip, mdi_pathinfo_state_t state, int flag)
 				    ((state == MDI_PATHINFO_STATE_ONLINE) ||
 				    (state == MDI_PATHINFO_STATE_STANDBY))) {
 
-					i_mdi_client_unlock(ct);
 					/*
 					 * Must do ndi_devi_online() through
 					 * hotplug thread for deferred
 					 * attach mechanism to work
 					 */
+					MDI_CLIENT_UNLOCK(ct);
 					rv = ndi_devi_online(cdip, 0);
-					i_mdi_client_lock(ct, NULL);
+					MDI_CLIENT_LOCK(ct);
 					if ((rv != NDI_SUCCESS) &&
 					    (MDI_CLIENT_STATE(ct) ==
 					    MDI_CLIENT_STATE_DEGRADED)) {
@@ -3274,9 +3426,9 @@ i_mdi_pi_state_change(mdi_pathinfo_t *pip, mdi_pathinfo_state_t state, int flag)
 				if (((flag & NDI_DEVI_REMOVE) == 0) &&
 				    cdip && (i_ddi_node_state(cdip) >=
 				    DS_INITIALIZED)) {
-					i_mdi_client_unlock(ct);
+					MDI_CLIENT_UNLOCK(ct);
 					rv = ndi_devi_offline(cdip, 0);
-					i_mdi_client_lock(ct, NULL);
+					MDI_CLIENT_LOCK(ct);
 
 					if (rv != NDI_SUCCESS) {
 						/*
@@ -3348,7 +3500,7 @@ mdi_pi_online(mdi_pathinfo_t *pip, int flags)
 	MDI_PI_LOCK(pip);
 	if (MDI_PI(pip)->pi_pm_held == 0) {
 		MDI_DEBUG(4, (CE_NOTE, ct->ct_dip, "mdi_pi_online "
-		    "i_mdi_pm_hold_pip\n"));
+		    "i_mdi_pm_hold_pip %p\n", (void *)pip));
 		i_mdi_pm_hold_pip(pip);
 		client_held = 1;
 	}
@@ -3361,7 +3513,7 @@ mdi_pi_online(mdi_pathinfo_t *pip, int flags)
 		}
 
 		MDI_DEBUG(4, (CE_NOTE, ct->ct_dip, "mdi_pi_online "
-		    "i_mdi_pm_hold_client\n"));
+		    "i_mdi_pm_hold_client %p\n", (void *)ct));
 		i_mdi_pm_hold_client(ct, 1);
 		MDI_CLIENT_UNLOCK(ct);
 	}
@@ -3473,9 +3625,9 @@ i_mdi_pi_offline(mdi_pathinfo_t *pip, int flags)
 		/*
 		 * Give a chance for pending I/Os to complete.
 		 */
-		MDI_DEBUG(1, (CE_NOTE, vdip, "!i_mdi_pi_offline: "
+		MDI_DEBUG(1, (CE_NOTE, ct->ct_dip, "!i_mdi_pi_offline: "
 		    "%d cmds still pending on path: %p\n",
-		    MDI_PI(pip)->pi_ref_cnt, pip));
+		    MDI_PI(pip)->pi_ref_cnt, (void *)pip));
 		if (cv_timedwait(&MDI_PI(pip)->pi_ref_cv,
 		    &MDI_PI(pip)->pi_mutex,
 		    ddi_get_lbolt() + drv_usectohz(60 * 1000000)) == -1) {
@@ -3483,12 +3635,12 @@ i_mdi_pi_offline(mdi_pathinfo_t *pip, int flags)
 			 * The timeout time reached without ref_cnt being zero
 			 * being signaled.
 			 */
-			MDI_DEBUG(1, (CE_NOTE, vdip, "!i_mdi_pi_offline: "
+			MDI_DEBUG(1, (CE_NOTE, ct->ct_dip, "!i_mdi_pi_offline: "
 			    "Timeout reached on path %p without the cond\n",
-			    pip));
-			MDI_DEBUG(1, (CE_NOTE, vdip, "!i_mdi_pi_offline: "
+			    (void *)pip));
+			MDI_DEBUG(1, (CE_NOTE, ct->ct_dip, "!i_mdi_pi_offline: "
 			    "%d cmds still pending on path: %p\n",
-			    MDI_PI(pip)->pi_ref_cnt, pip));
+			    MDI_PI(pip)->pi_ref_cnt, (void *)pip));
 		}
 	}
 	vh = ct->ct_vhci;
@@ -3504,8 +3656,9 @@ i_mdi_pi_offline(mdi_pathinfo_t *pip, int flags)
 		MDI_PI_UNLOCK(pip);
 		if ((rv = (*f)(vdip, pip, MDI_PATHINFO_STATE_OFFLINE, 0,
 		    flags)) != MDI_SUCCESS) {
-			MDI_DEBUG(1, (CE_WARN, vdip, "!vo_path_offline failed "
-			    "vdip 0x%x, pip 0x%x", vdip, pip));
+			MDI_DEBUG(1, (CE_WARN, ct->ct_dip,
+			    "!vo_path_offline failed "
+			    "vdip %p, pip %p", (void *)vdip, (void *)pip));
 		}
 		MDI_PI_LOCK(pip);
 	}
@@ -3574,7 +3727,7 @@ i_mdi_pi_offline(mdi_pathinfo_t *pip, int flags)
 	 * Change in the mdi_pathinfo node state will impact the client state
 	 */
 	MDI_DEBUG(2, (CE_NOTE, NULL, "!i_mdi_pi_offline ct = %p pip = %p",
-	    ct, pip));
+	    (void *)ct, (void *)pip));
 	return (rv);
 }
 
@@ -3757,7 +3910,6 @@ mdi_pi_set_preferred(mdi_pathinfo_t *pip, int preferred)
 	}
 }
 
-
 /*
  * mdi_pi_set_state():
  *		Set the mdi_pathinfo node state
@@ -3777,7 +3929,6 @@ mdi_pi_set_state(mdi_pathinfo_t *pip, mdi_pathinfo_state_t state)
 /*
  * Property functions:
  */
-
 int
 i_map_nvlist_error_to_mdi(int val)
 {
@@ -3807,14 +3958,13 @@ i_map_nvlist_error_to_mdi(int val)
  *		and release by calling mdi_pi_unlock() at the end of walk to
  *		get a consistent value.
  */
-
 nvpair_t *
 mdi_pi_get_next_prop(mdi_pathinfo_t *pip, nvpair_t *prev)
 {
 	if ((pip == NULL) || (MDI_PI(pip)->pi_prop == NULL)) {
 		return (NULL);
 	}
-	ASSERT(MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(MDI_PI_LOCKED(pip));
 	return (nvlist_next_nvpair(MDI_PI(pip)->pi_prop, prev));
 }
 
@@ -3822,14 +3972,13 @@ mdi_pi_get_next_prop(mdi_pathinfo_t *pip, nvpair_t *prev)
  * mdi_prop_remove():
  * 		Remove the named property from the named list.
  */
-
 int
 mdi_prop_remove(mdi_pathinfo_t *pip, char *name)
 {
 	if (pip == NULL) {
 		return (DDI_PROP_NOT_FOUND);
 	}
-	ASSERT(!MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(!MDI_PI_LOCKED(pip));
 	MDI_PI_LOCK(pip);
 	if (MDI_PI(pip)->pi_prop == NULL) {
 		MDI_PI_UNLOCK(pip);
@@ -3861,7 +4010,6 @@ mdi_prop_remove(mdi_pathinfo_t *pip, char *name)
  * 		Caller should hold the mdi_pathinfo_t lock to get a consistent
  *		buffer size.
  */
-
 int
 mdi_prop_size(mdi_pathinfo_t *pip, size_t *buflenp)
 {
@@ -3872,7 +4020,7 @@ mdi_prop_size(mdi_pathinfo_t *pip, size_t *buflenp)
 	if ((pip == NULL) || (MDI_PI(pip)->pi_prop == NULL)) {
 		return (DDI_PROP_NOT_FOUND);
 	}
-	ASSERT(MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(MDI_PI_LOCKED(pip));
 	rv = nvlist_size(MDI_PI(pip)->pi_prop,
 	    &bufsize, NV_ENCODE_NATIVE);
 	*buflenp = bufsize;
@@ -3884,7 +4032,6 @@ mdi_prop_size(mdi_pathinfo_t *pip, size_t *buflenp)
  * 		pack the property list.  The caller should hold the
  *		mdi_pathinfo_t node to get a consistent data
  */
-
 int
 mdi_prop_pack(mdi_pathinfo_t *pip, char **bufp, uint_t buflen)
 {
@@ -3895,7 +4042,7 @@ mdi_prop_pack(mdi_pathinfo_t *pip, char **bufp, uint_t buflen)
 		return (DDI_PROP_NOT_FOUND);
 	}
 
-	ASSERT(MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(MDI_PI_LOCKED(pip));
 
 	bufsize = buflen;
 	rv = nvlist_pack(MDI_PI(pip)->pi_prop, bufp, (size_t *)&bufsize,
@@ -3916,7 +4063,7 @@ mdi_prop_update_byte(mdi_pathinfo_t *pip, char *name, uchar_t data)
 	if (pip == NULL) {
 		return (DDI_PROP_INVAL_ARG);
 	}
-	ASSERT(!MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(!MDI_PI_LOCKED(pip));
 	MDI_PI_LOCK(pip);
 	if (MDI_PI(pip)->pi_prop == NULL) {
 		MDI_PI_UNLOCK(pip);
@@ -3940,7 +4087,7 @@ mdi_prop_update_byte_array(mdi_pathinfo_t *pip, char *name, uchar_t *data,
 	if (pip == NULL) {
 		return (DDI_PROP_INVAL_ARG);
 	}
-	ASSERT(!MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(!MDI_PI_LOCKED(pip));
 	MDI_PI_LOCK(pip);
 	if (MDI_PI(pip)->pi_prop == NULL) {
 		MDI_PI_UNLOCK(pip);
@@ -3963,7 +4110,7 @@ mdi_prop_update_int(mdi_pathinfo_t *pip, char *name, int data)
 	if (pip == NULL) {
 		return (DDI_PROP_INVAL_ARG);
 	}
-	ASSERT(!MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(!MDI_PI_LOCKED(pip));
 	MDI_PI_LOCK(pip);
 	if (MDI_PI(pip)->pi_prop == NULL) {
 		MDI_PI_UNLOCK(pip);
@@ -3986,7 +4133,7 @@ mdi_prop_update_int64(mdi_pathinfo_t *pip, char *name, int64_t data)
 	if (pip == NULL) {
 		return (DDI_PROP_INVAL_ARG);
 	}
-	ASSERT(!MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(!MDI_PI_LOCKED(pip));
 	MDI_PI_LOCK(pip);
 	if (MDI_PI(pip)->pi_prop == NULL) {
 		MDI_PI_UNLOCK(pip);
@@ -4010,7 +4157,7 @@ mdi_prop_update_int_array(mdi_pathinfo_t *pip, char *name, int *data,
 	if (pip == NULL) {
 		return (DDI_PROP_INVAL_ARG);
 	}
-	ASSERT(!MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(!MDI_PI_LOCKED(pip));
 	MDI_PI_LOCK(pip);
 	if (MDI_PI(pip)->pi_prop == NULL) {
 		MDI_PI_UNLOCK(pip);
@@ -4034,7 +4181,7 @@ mdi_prop_update_string(mdi_pathinfo_t *pip, char *name, char *data)
 	if (pip == NULL) {
 		return (DDI_PROP_INVAL_ARG);
 	}
-	ASSERT(!MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(!MDI_PI_LOCKED(pip));
 	MDI_PI_LOCK(pip);
 	if (MDI_PI(pip)->pi_prop == NULL) {
 		MDI_PI_UNLOCK(pip);
@@ -4058,7 +4205,7 @@ mdi_prop_update_string_array(mdi_pathinfo_t *pip, char *name, char **data,
 	if (pip == NULL) {
 		return (DDI_PROP_INVAL_ARG);
 	}
-	ASSERT(!MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(!MDI_PI_LOCKED(pip));
 	MDI_PI_LOCK(pip);
 	if (MDI_PI(pip)->pi_prop == NULL) {
 		MDI_PI_UNLOCK(pip);
@@ -4188,7 +4335,6 @@ mdi_prop_lookup_string(mdi_pathinfo_t *pip, char *name, char **data)
  *		returned is the actual property and valid as long as
  *		mdi_pathinfo_t node is alive.
  */
-
 int
 mdi_prop_lookup_string_array(mdi_pathinfo_t *pip, char *name, char ***data,
     uint_t *nelements)
@@ -4210,7 +4356,6 @@ mdi_prop_lookup_string_array(mdi_pathinfo_t *pip, char *name, char ***data,
  *		copy of it.  So the data returned is valid as long as
  *		mdi_pathinfo_t node is valid.
  */
-
 /*ARGSUSED*/
 int
 mdi_prop_free(void *data)
@@ -4228,7 +4373,7 @@ i_mdi_report_path_state(mdi_client_t *ct, mdi_pathinfo_t *pip)
 	dev_info_t	*dip = ct->ct_dip;
 	char		lb_buf[64];
 
-	ASSERT(MUTEX_HELD(&ct->ct_mutex));
+	ASSERT(MDI_CLIENT_LOCKED(ct));
 	if ((dip == NULL) || (ddi_get_instance(dip) == -1) ||
 	    (MDI_CLIENT_IS_REPORT_DEV_NEEDED(ct) == 0)) {
 		return;
@@ -4291,28 +4436,23 @@ i_mdi_report_path_state(mdi_client_t *ct, mdi_pathinfo_t *pip)
  *		Utility function for error message management
  *
  */
-
-/*VARARGS3*/
+/*PRINTFLIKE3*/
 static void
 i_mdi_log(int level, dev_info_t *dip, const char *fmt, ...)
 {
-	char		buf[MAXNAMELEN];
 	char		name[MAXNAMELEN];
+	char		buf[MAXNAMELEN];
+	char		*bp;
 	va_list		ap;
 	int		log_only = 0;
 	int		boot_only = 0;
 	int		console_only = 0;
 
 	if (dip) {
-		if (level == CE_PANIC || level == CE_WARN || level == CE_NOTE) {
-			(void) snprintf(name, MAXNAMELEN, "%s%d:\n",
-			    ddi_node_name(dip), ddi_get_instance(dip));
-		} else {
-			(void) snprintf(name, MAXNAMELEN, "%s%d:",
-			    ddi_node_name(dip), ddi_get_instance(dip));
-		}
+		(void) snprintf(name, MAXNAMELEN, "%s%d: ",
+		    ddi_node_name(dip), ddi_get_instance(dip));
 	} else {
-		name[0] = '\0';
+		name[0] = 0;
 	}
 
 	va_start(ap, fmt);
@@ -4321,14 +4461,25 @@ i_mdi_log(int level, dev_info_t *dip, const char *fmt, ...)
 
 	switch (buf[0]) {
 	case '!':
+		bp = &buf[1];
 		log_only = 1;
 		break;
 	case '?':
+		bp = &buf[1];
 		boot_only = 1;
 		break;
 	case '^':
+		bp = &buf[1];
 		console_only = 1;
 		break;
+	default:
+		bp = buf;
+		break;
+	}
+	if (mdi_debug_logonly) {
+		log_only = 1;
+		boot_only = 0;
+		console_only = 0;
 	}
 
 	switch (level) {
@@ -4339,17 +4490,17 @@ i_mdi_log(int level, dev_info_t *dip, const char *fmt, ...)
 	case CE_WARN:
 	case CE_PANIC:
 		if (boot_only) {
-			cmn_err(level, "?%s\t%s", name, &buf[1]);
+			cmn_err(level, "?mdi: %s%s", name, bp);
 		} else if (console_only) {
-			cmn_err(level, "^%s\t%s", name, &buf[1]);
+			cmn_err(level, "^mdi: %s%s", name, bp);
 		} else if (log_only) {
-			cmn_err(level, "!%s\t%s", name, &buf[1]);
+			cmn_err(level, "!mdi: %s%s", name, bp);
 		} else {
-			cmn_err(level, "%s\t%s", name, buf);
+			cmn_err(level, "mdi: %s%s", name, bp);
 		}
 		break;
 	default:
-		cmn_err(level, "%s\t%s", name, buf);
+		cmn_err(level, "mdi: %s%s", name, bp);
 		break;
 	}
 }
@@ -4376,7 +4527,7 @@ i_mdi_client_online(dev_info_t *ct_dip)
 		(void) i_mdi_power_all_phci(ct);
 
 	MDI_DEBUG(4, (CE_NOTE, ct_dip, "i_mdi_client_online "
-	    "i_mdi_pm_hold_client\n"));
+	    "i_mdi_pm_hold_client %p\n", (void *)ct));
 	i_mdi_pm_hold_client(ct, 1);
 
 	MDI_CLIENT_UNLOCK(ct);
@@ -4403,7 +4554,6 @@ i_mdi_phci_online(dev_info_t *ph_dip)
  *		NDI_SUCCESS
  *		MDI_FAILURE
  */
-
 /*ARGSUSED*/
 int
 mdi_devi_online(dev_info_t *dip, uint_t flags)
@@ -4427,7 +4577,6 @@ mdi_devi_online(dev_info_t *dip, uint_t flags)
  *		NDI_SUCCESS
  *		NDI_FAILURE
  */
-
 /*ARGSUSED*/
 int
 mdi_devi_offline(dev_info_t *dip, uint_t flags)
@@ -4442,6 +4591,7 @@ mdi_devi_offline(dev_info_t *dip, uint_t flags)
 
 	if (MDI_PHCI(dip)) {
 		rv = i_mdi_phci_offline(dip, flags);
+
 		if ((rv != NDI_SUCCESS) && MDI_CLIENT(dip)) {
 			/* set client back online */
 			i_mdi_client_online(dip);
@@ -4471,10 +4621,9 @@ i_mdi_phci_offline(dev_info_t *dip, uint_t flags)
 	 * corresponding client devices, for which this pHCI provides
 	 * critical services.
 	 */
-	MDI_DEBUG(2, (CE_NOTE, dip, "!mdi_phci_offline called %p\n",
-	    dip));
-
 	ph = i_devi_get_phci(dip);
+	MDI_DEBUG(2, (CE_NOTE, dip, "!mdi_phci_offline called %p %p\n",
+	    (void *)dip, (void *)ph));
 	if (ph == NULL) {
 		return (rv);
 	}
@@ -4482,7 +4631,8 @@ i_mdi_phci_offline(dev_info_t *dip, uint_t flags)
 	MDI_PHCI_LOCK(ph);
 
 	if (MDI_PHCI_IS_OFFLINE(ph)) {
-		MDI_DEBUG(1, (CE_WARN, dip, "!pHCI %p already offlined", ph));
+		MDI_DEBUG(1, (CE_WARN, dip, "!pHCI %p already offlined",
+		    (void *)ph));
 		MDI_PHCI_UNLOCK(ph);
 		return (NDI_SUCCESS);
 	}
@@ -4503,6 +4653,7 @@ i_mdi_phci_offline(dev_info_t *dip, uint_t flags)
 	while (pip != NULL) {
 		MDI_PI_LOCK(pip);
 		next = (mdi_pathinfo_t *)MDI_PI(pip)->pi_phci_link;
+
 		/*
 		 * The mdi_pathinfo state is OK. Check the client state.
 		 * If failover in progress fail the pHCI from offlining
@@ -4520,7 +4671,7 @@ i_mdi_phci_offline(dev_info_t *dip, uint_t flags)
 			    "This device can not be removed at "
 			    "this moment. Please try again later."));
 			MDI_PI_UNLOCK(pip);
-			MDI_CLIENT_UNLOCK(ct);
+			i_mdi_client_unlock(ct);
 			MDI_PHCI_UNLOCK(ph);
 			return (NDI_BUSY);
 		}
@@ -4662,9 +4813,9 @@ i_mdi_client_offline(dev_info_t *dip, uint_t flags)
 	 * not in failing over state and update client state
 	 * accordingly
 	 */
-	MDI_DEBUG(2, (CE_NOTE, dip, "!i_mdi_client_offline called %p\n",
-	    dip));
 	ct = i_devi_get_client(dip);
+	MDI_DEBUG(2, (CE_NOTE, dip, "!i_mdi_client_offline called %p %p\n",
+	    (void *)dip, (void *)ct));
 	if (ct != NULL) {
 		MDI_CLIENT_LOCK(ct);
 		if (ct->ct_unstable) {
@@ -4710,7 +4861,6 @@ i_mdi_client_offline(dev_info_t *dip, uint_t flags)
  * mdi_pre_attach():
  *		Pre attach() notification handler
  */
-
 /*ARGSUSED*/
 int
 mdi_pre_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
@@ -4727,7 +4877,6 @@ mdi_pre_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
  * mdi_post_attach():
  *		Post attach() notification handler
  */
-
 /*ARGSUSED*/
 void
 mdi_post_attach(dev_info_t *dip, ddi_attach_cmd_t cmd, int error)
@@ -4744,7 +4893,7 @@ mdi_post_attach(dev_info_t *dip, ddi_attach_cmd_t cmd, int error)
 		switch (cmd) {
 		case DDI_ATTACH:
 			MDI_DEBUG(2, (CE_NOTE, dip,
-			    "!pHCI post_attach: called %p\n", ph));
+			    "!pHCI post_attach: called %p\n", (void *)ph));
 			if (error == DDI_SUCCESS) {
 				MDI_PHCI_SET_ATTACH(ph);
 			} else {
@@ -4757,7 +4906,7 @@ mdi_post_attach(dev_info_t *dip, ddi_attach_cmd_t cmd, int error)
 
 		case DDI_RESUME:
 			MDI_DEBUG(2, (CE_NOTE, dip,
-			    "!pHCI post_resume: called %p\n", ph));
+			    "!pHCI post_resume: called %p\n", (void *)ph));
 			if (error == DDI_SUCCESS) {
 				MDI_PHCI_SET_RESUME(ph);
 			} else {
@@ -4779,7 +4928,7 @@ mdi_post_attach(dev_info_t *dip, ddi_attach_cmd_t cmd, int error)
 		switch (cmd) {
 		case DDI_ATTACH:
 			MDI_DEBUG(2, (CE_NOTE, dip,
-			    "!Client post_attach: called %p\n", ct));
+			    "!Client post_attach: called %p\n", (void *)ct));
 			if (error != DDI_SUCCESS) {
 				MDI_DEBUG(1, (CE_NOTE, dip,
 				    "!Client post_attach: failed error=%d\n",
@@ -4799,15 +4948,17 @@ mdi_post_attach(dev_info_t *dip, ddi_attach_cmd_t cmd, int error)
 			for (pip = ct->ct_path_head; pip != NULL;
 			    pip = (mdi_pathinfo_t *)
 			    MDI_PI(pip)->pi_client_link) {
-				(void) i_mdi_pi_kstat_create(pip);
-				i_mdi_report_path_state(ct, pip);
+				if (!MDI_PI_IS_OFFLINE(pip)) {
+					(void) i_mdi_pi_kstat_create(pip);
+					i_mdi_report_path_state(ct, pip);
+				}
 			}
 			MDI_CLIENT_SET_ATTACH(ct);
 			break;
 
 		case DDI_RESUME:
 			MDI_DEBUG(2, (CE_NOTE, dip,
-			    "!Client post_attach: called %p\n", ct));
+			    "!Client post_attach: called %p\n", (void *)ct));
 			if (error == DDI_SUCCESS) {
 				MDI_CLIENT_SET_RESUME(ct);
 			} else {
@@ -4826,7 +4977,6 @@ mdi_post_attach(dev_info_t *dip, ddi_attach_cmd_t cmd, int error)
  * mdi_pre_detach():
  *		Pre detach notification handler
  */
-
 /*ARGSUSED*/
 int
 mdi_pre_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
@@ -4864,7 +5014,7 @@ i_mdi_phci_pre_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 	switch (cmd) {
 	case DDI_DETACH:
 		MDI_DEBUG(2, (CE_NOTE, dip,
-		    "!pHCI pre_detach: called %p\n", ph));
+		    "!pHCI pre_detach: called %p\n", (void *)ph));
 		if (!MDI_PHCI_IS_OFFLINE(ph)) {
 			/*
 			 * mdi_pathinfo nodes are still attached to
@@ -4873,7 +5023,7 @@ i_mdi_phci_pre_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 			MDI_DEBUG(2, (CE_WARN, dip,
 			    "!pHCI pre_detach: "
 			    "mdi_pathinfo nodes are still attached "
-			    "%p\n", ph));
+			    "%p\n", (void *)ph));
 			rv = DDI_FAILURE;
 			break;
 		}
@@ -4889,7 +5039,7 @@ i_mdi_phci_pre_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 		 */
 
 		MDI_DEBUG(2, (CE_NOTE, dip,
-		    "!pHCI pre_suspend: called %p\n", ph));
+		    "!pHCI pre_suspend: called %p\n", (void *)ph));
 		/*
 		 * Suspend all the client devices accessible through this pHCI
 		 */
@@ -4981,13 +5131,13 @@ i_mdi_client_pre_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
 	switch (cmd) {
 	case DDI_DETACH:
 		MDI_DEBUG(2, (CE_NOTE, dip,
-		    "!Client pre_detach: called %p\n", ct));
+		    "!Client pre_detach: called %p\n", (void *)ct));
 		MDI_CLIENT_SET_DETACH(ct);
 		break;
 
 	case DDI_SUSPEND:
 		MDI_DEBUG(2, (CE_NOTE, dip,
-		    "!Client pre_suspend: called %p\n", ct));
+		    "!Client pre_suspend: called %p\n", (void *)ct));
 		MDI_CLIENT_SET_SUSPEND(ct);
 		break;
 
@@ -5003,7 +5153,6 @@ i_mdi_client_pre_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
  * mdi_post_detach():
  *		Post detach notification handler
  */
-
 /*ARGSUSED*/
 void
 mdi_post_detach(dev_info_t *dip, ddi_detach_cmd_t cmd, int error)
@@ -5042,14 +5191,14 @@ i_mdi_phci_post_detach(dev_info_t *dip, ddi_detach_cmd_t cmd, int error)
 	switch (cmd) {
 	case DDI_DETACH:
 		MDI_DEBUG(2, (CE_NOTE, dip,
-		    "!pHCI post_detach: called %p\n", ph));
+		    "!pHCI post_detach: called %p\n", (void *)ph));
 		if (error != DDI_SUCCESS)
 			MDI_PHCI_SET_ATTACH(ph);
 		break;
 
 	case DDI_SUSPEND:
 		MDI_DEBUG(2, (CE_NOTE, dip,
-		    "!pHCI post_suspend: called %p\n", ph));
+		    "!pHCI post_suspend: called %p\n", (void *)ph));
 		if (error != DDI_SUCCESS)
 			MDI_PHCI_SET_RESUME(ph);
 		break;
@@ -5075,7 +5224,7 @@ i_mdi_client_post_detach(dev_info_t *dip, ddi_detach_cmd_t cmd, int error)
 	switch (cmd) {
 	case DDI_DETACH:
 		MDI_DEBUG(2, (CE_NOTE, dip,
-		    "!Client post_detach: called %p\n", ct));
+		    "!Client post_detach: called %p\n", (void *)ct));
 		if (DEVI_IS_ATTACHING(ct->ct_dip)) {
 			MDI_DEBUG(4, (CE_NOTE, dip, "i_mdi_client_post_detach "
 			    "i_mdi_pm_rele_client\n"));
@@ -5091,7 +5240,7 @@ i_mdi_client_post_detach(dev_info_t *dip, ddi_detach_cmd_t cmd, int error)
 
 	case DDI_SUSPEND:
 		MDI_DEBUG(2, (CE_NOTE, dip,
-		    "!Client post_suspend: called %p\n", ct));
+		    "!Client post_suspend: called %p\n", (void *)ct));
 		if (error != DDI_SUCCESS)
 			MDI_CLIENT_SET_RESUME(ct);
 		break;
@@ -5119,14 +5268,14 @@ i_mdi_pi_kstat_create(mdi_pathinfo_t *pip)
 
 	ASSERT(client != NULL && ppath != NULL);
 
-	ASSERT(mutex_owned(&(MDI_PI(pip)->pi_client->ct_mutex)));
+	ASSERT(MDI_CLIENT_LOCKED(MDI_PI(pip)->pi_client));
 
 	if (MDI_PI(pip)->pi_kstats != NULL)
 		return (MDI_SUCCESS);
 
 	for (cpip = MDI_PI(pip)->pi_client->ct_path_head; cpip != NULL;
 	    cpip = (mdi_pathinfo_t *)(MDI_PI(cpip)->pi_client_link)) {
-		if (cpip == pip)
+		if ((cpip == pip) || MDI_PI_IS_OFFLINE(pip))
 			continue;
 		/*
 		 * We have found a different path with same parent
@@ -5263,14 +5412,15 @@ mdi_pi_enable_path(mdi_pathinfo_t *pip, int flags)
 	ph = i_devi_get_phci(mdi_pi_get_phci(pip));
 	if (ph == NULL) {
 		MDI_DEBUG(1, (CE_NOTE, NULL, "!mdi_pi_enable_path:"
-			" failed. pip: %p ph = NULL\n", pip));
+			" failed. pip: %p ph = NULL\n", (void *)pip));
 		return (MDI_FAILURE);
 	}
 
 	(void) i_mdi_enable_disable_path(pip, ph->ph_vhci, flags,
 		MDI_ENABLE_OP);
 	MDI_DEBUG(5, (CE_NOTE, NULL, "!mdi_pi_enable_path:"
-		" Returning success pip = %p. ph = %p\n", pip, ph));
+		" Returning success pip = %p. ph = %p\n",
+		(void *)pip, (void *)ph));
 	return (MDI_SUCCESS);
 
 }
@@ -5288,14 +5438,15 @@ mdi_pi_disable_path(mdi_pathinfo_t *pip, int flags)
 	ph = i_devi_get_phci(mdi_pi_get_phci(pip));
 	if (ph == NULL) {
 		MDI_DEBUG(1, (CE_NOTE, NULL, "!mdi_pi_disable_path:"
-			" failed. pip: %p ph = NULL\n", pip));
+			" failed. pip: %p ph = NULL\n", (void *)pip));
 		return (MDI_FAILURE);
 	}
 
 	(void) i_mdi_enable_disable_path(pip,
 			ph->ph_vhci, flags, MDI_DISABLE_OP);
 	MDI_DEBUG(5, (CE_NOTE, NULL, "!mdi_pi_disable_path:"
-		"Returning success pip = %p. ph = %p", pip, ph));
+		"Returning success pip = %p. ph = %p",
+		(void *)pip, (void *)ph));
 	return (MDI_SUCCESS);
 }
 
@@ -5363,22 +5514,25 @@ i_mdi_enable_disable_path(mdi_pathinfo_t *pip, mdi_vhci_t *vh, int flags,
 
 	switch (flags) {
 		case USER_DISABLE:
-			if (op == MDI_DISABLE_OP)
+			if (op == MDI_DISABLE_OP) {
 				MDI_PI_SET_USER_DISABLE(pip);
-			else
+			} else {
 				MDI_PI_SET_USER_ENABLE(pip);
+			}
 			break;
 		case DRIVER_DISABLE:
-			if (op == MDI_DISABLE_OP)
+			if (op == MDI_DISABLE_OP) {
 				MDI_PI_SET_DRV_DISABLE(pip);
-			else
+			} else {
 				MDI_PI_SET_DRV_ENABLE(pip);
+			}
 			break;
 		case DRIVER_DISABLE_TRANSIENT:
-			if (op == MDI_DISABLE_OP && rv == MDI_SUCCESS)
+			if (op == MDI_DISABLE_OP && rv == MDI_SUCCESS) {
 				MDI_PI_SET_DRV_DISABLE_TRANS(pip);
-			else
+			} else {
 				MDI_PI_SET_DRV_ENABLE_TRANS(pip);
+			}
 			break;
 	}
 	MDI_PI_UNLOCK(pip);
@@ -5415,17 +5569,18 @@ i_mdi_pi_enable_disable(dev_info_t *cdip, dev_info_t *pdip, int flags, int op)
 	int		found_it;
 
 	ph = i_devi_get_phci(pdip);
-	MDI_DEBUG(5, (CE_NOTE, NULL, "!i_mdi_pi_enable_disable:"
-		" Operation = %d pdip = %p cdip = %p\n", op, pdip, cdip));
+	MDI_DEBUG(5, (CE_NOTE, NULL, "!i_mdi_pi_enable_disable: "
+		"Op = %d pdip = %p cdip = %p\n", op, (void *)pdip,
+		(void *)cdip));
 	if (ph == NULL) {
 		MDI_DEBUG(1, (CE_NOTE, NULL, "!i_mdi_pi_enable_disable:"
-			" failed. ph = NULL operation = %d\n", op));
+			"Op %d failed. ph = NULL\n", op));
 		return (MDI_FAILURE);
 	}
 
 	if ((op != MDI_ENABLE_OP) && (op != MDI_DISABLE_OP)) {
-		MDI_DEBUG(1, (CE_NOTE, NULL, "!i_mdi_pi_enable_disable:"
-			" Invalid operation = %d\n", op));
+		MDI_DEBUG(1, (CE_NOTE, NULL, "!i_mdi_pi_enable_disable: "
+			"Op Invalid operation = %d\n", op));
 		return (MDI_FAILURE);
 	}
 
@@ -5435,27 +5590,30 @@ i_mdi_pi_enable_disable(dev_info_t *cdip, dev_info_t *pdip, int flags, int op)
 		/*
 		 * Need to mark the Phci as enabled/disabled.
 		 */
-		MDI_DEBUG(3, (CE_NOTE, NULL, "!i_mdi_pi_enable_disable:"
-		"Operation %d for the phci\n", op));
+		MDI_DEBUG(3, (CE_NOTE, NULL, "!i_mdi_pi_enable_disable: "
+		"Op %d for the phci\n", op));
 		MDI_PHCI_LOCK(ph);
 		switch (flags) {
 			case USER_DISABLE:
-				if (op == MDI_DISABLE_OP)
+				if (op == MDI_DISABLE_OP) {
 					MDI_PHCI_SET_USER_DISABLE(ph);
-				else
+				} else {
 					MDI_PHCI_SET_USER_ENABLE(ph);
+				}
 				break;
 			case DRIVER_DISABLE:
-				if (op == MDI_DISABLE_OP)
+				if (op == MDI_DISABLE_OP) {
 					MDI_PHCI_SET_DRV_DISABLE(ph);
-				else
+				} else {
 					MDI_PHCI_SET_DRV_ENABLE(ph);
+				}
 				break;
 			case DRIVER_DISABLE_TRANSIENT:
-				if (op == MDI_DISABLE_OP)
+				if (op == MDI_DISABLE_OP) {
 					MDI_PHCI_SET_DRV_DISABLE_TRANSIENT(ph);
-				else
+				} else {
 					MDI_PHCI_SET_DRV_ENABLE_TRANSIENT(ph);
+				}
 				break;
 			default:
 				MDI_PHCI_UNLOCK(ph);
@@ -5513,56 +5671,9 @@ i_mdi_pi_enable_disable(dev_info_t *cdip, dev_info_t *pdip, int flags, int op)
 		(void) i_mdi_enable_disable_path(pip, vh, flags, op);
 	}
 
-	MDI_DEBUG(5, (CE_NOTE, NULL, "!i_mdi_pi_enable_disable:"
-		" Returning success op: %x pdip = %p cdip = %p\n", op,
-			pdip, cdip));
-	return (MDI_SUCCESS);
-}
-
-/*ARGSUSED3*/
-int
-mdi_devi_config_one(dev_info_t *pdip, char *devnm, dev_info_t **cdipp,
-    int flags, clock_t timeout)
-{
-	mdi_pathinfo_t *pip;
-	dev_info_t *dip;
-	clock_t interval = drv_usectohz(100000);	/* 0.1 sec */
-	char *paddr;
-
-	MDI_DEBUG(2, (CE_NOTE, NULL, "configure device %s", devnm));
-
-	if (!MDI_PHCI(pdip))
-		return (MDI_FAILURE);
-
-	paddr = strchr(devnm, '@');
-	if (paddr == NULL)
-		return (MDI_FAILURE);
-
-	paddr++;	/* skip '@' */
-	pip = mdi_pi_find(pdip, NULL, paddr);
-	while (pip == NULL && timeout > 0) {
-		if (interval > timeout)
-			interval = timeout;
-		if (flags & NDI_DEVI_DEBUG) {
-			cmn_err(CE_CONT, "%s%d: %s timeout %ld %ld\n",
-			    ddi_driver_name(pdip), ddi_get_instance(pdip),
-			    paddr, interval, timeout);
-		}
-		delay(interval);
-		timeout -= interval;
-		interval += interval;
-		pip = mdi_pi_find(pdip, NULL, paddr);
-	}
-
-	if (pip == NULL)
-		return (MDI_FAILURE);
-	dip = mdi_pi_get_client(pip);
-	if (ndi_devi_online(dip, flags) != NDI_SUCCESS)
-		return (MDI_FAILURE);
-	*cdipp = dip;
-
-	/* TODO: holding should happen inside search functions */
-	ndi_hold_devi(dip);
+	MDI_DEBUG(5, (CE_NOTE, NULL, "!i_mdi_pi_enable_disable: "
+		"Op %d Returning success pdip = %p cdip = %p\n",
+		op, (void *)pdip, (void *)cdip));
 	return (MDI_SUCCESS);
 }
 
@@ -5575,15 +5686,15 @@ i_mdi_pm_hold_pip(mdi_pathinfo_t *pip)
 	dev_info_t	*ph_dip;
 
 	ASSERT(pip != NULL);
-	ASSERT(MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(MDI_PI_LOCKED(pip));
 
 	if (MDI_PI(pip)->pi_pm_held) {
 		return;
 	}
 
 	ph_dip = mdi_pi_get_phci(pip);
-	MDI_DEBUG(4, (CE_NOTE, ph_dip, "i_mdi_pm_hold_pip for %s%d\n",
-	    ddi_get_name(ph_dip), ddi_get_instance(ph_dip)));
+	MDI_DEBUG(4, (CE_NOTE, ph_dip, "i_mdi_pm_hold_pip for %s%d %p\n",
+	    ddi_get_name(ph_dip), ddi_get_instance(ph_dip), (void *)pip));
 	if (ph_dip == NULL) {
 		return;
 	}
@@ -5591,7 +5702,9 @@ i_mdi_pm_hold_pip(mdi_pathinfo_t *pip)
 	MDI_PI_UNLOCK(pip);
 	MDI_DEBUG(4, (CE_NOTE, ph_dip, "kidsupcnt was %d\n",
 	    DEVI(ph_dip)->devi_pm_kidsupcnt));
+
 	pm_hold_power(ph_dip);
+
 	MDI_DEBUG(4, (CE_NOTE, ph_dip, "kidsupcnt is %d\n",
 	    DEVI(ph_dip)->devi_pm_kidsupcnt));
 	MDI_PI_LOCK(pip);
@@ -5608,7 +5721,7 @@ i_mdi_pm_rele_pip(mdi_pathinfo_t *pip)
 	dev_info_t	*ph_dip = NULL;
 
 	ASSERT(pip != NULL);
-	ASSERT(MUTEX_HELD(&MDI_PI(pip)->pi_mutex));
+	ASSERT(MDI_PI_LOCKED(pip));
 
 	if (MDI_PI(pip)->pi_pm_held == 0) {
 		return;
@@ -5618,8 +5731,8 @@ i_mdi_pm_rele_pip(mdi_pathinfo_t *pip)
 	ASSERT(ph_dip != NULL);
 
 	MDI_PI_UNLOCK(pip);
-	MDI_DEBUG(4, (CE_NOTE, ph_dip, "i_mdi_pm_rele_pip for %s%d\n",
-	    ddi_get_name(ph_dip), ddi_get_instance(ph_dip)));
+	MDI_DEBUG(4, (CE_NOTE, ph_dip, "i_mdi_pm_rele_pip for %s%d %p\n",
+	    ddi_get_name(ph_dip), ddi_get_instance(ph_dip), (void *)pip));
 
 	MDI_DEBUG(4, (CE_NOTE, ph_dip, "kidsupcnt was %d\n",
 	    DEVI(ph_dip)->devi_pm_kidsupcnt));
@@ -5634,11 +5747,12 @@ i_mdi_pm_rele_pip(mdi_pathinfo_t *pip)
 static void
 i_mdi_pm_hold_client(mdi_client_t *ct, int incr)
 {
-	ASSERT(ct);
+	ASSERT(MDI_CLIENT_LOCKED(ct));
 
 	ct->ct_power_cnt += incr;
-	MDI_DEBUG(4, (CE_NOTE, ct->ct_dip, "i_mdi_pm_hold_client "
-	    "ct_power_cnt = %d incr = %d\n", ct->ct_power_cnt, incr));
+	MDI_DEBUG(4, (CE_NOTE, ct->ct_dip, "i_mdi_pm_hold_client %p "
+	    "ct_power_cnt = %d incr = %d\n", (void *)ct,
+	    ct->ct_power_cnt, incr));
 	ASSERT(ct->ct_power_cnt >= 0);
 }
 
@@ -5647,7 +5761,7 @@ i_mdi_rele_all_phci(mdi_client_t *ct)
 {
 	mdi_pathinfo_t  *pip;
 
-	ASSERT(mutex_owned(&ct->ct_mutex));
+	ASSERT(MDI_CLIENT_LOCKED(ct));
 	pip = (mdi_pathinfo_t *)ct->ct_path_head;
 	while (pip != NULL) {
 		mdi_hold_path(pip);
@@ -5662,12 +5776,13 @@ i_mdi_rele_all_phci(mdi_client_t *ct)
 static void
 i_mdi_pm_rele_client(mdi_client_t *ct, int decr)
 {
-	ASSERT(ct);
+	ASSERT(MDI_CLIENT_LOCKED(ct));
 
 	if (i_ddi_devi_attached(ct->ct_dip)) {
 		ct->ct_power_cnt -= decr;
-		MDI_DEBUG(4, (CE_NOTE, ct->ct_dip, "i_mdi_pm_rele_client "
-		    "ct_power_cnt = %d decr = %d\n", ct->ct_power_cnt, decr));
+		MDI_DEBUG(4, (CE_NOTE, ct->ct_dip, "i_mdi_pm_rele_client %p "
+		    "ct_power_cnt = %d decr = %d\n",
+		    (void *)ct, ct->ct_power_cnt, decr));
 	}
 
 	ASSERT(ct->ct_power_cnt >= 0);
@@ -5680,8 +5795,9 @@ i_mdi_pm_rele_client(mdi_client_t *ct, int decr)
 static void
 i_mdi_pm_reset_client(mdi_client_t *ct)
 {
-	MDI_DEBUG(4, (CE_NOTE, ct->ct_dip, "i_mdi_pm_reset_client "
-	    "ct_power_cnt = %d\n", ct->ct_power_cnt));
+	MDI_DEBUG(4, (CE_NOTE, ct->ct_dip, "i_mdi_pm_reset_client %p "
+	    "ct_power_cnt = %d\n", (void *)ct, ct->ct_power_cnt));
+	ASSERT(MDI_CLIENT_LOCKED(ct));
 	ct->ct_power_cnt = 0;
 	i_mdi_rele_all_phci(ct);
 	ct->ct_powercnt_config = 0;
@@ -5689,23 +5805,6 @@ i_mdi_pm_reset_client(mdi_client_t *ct)
 	ct->ct_powercnt_reset = 1;
 }
 
-static void
-i_mdi_pm_hold_all_phci(mdi_client_t *ct)
-{
-	mdi_pathinfo_t  *pip;
-	ASSERT(mutex_owned(&ct->ct_mutex));
-
-	pip = (mdi_pathinfo_t *)ct->ct_path_head;
-	while (pip != NULL) {
-		mdi_hold_path(pip);
-		MDI_PI_LOCK(pip);
-		i_mdi_pm_hold_pip(pip);
-		MDI_PI_UNLOCK(pip);
-		mdi_rele_path(pip);
-		pip = (mdi_pathinfo_t *)MDI_PI(pip)->pi_client_link;
-	}
-}
-
 static int
 i_mdi_power_one_phci(mdi_pathinfo_t *pip)
 {
@@ -5720,15 +5819,16 @@ i_mdi_power_one_phci(mdi_pathinfo_t *pip)
 
 	/* bring all components of phci to full power */
 	MDI_DEBUG(4, (CE_NOTE, ph_dip, "i_mdi_power_one_phci "
-	    "pm_powerup for %s%d\n", ddi_get_name(ph_dip),
-	    ddi_get_instance(ph_dip)));
+	    "pm_powerup for %s%d %p\n", ddi_get_name(ph_dip),
+	    ddi_get_instance(ph_dip), (void *)pip));
 
 	ret = pm_powerup(ph_dip);
 
 	if (ret == DDI_FAILURE) {
 		MDI_DEBUG(4, (CE_NOTE, ph_dip, "i_mdi_power_one_phci "
-		    "pm_powerup FAILED for %s%d\n",
-		    ddi_get_name(ph_dip), ddi_get_instance(ph_dip)));
+		    "pm_powerup FAILED for %s%d %p\n",
+		    ddi_get_name(ph_dip), ddi_get_instance(ph_dip),
+		    (void *)pip));
 
 		MDI_PI_LOCK(pip);
 		i_mdi_pm_rele_pip(pip);
@@ -5745,16 +5845,19 @@ i_mdi_power_all_phci(mdi_client_t *ct)
 	mdi_pathinfo_t  *pip;
 	int		succeeded = 0;
 
+	ASSERT(MDI_CLIENT_LOCKED(ct));
 	pip = (mdi_pathinfo_t *)ct->ct_path_head;
 	while (pip != NULL) {
-		mdi_hold_path(pip);
-		MDI_CLIENT_UNLOCK(ct);
-		if (i_mdi_power_one_phci(pip) == MDI_SUCCESS)
-			succeeded = 1;
+		if (MDI_PI_IS_ONLINE(pip) || MDI_PI_IS_STANDBY(pip)) {
+			mdi_hold_path(pip);
+			MDI_CLIENT_UNLOCK(ct);
+			if (i_mdi_power_one_phci(pip) == MDI_SUCCESS)
+				succeeded = 1;
 
-		ASSERT(ct == MDI_PI(pip)->pi_client);
-		MDI_CLIENT_LOCK(ct);
-		mdi_rele_path(pip);
+			ASSERT(ct == MDI_PI(pip)->pi_client);
+			MDI_CLIENT_LOCK(ct);
+			mdi_rele_path(pip);
+		}
 		pip = (mdi_pathinfo_t *)MDI_PI(pip)->pi_client_link;
 	}
 
@@ -5960,22 +6063,23 @@ i_mdi_pm_pre_config_one(dev_info_t *child)
 }
 
 static int
-i_mdi_pm_pre_config(dev_info_t *parent, dev_info_t *child)
+i_mdi_pm_pre_config(dev_info_t *vdip, dev_info_t *child)
 {
 	int			ret = MDI_SUCCESS;
 	dev_info_t		*cdip;
 	int			circ;
 
-	ASSERT(MDI_VHCI(parent));
+	ASSERT(MDI_VHCI(vdip));
 
 	/* ndi_devi_config_one */
 	if (child) {
+		ASSERT(DEVI_BUSY_OWNED(vdip));
 		return (i_mdi_pm_pre_config_one(child));
 	}
 
 	/* devi_config_common */
-	ndi_devi_enter(parent, &circ);
-	cdip = ddi_get_child(parent);
+	ndi_devi_enter(vdip, &circ);
+	cdip = ddi_get_child(vdip);
 	while (cdip) {
 		dev_info_t *next = ddi_get_next_sibling(cdip);
 
@@ -5984,7 +6088,7 @@ i_mdi_pm_pre_config(dev_info_t *parent, dev_info_t *child)
 			break;
 		cdip = next;
 	}
-	ndi_devi_exit(parent, circ);
+	ndi_devi_exit(vdip, circ);
 	return (ret);
 }
 
@@ -6040,31 +6144,32 @@ i_mdi_pm_pre_unconfig_one(dev_info_t *child, int *held, int flags)
 }
 
 static int
-i_mdi_pm_pre_unconfig(dev_info_t *parent, dev_info_t *child, int *held,
+i_mdi_pm_pre_unconfig(dev_info_t *vdip, dev_info_t *child, int *held,
     int flags)
 {
 	int			ret = MDI_SUCCESS;
 	dev_info_t		*cdip;
 	int			circ;
 
-	ASSERT(MDI_VHCI(parent));
+	ASSERT(MDI_VHCI(vdip));
 	*held = 0;
 
 	/* ndi_devi_unconfig_one */
 	if (child) {
+		ASSERT(DEVI_BUSY_OWNED(vdip));
 		return (i_mdi_pm_pre_unconfig_one(child, held, flags));
 	}
 
 	/* devi_unconfig_common */
-	ndi_devi_enter(parent, &circ);
-	cdip = ddi_get_child(parent);
+	ndi_devi_enter(vdip, &circ);
+	cdip = ddi_get_child(vdip);
 	while (cdip) {
 		dev_info_t *next = ddi_get_next_sibling(cdip);
 
 		ret = i_mdi_pm_pre_unconfig_one(cdip, held, flags);
 		cdip = next;
 	}
-	ndi_devi_exit(parent, circ);
+	ndi_devi_exit(vdip, circ);
 
 	if (*held)
 		ret = MDI_SUCCESS;
@@ -6130,28 +6235,30 @@ i_mdi_pm_post_config_one(dev_info_t *child)
 }
 
 static void
-i_mdi_pm_post_config(dev_info_t *parent, dev_info_t *child)
+i_mdi_pm_post_config(dev_info_t *vdip, dev_info_t *child)
 {
 	int		circ;
 	dev_info_t	*cdip;
-	ASSERT(MDI_VHCI(parent));
+
+	ASSERT(MDI_VHCI(vdip));
 
 	/* ndi_devi_config_one */
 	if (child) {
+		ASSERT(DEVI_BUSY_OWNED(vdip));
 		i_mdi_pm_post_config_one(child);
 		return;
 	}
 
 	/* devi_config_common */
-	ndi_devi_enter(parent, &circ);
-	cdip = ddi_get_child(parent);
+	ndi_devi_enter(vdip, &circ);
+	cdip = ddi_get_child(vdip);
 	while (cdip) {
 		dev_info_t *next = ddi_get_next_sibling(cdip);
 
 		i_mdi_pm_post_config_one(cdip);
 		cdip = next;
 	}
-	ndi_devi_exit(parent, circ);
+	ndi_devi_exit(vdip, circ);
 }
 
 static void
@@ -6205,33 +6312,34 @@ i_mdi_pm_post_unconfig_one(dev_info_t *child)
 }
 
 static void
-i_mdi_pm_post_unconfig(dev_info_t *parent, dev_info_t *child, int held)
+i_mdi_pm_post_unconfig(dev_info_t *vdip, dev_info_t *child, int held)
 {
 	int			circ;
 	dev_info_t		*cdip;
 
-	ASSERT(MDI_VHCI(parent));
+	ASSERT(MDI_VHCI(vdip));
 
 	if (!held) {
-		MDI_DEBUG(4, (CE_NOTE, parent,
+		MDI_DEBUG(4, (CE_NOTE, vdip,
 		    "i_mdi_pm_post_unconfig held = %d\n", held));
 		return;
 	}
 
 	if (child) {
+		ASSERT(DEVI_BUSY_OWNED(vdip));
 		i_mdi_pm_post_unconfig_one(child);
 		return;
 	}
 
-	ndi_devi_enter(parent, &circ);
-	cdip = ddi_get_child(parent);
+	ndi_devi_enter(vdip, &circ);
+	cdip = ddi_get_child(vdip);
 	while (cdip) {
 		dev_info_t *next = ddi_get_next_sibling(cdip);
 
 		i_mdi_pm_post_unconfig_one(cdip);
 		cdip = next;
 	}
-	ndi_devi_exit(parent, circ);
+	ndi_devi_exit(vdip, circ);
 }
 
 int
@@ -6251,29 +6359,29 @@ mdi_power(dev_info_t *vdip, mdi_pm_op_t op, void *args, char *devnm, int flags)
 	if (devnm != NULL) {
 		ndi_devi_enter(vdip, &circ);
 		client_dip = ndi_devi_findchild(vdip, devnm);
-		ndi_devi_exit(vdip, circ);
 	}
 
-	MDI_DEBUG(4, (CE_NOTE, vdip, "mdi_power op = %d\n", op));
+	MDI_DEBUG(4, (CE_NOTE, vdip, "mdi_power op = %d %s %p\n",
+	    op, devnm ? devnm : "NULL", (void *)client_dip));
 
 	switch (op) {
 	case MDI_PM_PRE_CONFIG:
 		ret = i_mdi_pm_pre_config(vdip, client_dip);
-
 		break;
+
 	case MDI_PM_PRE_UNCONFIG:
 		ret = i_mdi_pm_pre_unconfig(vdip, client_dip, (int *)args,
 		    flags);
-
 		break;
+
 	case MDI_PM_POST_CONFIG:
 		i_mdi_pm_post_config(vdip, client_dip);
-
 		break;
+
 	case MDI_PM_POST_UNCONFIG:
 		i_mdi_pm_post_unconfig(vdip, client_dip, *(int *)args);
-
 		break;
+
 	case MDI_PM_HOLD_POWER:
 	case MDI_PM_RELE_POWER:
 		ASSERT(args);
@@ -6305,10 +6413,14 @@ mdi_power(dev_info_t *vdip, mdi_pm_op_t op, void *args, char *devnm, int flags)
 
 		MDI_CLIENT_UNLOCK(ct);
 		break;
+
 	default:
 		break;
 	}
 
+	if (devnm)
+		ndi_devi_exit(vdip, circ);
+
 	return (ret);
 }
 
@@ -8276,19 +8388,17 @@ mdi_vhci_bus_config(dev_info_t *vdip, uint_t flags, ddi_bus_config_op_t op,
 	char *cp;
 
 	/*
-	 * While bus configuring phcis, the phci driver interactions with MDI
-	 * cause child nodes to be enumerated under the vhci node for which
-	 * they need to ndi_devi_enter the vhci node.
-	 *
-	 * Unfortunately, to avoid the deadlock, we ourself can not wait for
-	 * for the bus config operations on phcis to finish while holding the
-	 * ndi_devi_enter lock. To avoid this deadlock, skip bus configs on
-	 * phcis and call the default framework provided bus config function
-	 * if we are called with ndi_devi_enter lock held.
+	 * To bus config vhcis we relay operation, possibly using another
+	 * thread, to phcis. The phci driver then interacts with MDI to cause
+	 * vhci child nodes to be enumerated under the vhci node.  Adding a
+	 * vhci child requires an ndi_devi_enter of the vhci. Since another
+	 * thread may be adding the child, to avoid deadlock we can't wait
+	 * for the relayed operations to complete if we have already entered
+	 * the vhci node.
 	 */
 	if (DEVI_BUSY_OWNED(vdip)) {
-		MDI_DEBUG(2, (CE_NOTE, vdip,
-		    "!MDI: vhci bus config: vhci dip is busy owned\n"));
+		MDI_DEBUG(2, (CE_NOTE, vdip, "!MDI: vhci bus config: "
+		    "vhci dip is busy owned %p\n", (void *)vdip));
 		goto default_bus_config;
 	}
 
@@ -8479,31 +8589,24 @@ void
 mdi_vhci_walk_clients(dev_info_t *vdip,
     int (*f)(dev_info_t *, void *), void *arg)
 {
+	mdi_vhci_t	*vh = i_devi_get_vhci(vdip);
 	dev_info_t	*cdip;
 	mdi_client_t	*ct;
 
-	mutex_enter(&mdi_mutex);
-
+	MDI_VHCI_CLIENT_LOCK(vh);
 	cdip = ddi_get_child(vdip);
-
 	while (cdip) {
 		ct = i_devi_get_client(cdip);
 		MDI_CLIENT_LOCK(ct);
 
-		switch ((*f)(cdip, arg)) {
-		case DDI_WALK_CONTINUE:
+		if (((*f)(cdip, arg)) == DDI_WALK_CONTINUE)
 			cdip = ddi_get_next_sibling(cdip);
-			MDI_CLIENT_UNLOCK(ct);
-			break;
+		else
+			cdip = NULL;
 
-		default:
-			MDI_CLIENT_UNLOCK(ct);
-			mutex_exit(&mdi_mutex);
-			return;
-		}
+		MDI_CLIENT_UNLOCK(ct);
 	}
-
-	mutex_exit(&mdi_mutex);
+	MDI_VHCI_CLIENT_UNLOCK(vh);
 }
 
 /*
@@ -8514,31 +8617,23 @@ void
 mdi_vhci_walk_phcis(dev_info_t *vdip,
     int (*f)(dev_info_t *, void *), void *arg)
 {
-	mdi_vhci_t	*vh = NULL;
-	mdi_phci_t	*ph = NULL;
+	mdi_vhci_t	*vh = i_devi_get_vhci(vdip);
+	mdi_phci_t	*ph, *next;
 
-	mutex_enter(&mdi_mutex);
-
-	vh = i_devi_get_vhci(vdip);
+	MDI_VHCI_PHCI_LOCK(vh);
 	ph = vh->vh_phci_head;
-
 	while (ph) {
 		MDI_PHCI_LOCK(ph);
 
-		switch ((*f)(ph->ph_dip, arg)) {
-		case DDI_WALK_CONTINUE:
-			MDI_PHCI_UNLOCK(ph);
-			ph = ph->ph_next;
-			break;
+		if (((*f)(ph->ph_dip, arg)) == DDI_WALK_CONTINUE)
+			next = ph->ph_next;
+		else
+			next = NULL;
 
-		default:
-			MDI_PHCI_UNLOCK(ph);
-			mutex_exit(&mdi_mutex);
-			return;
-		}
+		MDI_PHCI_UNLOCK(ph);
+		ph = next;
 	}
-
-	mutex_exit(&mdi_mutex);
+	MDI_VHCI_PHCI_UNLOCK(vh);
 }
 
 
diff --git a/usr/src/uts/common/os/sunpm.c b/usr/src/uts/common/os/sunpm.c
index cd77e635d6..3a38cb1b96 100644
--- a/usr/src/uts/common/os/sunpm.c
+++ b/usr/src/uts/common/os/sunpm.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -2024,9 +2023,9 @@ e_pm_hold_rele_power(dev_info_t *dip, int cnt)
 	PMD_FUNC(pmf, "hold_rele_power")
 	int circ;
 
-	if ((dip == NULL) ||
-	    (PM_GET_PM_INFO(dip) == NULL) || PM_ISBC(dip))
+	if ((dip == NULL) || PM_ISBC(dip))
 		return;
+
 	PM_LOCK_POWER(dip, &circ);
 	ASSERT(cnt >= 0 && PM_KUC(dip) >= 0 || cnt < 0 && PM_KUC(dip) > 0);
 	PMD(PMD_KIDSUP, ("%s: kidsupcnt for %s@%s(%s#%d) %d->%d\n", pmf,
@@ -4605,7 +4604,8 @@ pm_stop(dev_info_t *dip)
 				DEVI(dip)->devi_pm_flags &= ~PMC_NOPMKID;
 				if (pdip && !PM_WANTS_NOTIFICATION(pdip)) {
 					pm_rele_power(pdip);
-				} else if (pdip && MDI_VHCI(pdip)) {
+				} else if (pdip &&
+				    MDI_VHCI(pdip) && MDI_CLIENT(dip)) {
 					(void) mdi_power(pdip,
 					    MDI_PM_RELE_POWER,
 					    (void *)dip, NULL, 0);
@@ -4898,7 +4898,7 @@ pm_start(dev_info_t *dip)
 		DEVI(dip)->devi_pm_flags |= PMC_NOPMKID;
 		if (pdip && !PM_WANTS_NOTIFICATION(pdip)) {
 			pm_hold_power(pdip);
-		} else if (pdip && MDI_VHCI(pdip)) {
+		} else if (pdip && MDI_VHCI(pdip) && MDI_CLIENT(dip)) {
 			(void) mdi_power(pdip, MDI_PM_HOLD_POWER,
 			    (void *)dip, NULL, 0);
 		}
diff --git a/usr/src/uts/common/sys/ddi_impldefs.h b/usr/src/uts/common/sys/ddi_impldefs.h
index e5a46a2020..e65023d1fc 100644
--- a/usr/src/uts/common/sys/ddi_impldefs.h
+++ b/usr/src/uts/common/sys/ddi_impldefs.h
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -140,7 +139,7 @@ struct dev_info  {
 	struct dev_info *devi_pm_ppm;		/* ppm attached to this one */
 	void		*devi_pm_ppm_private;	/* for use by ppm driver */
 	int		devi_pm_dev_thresh;	/* "device" threshold */
-	uint_t		devi_pm_kidsupcnt;	/* # of kids powered up */
+	int		devi_pm_kidsupcnt;	/* # of kids powered up */
 	struct pm_scan	*devi_pm_scan;		/* pm scan info */
 	uint_t		devi_pm_noinvolpm;	/* # of descendents no-invol */
 	uint_t		devi_pm_volpmd;		/* # of voluntarily pm'ed */
@@ -497,6 +496,7 @@ void	i_devi_exit(dev_info_t *, uint_t c_mask, int has_lock);
 #define	DEVI_BRANCH_HELD	0x00000008 /* branch rooted at this dip held */
 #define	DEVI_NO_BIND		0x00000010 /* prevent driver binding */
 #define	DEVI_REGISTERED_DEVID	0x00000020 /* device registered a devid */
+#define	DEVI_PHCI_SIGNALS_VHCI	0x00000040 /* pHCI ndi_devi_exit signals vHCI */
 
 #define	DEVI_BUSY_CHANGING(dip)	(DEVI(dip)->devi_flags & DEVI_BUSY)
 #define	DEVI_BUSY_OWNED(dip)	(DEVI_BUSY_CHANGING(dip) &&	\
diff --git a/usr/src/uts/common/sys/mdi_impldefs.h b/usr/src/uts/common/sys/mdi_impldefs.h
index b861b2d779..32d26a4a3a 100644
--- a/usr/src/uts/common/sys/mdi_impldefs.h
+++ b/usr/src/uts/common/sys/mdi_impldefs.h
@@ -73,68 +73,90 @@ extern "C" {
  * With the scsi_vhci, a QLC card, and mpxio enabled, the device tree might
  * look like this:
  *
- *	+-----------+   +-----------+
+ *              /\
+ *             /  ............
+ *     <vHCI>:/               \
+ *      +-----------+   +-----------+
  *      | scsi_vhci |   |  pci@1f,0 |
  *      +-----------+   +-----------+
- *         /     \               \
- * +----------+ +-----------+    +-------------+
- * | ssd1     | | ssd2	    |    | qlc@0,0     |
- * +----------+ +-----------+    +-------------+
- *   |          |                  /        \
- *   |          |        +-------------+   +------------+
- *   |          |        | pHCI 1      |   |  pHCI 2    |
- *   |          |        +-------------+   +------------+
- *   |          |          /        |      /          |
- *   |          |    +------+       |    +------+     |
- *   |          |    |  ssd |       |    |  ssd |     |
- *   |          |    | (OBP)|       |    | (OBP)|     |
- *   |          |    +------+       |    +------+     |
- *   |          |                   |                 |
- *   |          |               +-------+           +--------+
- *   |          +-------------->| path  |---------->| path   |
- *   |                          | info  |           | info   |
- *   |                          | node 1|           | node 3 |
- *   |                          +-------+           +--------+
- *   |                              |                 |
- *   |                          +-------+           +--------+
- *   +------------------------->| path  |---------->| path   |
- *                              | info  |           | info   |
- *                              | node 2|           | node 4 |
- *                              +-------+           +--------+
+ *            /   \               \
+ * <Client>: /     \ :<Client>     \ :parent(pHCI)
+ *  +----------+ +-----------+    +-------------+
+ *  | ssd 1    | | ssd 2     |    | qlc@0,0     |
+ *  +----------+ +-----------+    +-------------+
+ *   |            |                /        \
+ *   |            |       <pHCI>: /          \ :<pHCI>
+ *   |            |      +-------------+   +-------------+
+ *   |            |      | pHCI 1 (fp) |   | pHCI 2 (fp) |
+ *   |            |      +-------------+   +-------------+
+ *   |            |          /        |      /          |
+ *   |            |    +------+       |    +------+     |
+ *   |            |    | ssd 3|       |    | ssd  |     |
+ *   |            |    |!mpxio|       |    | (OBP)|     |
+ *   |            |    +------+       |    +------+     |
+ *   |            |                   |                 |
+ *   |            |       <pathinfo>: |                 |
+ *   |            |               +-------+         +--------+
+ *   |            +-------------->| path  |-------->| path   |
+ *   |                            | info  |         | info   |
+ *   |                            | node 1|         | node 3 |
+ *   |                            +-------+         +--------+
+ *   |                                |               |
+ *   |                                |            +~~~~~~~~+
+ *   |                            +-------+        :+--------+
+ *   +--------------------------->| path  |-------->| path   |
+ *                                | info  |        :| info   |
+ *                                | node 2|        +| node 4 |
+ *                                +-------+         +--------+
  *
  * The multipath information nodes (mdi_pathinfo nodes) establish the
- * relationship between the pseudo client driver instance nodes and the
- * physical host controller interconnect (pHCI drivers) forming a matrix
- * structure.
+ * relationship between the pseudo client driver instance nodes (children
+ * of the vHCI) and the physical host controller interconnect (pHCI
+ * drivers) forming a matrix structure.
  *
  * The mpxio module implements locking at multiple granularity levels to
  * support the needs of various consumers.  The multipath matrix can be
- * globally locked, column locked, or row locked depending on the consumer.
- * The intention is to balance simplicity and performance.
+ * column locked, or row locked depending on the consumer. The intention
+ * is to balance simplicity and performance.
  *
  * Locking:
  *
- * The current implementation utilizes the following locks:
+ * The devinfo locking still applies:
+ *
+ *   1) An ndi_devi_enter of a parent protects linkage/state of children.
+ *   2) state >= DS_INITIALIZED adds devi_ref of parent
+ *   3) devi_ref at state >= DS_ATTACHED prevents detach(9E).
+ *
+ * The ordering of 1) is (vHCI, pHCI). For a DEBUG kernel this ordering
+ * is asserted by the ndi_devi_enter() implementation.  There is also an
+ * ndi_devi_enter(Client), which is atypical since the client is a leaf.
+ * This is done to synchronize pathinfo nodes during devinfo snapshot (see
+ * di_register_pip) by pretending that the pathinfo nodes are children
+ * of the client.
+ *
+ * In addition to devinfo locking the current implementation utilizes
+ * the following locks:
  *
- *   mdi_mutex: protects the vHCI list, per-vHCI structure and the
- *   list of pHCIs and Client devices registered against them (protection
- *   against multi-threaded add/remove).
+ *   mdi_mutex: protects the global list of vHCIs.
  *
- *   devinfo_tree_lock (rw): protects system wide creation/removal of
- *   mdi_pathinfo nodes into the multipath matrix.  Consumers (like the devinfo
- *   driver) can freeze the configuration by acquiring this as a reader.
+ *   vh_phci_mutex: per-vHCI (mutex) lock: protects list of pHCIs registered
+ *   with vHCI.
  *
- *   per-pHCI (mutex) lock: protects the column (pHCI-mdi_pathinfo node list)
- *   and per-pHCI structure fields.  mdi_pathinfo node creation, deletion and
- *   child mdi_pathinfo node state changes are serialized on per pHCI basis
- *   (Protection against DR).
+ *   vh_client_mutex: per-vHCI (mutex) lock: protects list/hash of Clients
+ *   associated with vHCI.
  *
- *   per-client (mutex) lock: protects the row (client-mdi_pathinfo node list)
- *   and per-client structure fields.  The client-mdi_pathinfo node list is
- *   typically walked to select an optimal path when routing I/O requests.
+ *   ph_mutex: per-pHCI (mutex) lock: protects the column (pHCI-mdi_pathinfo
+ *   node list) and per-pHCI structure fields.  mdi_pathinfo node creation,
+ *   deletion and child mdi_pathinfo node state changes are serialized on per
+ *   pHCI basis (Protection against DR).
  *
- *   per-mdi_pathinfo (mutex) lock: protects the mdi_pathinfo node structure
- *   fields.
+ *   ct_mutex: per-client (mutex) lock: protects the row (client-mdi_pathinfo
+ *   node list) and per-client structure fields.  The client-mdi_pathinfo node
+ *   list is typically walked to select an optimal path when routing I/O
+ *   requests.
+ *
+ *   pi_mutex: per-mdi_pathinfo (mutex) lock: protects the mdi_pathinfo node
+ *   structure fields.
  *
  * Note that per-Client structure and per-pHCI fields are freely readable when
  * corresponding mdi_pathinfo locks are held, since holding an mdi_pathinfo
@@ -227,29 +249,56 @@ typedef struct mdi_vhci_ops {
  * Each vHCI driver is associated with a vHCI class name; this is the handle
  * used to register and unregister pHCI drivers for a given transport.
  *
- * Locking: This structure is guarded by the mdi_mutex; however, depending
- * on the context, some of the fields can be freely read without holding any
- * locks (ex. holding a child's lock also guarantees that the vHCI (parent)
- * cannot be unexpectedly freed).
+ * Locking: Different parts of this structure are guarded by different
+ * locks: global threading of multiple vHCIs and initialization is protected
+ * by mdi_mutex, the list of pHCIs associated with a vHCI is protected by
+ * vh_phci_mutex, and Clients are protected by vh_client_mutex.
+ *
+ * XXX Depending on the context, some of the fields can be freely read without
+ * holding any locks (ex. holding vh_client_mutex lock also guarantees that
+ * the vHCI (parent) cannot be unexpectedly freed).
  */
 typedef struct mdi_vhci {
-	struct mdi_vhci		*vh_next;	/* next link		*/
-	struct mdi_vhci		*vh_prev;	/* prev link		*/
-	int			vh_flags;	/* Operation flags	*/
-	dev_info_t		*vh_dip;	/* devi handle		*/
-	char			*vh_class;	/* Class name		*/
-	struct mdi_vhci_ops	*vh_ops;	/* Callback vectors	*/
-	client_lb_t		vh_lb;		/* Global cache		*/
+	/* protected by mdi_mutex... */
+	struct mdi_vhci		*vh_next;	/* next vHCI link	*/
+	struct mdi_vhci		*vh_prev;	/* prev vHCI link	*/
+	char			*vh_class;	/* vHCI class name	*/
+	dev_info_t		*vh_dip;	/* vHCI devi handle	*/
+	int			vh_refcnt;	/* vHCI reference count	*/
+	struct mdi_vhci_config	*vh_config;	/* vHCI config		*/
+	client_lb_t		vh_lb;		/* vHCI load-balancing	*/
+	struct mdi_vhci_ops	*vh_ops;	/* vHCI callback vectors */
+
+	/* protected by MDI_VHCI_PHCI_LOCK vh_phci_mutex... */
+	kmutex_t		vh_phci_mutex;	/* pHCI mutex		*/
 	int			vh_phci_count;	/* pHCI device count	*/
 	struct mdi_phci		*vh_phci_head;	/* pHCI list head	*/
 	struct mdi_phci		*vh_phci_tail;	/* pHCI list tail	*/
-	int			vh_client_count;	/* Client count	*/
-	struct client_hash	*vh_client_table;	/* Client hash	*/
-	int			vh_refcnt;	/* reference count */
-	struct mdi_vhci_config	*vh_config;	/* vhci config */
+
+	/* protected by MDI_VHCI_CLIENT_LOCK vh_client_mutex... */
+	kmutex_t		vh_client_mutex; /* Client mutex	*/
+	int			vh_client_count; /* Client count	*/
+	struct client_hash	*vh_client_table; /* Client hash	*/
 } mdi_vhci_t;
 
 /*
+ * per-vHCI lock macros
+ */
+#define	MDI_VHCI_PHCI_LOCK(vh)		mutex_enter(&(vh)->vh_phci_mutex)
+#define	MDI_VHCI_PHCI_TRYLOCK(vh)	mutex_tryenter(&(vh)->vh_phci_mutex)
+#define	MDI_VHCI_PHCI_UNLOCK(vh)	mutex_exit(&(vh)->vh_phci_mutex)
+#ifdef	DEBUG
+#define	MDI_VHCI_PCHI_LOCKED(vh)	MUTEX_HELD(&(vh)->vh_phci_mutex)
+#endif	/* DEBUG */
+#define	MDI_VHCI_CLIENT_LOCK(vh)	mutex_enter(&(vh)->vh_client_mutex)
+#define	MDI_VHCI_CLIENT_TRYLOCK(vh)	mutex_tryenter(&(vh)->vh_client_mutex)
+#define	MDI_VHCI_CLIENT_UNLOCK(vh)	mutex_exit(&(vh)->vh_client_mutex)
+#ifdef	DEBUG
+#define	MDI_VHCI_CLIENT_LOCKED(vh)	MUTEX_HELD(&(vh)->vh_client_mutex)
+#endif	/* DEBUG */
+
+
+/*
  * GUID Hash definitions
  *
  * Since all the mpxio managed devices for a given class are enumerated under
@@ -277,29 +326,31 @@ struct client_hash {
  * similarly call mdi_phci_unregister().
  *
  * The framework maintains a list of registered pHCI device instances for each
- * vHCI.  This list is vHCI->vh_phci_count, vHCI->vh_phci_head,
- * vHCI->vh_phci_tail and pHCI->ph_next.  This list is protected by the global
- * mdi_mutex.
+ * vHCI.  This list involves (vh_phci_count, vh_phci_head, vh_phci_tail) and
+ * (ph_next, ph_prev, ph_vhci) and is protected by vh_phci_mutex.
  *
  * Locking order:
  *
- * _NOTE(LOCK_ORDER(mdi_mutex, mdi_phci::ph_mutex))
- * _NOTE(LOCK_ORDER(mdi_phci::ph_mutex devinfo_tree_lock))
+ * _NOTE(LOCK_ORDER(mdi_mutex, mdi_phci::ph_mutex))		XXX
+ * _NOTE(LOCK_ORDER(mdi_phci::ph_mutex devinfo_tree_lock))		XXX
  */
 typedef struct mdi_phci {
+	/* protected by MDI_VHCI_PHCI_LOCK vh_phci_mutex... */
+	struct mdi_phci		*ph_next;	/* next pHCI link	*/
+	struct mdi_phci		*ph_prev;	/* prev pHCI link	*/
+	dev_info_t		*ph_dip;	/* pHCI devi handle	*/
+	struct mdi_vhci 	*ph_vhci;	/* pHCI back ref. to vHCI */
+
+	/* protected by MDI_PHCI_LOCK ph_mutex... */
 	kmutex_t		ph_mutex;	/* per-pHCI mutex	*/
-	struct mdi_phci		*ph_next;	/* next link		*/
-	struct mdi_phci		*ph_prev;	/* prev link		*/
-	dev_info_t		*ph_dip;	/* devi handle		*/
-	struct mdi_vhci 	*ph_vhci;	/* back ref. to vHCI	*/
-	int			ph_flags;	/* pHCI operation flags	*/
-	int			ph_path_count;	/* child pi count	*/
+	int			ph_path_count;	/* pi count		*/
 	mdi_pathinfo_t		*ph_path_head;	/* pi list head		*/
 	mdi_pathinfo_t		*ph_path_tail;	/* pi list tail		*/
+	int			ph_flags;	/* pHCI operation flags	*/
 	int			ph_unstable;	/* Paths in transient state */
 	kcondvar_t		ph_unstable_cv;	/* Paths in transient state */
-	kcondvar_t		ph_powerchange_cv;
-						/* Paths in transient state */
+
+	/* protected by mdi_phci_[gs]et_vhci_private caller... */
 	void			*ph_vprivate;	/* vHCI driver private	*/
 } mdi_phci_t;
 
@@ -318,9 +369,12 @@ typedef struct mdi_phci {
 /*
  * per-pHCI lock macros
  */
-#define	MDI_PHCI_LOCK(ph)		mutex_enter(&((ph))->ph_mutex)
-#define	MDI_PHCI_TRYLOCK(ph)		mutex_tryenter(&((ph))->ph_mutex)
-#define	MDI_PHCI_UNLOCK(ph)		mutex_exit(&((ph))->ph_mutex)
+#define	MDI_PHCI_LOCK(ph)		mutex_enter(&(ph)->ph_mutex)
+#define	MDI_PHCI_TRYLOCK(ph)		mutex_tryenter(&(ph)->ph_mutex)
+#define	MDI_PHCI_UNLOCK(ph)		mutex_exit(&(ph)->ph_mutex)
+#ifdef	DEBUG
+#define	MDI_PHCI_LOCKED(vh)		MUTEX_HELD(&(ph)->ph_mutex)
+#endif	/* DEBUG */
 
 /*
  * pHCI state definitions and macros to track the pHCI driver instance state
@@ -334,79 +388,81 @@ typedef struct mdi_phci {
 #define	MDI_PHCI_FLAGS_D_DISABLE_TRANS	0x40	/* pHCI is disabled,transient */
 #define	MDI_PHCI_FLAGS_POWER_TRANSITION	0x80	/* pHCI is power transition */
 
-#define	MDI_PHCI_DISABLE_MASK	(~(MDI_PHCI_FLAGS_USER_DISABLE | \
-				MDI_PHCI_FLAGS_D_DISABLE | \
-				MDI_PHCI_FLAGS_D_DISABLE_TRANS))
-#define	MDI_PHCI_IS_READY(ph) \
-	(((ph)->ph_flags &  (MDI_PHCI_DISABLE_MASK)) == 0)
-
-#define	MDI_PHCI_SET_OFFLINE(ph) \
-	    ((ph)->ph_flags |= MDI_PHCI_FLAGS_OFFLINE)
-
-#define	MDI_PHCI_SET_ONLINE(ph) \
-	    ((ph)->ph_flags &= ~MDI_PHCI_FLAGS_OFFLINE)
-
-#define	MDI_PHCI_SET_SUSPEND(ph) \
-	    ((ph)->ph_flags |= MDI_PHCI_FLAGS_SUSPEND)
+#define	MDI_PHCI_DISABLE_MASK						\
+	    (MDI_PHCI_FLAGS_USER_DISABLE | MDI_PHCI_FLAGS_D_DISABLE |	\
+	    MDI_PHCI_FLAGS_D_DISABLE_TRANS)
 
-#define	MDI_PHCI_SET_RESUME(ph) \
-	    ((ph)->ph_flags &= ~MDI_PHCI_FLAGS_SUSPEND)
+#define	MDI_PHCI_IS_READY(ph)						\
+	    (((ph)->ph_flags & MDI_PHCI_DISABLE_MASK) == 0)
 
-#define	MDI_PHCI_IS_OFFLINE(ph) \
+#define	MDI_PHCI_SET_OFFLINE(ph) 					{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags |= MDI_PHCI_FLAGS_OFFLINE;			}
+#define	MDI_PHCI_SET_ONLINE(ph)						{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags &= ~MDI_PHCI_FLAGS_OFFLINE;			}
+#define	MDI_PHCI_IS_OFFLINE(ph)						\
 	    ((ph)->ph_flags & MDI_PHCI_FLAGS_OFFLINE)
 
-#define	MDI_PHCI_IS_SUSPENDED(ph) \
+#define	MDI_PHCI_SET_SUSPEND(ph) 					{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags |= MDI_PHCI_FLAGS_SUSPEND;			}
+#define	MDI_PHCI_SET_RESUME(ph)						{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags &= ~MDI_PHCI_FLAGS_SUSPEND;			}
+#define	MDI_PHCI_IS_SUSPENDED(ph)					\
 	    ((ph)->ph_flags & MDI_PHCI_FLAGS_SUSPEND)
 
-#define	MDI_PHCI_SET_DETACH(ph) \
-	    ((ph)->ph_flags |= MDI_PHCI_FLAGS_DETACH)
-
-#define	MDI_PHCI_SET_ATTACH(ph) \
-	    ((ph)->ph_flags &= ~MDI_PHCI_FLAGS_DETACH)
-
-#define	MDI_PHCI_SET_POWER_DOWN(ph) \
-	    ((ph)->ph_flags |= MDI_PHCI_FLAGS_POWER_DOWN)
-
-#define	MDI_PHCI_SET_POWER_UP(ph) \
-	    ((ph)->ph_flags &= ~MDI_PHCI_FLAGS_POWER_DOWN)
-
-#define	MDI_PHCI_SET_USER_ENABLE(ph) \
-		((ph)->ph_flags &= ~MDI_PHCI_FLAGS_USER_DISABLE)
-
-#define	MDI_PHCI_SET_USER_DISABLE(ph) \
-		((ph)->ph_flags |= MDI_PHCI_FLAGS_USER_DISABLE)
-
-#define	MDI_PHCI_SET_DRV_ENABLE(ph)	\
-		((ph)->ph_flags &= ~MDI_PHCI_FLAGS_D_DISABLE)
-
-#define	MDI_PHCI_SET_DRV_DISABLE(ph)	\
-		((ph)->ph_flags |= MDI_PHCI_FLAGS_D_DISABLE)
-
-#define	MDI_PHCI_SET_DRV_ENABLE_TRANSIENT(ph)	\
-		((ph)->ph_flags &= ~MDI_PHCI_FLAGS_D_DISABLE_TRANS)
-
-#define	MDI_PHCI_SET_DRV_DISABLE_TRANSIENT(ph)	\
-		((ph)->ph_flags |= MDI_PHCI_FLAGS_D_DISABLE_TRANS)
-
-#define	MDI_PHCI_IS_USER_DISABLED(ph) \
-		((ph)->ph_flags & MDI_PHCI_FLAGS_USER_DISABLE)
-
-#define	MDI_PHCI_IS_DRV_DISABLED_TRANSIENT(ph)	\
-		((ph)->ph_flags & MDI_PHCI_FLAGS_D_DISABLE_TRANS)
-
-#define	MDI_PHCI_IS_DRV_DISABLED(ph)	\
-		((ph)->ph_flags & MDI_PHCI_FLAGS_D_DISABLE)
-
-#define	MDI_PHCI_IS_POWERED_DOWN(ph) \
+#define	MDI_PHCI_SET_DETACH(ph)						{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags |= MDI_PHCI_FLAGS_DETACH;			}
+#define	MDI_PHCI_SET_ATTACH(ph)						{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags &= ~MDI_PHCI_FLAGS_DETACH;			}
+
+#define	MDI_PHCI_SET_POWER_DOWN(ph)					{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags |= MDI_PHCI_FLAGS_POWER_DOWN;		}
+#define	MDI_PHCI_SET_POWER_UP(ph)					{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags &= ~MDI_PHCI_FLAGS_POWER_DOWN;		}
+#define	MDI_PHCI_IS_POWERED_DOWN(ph)					\
 	    ((ph)->ph_flags & MDI_PHCI_FLAGS_POWER_DOWN)
 
-#define	MDI_PHCI_SET_POWER_TRANSITION(ph) \
-	    ((ph)->ph_flags |= MDI_PHCI_FLAGS_POWER_TRANSITION)
-
-#define	MDI_PHCI_CLEAR_POWER_TRANSITION(ph) \
-	    ((ph)->ph_flags &= ~MDI_PHCI_FLAGS_POWER_TRANSITION)
-
-#define	MDI_PHCI_IS_POWER_TRANSITION(ph) \
+#define	MDI_PHCI_SET_USER_ENABLE(ph)					{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags &= ~MDI_PHCI_FLAGS_USER_DISABLE;		}
+#define	MDI_PHCI_SET_USER_DISABLE(ph)					{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags |= MDI_PHCI_FLAGS_USER_DISABLE;		}
+#define	MDI_PHCI_IS_USER_DISABLED(ph)					\
+	    ((ph)->ph_flags & MDI_PHCI_FLAGS_USER_DISABLE)
+
+#define	MDI_PHCI_SET_DRV_ENABLE(ph)					{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags &= ~MDI_PHCI_FLAGS_D_DISABLE;		}
+#define	MDI_PHCI_SET_DRV_DISABLE(ph)					{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags |= MDI_PHCI_FLAGS_D_DISABLE;			}
+#define	MDI_PHCI_IS_DRV_DISABLED(ph)					\
+	    ((ph)->ph_flags & MDI_PHCI_FLAGS_D_DISABLE)
+
+#define	MDI_PHCI_SET_DRV_ENABLE_TRANSIENT(ph)				{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags &= ~MDI_PHCI_FLAGS_D_DISABLE_TRANS;		}
+#define	MDI_PHCI_SET_DRV_DISABLE_TRANSIENT(ph)				{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags |= MDI_PHCI_FLAGS_D_DISABLE_TRANS;		}
+#define	MDI_PHCI_IS_DRV_DISABLED_TRANSIENT(ph)				\
+	    ((ph)->ph_flags & MDI_PHCI_FLAGS_D_DISABLE_TRANS)
+
+#define	MDI_PHCI_SET_POWER_TRANSITION(ph)				{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags |= MDI_PHCI_FLAGS_POWER_TRANSITION;		}
+#define	MDI_PHCI_CLEAR_POWER_TRANSITION(ph)				{\
+	    ASSERT(MDI_PHCI_LOCKED(ph));				\
+	    (ph)->ph_flags &= ~MDI_PHCI_FLAGS_POWER_TRANSITION;		}
+#define	MDI_PHCI_IS_POWER_TRANSITION(ph)				\
 	    ((ph)->ph_flags & MDI_PHCI_FLAGS_POWER_TRANSITION)
 
 /*
@@ -425,12 +481,11 @@ typedef struct mdi_phci {
  * Multipath client devices are instantiated as children of corresponding vHCI
  * driver instance. Each client device is uniquely identified by a GUID
  * provided by target device itself.  The parent vHCI device also maintains a
- * hashed list of client devices, protected by the global mdi_mutex.
+ * hashed list of client devices, protected by vh_client_mutex.
  *
  * Typically pHCI devices self-enumerate their child devices using taskq,
  * resulting in multiple paths to the same client device to be enumerated by
- * competing threads.  mdi_mutex is also used to serialize the client device
- * creation.
+ * competing threads.
  *
  * Currently this framework supports two kinds of load-balancing policy
  * configurable through the vHCI driver configuration files.
@@ -442,36 +497,40 @@ typedef struct mdi_phci {
  *
  * OPTIMAL	- Client device has atleast one redundant path.
  * DEGRADED	- No redundant paths (critical).  Failure in the current active
- *                path would result in data access failures.
+ *		  path would result in data access failures.
  * FAILED 	- No paths are available to access this device.
  *
  * Locking order:
  *
- * _NOTE(LOCK_ORDER(mdi_mutex, mdi_client::ct_mutex))
- * _NOTE(LOCK_ORDER(mdi_client::ct_mutex devinfo_tree_lock))
+ * _NOTE(LOCK_ORDER(mdi_mutex, mdi_client::ct_mutex))			XXX
+ * _NOTE(LOCK_ORDER(mdi_client::ct_mutex devinfo_tree_lock))		XXX
  */
 typedef struct mdi_client {
-	kmutex_t		ct_mutex;	/* per-client mutex	*/
+	/* protected by MDI_VHCI_CLIENT_LOCK vh_client_mutex... */
 	struct mdi_client	*ct_hnext;	/* next client		*/
 	struct mdi_client	*ct_hprev;	/* prev client		*/
 	dev_info_t		*ct_dip;	/* client devi handle	*/
 	struct mdi_vhci		*ct_vhci;	/* vHCI back ref	*/
 	char			*ct_drvname;	/* client driver name	*/
 	char			*ct_guid;	/* client guid		*/
-	void			*ct_cprivate;	/* client driver private */
 	client_lb_t		ct_lb;		/* load balancing scheme */
 	client_lb_args_t	*ct_lb_args; 	/* load balancing args */
-	int			ct_flags;	/* Driver op. flags	*/
-	int			ct_state;	/* state information	*/
-	int			ct_failover_flags;	/* Failover args */
-	int			ct_failover_status;	/* last fo status */
-	kcondvar_t		ct_failover_cv;	/* Failover status cv	*/
+
+
+	/* protected by MDI_CLIENT_LOCK ct_mutex... */
+	kmutex_t		ct_mutex;	/* per-client mutex	*/
 	int			ct_path_count;	/* multi path count	*/
 	mdi_pathinfo_t		*ct_path_head;	/* multi path list head	*/
 	mdi_pathinfo_t		*ct_path_tail;	/* multi path list tail	*/
 	mdi_pathinfo_t		*ct_path_last;	/* last path used for i/o */
+	int			ct_state;	/* state information	*/
+	int			ct_flags;	/* Driver op. flags	*/
+	int			ct_failover_flags;	/* Failover args */
+	int			ct_failover_status;	/* last fo status */
+	kcondvar_t		ct_failover_cv;	/* Failover status cv	*/
 	int			ct_unstable;	/* Paths in transient state */
 	kcondvar_t		ct_unstable_cv;	/* Paths in transient state */
+
 	int			ct_power_cnt;	/* Hold count on parent power */
 	kcondvar_t		ct_powerchange_cv;
 					/* Paths in power transient state */
@@ -481,15 +540,20 @@ typedef struct mdi_client {
 					/* held in pre/post unconfig */
 	int			ct_powercnt_reset;
 					/* ct_power_cnt was resetted */
+
+	void			*ct_cprivate;	/* client driver private */
 	void			*ct_vprivate;	/* vHCI driver private	*/
 } mdi_client_t;
 
 /*
  * per-Client device locking definitions
  */
-#define	MDI_CLIENT_LOCK(ct)		mutex_enter(&((ct))->ct_mutex)
-#define	MDI_CLIENT_TRYLOCK(ct)		mutex_tryenter(&((ct))->ct_mutex)
-#define	MDI_CLIENT_UNLOCK(ct)		mutex_exit(&((ct))->ct_mutex)
+#define	MDI_CLIENT_LOCK(ct)		mutex_enter(&(ct)->ct_mutex)
+#define	MDI_CLIENT_TRYLOCK(ct)		mutex_tryenter(&(ct)->ct_mutex)
+#define	MDI_CLIENT_UNLOCK(ct)		mutex_exit(&(ct)->ct_mutex)
+#ifdef	DEBUG
+#define	MDI_CLIENT_LOCKED(ct)		MUTEX_HELD(&(ct)->ct_mutex)
+#endif	/* DEBUG */
 
 /*
  * A Client device is in unstable while one or more paths are in transitional
@@ -519,81 +583,81 @@ typedef struct mdi_client {
 #define	MDI_CLIENT_FLAGS_DEV_NOT_SUPPORTED	0x00000100
 #define	MDI_CLIENT_FLAGS_POWER_TRANSITION	0x00000200
 
-#define	MDI_CLIENT_SET_OFFLINE(ct) \
-	    ((ct)->ct_flags |= MDI_CLIENT_FLAGS_OFFLINE)
-
-#define	MDI_CLIENT_SET_ONLINE(ct) \
-	    ((ct)->ct_flags &= ~MDI_CLIENT_FLAGS_OFFLINE)
-
+#define	MDI_CLIENT_SET_OFFLINE(ct)					{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags |= MDI_CLIENT_FLAGS_OFFLINE;			}
+#define	MDI_CLIENT_SET_ONLINE(ct)					{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags &= ~MDI_CLIENT_FLAGS_OFFLINE;		}
 #define	MDI_CLIENT_IS_OFFLINE(ct) \
 	    ((ct)->ct_flags & MDI_CLIENT_FLAGS_OFFLINE)
 
-#define	MDI_CLIENT_SET_SUSPEND(ct) \
-	    ((ct)->ct_flags |= MDI_CLIENT_FLAGS_SUSPEND)
-
-#define	MDI_CLIENT_SET_RESUME(ct) \
-	    ((ct)->ct_flags &= ~MDI_CLIENT_FLAGS_SUSPEND)
-
+#define	MDI_CLIENT_SET_SUSPEND(ct)					{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags |= MDI_CLIENT_FLAGS_SUSPEND;			}
+#define	MDI_CLIENT_SET_RESUME(ct)					{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags &= ~MDI_CLIENT_FLAGS_SUSPEND;		}
 #define	MDI_CLIENT_IS_SUSPENDED(ct) \
 	    ((ct)->ct_flags & MDI_CLIENT_FLAGS_SUSPEND)
 
-#define	MDI_CLIENT_SET_POWER_DOWN(ct) \
-	    ((ct)->ct_flags |= MDI_CLIENT_FLAGS_POWER_DOWN)
-
-#define	MDI_CLIENT_SET_POWER_UP(ct) \
-	    ((ct)->ct_flags &= ~MDI_CLIENT_FLAGS_POWER_DOWN)
-
+#define	MDI_CLIENT_SET_POWER_DOWN(ct)					{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags |= MDI_CLIENT_FLAGS_POWER_DOWN;		}
+#define	MDI_CLIENT_SET_POWER_UP(ct)					{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags &= ~MDI_CLIENT_FLAGS_POWER_DOWN;		}
 #define	MDI_CLIENT_IS_POWERED_DOWN(ct) \
 	    ((ct)->ct_flags & MDI_CLIENT_FLAGS_POWER_DOWN)
 
-#define	MDI_CLIENT_SET_POWER_TRANSITION(ct) \
-	    ((ct)->ct_flags |= MDI_CLIENT_FLAGS_POWER_TRANSITION)
-
-#define	MDI_CLIENT_CLEAR_POWER_TRANSITION(ct) \
-	    ((ct)->ct_flags &= ~MDI_CLIENT_FLAGS_POWER_TRANSITION)
-
+#define	MDI_CLIENT_SET_POWER_TRANSITION(ct)				{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags |= MDI_CLIENT_FLAGS_POWER_TRANSITION;	}
+#define	MDI_CLIENT_CLEAR_POWER_TRANSITION(ct)				{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags &= ~MDI_CLIENT_FLAGS_POWER_TRANSITION;	}
 #define	MDI_CLIENT_IS_POWER_TRANSITION(ct) \
 	    ((ct)->ct_flags & MDI_CLIENT_FLAGS_POWER_TRANSITION)
 
-#define	MDI_CLIENT_SET_DETACH(ct) \
-	    ((ct)->ct_flags |= MDI_CLIENT_FLAGS_DETACH)
-
-#define	MDI_CLIENT_SET_ATTACH(ct) \
-	    ((ct)->ct_flags &= ~MDI_CLIENT_FLAGS_DETACH)
-
+#define	MDI_CLIENT_SET_DETACH(ct)					{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags |= MDI_CLIENT_FLAGS_DETACH;			}
+#define	MDI_CLIENT_SET_ATTACH(ct)					{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags &= ~MDI_CLIENT_FLAGS_DETACH;			}
 #define	MDI_CLIENT_IS_DETACHED(ct) \
 	    ((ct)->ct_flags & MDI_CLIENT_FLAGS_DETACH)
 
-#define	MDI_CLIENT_SET_FAILOVER_IN_PROGRESS(ct) \
-	    ((ct)->ct_flags |= MDI_CLIENT_FLAGS_FAILOVER)
-
-#define	MDI_CLIENT_CLEAR_FAILOVER_IN_PROGRESS(ct) \
-	    ((ct)->ct_flags &= ~MDI_CLIENT_FLAGS_FAILOVER)
-
+#define	MDI_CLIENT_SET_FAILOVER_IN_PROGRESS(ct)				{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags |= MDI_CLIENT_FLAGS_FAILOVER;		}
+#define	MDI_CLIENT_CLEAR_FAILOVER_IN_PROGRESS(ct)			{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags &= ~MDI_CLIENT_FLAGS_FAILOVER;		}
 #define	MDI_CLIENT_IS_FAILOVER_IN_PROGRESS(ct) \
 	    ((ct)->ct_flags & MDI_CLIENT_FLAGS_FAILOVER)
 
-#define	MDI_CLIENT_SET_REPORT_DEV_NEEDED(ct) \
-	    ((ct)->ct_flags |= MDI_CLIENT_FLAGS_REPORT_DEV)
-
-#define	MDI_CLIENT_CLEAR_REPORT_DEV_NEEDED(ct) \
-	    ((ct)->ct_flags &= ~MDI_CLIENT_FLAGS_REPORT_DEV)
-
+#define	MDI_CLIENT_SET_REPORT_DEV_NEEDED(ct)				{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags |= MDI_CLIENT_FLAGS_REPORT_DEV;		}
+#define	MDI_CLIENT_CLEAR_REPORT_DEV_NEEDED(ct)				{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags &= ~MDI_CLIENT_FLAGS_REPORT_DEV;		}
 #define	MDI_CLIENT_IS_REPORT_DEV_NEEDED(ct) \
 	    ((ct)->ct_flags & MDI_CLIENT_FLAGS_REPORT_DEV)
 
-#define	MDI_CLIENT_SET_PATH_FREE_IN_PROGRESS(ct) \
-	    ((ct)->ct_flags |= MDI_CLIENT_FLAGS_PATH_FREE_IN_PROGRESS)
-
-#define	MDI_CLIENT_CLEAR_PATH_FREE_IN_PROGRESS(ct) \
-	    ((ct)->ct_flags &= ~MDI_CLIENT_FLAGS_PATH_FREE_IN_PROGRESS)
-
+#define	MDI_CLIENT_SET_PATH_FREE_IN_PROGRESS(ct)			{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags |= MDI_CLIENT_FLAGS_PATH_FREE_IN_PROGRESS;	}
+#define	MDI_CLIENT_CLEAR_PATH_FREE_IN_PROGRESS(ct)			{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags &= ~MDI_CLIENT_FLAGS_PATH_FREE_IN_PROGRESS;	}
 #define	MDI_CLIENT_IS_PATH_FREE_IN_PROGRESS(ct) \
 	    ((ct)->ct_flags & MDI_CLIENT_FLAGS_PATH_FREE_IN_PROGRESS)
 
-#define	MDI_CLIENT_SET_DEV_NOT_SUPPORTED(ct) \
-	    ((ct)->ct_flags |= MDI_CLIENT_FLAGS_DEV_NOT_SUPPORTED)
-
+#define	MDI_CLIENT_SET_DEV_NOT_SUPPORTED(ct)				{\
+	    ASSERT(MDI_CLIENT_LOCKED(ct));				\
+	    (ct)->ct_flags |= MDI_CLIENT_FLAGS_DEV_NOT_SUPPORTED;	}
 #define	MDI_CLIENT_IS_DEV_NOT_SUPPORTED(ct) \
 	    ((ct)->ct_flags & MDI_CLIENT_FLAGS_DEV_NOT_SUPPORTED)
 
@@ -627,26 +691,33 @@ typedef struct mdi_client {
  *
  * Locking order:
  *
- * _NOTE(LOCK_ORDER(mdi_phci::ph_mutex mdi_pathinfo::pi_mutex))
- * _NOTE(LOCK_ORDER(mdi_client::ct_mutex mdi_pathinfo::pi_mutex))
- * _NOTE(LOCK_ORDER(mdi_phci::ph_mutex mdi_client::ct_mutex))
- * _NOTE(LOCK_ORDER(devinfo_tree_lock mdi_pathinfo::pi_mutex))
+ * _NOTE(LOCK_ORDER(mdi_phci::ph_mutex mdi_pathinfo::pi_mutex))		XXX
+ * _NOTE(LOCK_ORDER(mdi_client::ct_mutex mdi_pathinfo::pi_mutex))	XXX
+ * _NOTE(LOCK_ORDER(mdi_phci::ph_mutex mdi_client::ct_mutex))		XXX
+ * _NOTE(LOCK_ORDER(devinfo_tree_lock mdi_pathinfo::pi_mutex))		XXX
  *
  * mdi_pathinfo node structure definition
  */
 struct mdi_pathinfo {
+	/* protected by MDI_PHCI_LOCK ph_mutex... */
+	struct mdi_pathinfo	*pi_phci_link;	 /* next path in phci list */
+	mdi_phci_t		*pi_phci;	/* pHCI dev_info node	*/
+
+	/* protected by MDI_CLIENT_LOCK ct_mutex... */
+	struct mdi_pathinfo	*pi_client_link; /* next path in client list */
+	mdi_client_t		*pi_client;	/* client		*/
+
+	/* protected by MDI_VHCI_CLIENT_LOCK vh_client_mutex... */
+	char			*pi_addr;	/* path unit address	*/
+
+	/* protected by MDI_PI_LOCK pi_mutex... */
 	kmutex_t		pi_mutex;	/* per path mutex	*/
 	mdi_pathinfo_state_t	pi_state;	/* path state		*/
 	mdi_pathinfo_state_t	pi_old_state;	/* path state		*/
 	kcondvar_t		pi_state_cv;	/* path state condvar	*/
-	mdi_client_t		*pi_client;	/* client		*/
-	mdi_phci_t		*pi_phci;	/* pHCI dev_info node	*/
-	char			*pi_addr;	/* path unit address	*/
 	nvlist_t		*pi_prop;	/* Properties		*/
 	void			*pi_cprivate;	/* client private info	*/
 	void			*pi_pprivate;	/* phci private info	*/
-	struct mdi_pathinfo	*pi_client_link; /* next path in client list */
-	struct mdi_pathinfo	*pi_phci_link;	 /* next path in phci list */
 	int			pi_ref_cnt;	/* pi reference count	*/
 	kcondvar_t		pi_ref_cv;	/* condition variable	*/
 	struct mdi_pi_kstats	*pi_kstats;	/* aggregate kstats */
@@ -711,10 +782,15 @@ struct pi_errs {
 
 #define	MDI_PI(type)			((struct mdi_pathinfo *)(type))
 
-#define	MDI_PI_LOCK(pip)		mutex_enter(&MDI_PI((pip))->pi_mutex)
-#define	MDI_PI_UNLOCK(pip)		mutex_exit(&MDI_PI((pip))->pi_mutex)
-#define	MDI_PI_HOLD(pip)		(++MDI_PI((pip))->pi_ref_cnt)
-#define	MDI_PI_RELE(pip)		(--MDI_PI((pip))->pi_ref_cnt)
+#define	MDI_PI_LOCK(pip)		mutex_enter(&MDI_PI(pip)->pi_mutex)
+#define	MDI_PI_TRYLOCK(pip)		mutex_tryenter(&MDI_PI(pip)->pi_mutex)
+#define	MDI_PI_UNLOCK(pip)		mutex_exit(&MDI_PI(pip)->pi_mutex)
+#ifdef	DEBUG
+#define	MDI_PI_LOCKED(pip)		MUTEX_HELD(&MDI_PI(pip)->pi_mutex)
+#endif	/* DEBUG */
+
+#define	MDI_PI_HOLD(pip)		(++MDI_PI(pip)->pi_ref_cnt)
+#define	MDI_PI_RELE(pip)		(--MDI_PI(pip)->pi_ref_cnt)
 
 #define	MDI_EXT_STATE_CHANGE		0x10000000
 
@@ -725,161 +801,159 @@ struct pi_errs {
 #define	MDI_AFTER_STATE_CHANGE		0x8
 #define	MDI_SYNC_FLAG			0x10
 
-#define	MDI_PI_STATE(pip) \
-	    (MDI_PI((pip))->pi_state & MDI_PATHINFO_STATE_MASK)
-
-#define	MDI_PI_OLD_STATE(pip) \
-	    (MDI_PI((pip))->pi_old_state & MDI_PATHINFO_STATE_MASK)
-
-#define	MDI_PI_EXT_STATE(pip) \
-		(MDI_PI((pip))->pi_state & MDI_PATHINFO_EXT_STATE_MASK)
-
-#define	MDI_PI_OLD_EXT_STATE(pip) \
-		(MDI_PI((pip))->pi_old_state & MDI_PATHINFO_EXT_STATE_MASK)
-
-#define	MDI_PI_SET_TRANSIENT(pip) \
-	    (MDI_PI(pip)->pi_state |= MDI_PATHINFO_STATE_TRANSIENT)
-
-#define	MDI_PI_CLEAR_TRANSIENT(pip) \
-	    (MDI_PI(pip)->pi_state &= ~MDI_PATHINFO_STATE_TRANSIENT)
-
+#define	MDI_PI_STATE(pip)						\
+	(MDI_PI((pip))->pi_state & MDI_PATHINFO_STATE_MASK)
+#define	MDI_PI_OLD_STATE(pip)						\
+	(MDI_PI((pip))->pi_old_state & MDI_PATHINFO_STATE_MASK)
+
+#define	MDI_PI_EXT_STATE(pip)						\
+	(MDI_PI((pip))->pi_state & MDI_PATHINFO_EXT_STATE_MASK)
+#define	MDI_PI_OLD_EXT_STATE(pip)					\
+	(MDI_PI((pip))->pi_old_state & MDI_PATHINFO_EXT_STATE_MASK)
+
+#define	MDI_PI_SET_TRANSIENT(pip)					{\
+	ASSERT(MDI_PI_LOCKED(pip));					\
+	MDI_PI(pip)->pi_state |= MDI_PATHINFO_STATE_TRANSIENT;		}
+#define	MDI_PI_CLEAR_TRANSIENT(pip)					{\
+	ASSERT(MDI_PI_LOCKED(pip));					\
+	MDI_PI(pip)->pi_state &= ~MDI_PATHINFO_STATE_TRANSIENT;		}
 #define	MDI_PI_IS_TRANSIENT(pip) \
 	(MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_TRANSIENT)
 
-#define	MDI_PI_SET_USER_DISABLE(pip) \
-	(MDI_PI(pip)->pi_state |= MDI_PATHINFO_STATE_USER_DISABLE)
-
-#define	MDI_PI_SET_DRV_DISABLE(pip) \
-	(MDI_PI(pip)->pi_state |= MDI_PATHINFO_STATE_DRV_DISABLE)
-
-#define	MDI_PI_SET_DRV_DISABLE_TRANS(pip) \
-	(MDI_PI(pip)->pi_state |= MDI_PATHINFO_STATE_DRV_DISABLE_TRANSIENT)
-
-#define	MDI_PI_SET_USER_ENABLE(pip) \
-	(MDI_PI(pip)->pi_state &= ~MDI_PATHINFO_STATE_USER_DISABLE)
-
-#define	MDI_PI_SET_DRV_ENABLE(pip) \
-	(MDI_PI(pip)->pi_state &= ~MDI_PATHINFO_STATE_DRV_DISABLE)
-
-#define	MDI_PI_SET_DRV_ENABLE_TRANS(pip) \
-	(MDI_PI(pip)->pi_state &= ~MDI_PATHINFO_STATE_DRV_DISABLE_TRANSIENT)
-
-#define	MDI_PI_IS_USER_DISABLE(pip)	\
+#define	MDI_PI_SET_USER_DISABLE(pip)					{\
+	ASSERT(MDI_PI_LOCKED(pip));					\
+	MDI_PI(pip)->pi_state |= MDI_PATHINFO_STATE_USER_DISABLE;	}
+#define	MDI_PI_SET_DRV_DISABLE(pip)					{\
+	ASSERT(MDI_PI_LOCKED(pip));					\
+	MDI_PI(pip)->pi_state |= MDI_PATHINFO_STATE_DRV_DISABLE;	}
+#define	MDI_PI_SET_DRV_DISABLE_TRANS(pip)				{\
+	ASSERT(MDI_PI_LOCKED(pip));					\
+	MDI_PI(pip)->pi_state |= MDI_PATHINFO_STATE_DRV_DISABLE_TRANSIENT; }
+
+#define	MDI_PI_SET_USER_ENABLE(pip)					{\
+	ASSERT(MDI_PI_LOCKED(pip));					\
+	MDI_PI(pip)->pi_state &= ~MDI_PATHINFO_STATE_USER_DISABLE;	}
+#define	MDI_PI_SET_DRV_ENABLE(pip)					{\
+	ASSERT(MDI_PI_LOCKED(pip));					\
+	MDI_PI(pip)->pi_state &= ~MDI_PATHINFO_STATE_DRV_DISABLE;	}
+#define	MDI_PI_SET_DRV_ENABLE_TRANS(pip)				{\
+	ASSERT(MDI_PI_LOCKED(pip));					\
+	MDI_PI(pip)->pi_state &= ~MDI_PATHINFO_STATE_DRV_DISABLE_TRANSIENT; }
+
+#define	MDI_PI_IS_USER_DISABLE(pip)					\
 	(MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_USER_DISABLE)
-
-#define	MDI_PI_IS_DRV_DISABLE(pip)	\
+#define	MDI_PI_IS_DRV_DISABLE(pip)					\
 	(MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_DRV_DISABLE)
-
-#define	MDI_PI_IS_DRV_DISABLE_TRANSIENT(pip)	\
+#define	MDI_PI_IS_DRV_DISABLE_TRANSIENT(pip)				\
 	(MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_DRV_DISABLE_TRANSIENT)
 
-#define	MDI_PI_IS_DISABLE(pip)	\
-	(MDI_PI_IS_USER_DISABLE(pip) || \
-	MDI_PI_IS_DRV_DISABLE(pip) || \
+#define	MDI_PI_IS_DISABLE(pip)						\
+	(MDI_PI_IS_USER_DISABLE(pip) ||					\
+	MDI_PI_IS_DRV_DISABLE(pip) ||					\
 	MDI_PI_IS_DRV_DISABLE_TRANSIENT(pip))
 
-#define	MDI_PI_IS_INIT(pip) \
-	    ((MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_MASK) == \
+#define	MDI_PI_IS_INIT(pip)						\
+	((MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_MASK) ==		\
 		MDI_PATHINFO_STATE_INIT)
 
-#define	MDI_PI_IS_INITING(pip) \
-	    ((MDI_PI(pip)->pi_state & ~MDI_PATHINFO_EXT_STATE_MASK) == \
+#define	MDI_PI_IS_INITING(pip)						\
+	((MDI_PI(pip)->pi_state & ~MDI_PATHINFO_EXT_STATE_MASK) ==	\
 		(MDI_PATHINFO_STATE_INIT | MDI_PATHINFO_STATE_TRANSIENT))
 
-#define	MDI_PI_SET_INIT(pip) \
-	    (MDI_PI(pip)->pi_state = MDI_PATHINFO_STATE_INIT)
+#define	MDI_PI_SET_INIT(pip)						{\
+	ASSERT(MDI_PI_LOCKED(pip));					\
+	MDI_PI(pip)->pi_state = MDI_PATHINFO_STATE_INIT;		}
 
-#define	MDI_PI_SET_ONLINING(pip) { \
-	uint32_t	ext_state; \
+#define	MDI_PI_SET_ONLINING(pip)					{\
+	uint32_t	ext_state;					\
+	ASSERT(MDI_PI_LOCKED(pip));					\
 	ext_state = MDI_PI(pip)->pi_state & MDI_PATHINFO_EXT_STATE_MASK; \
-	MDI_PI(pip)->pi_old_state = MDI_PI_STATE(pip); \
-	MDI_PI(pip)->pi_state = \
-	(MDI_PATHINFO_STATE_ONLINE | MDI_PATHINFO_STATE_TRANSIENT); \
-	MDI_PI(pip)->pi_state |= ext_state; \
-}
+	MDI_PI(pip)->pi_old_state = MDI_PI_STATE(pip);			\
+	MDI_PI(pip)->pi_state =						\
+	(MDI_PATHINFO_STATE_ONLINE | MDI_PATHINFO_STATE_TRANSIENT);	\
+	MDI_PI(pip)->pi_state |= ext_state;				}
 
-#define	MDI_PI_IS_ONLINING(pip) \
-	((MDI_PI(pip)->pi_state & ~MDI_PATHINFO_EXT_STATE_MASK) == \
+#define	MDI_PI_IS_ONLINING(pip)						\
+	((MDI_PI(pip)->pi_state & ~MDI_PATHINFO_EXT_STATE_MASK) ==	\
 	(MDI_PATHINFO_STATE_ONLINE | MDI_PATHINFO_STATE_TRANSIENT))
 
-#define	MDI_PI_SET_ONLINE(pip) { \
-	uint32_t	ext_state; \
+#define	MDI_PI_SET_ONLINE(pip)						{\
+	uint32_t	ext_state;					\
+	ASSERT(MDI_PI_LOCKED(pip));					\
 	ext_state = MDI_PI(pip)->pi_state & MDI_PATHINFO_EXT_STATE_MASK; \
-	MDI_PI(pip)->pi_state = MDI_PATHINFO_STATE_ONLINE; \
-	MDI_PI(pip)->pi_state |= ext_state; \
-}
+	MDI_PI(pip)->pi_state = MDI_PATHINFO_STATE_ONLINE;		\
+	MDI_PI(pip)->pi_state |= ext_state;				}
 
-
-#define	MDI_PI_IS_ONLINE(pip) \
-	((MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_MASK) == \
+#define	MDI_PI_IS_ONLINE(pip)						\
+	((MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_MASK) ==		\
 	MDI_PATHINFO_STATE_ONLINE)
 
-#define	MDI_PI_SET_OFFLINING(pip) { \
-	uint32_t	ext_state; \
+#define	MDI_PI_SET_OFFLINING(pip)					{\
+	uint32_t	ext_state;					\
+	ASSERT(MDI_PI_LOCKED(pip));					\
 	ext_state = MDI_PI(pip)->pi_state & MDI_PATHINFO_EXT_STATE_MASK; \
-	MDI_PI(pip)->pi_old_state = MDI_PI_STATE(pip); \
-	MDI_PI(pip)->pi_state = \
-	(MDI_PATHINFO_STATE_OFFLINE | MDI_PATHINFO_STATE_TRANSIENT); \
-	MDI_PI(pip)->pi_state |= ext_state; \
-}
-
-#define	MDI_PI_IS_OFFLINING(pip) \
-	    ((MDI_PI(pip)->pi_state & ~MDI_PATHINFO_EXT_STATE_MASK) == \
-	    (MDI_PATHINFO_STATE_OFFLINE | MDI_PATHINFO_STATE_TRANSIENT))
-
-#define	MDI_PI_SET_OFFLINE(pip) { \
-	uint32_t	ext_state; \
+	MDI_PI(pip)->pi_old_state = MDI_PI_STATE(pip);			\
+	MDI_PI(pip)->pi_state =						\
+	(MDI_PATHINFO_STATE_OFFLINE | MDI_PATHINFO_STATE_TRANSIENT);	\
+	MDI_PI(pip)->pi_state |= ext_state;				}
+
+#define	MDI_PI_IS_OFFLINING(pip)					\
+	((MDI_PI(pip)->pi_state & ~MDI_PATHINFO_EXT_STATE_MASK) ==	\
+	(MDI_PATHINFO_STATE_OFFLINE | MDI_PATHINFO_STATE_TRANSIENT))
+
+#define	MDI_PI_SET_OFFLINE(pip)						{\
+	uint32_t	ext_state;					\
+	ASSERT(MDI_PI_LOCKED(pip));					\
 	ext_state = MDI_PI(pip)->pi_state & MDI_PATHINFO_EXT_STATE_MASK; \
-	MDI_PI(pip)->pi_state = MDI_PATHINFO_STATE_OFFLINE; \
-	MDI_PI(pip)->pi_state |= ext_state; \
-}
+	MDI_PI(pip)->pi_state = MDI_PATHINFO_STATE_OFFLINE;		\
+	MDI_PI(pip)->pi_state |= ext_state;				}
 
-#define	MDI_PI_IS_OFFLINE(pip) \
-	    ((MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_MASK) == \
-		MDI_PATHINFO_STATE_OFFLINE)
+#define	MDI_PI_IS_OFFLINE(pip)						\
+	((MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_MASK) ==		\
+	MDI_PATHINFO_STATE_OFFLINE)
 
-#define	MDI_PI_SET_STANDBYING(pip) { \
-	uint32_t	ext_state; \
+#define	MDI_PI_SET_STANDBYING(pip)					{\
+	uint32_t	ext_state;					\
+	ASSERT(MDI_PI_LOCKED(pip));					\
 	ext_state = MDI_PI(pip)->pi_state & MDI_PATHINFO_EXT_STATE_MASK; \
-	MDI_PI(pip)->pi_old_state = MDI_PI_STATE(pip); \
-	MDI_PI(pip)->pi_state = \
-	(MDI_PATHINFO_STATE_STANDBY | MDI_PATHINFO_STATE_TRANSIENT); \
-	MDI_PI(pip)->pi_state |= ext_state; \
-}
-
-#define	MDI_PI_SET_STANDBY(pip) { \
-	uint32_t	ext_state; \
+	MDI_PI(pip)->pi_old_state = MDI_PI_STATE(pip);			\
+	MDI_PI(pip)->pi_state =						\
+	(MDI_PATHINFO_STATE_STANDBY | MDI_PATHINFO_STATE_TRANSIENT);	\
+	MDI_PI(pip)->pi_state |= ext_state;				}
+
+#define	MDI_PI_SET_STANDBY(pip)						{\
+	uint32_t	ext_state;					\
+	ASSERT(MDI_PI_LOCKED(pip));					\
 	ext_state = MDI_PI(pip)->pi_state & MDI_PATHINFO_EXT_STATE_MASK; \
-	MDI_PI(pip)->pi_state = MDI_PATHINFO_STATE_STANDBY; \
-	MDI_PI(pip)->pi_state |= ext_state; \
-}
+	MDI_PI(pip)->pi_state = MDI_PATHINFO_STATE_STANDBY;		\
+	MDI_PI(pip)->pi_state |= ext_state;				}
 
-#define	MDI_PI_IS_STANDBY(pip) \
-	((MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_MASK) == \
+#define	MDI_PI_IS_STANDBY(pip)						\
+	((MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_MASK) ==		\
 	MDI_PATHINFO_STATE_STANDBY)
 
-#define	MDI_PI_SET_FAULTING(pip) { \
-	uint32_t	ext_state; \
+#define	MDI_PI_SET_FAULTING(pip)					{\
+	uint32_t	ext_state;					\
+	ASSERT(MDI_PI_LOCKED(pip));					\
 	ext_state = MDI_PI(pip)->pi_state & MDI_PATHINFO_EXT_STATE_MASK; \
-	MDI_PI(pip)->pi_old_state = MDI_PI_STATE(pip); \
-	MDI_PI(pip)->pi_state = \
-	    (MDI_PATHINFO_STATE_FAULT | MDI_PATHINFO_STATE_TRANSIENT); \
-	MDI_PI(pip)->pi_state |= ext_state; \
-}
-
-#define	MDI_PI_SET_FAULT(pip) { \
-	uint32_t	ext_state; \
+	MDI_PI(pip)->pi_old_state = MDI_PI_STATE(pip);			\
+	MDI_PI(pip)->pi_state =						\
+	    (MDI_PATHINFO_STATE_FAULT | MDI_PATHINFO_STATE_TRANSIENT);	\
+	MDI_PI(pip)->pi_state |= ext_state;				}
+
+#define	MDI_PI_SET_FAULT(pip)						{\
+	uint32_t	ext_state;					\
+	ASSERT(MDI_PI_LOCKED(pip));					\
 	ext_state = MDI_PI(pip)->pi_state & MDI_PATHINFO_EXT_STATE_MASK; \
-	MDI_PI(pip)->pi_state = MDI_PATHINFO_STATE_FAULT; \
-	MDI_PI(pip)->pi_state |= ext_state; \
-}
+	MDI_PI(pip)->pi_state = MDI_PATHINFO_STATE_FAULT;		\
+	MDI_PI(pip)->pi_state |= ext_state;				}
 
-#define	MDI_PI_IS_FAULT(pip) \
-	((MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_MASK) == \
+#define	MDI_PI_IS_FAULT(pip)						\
+	((MDI_PI(pip)->pi_state & MDI_PATHINFO_STATE_MASK) ==		\
 	MDI_PATHINFO_STATE_FAULT)
 
-#define	MDI_PI_IS_SUSPENDED(pip) \
-	    ((MDI_PI(pip))->pi_phci->ph_flags & MDI_PHCI_FLAGS_SUSPEND)
+#define	MDI_PI_IS_SUSPENDED(pip)					\
+	((MDI_PI(pip))->pi_phci->ph_flags & MDI_PHCI_FLAGS_SUSPEND)
 
 /*
  * mdi_vhcache_client, mdi_vhcache_pathinfo, and mdi_vhcache_phci structures
@@ -1062,13 +1136,12 @@ dev_info_t	*mdi_phci_path2devinfo(dev_info_t *, caddr_t);
  * 	MDI_SELECT_USER_DISABLE_PATH: select user disable for failover and
  *		auto_failback
  *
- * The selected paths are returned in a held state (ref_cnt) and caller should
- * release the hold by calling mdi_rele_path() at the end of operation.
+ * The selected paths are returned in an mdi_hold_path() state (pi_ref_cnt),
+ * caller should release the hold by calling mdi_rele_path() at the end of
+ * operation.
  */
 int		mdi_select_path(dev_info_t *, struct buf *, int,
 		    mdi_pathinfo_t *, mdi_pathinfo_t **);
-void		mdi_hold_path(mdi_pathinfo_t *);
-void		mdi_rele_path(mdi_pathinfo_t *);
 int		mdi_set_lb_policy(dev_info_t *, client_lb_t);
 int		mdi_set_lb_region_size(dev_info_t *, int);
 client_lb_t	mdi_get_lb_policy(dev_info_t *);
diff --git a/usr/src/uts/common/sys/sunmdi.h b/usr/src/uts/common/sys/sunmdi.h
index 31897d84bc..e524523703 100644
--- a/usr/src/uts/common/sys/sunmdi.h
+++ b/usr/src/uts/common/sys/sunmdi.h
@@ -133,6 +133,20 @@ int mdi_devi_online(dev_info_t *, uint_t);
 int mdi_devi_offline(dev_info_t *, uint_t);
 
 /*
+ * MDI devinfo locking functions.
+ */
+void mdi_devi_enter(dev_info_t *, int *);
+void mdi_devi_exit_phci(dev_info_t *, int);
+void mdi_devi_enter_phci(dev_info_t *, int *);
+void mdi_devi_exit(dev_info_t *, int);
+
+/*
+ * MDI device support functions.
+ */
+dev_info_t *mdi_devi_get_vdip(dev_info_t *);
+int mdi_devi_pdip_entered(dev_info_t *);
+
+/*
  * MDI component device instance attach/detach notification
  */
 int mdi_pre_attach(dev_info_t *, ddi_attach_cmd_t);
@@ -140,8 +154,6 @@ void mdi_post_attach(dev_info_t *, ddi_attach_cmd_t, int);
 int mdi_pre_detach(dev_info_t *, ddi_detach_cmd_t);
 void mdi_post_detach(dev_info_t *, ddi_detach_cmd_t, int);
 
-int mdi_devi_config_one(dev_info_t *, char *, dev_info_t **, int, clock_t);
-
 /*
  * mdi_pathinfo management functions.
  *
@@ -153,6 +165,9 @@ int mdi_pi_alloc_compatible(dev_info_t *, char *, char *, char *,
 	char **, int, int, mdi_pathinfo_t **);
 int mdi_pi_free(mdi_pathinfo_t *, int);
 
+void mdi_hold_path(mdi_pathinfo_t *);
+void mdi_rele_path(mdi_pathinfo_t *);
+
 /*
  * mdi_pathinfo node state change functions.
  */
diff --git a/usr/src/uts/common/sys/sunndi.h b/usr/src/uts/common/sys/sunndi.h
index 09b8c83bb6..dede7d8f10 100644
--- a/usr/src/uts/common/sys/sunndi.h
+++ b/usr/src/uts/common/sys/sunndi.h
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -281,26 +280,27 @@ ndi_devi_unconfig_driver(dev_info_t *dip, int flags, major_t major);
  *			  first taking it Offline.
  */
 
-#define	NDI_DEVI_REMOVE		0x00001 /* remove after unconfig */
-#define	NDI_ONLINE_ATTACH	0x00002 /* online/attach after config */
-#define	NDI_MDI_FALLBACK	0x00004 /* Leadville to fallback to phci */
-#define	NDI_CONFIG		0x00008 /* recursively config descendants */
-#define	NDI_UNCONFIG		0x00010 /* unconfig to uninitialized state */
-#define	NDI_DEVI_BIND		0x00020 /* transition to DS_BOUND state */
-#define	NDI_DEVI_PERSIST	0x00040 /* do not config offlined nodes */
-#define	NDI_PROMNAME		0x00080 /* name comes from prom */
-#define	NDI_DEVFS_CLEAN		0x01000 /* clean up dv_nodes only, no detach */
-#define	NDI_AUTODETACH		0x02000 /* moduninstall daemon */
-#define	NDI_NO_EVENT		0x04000 /* don't devfs add/remove events */
-#define	NDI_DEVI_DEBUG		0x08000 /* turn on observability */
-#define	NDI_CONFIG_REPROBE	0x10000 /* force a reprobe (deferred attach) */
-#define	NDI_DEVI_ONLINE		0x20000 /* force offlined device to online */
-#define	NDI_DEVI_OFFLINE	0x40000 /* set detached device to offline */
-#define	NDI_POST_EVENT		0x80000 /* Post NDI events before remove */
-#define	NDI_BRANCH_EVENT_OP	0x1000000 /* branch op needing a branch event */
-#define	NDI_NO_EVENT_STATE_CHNG	0x2000000 /* don't change the event state */
-#define	NDI_DRV_CONF_REPROBE	0x4000000 /* reprobe conf-enum'd nodes only */
-#define	NDI_DETACH_DRIVER	0x8000000 /* performing driver_detach */
+#define	NDI_DEVI_REMOVE		0x00000001 /* remove after unconfig */
+#define	NDI_ONLINE_ATTACH	0x00000002 /* online/attach after config */
+#define	NDI_MDI_FALLBACK	0x00000004 /* Leadville to fallback to phci */
+#define	NDI_CONFIG		0x00000008 /* recursively config descendants */
+#define	NDI_UNCONFIG		0x00000010 /* unconfig to uninitialized state */
+#define	NDI_DEVI_BIND		0x00000020 /* transition to DS_BOUND state */
+#define	NDI_DEVI_PERSIST	0x00000040 /* do not config offlined nodes */
+#define	NDI_PROMNAME		0x00000080 /* name comes from prom */
+#define	NDI_DEVFS_CLEAN		0x00001000 /* clean dv_nodes only, no detach */
+#define	NDI_AUTODETACH		0x00002000 /* moduninstall daemon */
+#define	NDI_NO_EVENT		0x00004000 /* don't devfs add/remove events */
+#define	NDI_DEVI_DEBUG		0x00008000 /* turn on observability */
+#define	NDI_CONFIG_REPROBE	0x00010000 /* force reprobe (deferred attach) */
+#define	NDI_DEVI_ONLINE		0x00020000 /* force offlined device to online */
+#define	NDI_DEVI_OFFLINE	0x00040000 /* set detached device to offline */
+#define	NDI_POST_EVENT		0x00080000 /* Post NDI events before remove */
+#define	NDI_BRANCH_EVENT_OP	0x01000000 /* branch op needs branch event */
+#define	NDI_NO_EVENT_STATE_CHNG	0x02000000 /* don't change the event state */
+#define	NDI_DRV_CONF_REPROBE	0x04000000 /* reprobe conf-enum'd nodes only */
+#define	NDI_DETACH_DRIVER	0x08000000 /* performing driver_detach */
+#define	NDI_MTC_OFF		0x10000000 /* disable multi-threading */
 
 /* ndi interface flag values */
 #define	NDI_SLEEP		0x000000
diff --git a/usr/src/uts/sun4u/io/ppm/ppm.c b/usr/src/uts/sun4u/io/ppm/ppm.c
index c628f3466b..8ec6c27c6b 100644
--- a/usr/src/uts/sun4u/io/ppm/ppm.c
+++ b/usr/src/uts/sun4u/io/ppm/ppm.c
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -1606,25 +1605,37 @@ ppm_dev_init(ppm_dev_t *ppmd)
 		ppmd->domp->pwr_cnt++;
 
 	dip = ppmd->dip;
-	dcomps = DEVI(dip)->devi_pm_components;
-	pm_comp = &dcomps[ppmd->cmpt].pmc_comp;
-
-	ppmd->lowest = pm_comp->pmc_lvals[0];
-	ASSERT(ppmd->lowest >= 0);
-	maxi = pm_comp->pmc_numlevels - 1;
-	ppmd->highest = pm_comp->pmc_lvals[maxi];
 
 	/*
-	 * If 66mhz PCI device on pci 66mhz bus supports D2 state
-	 * (config reg PMC bit 10 set), ppm could turn off its bus
-	 * clock once it is at D3hot.
+	 * ppm exists to handle power-manageable devices which require
+	 * special handling on the current platform.  However, a
+	 * driver for such a device may choose not to support power
+	 * management on a particular load/attach.  In this case we
+	 * we create a structure to represent a single-component device
+	 * for which "level" = PM_LEVEL_UNKNOWN and "lowest" = 0
+	 * are effectively constant.
 	 */
-	if (ppmd->domp->dflags & PPMD_PCI66MHZ) {
-		for (i = 0; i < maxi; i++)
-			if (pm_comp->pmc_lvals[i] == PM_LEVEL_D2) {
-				ppmd->flags |= PPMDEV_PCI66_D2;
-				break;
-			}
+	if (PM_GET_PM_INFO(dip)) {
+		dcomps = DEVI(dip)->devi_pm_components;
+		pm_comp = &dcomps[ppmd->cmpt].pmc_comp;
+
+		ppmd->lowest = pm_comp->pmc_lvals[0];
+		ASSERT(ppmd->lowest >= 0);
+		maxi = pm_comp->pmc_numlevels - 1;
+		ppmd->highest = pm_comp->pmc_lvals[maxi];
+
+		/*
+		 * If 66mhz PCI device on pci 66mhz bus supports D2 state
+		 * (config reg PMC bit 10 set), ppm could turn off its bus
+		 * clock once it is at D3hot.
+		 */
+		if (ppmd->domp->dflags & PPMD_PCI66MHZ) {
+			for (i = 0; i < maxi; i++)
+				if (pm_comp->pmc_lvals[i] == PM_LEVEL_D2) {
+					ppmd->flags |= PPMDEV_PCI66_D2;
+					break;
+				}
+		}
 	}
 
 	/*