6583365 add PortChange event

4981612 IBTF support for re-registration handler
6839683 Add support for port change event

20 files changed, 920 insertions, 42 deletions

diff --git a/usr/src/uts/common/io/ib/adapters/hermon/hermon_agents.c b/usr/src/uts/common/io/ib/adapters/hermon/hermon_agents.c
index b125855462..c1b84de476 100644
--- a/usr/src/uts/common/io/ib/adapters/hermon/hermon_agents.c
+++ b/usr/src/uts/common/io/ib/adapters/hermon/hermon_agents.c
@@ -222,6 +222,88 @@ hermon_get_smlid(hermon_state_t *state, uint_t port)
 }
 
 /*
+ * hermon_get_port_change_flags()
+ * 	Helper function to determine the changes in the incoming MAD's portinfo
+ * 	for the Port Change event.
+ */
+static ibt_port_change_t
+hermon_port_change_flags(sm_portinfo_t *curpinfo, sm_portinfo_t *madpinfo)
+{
+	int SMDisabled, ReregSuppd;
+	ibt_port_change_t flags = 0;
+
+	SMDisabled = curpinfo->CapabilityMask & SM_CAP_MASK_IS_SM_DISABLED;
+	ReregSuppd = curpinfo->CapabilityMask & SM_CAP_MASK_IS_CLNT_REREG_SUPPD;
+
+	if (curpinfo->MasterSMLID != madpinfo->MasterSMLID) {
+		flags |= IBT_PORT_CHANGE_SM_LID;
+	}
+	if (curpinfo->MasterSMSL != madpinfo->MasterSMSL) {
+		flags |= IBT_PORT_CHANGE_SM_SL;
+	}
+	if (curpinfo->SubnetTimeOut != madpinfo->SubnetTimeOut) {
+		flags |= IBT_PORT_CHANGE_SUB_TIMEOUT;
+	}
+	if ((madpinfo->CapabilityMask & SM_CAP_MASK_IS_SM_DISABLED)
+	    ^ SMDisabled) {
+		flags |= IBT_PORT_CHANGE_SM_FLAG;
+	}
+	if ((madpinfo->CapabilityMask & SM_CAP_MASK_IS_CLNT_REREG_SUPPD)
+	    ^ ReregSuppd) {
+		flags |= IBT_PORT_CHANGE_REREG;
+	}
+	return (flags);
+}
+
+int
+hermon_set_port_capability(hermon_state_t *state, uint8_t port,
+    sm_portinfo_t *portinfo, ibt_port_change_t flags)
+{
+	uint32_t		capmask;
+	int			status;
+	hermon_hw_set_port_t	set_port;
+
+	bzero(&set_port, sizeof (set_port));
+
+	/* Validate that specified port number is legal */
+	if (!hermon_portnum_is_valid(state, port)) {
+		return (IBT_HCA_PORT_INVALID);
+	}
+
+	/*
+	 * Convert InfiniBand-defined port capability flags to the format
+	 * specified by the IBTF.  Specifically, we modify the capability
+	 * mask based on the specified values.
+	 */
+	capmask = portinfo->CapabilityMask;
+
+	if (flags & IBT_PORT_CHANGE_SM_FLAG)
+		capmask ^= SM_CAP_MASK_IS_SM;
+
+	if (flags & IBT_PORT_CHANGE_REREG)
+		capmask ^= SM_CAP_MASK_IS_CLNT_REREG_SUPPD;
+	set_port.cap_mask = capmask;
+
+	/*
+	 * Use the Hermon SET_PORT command to update the capability mask and
+	 * (possibly) reset the QKey violation counter for the specified port.
+	 * Note: In general, this operation shouldn't fail.  If it does, then
+	 * it is an indication that something (probably in HW, but maybe in
+	 * SW) has gone seriously wrong.
+	 */
+	status = hermon_set_port_cmd_post(state, &set_port, port,
+	    HERMON_SLEEPFLAG_FOR_CONTEXT());
+	if (status != HERMON_CMD_SUCCESS) {
+		HERMON_WARNING(state, "failed to modify port capabilities");
+		cmn_err(CE_CONT, "Hermon: SET_IB (port %02d) command failed: "
+		    "%08x\n", port, status);
+		return (DDI_FAILURE);
+	}
+
+	return (DDI_SUCCESS);
+}
+
+/*
  * hermon_agent_handle_req()
  *    Context: Called with priority of taskQ thread
  */
@@ -230,15 +312,23 @@ hermon_agent_handle_req(void *cb_args)
 {
 	hermon_agent_handler_arg_t	*agent_args;
 	hermon_agent_list_t		*curr;
+	ibc_async_event_t		event;
+	ibt_async_code_t		type, code;
+	sm_portinfo_t			curpinfo, tmadpinfo;
+	sm_portinfo_t			*madpinfop;
 	hermon_state_t			*state;
 	ibmf_handle_t			ibmf_handle;
 	ibmf_msg_t			*msgp;
 	ibmf_msg_bufs_t			*recv_msgbufp;
 	ibmf_msg_bufs_t			*send_msgbufp;
+	ib_mad_hdr_t			*madhdrp;
 	ibmf_retrans_t			retrans;
 	uint_t				port;
 	int				status;
 
+	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*((sm_portinfo_t *)madpinfop)))
+	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(curpinfo))
+	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(tmadpinfo))
 	/* Extract the necessary info from the callback args parameter */
 	agent_args  = (hermon_agent_handler_arg_t *)cb_args;
 	ibmf_handle = agent_args->ahd_ibmfhdl;
@@ -275,6 +365,43 @@ hermon_agent_handle_req(void *cb_args)
 		msgp->im_local_addr.ia_remote_lid =
 		    hermon_get_smlid(state, port);
 	} else {
+		int isSMSet, isReregSuppd;
+		uint_t attr_id, method, mgmt_class;
+
+		madhdrp = recv_msgbufp->im_bufs_mad_hdr;
+		method = madhdrp->R_Method;
+		attr_id = b2h16(madhdrp->AttributeID);
+		mgmt_class = madhdrp->MgmtClass;
+
+		/*
+		 * Is this a Subnet Manager MAD with SET method ? If so
+		 * we will have to get the current portinfo to generate
+		 * events based on what has changed in portinfo.
+		 */
+		isSMSet = (((mgmt_class == MAD_MGMT_CLASS_SUBN_LID_ROUTED)||
+		    (mgmt_class == MAD_MGMT_CLASS_SUBN_DIRECT_ROUTE)) &&
+		    (method == MAD_METHOD_SET));
+
+		/*
+		 * Get the current portinfo to compare with the portinfo
+		 * received in the MAD for PortChange event.
+		 */
+		if (isSMSet && (attr_id == SM_PORTINFO_ATTRID) ||
+		    (attr_id == SM_PKEY_TABLE_ATTRID) ||
+		    (attr_id == SM_GUIDINFO_ATTRID)) {
+			madpinfop = recv_msgbufp->im_bufs_cl_data;
+			tmadpinfo = *madpinfop;
+			HERMON_GETPORTINFO_SWAP(&tmadpinfo);
+			status = hermon_getportinfo_cmd_post(state, port,
+			    HERMON_SLEEPFLAG_FOR_CONTEXT(), &curpinfo);
+			if (status != HERMON_CMD_SUCCESS) {
+				cmn_err(CE_CONT, "Hermon: GetPortInfo "
+				    "(port %02d) command failed: %08x\n", port,
+				    status);
+				goto hermon_agent_handle_req_skip_response;
+			}
+		}
+
 		/*
 		 * Post the command to the firmware (using the MAD_IFC
 		 * command).  Note: We also reuse the command that was passed
@@ -299,6 +426,96 @@ hermon_agent_handle_req(void *cb_args)
 			/* finish cleanup */
 			goto hermon_agent_handle_req_skip_response;
 		}
+
+		if (isSMSet) {
+			event.ev_port_flags = 0;
+			type = 0;
+			event.ev_port = (uint8_t)port;
+
+			switch (attr_id) {
+			case SM_PORTINFO_ATTRID:
+				/*
+				 * This is a SM SET method with portinfo
+				 * attribute. If ClientRereg bit was set in
+				 * the MADs portinfo this is a REREG event
+				 * (see section 14.4.11 in IB Spec 1.2.1). Else
+				 * compare the current (before MAD_IFC command)
+				 * portinfo with the portinfo in the MAD and
+				 * signal PORT_CHANGE event with the proper
+				 * ev_port_flags.
+				 *
+				 */
+				isReregSuppd = curpinfo.CapabilityMask &
+				    SM_CAP_MASK_IS_CLNT_REREG_SUPPD;
+
+				madpinfop = recv_msgbufp->im_bufs_cl_data;
+				if (tmadpinfo.ClientRereg && isReregSuppd) {
+					type |= IBT_CLNT_REREG_EVENT;
+				}
+
+				type |= IBT_PORT_CHANGE_EVENT;
+				event.ev_port_flags = hermon_port_change_flags(
+				    &curpinfo, &tmadpinfo);
+				if (event.ev_port_flags &
+				    (IBT_PORT_CHANGE_REREG |
+				    IBT_PORT_CHANGE_SM_FLAG)) {
+					if (hermon_set_port_capability(state,
+					    port, &curpinfo,
+					    event.ev_port_flags)
+					    != DDI_SUCCESS) {
+						cmn_err(CE_CONT, "HERMON: Port "
+						    "%d capability reset "
+						    "failed\n", port);
+					}
+				}
+
+				/*
+				 * If we have a SMLID change event but
+				 * capability mask doesn't have Rereg support
+				 * bit set, we have to do the Rereg part too.
+				 */
+				if ((event.ev_port_flags &
+				    IBT_PORT_CHANGE_SM_LID) && !isReregSuppd)
+					type |= IBT_CLNT_REREG_EVENT;
+				break;
+			case SM_PKEY_TABLE_ATTRID:
+				type |= IBT_PORT_CHANGE_EVENT;
+				event.ev_port_flags = IBT_PORT_CHANGE_PKEY;
+				break;
+			case SM_GUIDINFO_ATTRID:
+				type |= IBT_PORT_CHANGE_EVENT;
+				event.ev_port_flags = IBT_PORT_CHANGE_SGID;
+				break;
+			default:
+				break;
+
+			}
+
+			/*
+			 * NOTE: here we call ibc_async_handler directly without
+			 * using the HERMON_DO_IBTF_ASYNC_CALLB, since hermon
+			 * can not be unloaded till ibmf_unregiter is done and
+			 * this thread (hs_taskq_agents) will be destroyed
+			 * before ibmf_uregister is called.
+			 *
+			 * The hermon event queue based hs_in_evcallb flag
+			 * assumes that we will pick one event after another
+			 * and dispatch them sequentially. If we use
+			 * HERMON_DO_IBTF_ASYNC_CALLB, we will break this
+			 * assumption make hs_in_evcallb inconsistent.
+			 */
+			while (type != 0) {
+				if (type & IBT_PORT_CHANGE_EVENT) {
+					code = IBT_PORT_CHANGE_EVENT;
+					type &= ~IBT_PORT_CHANGE_EVENT;
+				} else {
+					code = IBT_CLNT_REREG_EVENT;
+					type = 0;
+				}
+				ibc_async_handler(state->hs_ibtfpriv, code,
+				    &event);
+			}
+		}
 	}
 
 	/*
diff --git a/usr/src/uts/common/io/ib/adapters/hermon/hermon_cfg.c b/usr/src/uts/common/io/ib/adapters/hermon/hermon_cfg.c
index 88ce58be61..ca3a370485 100644
--- a/usr/src/uts/common/io/ib/adapters/hermon/hermon_cfg.c
+++ b/usr/src/uts/common/io/ib/adapters/hermon/hermon_cfg.c
@@ -134,7 +134,7 @@ uint32_t hermon_max_vlcap		= HERMON_MAX_VLCAP;
  * Whether or not to use the built-in (i.e. in firmware) agents for QP0 and
  * QP1, respectively.
  */
-uint32_t hermon_qp0_agents_in_fw	= 1;
+uint32_t hermon_qp0_agents_in_fw	= 0;
 uint32_t hermon_qp1_agents_in_fw	= 0;
 
 /*
diff --git a/usr/src/uts/common/io/ib/adapters/hermon/hermon_cmd.c b/usr/src/uts/common/io/ib/adapters/hermon/hermon_cmd.c
index c505362b41..6e10d8bbb7 100644
--- a/usr/src/uts/common/io/ib/adapters/hermon/hermon_cmd.c
+++ b/usr/src/uts/common/io/ib/adapters/hermon/hermon_cmd.c
@@ -2038,6 +2038,7 @@ hermon_getportinfo_cmd_post(hermon_state_t *state, uint_t port,
 	 */
 	bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
 	    HERMON_CMD_MADDATA_OFFSET), portinfo, size);
+	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*portinfo))
 	HERMON_GETPORTINFO_SWAP(portinfo);
 
 getportinfo_fail:
@@ -2212,6 +2213,7 @@ hermon_getnodeinfo_cmd_post(hermon_state_t *state, uint_t sleepflag,
 	 */
 	bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
 	    HERMON_CMD_MADDATA_OFFSET), nodeinfo, size);
+	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*nodeinfo))
 	HERMON_GETNODEINFO_SWAP(nodeinfo);
 
 getnodeinfo_fail:
@@ -2350,6 +2352,7 @@ hermon_getguidinfo_cmd_post(hermon_state_t *state, uint_t port,
 	 */
 	bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
 	    HERMON_CMD_MADDATA_OFFSET), guidinfo, size);
+	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*guidinfo))
 	HERMON_GETGUIDINFO_SWAP(guidinfo);
 
 getguidinfo_fail:
@@ -2421,6 +2424,7 @@ hermon_getpkeytable_cmd_post(hermon_state_t *state, uint_t port,
 	 */
 	bcopy((void *)((uintptr_t)mbox_info.mbi_out->mb_addr +
 	    HERMON_CMD_MADDATA_OFFSET), pkeytable, size);
+	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pkeytable))
 	HERMON_GETPKEYTABLE_SWAP(pkeytable);
 
 getpkeytable_fail:
diff --git a/usr/src/uts/common/io/ib/adapters/hermon/hermon_misc.c b/usr/src/uts/common/io/ib/adapters/hermon/hermon_misc.c
index 7a51d5c5dd..e26c0071ae 100644
--- a/usr/src/uts/common/io/ib/adapters/hermon/hermon_misc.c
+++ b/usr/src/uts/common/io/ib/adapters/hermon/hermon_misc.c
@@ -1960,6 +1960,8 @@ hermon_port_query(hermon_state_t *state, uint_t port, ibt_hca_portinfo_t *pi)
 		pi->p_capabilities |= IBT_PORT_CAP_DM;
 	if (portinfo.CapabilityMask & SM_CAP_MASK_IS_VM_SUPPD)
 		pi->p_capabilities |= IBT_PORT_CAP_VENDOR;
+	if (portinfo.CapabilityMask & SM_CAP_MASK_IS_CLNT_REREG_SUPPD)
+		pi->p_capabilities |= IBT_PORT_CAP_CLNT_REREG;
 
 	/*
 	 * Fill in the SGID table.  Since the only access to the Hermon
diff --git a/usr/src/uts/common/io/ib/clients/ibd/ibd.c b/usr/src/uts/common/io/ib/clients/ibd/ibd.c
index 7c0cf75fb5..4725215681 100644
--- a/usr/src/uts/common/io/ib/clients/ibd/ibd.c
+++ b/usr/src/uts/common/io/ib/clients/ibd/ibd.c
@@ -1962,7 +1962,8 @@ ibd_link_mod(ibd_state_t *state, ibt_async_code_t code)
 		return;
 	}
 
-	if (code == IBT_EVENT_PORT_UP) {
+	if ((code == IBT_EVENT_PORT_UP) || (code == IBT_CLNT_REREG_EVENT) ||
+	    (code == IBT_PORT_CHANGE_EVENT)) {
 		uint8_t itreply;
 		boolean_t badup = B_FALSE;
 
@@ -2018,14 +2019,20 @@ ibd_link_mod(ibd_state_t *state, ibt_async_code_t code)
 		    SM_INIT_TYPE_PRESERVE_PRESENCE_REPLY)
 			opcode = IBD_LINK_UP;
 
-		if (badup)
-			code = IBT_ERROR_PORT_DOWN;
 		ibt_free_portinfo(port_infop, port_infosz);
+
+		if (badup) {
+			code = IBT_ERROR_PORT_DOWN;
+		} else if (code == IBT_PORT_CHANGE_EVENT) {
+			mutex_exit(&state->id_link_mutex);
+			return;
+		}
 	}
 
 	if (!ibd_async_safe(state)) {
-		state->id_link_state = ((code == IBT_EVENT_PORT_UP) ?
-		    LINK_STATE_UP : LINK_STATE_DOWN);
+		state->id_link_state = (((code == IBT_EVENT_PORT_UP) ||
+		    (code == IBT_CLNT_REREG_EVENT)) ?  LINK_STATE_UP :
+		    LINK_STATE_DOWN);
 		mutex_exit(&state->id_link_mutex);
 		return;
 	}
@@ -2058,7 +2065,26 @@ ibd_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
 	case IBT_ERROR_CQ:
 		ibd_print_warn(state, "completion queue error");
 		break;
+	case IBT_PORT_CHANGE_EVENT:
+		/*
+		 * Events will be delivered to all instances that have
+		 * done ibt_open_hca() but not yet done ibt_close_hca().
+		 * Only need to do work for our port; IBTF will deliver
+		 * events for other ports on the hca we have ibt_open_hca'ed
+		 * too. Note that ibd_drv_init() initializes id_port before
+		 * doing ibt_open_hca().
+		 */
+		ASSERT(state->id_hca_hdl == hca_hdl);
+		if (state->id_port != event->ev_port)
+			break;
+
+		if ((event->ev_port_flags & IBT_PORT_CHANGE_PKEY) ==
+		    IBT_PORT_CHANGE_PKEY) {
+			ibd_link_mod(state, code);
+		}
+		break;
 	case IBT_ERROR_PORT_DOWN:
+	case IBT_CLNT_REREG_EVENT:
 	case IBT_EVENT_PORT_UP:
 		/*
 		 * Events will be delivered to all instances that have
diff --git a/usr/src/uts/common/io/ib/ibtl/ibtl_handlers.c b/usr/src/uts/common/io/ib/ibtl/ibtl_handlers.c
index 1592d06e01..e7ee68b5ba 100644
--- a/usr/src/uts/common/io/ib/ibtl/ibtl_handlers.c
+++ b/usr/src/uts/common/io/ib/ibtl/ibtl_handlers.c
@@ -148,13 +148,28 @@ static int ibtl_cq_thread_exit = 0;	/* set if/when thread(s) should exit */
 
 /* value used to tell IBTL threads to exit */
 #define	IBTL_THREAD_EXIT 0x1b7fdead	/* IBTF DEAD */
+/* Cisco Topspin Vendor ID for Rereg hack */
+#define	IBT_VENDOR_CISCO 0x05ad
 
 int ibtl_eec_not_supported = 1;
 
 char *ibtl_last_client_name;	/* may help debugging */
+typedef ibt_status_t (*ibtl_node_info_cb_t)(ib_guid_t, uint8_t, ib_lid_t,
+    ibt_node_info_t *);
+
+ibtl_node_info_cb_t ibtl_node_info_cb;
 
 _NOTE(LOCK_ORDER(ibtl_clnt_list_mutex ibtl_async_mutex))
 
+void
+ibtl_cm_set_node_info_cb(ibt_status_t (*node_info_cb)(ib_guid_t, uint8_t,
+    ib_lid_t, ibt_node_info_t *))
+{
+	mutex_enter(&ibtl_clnt_list_mutex);
+	ibtl_node_info_cb = node_info_cb;
+	mutex_exit(&ibtl_clnt_list_mutex);
+}
+
 /*
  * ibc_async_handler()
  *
@@ -180,6 +195,8 @@ ibc_async_handler(ibc_clnt_hdl_t hca_devp, ibt_async_code_t code,
 	ibtl_eec_t	*ibtl_eec;
 	uint8_t		port_minus1;
 
+	ibtl_async_port_event_t	*portp;
+
 	IBTF_DPRINTF_L2(ibtf_handlers, "ibc_async_handler(%p, 0x%x, %p)",
 	    hca_devp, code, event_p);
 
@@ -309,9 +326,10 @@ ibc_async_handler(ibc_clnt_hdl_t hca_devp, ibt_async_code_t code,
 		/* FALLTHROUGH */
 
 	case IBT_EVENT_PORT_UP:
+	case IBT_PORT_CHANGE_EVENT:
+	case IBT_CLNT_REREG_EVENT:
 	case IBT_ERROR_PORT_DOWN:
-		if ((code == IBT_EVENT_PORT_UP) ||
-		    (code == IBT_ERROR_PORT_DOWN)) {
+		if ((code & IBT_PORT_EVENTS) != 0) {
 			if ((port_minus1 = event_p->ev_port - 1) >=
 			    hca_devp->hd_hca_attr->hca_nports) {
 				IBTF_DPRINTF_L2(ibtf_handlers,
@@ -319,9 +337,43 @@ ibc_async_handler(ibc_clnt_hdl_t hca_devp, ibt_async_code_t code,
 				    event_p->ev_port);
 				break;
 			}
-			hca_devp->hd_async_port[port_minus1] =
-			    ((code == IBT_EVENT_PORT_UP) ? IBTL_HCA_PORT_UP :
-			    IBTL_HCA_PORT_DOWN) | IBTL_HCA_PORT_CHANGED;
+			portp = &hca_devp->hd_async_port[port_minus1];
+			if (code == IBT_EVENT_PORT_UP) {
+				/*
+				 * The port is just coming UP we can't have any
+				 * valid older events.
+				 */
+				portp->status = IBTL_HCA_PORT_UP;
+			} else if (code == IBT_ERROR_PORT_DOWN) {
+				/*
+				 * The port is going DOWN older events don't
+				 * count.
+				 */
+				portp->status = IBTL_HCA_PORT_DOWN;
+			} else if (code == IBT_PORT_CHANGE_EVENT) {
+				/*
+				 * For port UP and DOWN events only the latest
+				 * event counts. If we get a UP after DOWN it
+				 * is sufficient to send just UP and vice versa.
+				 * In the case of port CHANGE event it is valid
+				 * only when the port is UP already but if we
+				 * receive it after UP but before UP is
+				 * delivered we still need to deliver CHANGE
+				 * after we deliver UP event.
+				 *
+				 * We will not get a CHANGE event when the port
+				 * is down or DOWN event is pending.
+				 */
+				portp->flags |= event_p->ev_port_flags;
+				portp->status |= IBTL_HCA_PORT_CHG;
+			} else if (code == IBT_CLNT_REREG_EVENT) {
+				/*
+				 * SM has requested a re-register of
+				 * subscription to SM events notification.
+				 */
+				portp->status |= IBTL_HCA_PORT_ASYNC_CLNT_REREG;
+			}
+
 			hca_devp->hd_async_codes |= code;
 		}
 
@@ -429,6 +481,73 @@ ibtl_do_mgr_async_task(void *arg)
 }
 
 static void
+ibt_cisco_embedded_sm_rereg_fix(void *arg)
+{
+	struct ibtl_mgr_s *mgrp = arg;
+	ibtl_hca_devinfo_t *hca_devp;
+	ibt_node_info_t node_info;
+	ibt_status_t ibt_status;
+	ibtl_async_port_event_t *portp;
+	ib_lid_t sm_lid;
+	ib_guid_t hca_guid;
+	ibt_async_event_t *event_p;
+	ibt_hca_portinfo_t *pinfop;
+	uint8_t	port;
+
+	hca_devp = mgrp->mgr_hca_devp;
+
+	mutex_enter(&ibtl_clnt_list_mutex);
+	event_p = &hca_devp->hd_async_event;
+	port = event_p->ev_port;
+	portp = &hca_devp->hd_async_port[port - 1];
+	pinfop = &hca_devp->hd_portinfop[port - 1];
+	sm_lid = pinfop->p_sm_lid;
+	hca_guid = hca_devp->hd_hca_attr->hca_node_guid;
+	mutex_exit(&ibtl_clnt_list_mutex);
+
+	ibt_status = ((ibtl_node_info_cb_t)mgrp->mgr_async_handler)(hca_guid,
+	    port, sm_lid, &node_info);
+	if (ibt_status == IBT_SUCCESS) {
+		if ((node_info.n_vendor_id == IBT_VENDOR_CISCO) &&
+		    (node_info.n_node_type == IBT_NODE_TYPE_SWITCH)) {
+			mutex_enter(&ibtl_async_mutex);
+			portp->status |= IBTL_HCA_PORT_ASYNC_CLNT_REREG;
+			hca_devp->hd_async_codes |= IBT_CLNT_REREG_EVENT;
+			mutex_exit(&ibtl_async_mutex);
+		}
+	}
+	kmem_free(mgrp, sizeof (*mgrp));
+
+	mutex_enter(&ibtl_clnt_list_mutex);
+	if (--hca_devp->hd_async_task_cnt == 0)
+		cv_signal(&hca_devp->hd_async_task_cv);
+	mutex_exit(&ibtl_clnt_list_mutex);
+}
+
+static void
+ibtl_cm_get_node_info(ibtl_hca_devinfo_t *hca_devp,
+    ibt_async_handler_t async_handler)
+{
+	struct ibtl_mgr_s *mgrp;
+
+	if (async_handler == NULL)
+		return;
+
+	_NOTE(NO_COMPETING_THREADS_NOW)
+	mgrp = kmem_alloc(sizeof (*mgrp), KM_SLEEP);
+	mgrp->mgr_hca_devp = hca_devp;
+	mgrp->mgr_async_handler = async_handler;
+	mgrp->mgr_clnt_private = NULL;
+	hca_devp->hd_async_task_cnt++;
+
+	(void) taskq_dispatch(ibtl_async_taskq,
+	    ibt_cisco_embedded_sm_rereg_fix, mgrp, TQ_SLEEP);
+#ifndef lint
+	_NOTE(COMPETING_THREADS_NOW)
+#endif
+}
+
+static void
 ibtl_tell_mgr(ibtl_hca_devinfo_t *hca_devp, ibt_async_handler_t async_handler,
     void *clnt_private)
 {
@@ -503,11 +622,12 @@ static void
 ibtl_do_hca_asyncs(ibtl_hca_devinfo_t *hca_devp)
 {
 	ibtl_hca_t			*ibt_hca;
+	ibt_async_event_t		*eventp;
 	ibt_async_code_t		code;
 	ibtl_async_port_status_t  	temp;
 	uint8_t				nports;
 	uint8_t				port_minus1;
-	ibtl_async_port_status_t	*portp;
+	ibtl_async_port_event_t		*portp;
 
 	mutex_exit(&ibtl_async_mutex);
 
@@ -527,11 +647,19 @@ ibtl_do_hca_asyncs(ibtl_hca_devinfo_t *hca_devp)
 			code = IBT_ERROR_LOCAL_CATASTROPHIC;
 			hca_devp->hd_async_event.ev_fma_ena =
 			    hca_devp->hd_fma_ena;
-		} else if (code & IBT_ERROR_PORT_DOWN)
+		} else if (code & IBT_ERROR_PORT_DOWN) {
 			code = IBT_ERROR_PORT_DOWN;
-		else if (code & IBT_EVENT_PORT_UP)
+			temp = IBTL_HCA_PORT_DOWN;
+		} else if (code & IBT_EVENT_PORT_UP) {
 			code = IBT_EVENT_PORT_UP;
-		else {
+			temp = IBTL_HCA_PORT_UP;
+		} else if (code & IBT_PORT_CHANGE_EVENT) {
+			code = IBT_PORT_CHANGE_EVENT;
+			temp = IBTL_HCA_PORT_CHG;
+		} else if (code & IBT_CLNT_REREG_EVENT) {
+			code = IBT_CLNT_REREG_EVENT;
+			temp = IBTL_HCA_PORT_ASYNC_CLNT_REREG;
+		} else {
 			hca_devp->hd_async_codes = 0;
 			code = 0;
 		}
@@ -542,17 +670,17 @@ ibtl_do_hca_asyncs(ibtl_hca_devinfo_t *hca_devp)
 		}
 		hca_devp->hd_async_codes &= ~code;
 
-		if ((code == IBT_EVENT_PORT_UP) ||
-		    (code == IBT_ERROR_PORT_DOWN)) {
-			/* PORT_UP or PORT_DOWN */
+		/* PORT_UP, PORT_CHANGE, PORT_DOWN or ASYNC_REREG */
+		if ((code & IBT_PORT_EVENTS) != 0) {
 			portp = hca_devp->hd_async_port;
 			nports = hca_devp->hd_hca_attr->hca_nports;
 			for (port_minus1 = 0; port_minus1 < nports;
 			    port_minus1++) {
-				temp = ((code == IBT_EVENT_PORT_UP) ?
-				    IBTL_HCA_PORT_UP : IBTL_HCA_PORT_DOWN) |
-				    IBTL_HCA_PORT_CHANGED;
-				if (portp[port_minus1] == temp)
+				/*
+				 * Matching event in this port, let's go handle
+				 * it.
+				 */
+				if ((portp[port_minus1].status & temp) != 0)
 					break;
 			}
 			if (port_minus1 >= nports) {
@@ -564,13 +692,24 @@ ibtl_do_hca_asyncs(ibtl_hca_devinfo_t *hca_devp)
 			/* mark it to check for other ports after we're done */
 			hca_devp->hd_async_codes |= code;
 
+			/*
+			 * Copy the event information into hca_devp and clear
+			 * event information from the per port data.
+			 */
 			hca_devp->hd_async_event.ev_port = port_minus1 + 1;
-			hca_devp->hd_async_port[port_minus1] &=
-			    ~IBTL_HCA_PORT_CHANGED;
+			if (temp == IBTL_HCA_PORT_CHG) {
+				hca_devp->hd_async_event.ev_port_flags =
+				    hca_devp->hd_async_port[port_minus1].flags;
+				hca_devp->hd_async_port[port_minus1].flags = 0;
+			}
+			hca_devp->hd_async_port[port_minus1].status &= ~temp;
 
 			mutex_exit(&ibtl_async_mutex);
 			ibtl_reinit_hca_portinfo(hca_devp, port_minus1 + 1);
 			mutex_enter(&ibtl_async_mutex);
+			eventp = &hca_devp->hd_async_event;
+			eventp->ev_hca_guid =
+			    hca_devp->hd_hca_attr->hca_node_guid;
 		}
 
 		hca_devp->hd_async_code = code;
@@ -592,6 +731,21 @@ ibtl_do_hca_asyncs(ibtl_hca_devinfo_t *hca_devp)
 			cv_wait(&hca_devp->hd_async_task_cv,
 			    &ibtl_clnt_list_mutex);
 
+		/*
+		 * Hack Alert:
+		 * The ibmf handler would have updated the Master SM LID if it
+		 * was SM LID change event. Now lets check if the new Master SM
+		 * is a Embedded Cisco Topspin SM.
+		 */
+		if ((code == IBT_PORT_CHANGE_EVENT) &&
+		    eventp->ev_port_flags & IBT_PORT_CHANGE_SM_LID)
+			ibtl_cm_get_node_info(hca_devp,
+			    (ibt_async_handler_t)ibtl_node_info_cb);
+		/* wait for node info task to complete */
+		while (hca_devp->hd_async_task_cnt != 0)
+			cv_wait(&hca_devp->hd_async_task_cv,
+			    &ibtl_clnt_list_mutex);
+
 		if (ibtl_dm_async_handler)
 			ibtl_tell_mgr(hca_devp, ibtl_dm_async_handler,
 			    ibtl_dm_clnt_private);
@@ -1921,3 +2075,10 @@ ibtl_thread_fini(void)
 	cv_destroy(&ibtl_async_cv);
 	cv_destroy(&ibtl_clnt_cv);
 }
+
+/* ARGSUSED */
+ibt_status_t ibtl_dummy_node_info_cb(ib_guid_t hca_guid, uint8_t port,
+    ib_lid_t lid, ibt_node_info_t *node_info)
+{
+	return (IBT_SUCCESS);
+}
diff --git a/usr/src/uts/common/io/ib/ibtl/ibtl_impl.c b/usr/src/uts/common/io/ib/ibtl/ibtl_impl.c
index 8f2d68781b..b83ea820cc 100644
--- a/usr/src/uts/common/io/ib/ibtl/ibtl_impl.c
+++ b/usr/src/uts/common/io/ib/ibtl/ibtl_impl.c
@@ -577,7 +577,7 @@ ibc_attach(ibc_clnt_hdl_t *ibc_hdl_p, ibc_hca_info_t *info_p)
 
 	/* Allocate the memory for per-client info structure */
 	hca_devp = kmem_zalloc(sizeof (ibtl_hca_devinfo_t) +
-	    (nports - 1) * sizeof (ibtl_async_port_status_t), KM_SLEEP);
+	    (nports - 1) * sizeof (ibtl_async_port_event_t), KM_SLEEP);
 
 	mutex_enter(&ibtl_clnt_list_mutex);
 
@@ -593,7 +593,7 @@ ibc_attach(ibc_clnt_hdl_t *ibc_hdl_p, ibc_hca_info_t *info_p)
 		IBTF_DPRINTF_L1(ibtf, "ibc_attach: call to ibc_query_hca_ports "
 		    "failed: status = %d", status);
 		kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) +
-		    (nports - 1) * sizeof (ibtl_async_port_status_t));
+		    (nports - 1) * sizeof (ibtl_async_port_event_t));
 		return (IBC_FAILURE);
 	}
 
@@ -602,7 +602,7 @@ ibc_attach(ibc_clnt_hdl_t *ibc_hdl_p, ibc_hca_info_t *info_p)
 		mutex_exit(&ibtl_clnt_list_mutex);
 		IBTF_DPRINTF_L1(ibtf, "ibc_attach: MPxIO register failed");
 		kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) +
-		    (nports - 1) * sizeof (ibtl_async_port_status_t));
+		    (nports - 1) * sizeof (ibtl_async_port_event_t));
 		return (IBC_FAILURE);
 	}
 
@@ -813,7 +813,7 @@ ibc_detach(ibc_clnt_hdl_t hca_devp)
 	/* Free up the memory of per-client info struct */
 	kmem_free(hca_devp, sizeof (ibtl_hca_devinfo_t) +
 	    (hca_devp->hd_hca_attr->hca_nports - 1) *
-	    sizeof (ibtl_async_port_status_t));
+	    sizeof (ibtl_async_port_event_t));
 	ibtl_clear_ibhw_status();
 }
 
diff --git a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_impl.c b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_impl.c
index ca564dc699..ace311ae33 100644
--- a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_impl.c
+++ b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_impl.c
@@ -626,6 +626,11 @@ ibcm_init(void)
 	ibcm_finit_state = IBCM_FINIT_IDLE;
 
 	ibcm_path_cache_init();
+	/*
+	 * This callback will be used by IBTL to get the Node record for a
+	 * given LID via the speccified HCA and port.
+	 */
+	ibtl_cm_set_node_info_cb(ibcm_ibtl_node_info);
 
 	/* Unblock any waiting HCA DR asyncs in CM */
 	mutex_exit(&ibcm_global_hca_lock);
@@ -735,6 +740,7 @@ ibcm_fini(void)
 
 	ibcm_stop_timeout_thread();
 
+	ibtl_cm_set_node_info_cb(NULL);
 	/*
 	 * Detach from IBTL. Waits until all pending asyncs are complete.
 	 * Above cv_broadcast wakes up any waiting hca attach/detach asyncs
@@ -2413,6 +2419,11 @@ ibcm_async_handler(void *clnt_hdl, ibt_hca_hdl_t hca_hdl,
 	}
 
 	switch (code) {
+	case IBT_PORT_CHANGE_EVENT:
+		if ((eventp->ev_port_flags & IBT_PORT_CHANGE_SM_LID) == 0)
+			break;
+	/* FALLTHROUGH */
+	case IBT_CLNT_REREG_EVENT:
 	case IBT_EVENT_PORT_UP:
 		mutex_exit(&ibcm_global_hca_lock);
 		pup = kmem_alloc(sizeof (ibcm_port_up_t), KM_SLEEP);
diff --git a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_ti.c b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_ti.c
index 5994112340..bdb1c0c515 100644
--- a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_ti.c
+++ b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_ti.c
@@ -51,7 +51,7 @@ static ibt_status_t	ibcm_process_rc_recycle_ret(void *recycle_arg);
 static ibt_status_t	ibcm_process_join_mcg(void *taskq_arg);
 static void		ibcm_process_async_join_mcg(void *tq_arg);
 
-static ibt_status_t ibcm_get_node_rec(ibmf_saa_handle_t, sa_node_record_t *,
+ibt_status_t ibcm_get_node_rec(ibmf_saa_handle_t, sa_node_record_t *,
     uint64_t c_mask, void *, size_t *);
 
 static ibt_status_t ibcm_close_rc_channel(ibt_channel_hdl_t channel,
@@ -5641,7 +5641,7 @@ gid_to_ni_exit:
 }
 
 
-static ibt_status_t
+ibt_status_t
 ibcm_get_node_rec(ibmf_saa_handle_t saa_handle, sa_node_record_t *nr_req,
     uint64_t component_mask, void *result_p, size_t *len)
 {
diff --git a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_utils.c b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_utils.c
index 173a460840..9a4d16813a 100644
--- a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_utils.c
+++ b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_utils.c
@@ -50,6 +50,8 @@ static void		ibcm_init_conn_trace(ibcm_state_data_t *statep);
 static void		ibcm_fini_conn_trace(ibcm_state_data_t *statep);
 static void		ibcm_dump_conn_trbuf(void *statep, char *line_prefix,
 			    char *buf, int buf_size);
+extern ibt_status_t	ibcm_get_node_rec(ibmf_saa_handle_t, sa_node_record_t *,
+			    uint64_t c_mask, void *, size_t *);
 
 /*
  * ibcm_lookup_msg:
@@ -1894,7 +1896,6 @@ ibcm_dump_pathrec(sa_path_record_t *path_rec)
 	IBTF_DPRINTF_L5(cmlog, "Preference Bit:    %02X", path_rec->Preference);
 }
 
-
 /*
  * ibcm_dump_node_rec:
  *	Dumps Node Records.
@@ -1927,6 +1928,103 @@ ibcm_dump_noderec(sa_node_record_t *nrec)
 }
 #endif
 
+/*
+ * ibcm_ibtl_node_info:
+ *	Get the node record of the destination specified by lid via the HCA
+ *	and port specified.
+ *
+ * Arguments:
+ *	hca_guid - GUID of the local HCA.
+ *	port     - port in the HCA to be used.
+ *	lid      - destination LID
+ *	node_info_p - pointer to the Node Info to be returned.
+ *
+ * Return values:
+ * 	IBT_SUCCESS : Got the node record sucessfully
+ * 	IBT_FILURE  : Failed to get the node record.
+ */
+ibt_status_t
+ibcm_ibtl_node_info(ib_guid_t hca_guid, uint8_t port, ib_lid_t lid,
+    ibt_node_info_t *node_info_p)
+{
+	sa_node_record_t	nr_req, *nr_resp;
+	void			*res_p;
+	ibmf_saa_handle_t	saa_handle;
+	ibt_status_t		ibt_status;
+	ibcm_hca_info_t		*hcap;
+	uint_t			num_rec;
+	size_t			len;
+
+	IBTF_DPRINTF_L3(cmlog, "ibcm_ibtl_node_info: ENTER: port %x "
+	    "guid %llx\n", port, hca_guid);
+
+	hcap = ibcm_find_hca_entry(hca_guid);
+	if (hcap == NULL) {
+		IBTF_DPRINTF_L2(cmlog, "ibcm_ibtl_node_info: "
+		    "HCA(%llX) info not found", hca_guid);
+		return (IBT_FAILURE);
+	}
+
+	/* Get SA Access Handle. */
+	saa_handle = ibcm_get_saa_handle(hcap, port);
+	if (saa_handle == NULL) {
+		IBTF_DPRINTF_L2(cmlog, "ibcm_ibtl_node_info: "
+		    "Port %d of HCA (%llX) is NOT ACTIVE", port, hca_guid);
+		ibcm_dec_hca_acc_cnt(hcap);
+		return (IBT_FAILURE);
+	}
+
+	/* Retrieve Node Records from SA Access. */
+	bzero(&nr_req, sizeof (sa_node_record_t));
+	nr_req.LID =  lid;
+
+	ibt_status = ibcm_get_node_rec(saa_handle, &nr_req,
+	    SA_NODEINFO_COMPMASK_NODELID, &res_p, &len);
+	if (ibt_status != IBT_SUCCESS) {
+		IBTF_DPRINTF_L2(cmlog, "ibcm_ibtl_node_info: "
+		    "failed (%d) to get Node records", ibt_status);
+		ibcm_dec_hca_acc_cnt(hcap);
+		return (IBT_FAILURE);
+	}
+
+	num_rec = len/sizeof (sa_node_record_t);
+	nr_resp = (sa_node_record_t *)(uchar_t *)res_p;
+
+	if ((nr_resp != NULL) && (num_rec > 0)) {
+		IBCM_DUMP_NODE_REC(nr_resp);
+
+		_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(
+		    *node_info_p))
+
+		node_info_p->n_sys_img_guid =
+		    nr_resp->NodeInfo.SystemImageGUID;
+		node_info_p->n_node_guid =
+		    nr_resp->NodeInfo.NodeGUID;
+		node_info_p->n_port_guid =
+		    nr_resp->NodeInfo.PortGUID;
+		node_info_p->n_dev_id =
+		    nr_resp->NodeInfo.DeviceID;
+		node_info_p->n_revision =
+		    nr_resp->NodeInfo.Revision;
+		node_info_p->n_vendor_id =
+		    nr_resp->NodeInfo.VendorID;
+		node_info_p->n_num_ports =
+		    nr_resp->NodeInfo.NumPorts;
+		node_info_p->n_port_num =
+		    nr_resp->NodeInfo.LocalPortNum;
+		node_info_p->n_node_type =
+		    nr_resp->NodeInfo.NodeType;
+		(void) strncpy(node_info_p->n_description,
+		    (char *)&nr_resp->NodeDescription, 64);
+		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(
+		    *node_info_p))
+
+
+		kmem_free(nr_resp, len);
+	}
+	ibcm_dec_hca_acc_cnt(hcap);
+	return (IBT_SUCCESS);
+}
 
 /*
  * ibcm_ibmf_analyze_error:
diff --git a/usr/src/uts/common/io/ib/mgt/ibdm/ibdm.c b/usr/src/uts/common/io/ib/mgt/ibdm/ibdm.c
index c39a0e794f..99910fbf96 100644
--- a/usr/src/uts/common/io/ib/mgt/ibdm/ibdm.c
+++ b/usr/src/uts/common/io/ib/mgt/ibdm/ibdm.c
@@ -81,6 +81,7 @@ static void	ibdm_process_incoming_mad(ibmf_handle_t, ibmf_msg_t *, void *);
 static void	ibdm_ibmf_send_cb(ibmf_handle_t, ibmf_msg_t *, void *);
 static void	ibdm_pkt_timeout_hdlr(void *arg);
 static void	ibdm_initialize_port(ibdm_port_attr_t *);
+static void	ibdm_update_port_pkeys(ibdm_port_attr_t *port);
 static void	ibdm_handle_diagcode(ibmf_msg_t *, ibdm_dp_gidinfo_t *, int *);
 static void	ibdm_probe_gid(ibdm_dp_gidinfo_t *);
 static void	ibdm_alloc_send_buffers(ibmf_msg_t *);
@@ -573,6 +574,23 @@ ibdm_event_hdlr(void *clnt_hdl,
 		ibdm_reset_all_dgids(port_sa_hdl);
 		break;
 
+	case IBT_PORT_CHANGE_EVENT:
+		IBTF_DPRINTF_L4("ibdm", "\tevent_hdlr: PORT_CHANGE");
+		if (event->ev_port_flags & IBT_PORT_CHANGE_PKEY) {
+			mutex_enter(&ibdm.ibdm_hl_mutex);
+			port = ibdm_get_port_attr(event, &hca_list);
+			if (port == NULL) {
+				IBTF_DPRINTF_L2("ibdm",
+				    "\tevent_hdlr: HCA not present");
+				mutex_exit(&ibdm.ibdm_hl_mutex);
+				break;
+			}
+			ibdm_update_port_pkeys(port);
+			cv_broadcast(&ibdm.ibdm_port_settle_cv);
+			mutex_exit(&ibdm.ibdm_hl_mutex);
+		}
+		break;
+
 	default:		/* Ignore all other events/errors */
 		break;
 	}
@@ -580,6 +598,92 @@ ibdm_event_hdlr(void *clnt_hdl,
 
 
 /*
+ * ibdm_update_port_pkeys()
+ *	Update the pkey table
+ *	Update the port attributes
+ */
+static void
+ibdm_update_port_pkeys(ibdm_port_attr_t *port)
+{
+	uint_t				nports, size;
+	uint_t				pkey_idx, opkey_idx;
+	uint16_t			npkeys;
+	ibt_hca_portinfo_t		*pinfop;
+	ib_pkey_t			pkey;
+	ibdm_pkey_tbl_t			*pkey_tbl;
+	ibdm_port_attr_t		newport;
+
+	IBTF_DPRINTF_L4("ibdm", "\tupdate_port_pkeys:");
+	ASSERT(MUTEX_HELD(&ibdm.ibdm_hl_mutex));
+
+	/* Check whether the port is active */
+	if (ibt_get_port_state(port->pa_hca_hdl, port->pa_port_num, NULL,
+	    NULL) != IBT_SUCCESS)
+		return;
+
+	if (ibt_query_hca_ports(port->pa_hca_hdl, port->pa_port_num,
+	    &pinfop, &nports, &size) != IBT_SUCCESS) {
+		/* This should not occur */
+		port->pa_npkeys = 0;
+		port->pa_pkey_tbl = NULL;
+		return;
+	}
+
+	npkeys = pinfop->p_pkey_tbl_sz;
+	pkey_tbl = kmem_zalloc(npkeys * sizeof (ibdm_pkey_tbl_t), KM_SLEEP);
+	newport.pa_pkey_tbl = pkey_tbl;
+	newport.pa_ibmf_hdl = port->pa_ibmf_hdl;
+
+	for (pkey_idx = 0; pkey_idx < npkeys; pkey_idx++) {
+		pkey = pkey_tbl[pkey_idx].pt_pkey =
+		    pinfop->p_pkey_tbl[pkey_idx];
+		/*
+		 * Is this pkey present in the current table ?
+		 */
+		for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) {
+			if (pkey == port->pa_pkey_tbl[opkey_idx].pt_pkey) {
+				pkey_tbl[pkey_idx].pt_qp_hdl =
+				    port->pa_pkey_tbl[opkey_idx].pt_qp_hdl;
+				port->pa_pkey_tbl[opkey_idx].pt_qp_hdl = NULL;
+				break;
+			}
+		}
+
+		if (opkey_idx == port->pa_npkeys) {
+			pkey = pkey_tbl[pkey_idx].pt_pkey;
+			if (IBDM_INVALID_PKEY(pkey)) {
+				pkey_tbl[pkey_idx].pt_qp_hdl = NULL;
+				continue;
+			}
+			ibdm_port_attr_ibmf_init(&newport, pkey, pkey_idx);
+		}
+	}
+
+	for (opkey_idx = 0; opkey_idx < port->pa_npkeys; opkey_idx++) {
+		if (port->pa_pkey_tbl[opkey_idx].pt_qp_hdl != NULL) {
+			if (ibdm_port_attr_ibmf_fini(port, opkey_idx) !=
+			    IBDM_SUCCESS) {
+				IBTF_DPRINTF_L2("ibdm", "\tupdate_port_pkeys: "
+				    "ibdm_port_attr_ibmf_fini failed for "
+				    "port pkey 0x%x",
+				    port->pa_pkey_tbl[opkey_idx].pt_pkey);
+			}
+		}
+	}
+
+	if (port->pa_pkey_tbl != NULL) {
+		kmem_free(port->pa_pkey_tbl,
+		    port->pa_npkeys * sizeof (ibdm_pkey_tbl_t));
+	}
+
+	port->pa_npkeys = npkeys;
+	port->pa_pkey_tbl = pkey_tbl;
+	port->pa_sn_prefix = pinfop->p_sgid_tbl[0].gid_prefix;
+	port->pa_state = pinfop->p_linkstate;
+	ibt_free_portinfo(pinfop, size);
+}
+
+/*
  * ibdm_initialize_port()
  *	Register with IBMF
  *	Register with SA access
diff --git a/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_saa_impl.c b/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_saa_impl.c
index 5e0cc85d1c..cd34904326 100644
--- a/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_saa_impl.c
+++ b/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_saa_impl.c
@@ -57,6 +57,8 @@ static void ibmf_saa_impl_async_event_cb(ibmf_handle_t ibmf_handle,
     void *clnt_private, ibmf_async_event_t event_type);
 static void ibmf_saa_impl_port_up(ib_guid_t ci_guid, uint8_t port_num);
 static void ibmf_saa_impl_port_down(ib_guid_t ci_guid, uint8_t port_num);
+static void ibmf_saa_impl_port_chg(ibt_async_event_t *event);
+static void ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num);
 static void ibmf_saa_impl_hca_detach(saa_port_t *saa_removed);
 static void ibmf_saa_impl_prepare_response(ibmf_handle_t ibmf_handle,
     ibmf_msg_t *msgp, boolean_t ignore_data, int *status, void **result,
@@ -2287,6 +2289,12 @@ ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event)
 	case IBT_ERROR_PORT_DOWN:
 		ibmf_saa_impl_port_down(event->ev_hca_guid, event->ev_port);
 		break;
+	case IBT_PORT_CHANGE_EVENT:
+		ibmf_saa_impl_port_chg(event);
+		break;
+	case IBT_CLNT_REREG_EVENT:
+		ibmf_saa_impl_client_rereg(event->ev_hca_guid, event->ev_port);
+		break;
 	default:
 		break;
 	}
@@ -2296,6 +2304,222 @@ ibmf_saa_impl_ibt_async_handler(ibt_async_code_t code, ibt_async_event_t *event)
 }
 
 /*
+ * ibmf_saa_impl_port_chg:
+ */
+static void
+ibmf_saa_impl_port_chg(ibt_async_event_t *event)
+{
+	saa_port_t		*saa_portp	= NULL;
+	boolean_t		is_ready = B_FALSE;
+	ibt_hca_portinfo_t	*ibt_portinfop;
+	uint_t			nports, size;
+	ibt_status_t		ibt_status;
+	ib_guid_t		ci_guid;
+	int			port_num;
+
+	ci_guid = event->ev_hca_guid;
+	port_num = event->ev_port;
+
+	IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_chg_start,
+	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg: Handling port chg"
+	    " guid %016" PRIx64 " port %d\n",
+	    tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
+
+	/* Get classportinfo of corresponding entry */
+	mutex_enter(&saa_statep->saa_port_list_mutex);
+
+	saa_portp = saa_statep->saa_port_list;
+	while (saa_portp != NULL) {
+		if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
+		    saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
+			mutex_enter(&saa_portp->saa_pt_mutex);
+
+			is_ready = (saa_portp->saa_pt_state
+			    == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
+
+			/*
+			 * increment reference count to account for cpi and
+			 * informinfos.  All 4 informinfo's sent are treated as
+			 * one port client reference
+			 */
+			if (is_ready)
+				saa_portp->saa_pt_reference_count ++;
+
+			mutex_exit(&saa_portp->saa_pt_mutex);
+
+			if (is_ready)
+				break; /* normally, only 1 port entry */
+		}
+		saa_portp = saa_portp->next;
+	}
+
+	mutex_exit(&saa_statep->saa_port_list_mutex);
+
+	if (saa_portp != NULL) {
+		/* first query the portinfo to see if the lid changed */
+		ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
+		    &ibt_portinfop, &nports, &size);
+
+		if (ibt_status != IBT_SUCCESS) {
+
+			IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
+			    ibmf_saa_impl_port_chg_err, IBMF_TNF_ERROR, "",
+			    "ibmf_saa_impl_port_chg: %s, ibmf_status ="
+			    " %d\n", tnf_string, msg,
+			    "ibt_query_hca_ports_byguid() failed",
+			    tnf_int, ibt_status, ibt_status);
+
+			goto bail;
+		}
+
+		mutex_enter(&saa_portp->saa_pt_mutex);
+		if (event->ev_port_flags & IBT_PORT_CHANGE_SM_LID) {
+			/* update the Master SM Lid value in ibmf_saa */
+			saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
+			    ibt_portinfop->p_sm_lid;
+		}
+		if (event->ev_port_flags & IBT_PORT_CHANGE_SM_SL) {
+			/* update the Master SM SL value in ibmf_saa */
+			saa_portp->saa_pt_ibmf_addr_info.ia_service_level =
+			    ibt_portinfop->p_sm_sl;
+		}
+		if (event->ev_port_flags & IBT_PORT_CHANGE_SUB_TIMEOUT) {
+			/* update the Subnet timeout value in ibmf_saa */
+			saa_portp->saa_pt_timeout =
+			    ibt_portinfop->p_subnet_timeout;
+		}
+		mutex_exit(&saa_portp->saa_pt_mutex);
+
+		ibt_free_portinfo(ibt_portinfop, size);
+
+		/* get the classportinfo again */
+		ibmf_saa_impl_get_classportinfo(saa_portp);
+	}
+bail:
+
+	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_chg_end,
+	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_port_chg() exit\n");
+}
+/*
+ * ibmf_saa_impl_client_rereg:
+ */
+static void
+ibmf_saa_impl_client_rereg(ib_guid_t ci_guid, uint8_t port_num)
+{
+	saa_port_t		*saa_portp	= NULL;
+	boolean_t		is_ready = B_FALSE;
+	ibt_hca_portinfo_t	*ibt_portinfop;
+	ib_lid_t		master_sm_lid;
+	uint_t			nports, size;
+	ibt_status_t		ibt_status;
+	boolean_t		event_subs = B_FALSE;
+
+	IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3, ibmf_saa_impl_port_rereg_start,
+	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg: Handling clnt "
+	    "rereg guid %016" PRIx64 " port %d\n",
+	    tnf_opaque, hca_guid, ci_guid, tnf_uint, port, port_num);
+
+	/* Get classportinfo of corresponding entry */
+	mutex_enter(&saa_statep->saa_port_list_mutex);
+
+	saa_portp = saa_statep->saa_port_list;
+	while (saa_portp != NULL) {
+
+		if (saa_portp->saa_pt_ibmf_reginfo.ir_ci_guid == ci_guid &&
+		    saa_portp->saa_pt_ibmf_reginfo.ir_port_num == port_num) {
+
+			mutex_enter(&saa_portp->saa_pt_mutex);
+
+			is_ready = (saa_portp->saa_pt_state
+			    == IBMF_SAA_PORT_STATE_READY) ? B_TRUE : B_FALSE;
+
+			/*
+			 * increment reference count to account for cpi and
+			 * informinfos.  All 4 informinfo's sent are treated as
+			 * one port client reference
+			 */
+			if (is_ready)
+				saa_portp->saa_pt_reference_count += 2;
+
+			mutex_exit(&saa_portp->saa_pt_mutex);
+
+			if (is_ready)
+				break; /* normally, only 1 port entry */
+		}
+		saa_portp = saa_portp->next;
+	}
+
+	mutex_exit(&saa_statep->saa_port_list_mutex);
+
+	if (saa_portp != NULL && is_ready == B_TRUE) {
+
+		/* verify whether master sm lid changed */
+
+		/* first query the portinfo to see if the lid changed */
+		ibt_status = ibt_query_hca_ports_byguid(ci_guid, port_num,
+		    &ibt_portinfop, &nports, &size);
+
+		if (ibt_status != IBT_SUCCESS) {
+
+			IBMF_TRACE_2(IBMF_TNF_NODEBUG, DPRINT_L1,
+			    ibmf_saa_impl_port_rereg_err, IBMF_TNF_ERROR, "",
+			    "ibmf_saa_impl_client_rereg: %s, ibmf_status ="
+			    " %d\n", tnf_string, msg,
+			    "ibt_query_hca_ports_byguid() failed",
+			    tnf_int, ibt_status, ibt_status);
+
+			goto bail;
+		}
+
+		master_sm_lid = ibt_portinfop->p_sm_lid;
+
+		ibt_free_portinfo(ibt_portinfop, size);
+
+		/* check whether we need to subscribe for events */
+		mutex_enter(&saa_portp->saa_pt_event_sub_mutex);
+
+		event_subs = (saa_portp->saa_pt_event_sub_client_list != NULL) ?
+		    B_TRUE : B_FALSE;
+
+		mutex_exit(&saa_portp->saa_pt_event_sub_mutex);
+
+		/* update the master smlid */
+		mutex_enter(&saa_portp->saa_pt_mutex);
+
+		/* update the master sm lid value in ibmf_saa */
+		saa_portp->saa_pt_ibmf_addr_info.ia_remote_lid =
+		    master_sm_lid;
+
+		/* if we're not subscribed for events, dec reference count */
+		if (event_subs == B_FALSE)
+			saa_portp->saa_pt_reference_count--;
+
+		mutex_exit(&saa_portp->saa_pt_mutex);
+
+		IBMF_TRACE_2(IBMF_TNF_DEBUG, DPRINT_L3,
+		    ibmf_saa_impl_port_rereg, IBMF_TNF_TRACE, "",
+		    "ibmf_saa_impl_client_rereg: %s, master_sm_lid = 0x%x\n",
+		    tnf_string, msg,
+		    "port is up.  Sending classportinfo request",
+		    tnf_opaque, master_sm_lid, master_sm_lid);
+
+		/* get the classportinfo again */
+		ibmf_saa_impl_get_classportinfo(saa_portp);
+
+		/*
+		 * resubscribe to events if there are subscribers since SA may
+		 * have removed our subscription records when the port went down
+		 */
+		if (event_subs == B_TRUE)
+			ibmf_saa_subscribe_events(saa_portp, B_TRUE, B_FALSE);
+	}
+
+bail:
+
+	IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4, ibmf_saa_impl_port_rereg_end,
+	    IBMF_TNF_TRACE, "", "ibmf_saa_impl_client_rereg() exit\n");
+}
+/*
  * ibmf_saa_impl_port_up:
  */
 static void
diff --git a/usr/src/uts/common/io/warlock/ibcm.wlcmd b/usr/src/uts/common/io/warlock/ibcm.wlcmd
index ae1004b1c4..a053986c89 100644
--- a/usr/src/uts/common/io/warlock/ibcm.wlcmd
+++ b/usr/src/uts/common/io/warlock/ibcm.wlcmd
@@ -18,10 +18,9 @@
 #
 # CDDL HEADER END
 #
-# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
-#ident	"%Z%%M%	%I%	%E% SMI"
 
 # ibcm api entry points
 root	ibt_open_rc_channel
@@ -92,6 +91,7 @@ root	ibcm_resend_post_rep_complete
 
 # Callback handlers from IBTL
 root	ibcm_async_handler
+root	ibcm_ibtl_node_info
 root	ibt_register_subnet_notices
 
 # Callback handlers from ibcm
diff --git a/usr/src/uts/common/io/warlock/ibtl.wlcmd b/usr/src/uts/common/io/warlock/ibtl.wlcmd
index 57cfbfc01d..7d48d1e0f2 100644
--- a/usr/src/uts/common/io/warlock/ibtl.wlcmd
+++ b/usr/src/uts/common/io/warlock/ibtl.wlcmd
@@ -194,7 +194,9 @@ root	ibtl_cm_free_active_plist
 root	ibtl_cm_get_1st_full_pkey_ix
 root	ibtl_cm_get_local_comp_gids
 root	ibtl_cm_is_multi_sm
+root	ibtl_cm_set_node_info_cb
 root	ibtl_cm_get_clnt_name
+root	ibtl_dummy_node_info_cb
 
 # IBTL-IBNEX private interface
 root	ibtl_ibnex_get_hca_info
@@ -211,6 +213,7 @@ root	ibtl_do_mgr_async_task
 root	ibtl_hca_client_async_task
 root	ibtl_tell_client_about_new_hca
 root	ibtl_sm_notice_task
+root	ibt_cisco_embedded_sm_rereg_fix
 
 # IBTL internal threads
 root	ibtl_async_thread
diff --git a/usr/src/uts/common/sys/ib/ibtl/ibtl_types.h b/usr/src/uts/common/sys/ib/ibtl/ibtl_types.h
index fc28bd979b..fdb19ce6ac 100644
--- a/usr/src/uts/common/sys/ib/ibtl/ibtl_types.h
+++ b/usr/src/uts/common/sys/ib/ibtl/ibtl_types.h
@@ -1515,6 +1515,9 @@ typedef enum ibt_async_code_e {
 	IBT_CLNT_REREG_EVENT			= 0x200000
 } ibt_async_code_t;
 
+#define	IBT_PORT_EVENTS (IBT_EVENT_PORT_UP|IBT_PORT_CHANGE_EVENT|\
+    IBT_ERROR_PORT_DOWN|IBT_CLNT_REREG_EVENT)
+
 typedef enum ibt_port_change_e {
 	IBT_PORT_CHANGE_SGID		= 0x000001, /* SGID table */
 	IBT_PORT_CHANGE_PKEY		= 0x000002, /* P_Key table */
diff --git a/usr/src/uts/common/sys/ib/ibtl/impl/ibtl.h b/usr/src/uts/common/sys/ib/ibtl/impl/ibtl.h
index bb8799d3a4..e07a67102a 100644
--- a/usr/src/uts/common/sys/ib/ibtl/impl/ibtl.h
+++ b/usr/src/uts/common/sys/ib/ibtl/impl/ibtl.h
@@ -87,13 +87,22 @@ typedef enum ibtl_hca_state_e {
  */
 
 typedef enum ibtl_async_port_status_e {
-	IBTL_HCA_PORT_UNKNOWN	= 0x0,	/* initial state */
-	IBTL_HCA_PORT_UP	= 0x1,
-	IBTL_HCA_PORT_DOWN	= 0x2,
-	IBTL_HCA_PORT_CHANGED	= 0x4
+	IBTL_HCA_PORT_UNKNOWN		= 0x000,	/* initial state */
+	IBTL_HCA_PORT_UP		= 0x001,
+	IBTL_HCA_PORT_DOWN		= 0x002,
+	IBTL_HCA_PORT_CHG		= 0x004,
+	IBTL_HCA_PORT_ASYNC_CLNT_REREG	= 0x008,
 } ibtl_async_port_status_t;
 
 /*
+ * Define a type to record the PORT async events and port change flags.
+ */
+typedef struct ibtl_async_port_event_s {
+	ibtl_async_port_status_t	status;
+	ibt_port_change_t		flags;
+} ibtl_async_port_event_t;
+
+/*
  * Bit definition(s) for {qp,cq,eec,hd,ha,srq}_async_flags.
  *
  *	IBTL_ASYNC_PENDING	This structure is known by the async_threads.
@@ -165,7 +174,7 @@ typedef struct ibtl_hca_devinfo_s {
 	ibtl_hca_port_kstat_t	*hd_hca_port_ks_info;	/* port kstat ptr */
 	uint_t			hd_hca_port_ks_info_len; /* port kstat size */
 		/* The following must be at the end of this struct */
-	ibtl_async_port_status_t hd_async_port[1]; /* per-port async data */
+	ibtl_async_port_event_t hd_async_port[1]; /* per-port async data */
 } ibtl_hca_devinfo_t;
 
 _NOTE(DATA_READABLE_WITHOUT_LOCK(ibtl_hca_devinfo_s::hd_ibc_ops))
diff --git a/usr/src/uts/common/sys/ib/ibtl/impl/ibtl_cm.h b/usr/src/uts/common/sys/ib/ibtl/impl/ibtl_cm.h
index 774e246cb1..630700fbcb 100644
--- a/usr/src/uts/common/sys/ib/ibtl/impl/ibtl_cm.h
+++ b/usr/src/uts/common/sys/ib/ibtl/impl/ibtl_cm.h
@@ -237,6 +237,14 @@ void ibtl_cm_sm_notice_init_failure(ibtl_cm_sm_init_fail_t *ifail);
 
 char *ibtl_cm_get_clnt_name(ibt_clnt_hdl_t ibt_hdl);
 
+/*
+ * ibtl_cm_set_node_info_cb: This is a private interface between IBTL and IBCM
+ * to let IBTL get the Node Record of a remote port. This interface is used by
+ * IBCM to register a callback which can be used by IBTL to get the Node record.
+ */
+void ibtl_cm_set_node_info_cb(ibt_status_t (*)(ib_guid_t, uint8_t, ib_lid_t,
+    ibt_node_info_t *));
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/usr/src/uts/common/sys/ib/mgt/ibcm/ibcm_impl.h b/usr/src/uts/common/sys/ib/mgt/ibcm/ibcm_impl.h
index ee4bf0a7bc..d42528a9ef 100644
--- a/usr/src/uts/common/sys/ib/mgt/ibcm/ibcm_impl.h
+++ b/usr/src/uts/common/sys/ib/mgt/ibcm/ibcm_impl.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -2179,6 +2179,9 @@ ibt_status_t ibcm_ibmf_analyze_error(int ibmf_status);
 ibt_status_t ibcm_contact_sa_access(ibmf_saa_handle_t saa_handle,
     ibmf_saa_access_args_t *access_args, size_t *length, void **results_p);
 
+ibt_status_t	ibcm_ibtl_node_info(ib_guid_t, uint8_t, ib_lid_t,
+    ibt_node_info_t *node_info);
+
 void ibcm_path_cache_init(void);
 void ibcm_path_cache_fini(void);
 void ibcm_path_cache_purge(void);
diff --git a/usr/src/uts/common/sys/ib/mgt/ibmf/ibmf_saa_impl.h b/usr/src/uts/common/sys/ib/mgt/ibmf/ibmf_saa_impl.h
index 472aa328db..3571d665f7 100644
--- a/usr/src/uts/common/sys/ib/mgt/ibmf/ibmf_saa_impl.h
+++ b/usr/src/uts/common/sys/ib/mgt/ibmf/ibmf_saa_impl.h
@@ -306,6 +306,8 @@ ibmf_saa_add_event_subscriber(saa_client_data_t *client,
 void ibmf_saa_subscribe_events(saa_port_t *saa_portp, boolean_t subscribe,
     boolean_t seq_unsubscribe);
 
+void ibmf_saa_subscribe_sm_events(saa_port_t *saa_portp);
+
 void
 ibmf_saa_notify_event_clients(saa_port_t *saa_portp,
     ibmf_saa_event_details_t *event_details,
diff --git a/usr/src/uts/common/sys/ib/mgt/sm_attr.h b/usr/src/uts/common/sys/ib/mgt/sm_attr.h
index f40a31618a..e845651b4b 100644
--- a/usr/src/uts/common/sys/ib/mgt/sm_attr.h
+++ b/usr/src/uts/common/sys/ib/mgt/sm_attr.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -447,7 +447,8 @@ typedef struct sm_portinfo_s {
 	uint16_t	P_KeyViolations;	/* count of P_key violations */
 	uint16_t	Q_KeyViolations;	/* count of Q_key violations */
 	uint8_t		GUIDCap;		/* number of GUIDs supported */
-	uint8_t		Reserved2	:3;
+	uint8_t		ClientRereg	:1;	/* Client ReReg supported */
+	uint8_t		Reserved2	:2;
 	uint8_t		SubnetTimeOut	:5;	/* defines subnet prop. dely */
 	uint8_t		Reserved3	:3;
 	uint8_t		RespTimeValue	:5;	/* defines resp time to SMPs */
@@ -497,7 +498,8 @@ typedef struct sm_portinfo_s {
 	uint16_t	Q_KeyViolations;	/* count of Q_key violations */
 	uint8_t		GUIDCap;		/* number of GUIDs supported */
 	uint8_t		SubnetTimeOut	:5;	/* defines subnet prop. dely */
-	uint8_t		Reserved2	:3;
+	uint8_t		Reserved2	:2;
+	uint8_t		ClientRereg	:1;	/* Client ReReg supported */
 	uint8_t		RespTimeValue	:5;	/* defines resp time to SMPs */
 	uint8_t		Reserved3	:3;
 	uint8_t		OverrunErrors	:4;	/* threshold for errors */
@@ -527,6 +529,7 @@ typedef struct sm_portinfo_s {
 #define	SM_CAP_MASK_IS_DR_NOTICE_SUPPD		0x00200000
 #define	SM_CAP_MASK_IS_CAP_MASK_NOTICE_SUPPD	0x00400000
 #define	SM_CAP_MASK_IS_BOOT_MGMT_SUPPD		0x00800000
+#define	SM_CAP_MASK_IS_CLNT_REREG_SUPPD		0x02000000
 
 /* Standard Encoding of DiagCode Bits 3-0: Table 133 */
 #define	SM_DIAG_CODE_PORT_READY			0x0