6860702 ibt_cm_ud_proceed(9f) does not populate SIDR response with private data

6879853 ibt_free_qp() should return ERROR when called when async open_rc_channel has not completed
6879858 Need mechanism to skip FLUSH QP when DREQ is recieved for OFUV clients
6901968 ibcm needs fixes for PSN and RNR retry count

8 files changed, 106 insertions, 20 deletions

diff --git a/usr/src/uts/common/io/ib/ibtl/ibtl_qp.c b/usr/src/uts/common/io/ib/ibtl/ibtl_qp.c
index 747f9810ec..9d7119fbd8 100644
--- a/usr/src/uts/common/io/ib/ibtl/ibtl_qp.c
+++ b/usr/src/uts/common/io/ib/ibtl/ibtl_qp.c
@@ -457,6 +457,26 @@ ibt_flush_qp(ibt_qp_hdl_t ibt_qp)
 
 
 /*
+ * ibtl_cm_chan_is_opening()
+ *
+ *	Inform IBTL that the connection established process is in progress
+ *	on this channel so that care need to be taken while free'ing when
+ *	open is NOT yet complete.
+ *
+ *	chan	Channel Handle
+ */
+void
+ibtl_cm_chan_is_opening(ibt_channel_hdl_t chan)
+{
+	IBTF_DPRINTF_L3(ibtf_qp, "ibtl_cm_chan_is_opening(%p)", chan);
+	ASSERT(chan->ch_qp.qp_type == IBT_RC_SRV);
+	mutex_enter(&ibtl_free_qp_mutex);
+	ASSERT(chan->ch_transport.rc.rc_free_flags == 0);
+	chan->ch_transport.rc.rc_free_flags |= IBTL_RC_QP_CONNECTING;
+	mutex_exit(&ibtl_free_qp_mutex);
+}
+
+/*
  * ibtl_cm_chan_is_open()
  *
  *	Inform IBTL that the connection has been established on this
@@ -471,7 +491,7 @@ ibtl_cm_chan_is_open(ibt_channel_hdl_t chan)
 	IBTF_DPRINTF_L3(ibtf_qp, "ibtl_cm_chan_is_open(%p)", chan);
 	ASSERT(chan->ch_qp.qp_type == IBT_RC_SRV);
 	mutex_enter(&ibtl_free_qp_mutex);
-	ASSERT(chan->ch_transport.rc.rc_free_flags == 0);
+	chan->ch_transport.rc.rc_free_flags &= ~IBTL_RC_QP_CONNECTING;
 	chan->ch_transport.rc.rc_free_flags |= IBTL_RC_QP_CONNECTED;
 	mutex_exit(&ibtl_free_qp_mutex);
 }
@@ -634,6 +654,14 @@ ibt_free_qp(ibt_qp_hdl_t ibt_qp)
 		ibtl_qp_flow_control_enter();
 		mutex_enter(&ibtl_free_qp_mutex);
 		if (ibt_qp->ch_transport.rc.rc_free_flags &
+		    IBTL_RC_QP_CONNECTING) {
+			IBTF_DPRINTF_L2(ibtf_qp, "ibt_free_qp: ERROR - "
+			    "Channel establishment is still in PROGRESS.");
+			mutex_exit(&ibtl_free_qp_mutex);
+			ibtl_qp_flow_control_exit();
+			return (IBT_CHAN_STATE_INVALID);
+		}
+		if (ibt_qp->ch_transport.rc.rc_free_flags &
 		    IBTL_RC_QP_CONNECTED) {
 			if ((ibt_qp->ch_transport.rc.rc_free_flags &
 			    IBTL_RC_QP_CLOSING) == 0) {
diff --git a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_sm.c b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_sm.c
index 2970c2f51f..ba4c56ad03 100644
--- a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_sm.c
+++ b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_sm.c
@@ -1050,6 +1050,8 @@ new_req:
 		statep->hcap		= hcap;
 		statep->remote_comid	= remote_comid;
 		statep->svcid		= b2h64(req_msgp->req_svc_id);
+		statep->local_qp_rnr_cnt =
+		    req_msgp->req_mtu_plus & 0x7;
 
 		/*
 		 * get the remote_ack_delay, etc.
@@ -1451,6 +1453,8 @@ ibcm_process_rep_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 	/* Lookup for an existing state structure */
 	rep_msgp = (ibcm_rep_msg_t *)(&input_madp[IBCM_MAD_HDR_SIZE]);
 
+	IBCM_DUMP_RAW_MSG((uchar_t *)input_madp);
+
 	IBTF_DPRINTF_L5(cmlog, "ibcm_process_rep_msg: active comid: %x",
 	    rep_msgp->rep_remote_comm_id);
 
@@ -1512,6 +1516,8 @@ ibcm_process_rep_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 		/* change state */
 		statep->state = IBCM_STATE_REP_RCVD;
 		statep->clnt_proceed = IBCM_BLOCK;
+		statep->local_qp_rnr_cnt =
+		    rep_msgp->rep_rnr_retry_cnt_plus >> 5;
 
 		/* cancel the REQ timer */
 		if (statep->timerid != 0) {
@@ -2467,6 +2473,8 @@ ibcm_process_dreq_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 				(void) untimeout(timer_val);
 			}
 		} else {	/* In ESTABLISHED State */
+			boolean_t	is_ofuv = statep->is_this_ofuv_chan;
+
 			statep->state = IBCM_STATE_DREQ_RCVD;
 			statep->clnt_proceed = IBCM_BLOCK;
 
@@ -2483,7 +2491,8 @@ ibcm_process_dreq_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 
 			close_event_type = IBT_CM_CLOSED_DREQ_RCVD;
 			/* Move CEP to error state */
-			(void) ibcm_cep_to_error_state(statep);
+			if (is_ofuv == B_FALSE) /* Skip for OFUV channel */
+				(void) ibcm_cep_to_error_state(statep);
 		}
 		mutex_enter(&statep->state_mutex);
 		statep->drep_in_progress = 0;
@@ -5609,7 +5618,8 @@ ibcm_invoke_qp_modify(ibcm_state_data_t *statep, ibcm_req_msg_t *req_msgp,
 
 		} else { /* Passive side CM */
 			/* Setting PSN on SQ and RQ */
-			IBCM_QPINFO_RC(qp_info).rc_rq_psn =
+			IBCM_QPINFO_RC(qp_info).rc_sq_psn =
+			    IBCM_QPINFO_RC(qp_info).rc_rq_psn =
 			    b2h32(rep_msgp->rep_starting_psn_plus) >> 8;
 
 			IBCM_QPINFO_RC(qp_info).rc_dst_qpn =
@@ -5654,12 +5664,13 @@ ibcm_invoke_qp_modify(ibcm_state_data_t *statep, ibcm_req_msg_t *req_msgp,
 
 	case IBT_UC_SRV:
 		if (statep->mode == IBCM_ACTIVE_MODE) { /* look at REP msg */
-			IBCM_QPINFO_UC(qp_info).uc_rq_psn =
+			IBCM_QPINFO_UC(qp_info).uc_sq_psn =
 			    b2h32(req_msgp->req_starting_psn_plus) >> 8;
 			IBCM_QPINFO_UC(qp_info).uc_dst_qpn =
 			    b2h32(rep_msgp->rep_local_qpn_plus) >> 8;
 		} else {
 			IBCM_QPINFO_UC(qp_info).uc_rq_psn =
+			    IBCM_QPINFO_UC(qp_info).uc_sq_psn =
 			    b2h32(rep_msgp->rep_starting_psn_plus) >> 8;
 			IBCM_QPINFO_UC(qp_info).uc_dst_qpn =
 			    b2h32(req_msgp->req_local_qpn_plus) >> 8;
@@ -6399,7 +6410,7 @@ ibcm_process_cep_req_cm_hdlr(ibcm_state_data_t *statep,
 		}
 
 		if (qp_attrs.qp_info.qp_state != IBT_STATE_INIT &&
-		    statep->skip_rtr == 0) {
+		    statep->is_this_ofuv_chan == B_FALSE) {
 			IBTF_DPRINTF_L3(cmlog, "ibcm_process_cep_req_cm_hdlr: "
 			    "qp state != INIT on server");
 			*reject_reason = IBT_CM_CHAN_INVALID_STATE;
@@ -6407,10 +6418,11 @@ ibcm_process_cep_req_cm_hdlr(ibcm_state_data_t *statep,
 			    IBT_CM_FAILURE_REQ, IBT_CM_CHAN_INVALID_STATE,
 			    NULL, 0);
 			return (IBCM_SEND_REJ);
-		} else if (qp_attrs.qp_info.qp_state != IBT_STATE_RTR &&
-		    statep->skip_rtr == 1) {
+		} else if (statep->is_this_ofuv_chan &&
+		    qp_attrs.qp_info.qp_state != IBT_STATE_RTR &&
+		    qp_attrs.qp_info.qp_state != IBT_STATE_INIT) {
 			IBTF_DPRINTF_L3(cmlog, "ibcm_process_cep_req_cm_hdlr: "
-			    "qp state != RTR on server");
+			    "qp state != INIT or RTR on server");
 			*reject_reason = IBT_CM_CHAN_INVALID_STATE;
 			ibcm_handler_conn_fail(statep, IBT_CM_FAILURE_REJ_SENT,
 			    IBT_CM_FAILURE_REQ, IBT_CM_CHAN_INVALID_STATE,
@@ -6418,7 +6430,8 @@ ibcm_process_cep_req_cm_hdlr(ibcm_state_data_t *statep,
 			return (IBCM_SEND_REJ);
 		}
 
-		if (statep->skip_rtr &&
+		if (statep->is_this_ofuv_chan &&
+		    qp_attrs.qp_info.qp_state == IBT_STATE_RTR &&
 		    qp_attrs.qp_info.qp_transport.rc.rc_path.cep_hca_port_num !=
 		    statep->prim_port) {
 			IBTF_DPRINTF_L2(cmlog, "ibcm_process_cep_req_cm_hdlr: "
@@ -6428,8 +6441,10 @@ ibcm_process_cep_req_cm_hdlr(ibcm_state_data_t *statep,
 			    IBT_CM_FAILURE_REQ, IBT_CM_CHAN_INVALID_STATE,
 			    NULL, 0);
 			return (IBCM_SEND_REJ);
-		} else if (statep->skip_rtr)
+		} else if (statep->is_this_ofuv_chan &&
+		    qp_attrs.qp_info.qp_state == IBT_STATE_RTR) {
 			goto skip_init_trans;
+		}
 
 		/* Init to Init, if required */
 		if (qp_attrs.qp_info.qp_transport.rc.rc_path.cep_hca_port_num !=
@@ -6549,7 +6564,8 @@ skip_init_trans:
 		bcopy(&local_ca_guid, rep_msgp->rep_local_ca_guid,
 		    sizeof (ib_guid_t));
 
-		if (statep->skip_rtr)
+		if (statep->is_this_ofuv_chan &&
+		    qp_attrs.qp_info.qp_state == IBT_STATE_RTR)
 			goto skip_rtr_trans;
 
 		/* Transition QP from Init to RTR state */
@@ -6594,7 +6610,7 @@ skip_rtr_trans:
 			break;
 		case IBT_RC_SRV:
 			rep_msgp->rep_starting_psn_plus =
-			    h2b32(IBCM_QP_RC(qp_attrs).rc_sq_psn << 8);
+			    h2b32(IBCM_QP_RC(qp_attrs).rc_rq_psn << 8);
 			break;
 		case IBT_UC_SRV:
 			rep_msgp->rep_starting_psn_plus =
@@ -6996,7 +7012,7 @@ ibcm_invoke_rtu_qp_modify(ibcm_state_data_t *statep, ib_time_t timeout,
 		IBCM_QPINFO_RC_PATH(qp_info).cep_timeout = timeout;
 		IBCM_QPINFO_RC(qp_info).rc_retry_cnt = statep->cep_retry_cnt;
 		IBCM_QPINFO_RC(qp_info).rc_rnr_retry_cnt =
-		    rep_msg->rep_rnr_retry_cnt_plus >> 5;
+		    statep->local_qp_rnr_cnt;
 		IBCM_QPINFO_RC(qp_info).rc_sq_psn = statep->starting_psn;
 
 		if (statep->mode == IBCM_ACTIVE_MODE) {
@@ -7652,6 +7668,12 @@ ibcm_process_sidr_req_cm_hdlr(ibcm_ud_state_data_t *ud_statep,
     ibt_cm_status_t cb_status, ibcm_ud_clnt_reply_info_t *ud_clnt_info,
     ibt_sidr_status_t *sidr_status, ibcm_sidr_rep_msg_t *sidr_repp)
 {
+	void	*sidr_rep_privp;
+
+	IBTF_DPRINTF_L5(cmlog, "ibcm_process_sidr_req_cm_hdlr(%p, %x, "
+	    "%p, %p, %p)", ud_statep, cb_status, ud_clnt_info,
+	    sidr_status, sidr_repp);
+
 	if (cb_status == IBT_CM_DEFAULT)
 		cb_status = IBT_CM_REJECT;
 
@@ -7666,6 +7688,20 @@ ibcm_process_sidr_req_cm_hdlr(ibcm_ud_state_data_t *ud_statep,
 		*sidr_status = IBT_CM_SREP_REDIRECT;
 	else *sidr_status = IBT_CM_SREP_REJ;
 
+	/*
+	 * For Accept and reject copy the private data, if ud_clnt_info
+	 * priv_data does not point to SIDR Response private data. This
+	 * copy is needed for ibt_cm_ud_proceed().
+	 */
+	sidr_rep_privp = (void *)(&(sidr_repp->sidr_rep_private_data[0]));
+	if ((cb_status == IBT_CM_ACCEPT || cb_status == IBT_CM_REJECT) &&
+	    (ud_clnt_info->priv_data != sidr_rep_privp) &&
+	    ud_clnt_info->priv_data_len) {
+		bcopy(ud_clnt_info->priv_data, sidr_rep_privp,
+		    min(ud_clnt_info->priv_data_len,
+		    IBT_SIDR_REP_PRIV_DATA_SZ));
+	}
+
 	if (*sidr_status != IBT_CM_SREP_CHAN_VALID) {
 		IBTF_DPRINTF_L2(cmlog, "ibcm_process_sidr_req_cm_hdlr: "
 		    "ud_handler return a failure: %d", cb_status);
diff --git a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_ti.c b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_ti.c
index 18532d818d..31a12310bc 100644
--- a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_ti.c
+++ b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_ti.c
@@ -858,6 +858,13 @@ ibt_open_rc_channel(ibt_channel_hdl_t channel, ibt_chan_open_flags_t flags,
 	IBCM_REF_CNT_INCR(statep);	/* Decremented after REQ is posted */
 	statep->send_mad_flags |= IBCM_REQ_POST_BUSY;
 
+	/*
+	 * Skip moving channel to error state during close, for OFUV clients.
+	 * OFUV clients transition the channel to error state by itself.
+	 */
+	if (flags & IBT_OCHAN_OFUV)
+		statep->is_this_ofuv_chan = B_TRUE;
+
 	IBCM_OUT_HDRP(statep->stored_msg)->AttributeID =
 	    h2b16(IBCM_INCOMING_REQ + IBCM_ATTR_BASE_ID);
 
@@ -867,6 +874,8 @@ ibt_open_rc_channel(ibt_channel_hdl_t channel, ibt_chan_open_flags_t flags,
 
 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*statep))
 
+	ibtl_cm_chan_is_opening(channel);
+
 	ibcm_open_enqueue(statep);
 
 	mutex_enter(&statep->state_mutex);
@@ -3879,7 +3888,7 @@ ibt_ofuvcm_proceed(ibt_cm_event_type_t event, void *session_id,
 		return (IBT_INVALID_PARAM);
 	}
 	mutex_enter(&statep->state_mutex);
-	statep->skip_rtr = 1;
+	statep->is_this_ofuv_chan = B_TRUE;
 	mutex_exit(&statep->state_mutex);
 
 	ret = ibt_cm_proceed(event, session_id, status, cm_event_data,
@@ -4060,10 +4069,6 @@ ibcm_proceed_via_taskq(void *targs)
 		ibcm_handle_cep_req_response(statep, response, reject_reason,
 		    arej_len);
 
-		mutex_enter(&statep->state_mutex);
-		statep->skip_rtr = 0;
-		mutex_exit(&statep->state_mutex);
-
 	} else if (proceed_targs->event == IBT_CM_EVENT_REP_RCV) {
 		response =
 		    ibcm_process_cep_rep_cm_hdlr(statep, proceed_targs->status,
@@ -6366,6 +6371,8 @@ ibt_get_src_ip(ib_gid_t gid, ib_pkey_t pkey, ibt_ip_addr_t *src_ip)
 
 	for (i = 0, ipp = ibds.ibcm_arp_ip; i < ibds.ibcm_arp_ibd_cnt;
 	    i++, ipp++) {
+		if (ipp->ip_inet_family == AF_UNSPEC)
+			continue;
 		if (ipp->ip_port_gid.gid_prefix == gid.gid_prefix &&
 		    ipp->ip_port_gid.gid_guid == gid.gid_guid) {
 			if (pkey) {
diff --git a/usr/src/uts/common/sys/ib/ibtl/ibti_common.h b/usr/src/uts/common/sys/ib/ibtl/ibti_common.h
index 442c4103e4..595ff2db1e 100644
--- a/usr/src/uts/common/sys/ib/ibtl/ibti_common.h
+++ b/usr/src/uts/common/sys/ib/ibtl/ibti_common.h
@@ -82,6 +82,7 @@ typedef enum ibt_clnt_class_e {
 	IBT_CM,			/* The CM Module */
 	IBT_DM,			/* The DM Module */
 	IBT_DM_AGENT,		/* DM Agent Module */
+	IBT_GENERIC_MISC,	/* Generic Misc Module */
 	IBT_CLASS_NUM		/* Place holder for class count */
 } ibt_clnt_class_t;
 
@@ -95,6 +96,7 @@ typedef enum ibt_clnt_class_e {
 					(class) == IBT_GENERIC || \
 					(class) == IBT_DM_AGENT || \
 					(class) == IBT_TEST_DEV || \
+					(class) == IBT_GENERIC_MISC ||	\
 					(class) == IBT_USER)
 
 #define	IBT_CLNT_MGMT_CLASS(class)	((class) == IBT_IBMA || \
@@ -110,6 +112,7 @@ typedef enum ibt_clnt_class_e {
 					(class) == IBT_CM || \
 					(class) == IBT_DM || \
 					(class) == IBT_DM_AGENT || \
+					(class) == IBT_GENERIC_MISC ||	\
 					(class) == IBT_TEST_DEV)
 
 /*
@@ -439,7 +442,8 @@ typedef enum ibt_chan_open_flags_e {
 	IBT_OCHAN_OPAQUE2		= 1 << 5,
 	IBT_OCHAN_OPAQUE3		= 1 << 6,
 	IBT_OCHAN_OPAQUE4		= 1 << 7,
-	IBT_OCHAN_OPAQUE5		= 1 << 8
+	IBT_OCHAN_OPAQUE5		= 1 << 8,
+	IBT_OCHAN_OPAQUE6		= 1 << 9
 } ibt_chan_open_flags_t;
 
 /*
diff --git a/usr/src/uts/common/sys/ib/ibtl/ibvti.h b/usr/src/uts/common/sys/ib/ibtl/ibvti.h
index 3765d4bfac..93dfbb68a9 100644
--- a/usr/src/uts/common/sys/ib/ibtl/ibvti.h
+++ b/usr/src/uts/common/sys/ib/ibtl/ibvti.h
@@ -48,6 +48,7 @@ extern "C" {
 #define	IBT_OCHAN_LOCAL_CM_TM	IBT_OCHAN_OPAQUE3 /* ibt_chan_open_flags_t */
 #define	IBT_OCHAN_REMOTE_CM_TM	IBT_OCHAN_OPAQUE4 /* ibt_chan_open_flags_t */
 #define	IBT_OCHAN_RDC_EXISTS	IBT_OCHAN_OPAQUE5 /* ibt_chan_open_flags_t */
+#define	IBT_OCHAN_OFUV		IBT_OCHAN_OPAQUE6 /* ibt_chan_open_flags_t */
 
 #define	oc_cm_retry_cnt		oc_opaque1	/* ibt_chan_open_args_t */
 						/* The number of times the */
diff --git a/usr/src/uts/common/sys/ib/ibtl/impl/ibtl.h b/usr/src/uts/common/sys/ib/ibtl/impl/ibtl.h
index e07a67102a..543886b7ac 100644
--- a/usr/src/uts/common/sys/ib/ibtl/impl/ibtl.h
+++ b/usr/src/uts/common/sys/ib/ibtl/impl/ibtl.h
@@ -426,6 +426,7 @@ typedef struct ibtl_rc_chan_s {
 #define	IBTL_RC_QP_CLOSING	0x2
 #define	IBTL_RC_QP_CLOSED	0x4
 #define	IBTL_RC_QP_FREED	0x8
+#define	IBTL_RC_QP_CONNECTING	0x10
 
 /*
  * Define a per Channel state structure.
diff --git a/usr/src/uts/common/sys/ib/ibtl/impl/ibtl_cm.h b/usr/src/uts/common/sys/ib/ibtl/impl/ibtl_cm.h
index 630700fbcb..7c14fe6350 100644
--- a/usr/src/uts/common/sys/ib/ibtl/impl/ibtl_cm.h
+++ b/usr/src/uts/common/sys/ib/ibtl/impl/ibtl_cm.h
@@ -115,6 +115,11 @@ uint16_t ibtl_cm_get_1st_full_pkey_ix(ib_guid_t hca_guid, uint8_t port);
  *	channel so that a later call to ibtl_cm_chan_is_closed()
  *	will be required to free the QPN used by this channel.
  *
+ * ibtl_cm_chan_is_opening()
+ *
+ *	Inform IBTL that the connection established on this channel is
+ *	in progress.
+ *
  * ibtl_cm_chan_is_closing()
  *
  *	Inform IBTL that the TIMEWAIT delay for the connection has been
@@ -143,6 +148,7 @@ uint16_t ibtl_cm_get_1st_full_pkey_ix(ib_guid_t hca_guid, uint8_t port);
  *	rc_chan		Channel Handle
  */
 void ibtl_cm_chan_is_open(ibt_channel_hdl_t rc_chan);
+void ibtl_cm_chan_is_opening(ibt_channel_hdl_t rc_chan);
 void ibtl_cm_chan_is_closing(ibt_channel_hdl_t rc_chan);
 void ibtl_cm_chan_is_closed(ibt_channel_hdl_t rc_chan);
 void ibtl_cm_chan_is_reused(ibt_channel_hdl_t rc_chan);
diff --git a/usr/src/uts/common/sys/ib/mgt/ibcm/ibcm_impl.h b/usr/src/uts/common/sys/ib/mgt/ibcm/ibcm_impl.h
index 873c10d6a6..fb5c4aed0d 100644
--- a/usr/src/uts/common/sys/ib/mgt/ibcm/ibcm_impl.h
+++ b/usr/src/uts/common/sys/ib/mgt/ibcm/ibcm_impl.h
@@ -421,7 +421,7 @@ typedef struct ibcm_state_data_s {
 	boolean_t		delete_mra_msg;
 	boolean_t		stale;
 	boolean_t		delete_state_data;
-	boolean_t		skip_rtr;
+	boolean_t		is_this_ofuv_chan;
 
 	boolean_t		open_done;
 	boolean_t		close_done;
@@ -469,6 +469,9 @@ typedef struct ibcm_state_data_s {
 	/* For ibt_ofuvcm_get_req_data() */
 	void			*req_msgp;
 
+	/* Stored RNR retry count from incoming REQ or REP */
+	ibt_rnr_retry_cnt_t	local_qp_rnr_cnt;
+
 } ibcm_state_data_t;
 
 _NOTE(MUTEX_PROTECTS_DATA(ibcm_state_data_s::state_mutex,