6293434 ibmf hangs when reusing message with stale im_pending_send_compls counter

4 files changed, 36 insertions, 15 deletions

diff --git a/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_impl.c b/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_impl.c
index db69a50eb7..86be7fde2d 100644
--- a/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_impl.c
+++ b/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_impl.c
@@ -2096,9 +2096,12 @@ ibmf_i_msg_transport(ibmf_client_t *clientp, ibmf_qp_handle_t ibmf_qp_handle,
 	msgimplp->im_mgt_class 	= madhdrp->MgmtClass;
 	msgimplp->im_unsolicited = B_FALSE;
 	msgimplp->im_trans_state_flags = IBMF_TRANS_STATE_FLAG_UNINIT;
+	bzero(&msgimplp->im_rmpp_ctx, sizeof (ibmf_rmpp_ctx_t));
 	msgimplp->im_rmpp_ctx.rmpp_state = IBMF_RMPP_STATE_UNDEFINED;
 	msgimplp->im_rmpp_ctx.rmpp_respt = IBMF_RMPP_DEFAULT_RRESPT;
 	msgimplp->im_rmpp_ctx.rmpp_retry_cnt = 0;
+	msgimplp->im_ref_count = 0;
+	msgimplp->im_pending_send_compls = 0;
 	IBMF_MSG_INCR_REFCNT(msgimplp);
 	if (msgimplp->im_retrans.retrans_retries == 0)
 		msgimplp->im_retrans.retrans_retries = IBMF_RETRANS_DEF_RETRIES;
@@ -2382,6 +2385,7 @@ ibmf_i_init_msg(ibmf_msg_impl_t *msgimplp, ibmf_msg_cb_t trans_cb,
 	msgimplp->im_trans_cb = trans_cb;
 	msgimplp->im_trans_cb_arg = trans_cb_arg;
 
+	bzero(&msgimplp->im_retrans, sizeof (ibmf_retrans_t));
 	if (retrans != NULL) {
 		bcopy((void *)retrans, (void *)&msgimplp->im_retrans,
 		    sizeof (ibmf_retrans_t));
diff --git a/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_send.c b/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_send.c
index 3892026781..dcdba756b2 100644
--- a/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_send.c
+++ b/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_send.c
@@ -747,6 +747,30 @@ ibmf_i_do_send_compl(ibmf_handle_t ibmf_handle, ibmf_msg_impl_t *msgimplp,
 			}
 		}
 
+		/*
+		 * If the transaction is a send-only RMPP, then
+		 * set the SEND_DONE flag on every send completion
+		 * as long as there are no outstanding ones.
+		 * This is needed so that the transaction can return
+		 * in the receive path, where ibmf_i_terminate_transaction
+		 * is called from ibmf_i_rmpp_sender_active_flow,
+		 * after checking if the SEND_DONE flag is set.
+		 * When a new MAD is sent as part of the RMPP transaction,
+		 * the SEND_DONE flag will get reset.
+		 * The RECV_DONE indicates that the last ACK was received.
+		 */
+		if ((msgimplp->im_flags & IBMF_MSG_FLAGS_SEQUENCED) == 0) {
+			if (msgimplp->im_pending_send_compls == 0) {
+				msgimplp->im_trans_state_flags |=
+				    IBMF_TRANS_STATE_FLAG_SEND_DONE;
+				if (msgimplp->im_trans_state_flags  &
+				    IBMF_TRANS_STATE_FLAG_RECV_DONE) {
+					msgimplp->im_trans_state_flags |=
+					    IBMF_TRANS_STATE_FLAG_DONE;
+				}
+			}
+		}
+
 		IBMF_TRACE_0(IBMF_TNF_DEBUG, DPRINT_L4,
 		    ibmf_i_do_send_compl_end, IBMF_TNF_TRACE, "",
 		    "ibmf_i_do_send_compl() exit\n");
@@ -812,6 +836,8 @@ ibmf_i_do_send_compl(ibmf_handle_t ibmf_handle, ibmf_msg_impl_t *msgimplp,
 		}
 	} else {
 		msgimplp->im_msg_status = IBMF_SUCCESS;
+		msgimplp->im_trans_state_flags |=
+		    IBMF_TRANS_STATE_FLAG_SEND_DONE;
 		msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_DONE;
 	}
 
diff --git a/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_timers.c b/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_timers.c
index d584b381ff..b737d7aecd 100644
--- a/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_timers.c
+++ b/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_timers.c
@@ -511,11 +511,6 @@ ibmf_i_send_timeout(void *argp)
 		ibmf_i_terminate_transaction(msgimplp->im_client,
 		    msgimplp, IBMF_TRANS_TIMEOUT);
 
-		/*
-		 * Force client notification from this point
-		 */
-		msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_DONE;
-
 		msg_flags = msgimplp->im_trans_state_flags;
 
 		mutex_exit(&msgimplp->im_mutex);
@@ -590,10 +585,6 @@ ibmf_i_send_timeout(void *argp)
 		ibmf_i_terminate_transaction(msgimplp->im_client,
 		    msgimplp, IBMF_TRANS_TIMEOUT);
 
-		/*
-		 * Force client notification from this point
-		 */
-		msgimplp->im_trans_state_flags |= IBMF_TRANS_STATE_FLAG_DONE;
 	} else {
 
 		if (rmpp_ctx->rmpp_state == IBMF_RMPP_STATE_SENDER_ACTIVE) {
diff --git a/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_trans.c b/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_trans.c
index ad9e71f080..027e244277 100644
--- a/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_trans.c
+++ b/usr/src/uts/common/io/ib/mgt/ibmf/ibmf_trans.c
@@ -20,7 +20,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -87,12 +87,12 @@ ibmf_i_terminate_transaction(ibmf_client_t *clientp, ibmf_msg_impl_t *msgimplp,
 		    IBMF_TRANS_STATE_FLAG_RECV_DONE;
 
 		/*
-		 * if it's a sequenced transaction, make sure send is done
-		 * before marking as done
+		 * Check if last send is done before marking as done.
+		 * We should get here for sequenced transactions and
+		 * non-sequenced send RMPP transaction.
 		 */
-		if (((msgimplp->im_flags & IBMF_MSG_FLAGS_SEQUENCED) == 0) ||
-		    (msgimplp->im_trans_state_flags &
-		    IBMF_TRANS_STATE_FLAG_SEND_DONE)) {
+		if (msgimplp->im_trans_state_flags &
+		    IBMF_TRANS_STATE_FLAG_SEND_DONE) {
 			msgimplp->im_trans_state_flags |=
 			    IBMF_TRANS_STATE_FLAG_DONE;
 		}