6832684 Panic in idm while running iser tests on snv_113 BAD TRAP: type=e (#pf Page fault)

6835844 iSCSI target server panic with RHEL 4.5.2 client, mutex_destroy
6755803 win2003 initiator numerous iscsi connection lost and connection retries mesgs to iscsi target
6823971 iscsit returns an overly pessimistic login error when the target is offline

9 files changed, 291 insertions, 34 deletions

diff --git a/usr/src/uts/common/io/comstar/port/iscsit/iscsit.c b/usr/src/uts/common/io/comstar/port/iscsit/iscsit.c
index cdaef9d414..77f6d19a0f 100644
--- a/usr/src/uts/common/io/comstar/port/iscsit/iscsit.c
+++ b/usr/src/uts/common/io/comstar/port/iscsit/iscsit.c
@@ -57,11 +57,6 @@ static int iscsit_drv_open(dev_t *, int, int, cred_t *);
 static int iscsit_drv_close(dev_t, int, int, cred_t *);
 static boolean_t iscsit_drv_busy(void);
 static int iscsit_drv_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
-static boolean_t iscsit_cmdsn_in_window(iscsit_conn_t *ict, uint32_t cmdsn);
-static void iscsit_send_direct_scsi_resp(iscsit_conn_t *ict, idm_pdu_t *rx_pdu,
-    uint8_t response, uint8_t cmd_status);
-static void iscsit_send_task_mgmt_resp(idm_pdu_t *tm_resp_pdu,
-    uint8_t tm_status);
 
 extern struct mod_ops mod_miscops;
 
@@ -200,6 +195,12 @@ iscsit_tm_task_alloc(iscsit_conn_t *ict);
 static void
 iscsit_tm_task_free(iscsit_task_t *itask);
 
+static idm_status_t
+iscsit_task_start(iscsit_task_t *itask);
+
+static void
+iscsit_task_done(iscsit_task_t *itask);
+
 static int
 iscsit_status_pdu_constructor(void *pdu_void, void *arg, int flags);
 
@@ -212,6 +213,14 @@ iscsit_config_merge(it_config_t *cfg);
 static idm_status_t
 iscsit_login_fail(idm_conn_t *ic);
 
+static boolean_t iscsit_cmdsn_in_window(iscsit_conn_t *ict, uint32_t cmdsn);
+static void iscsit_send_direct_scsi_resp(iscsit_conn_t *ict, idm_pdu_t *rx_pdu,
+    uint8_t response, uint8_t cmd_status);
+static void iscsit_send_task_mgmt_resp(idm_pdu_t *tm_resp_pdu,
+    uint8_t tm_status);
+static void iscsit_send_reject(iscsit_conn_t *ict, idm_pdu_t *rejected_pdu,
+    uint8_t reason);
+
 int
 _init(void)
 {
@@ -857,6 +866,11 @@ iscsit_task_aborted(idm_task_t *idt, idm_status_t status)
 		if (itask->it_stmf_abort) {
 			mutex_exit(&itask->it_mutex);
 			/*
+			 * Task is no longer active
+			 */
+			iscsit_task_done(itask);
+
+			/*
 			 * STMF has already asked for this task to be aborted
 			 *
 			 * STMF specification is wrong... says to return
@@ -967,6 +981,46 @@ iscsit_build_hdr(idm_task_t *idm_task, idm_pdu_t *pdu, uint8_t opcode)
 	 */
 }
 
+void
+iscsit_keepalive(idm_conn_t *ic)
+{
+	idm_pdu_t		*nop_in_pdu;
+	iscsi_nop_in_hdr_t	*nop_in;
+	iscsit_conn_t		*ict = ic->ic_handle;
+
+	/*
+	 * IDM noticed the connection has been idle for too long so it's
+	 * time to provoke some activity.  Build and transmit an iSCSI
+	 * nop-in PDU -- when the initiator responds it will be counted
+	 * as "activity" and keep the connection alive.
+	 *
+	 * We don't actually care about the response here at the iscsit level
+	 * so we will just throw it away without looking at it when it arrives.
+	 */
+	nop_in_pdu = idm_pdu_alloc(sizeof (*nop_in), 0);
+	idm_pdu_init(nop_in_pdu, ic, NULL, NULL);
+
+	nop_in = (iscsi_nop_in_hdr_t *)nop_in_pdu->isp_hdr;
+	bzero(nop_in, sizeof (*nop_in));
+	nop_in->opcode = ISCSI_OP_NOOP_IN;
+	nop_in->flags = ISCSI_FLAG_FINAL;
+	nop_in->itt = ISCSI_RSVD_TASK_TAG;
+	/*
+	 * This works because we don't currently allocate ttt's anywhere else
+	 * in iscsit so as long as we stay out of IDM's range we are safe.
+	 * If we need to allocate ttt's for other PDU's in the future this will
+	 * need to be improved.
+	 */
+	mutex_enter(&ict->ict_mutex);
+	nop_in->ttt = ict->ict_keepalive_ttt;
+	ict->ict_keepalive_ttt++;
+	if (ict->ict_keepalive_ttt == ISCSI_RSVD_TASK_TAG)
+		ict->ict_keepalive_ttt = IDM_TASKIDS_MAX;
+	mutex_exit(&ict->ict_mutex);
+
+	iscsit_pdu_tx(nop_in_pdu);
+}
+
 static idm_status_t
 iscsit_conn_accept(idm_conn_t *ic)
 {
@@ -994,6 +1048,7 @@ iscsit_conn_accept(idm_conn_t *ic)
 
 	ict->ict_ic = ic;
 	ict->ict_statsn = 1;
+	ict->ict_keepalive_ttt = IDM_TASKIDS_MAX; /* Avoid IDM TT range */
 	ic->ic_handle = ict;
 	mutex_init(&ict->ict_mutex, NULL, MUTEX_DRIVER, NULL);
 	idm_refcnt_init(&ict->ict_refcnt, ict);
@@ -1370,6 +1425,14 @@ iscsit_send_scsi_status(scsi_task_t *task, uint32_t ioflags)
 		return (STMF_SUCCESS);
 	}
 
+	/*
+	 * Remove the task from the session task list
+	 */
+	iscsit_task_done(itask);
+
+	/*
+	 * Send status
+	 */
 	mutex_enter(&itask->it_idm_task->idt_mutex);
 	if ((itask->it_idm_task->idt_state == TASK_ACTIVE) &&
 	    (task->task_completion_status == STMF_SUCCESS) &&
@@ -1539,6 +1602,11 @@ iscsit_abort(stmf_local_port_t *lport, int abort_cmd, void *arg, uint32_t flags)
 	if (iscsit_task->it_aborted) {
 		mutex_exit(&iscsit_task->it_mutex);
 		/*
+		 * Task is no longer active
+		 */
+		iscsit_task_done(iscsit_task);
+
+		/*
 		 * STMF specification is wrong... says to return
 		 * STMF_ABORTED, the code actually looks for
 		 * STMF_ABORT_SUCCESS.
@@ -1634,14 +1702,15 @@ iscsit_op_scsi_cmd(idm_conn_t *ic, idm_pdu_t *rx_pdu)
 
 	itask = iscsit_task_alloc(ict);
 	if (itask == NULL) {
+		/* Finish processing request */
+		iscsit_set_cmdsn(ict, rx_pdu);
+
 		iscsit_send_direct_scsi_resp(ict, rx_pdu,
 		    ISCSI_STATUS_CMD_COMPLETED, STATUS_BUSY);
 		idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 		return;
 	}
 
-	/* Finish processing request */
-	iscsit_set_cmdsn(ict, rx_pdu);
 
 	/*
 	 * Note CmdSN and ITT in task.  IDM will have already validated this
@@ -1669,6 +1738,37 @@ iscsit_op_scsi_cmd(idm_conn_t *ic, idm_pdu_t *rx_pdu)
 
 	ict = rx_pdu->isp_ic->ic_handle; /* IDM client private */
 
+	/*
+	 * Add task to session list.  This function will also check to
+	 * ensure that the task does not already exist.
+	 */
+	if (iscsit_task_start(itask) != IDM_STATUS_SUCCESS) {
+		/*
+		 * Task exists, free all resources and reject.  Don't
+		 * update expcmdsn in this case because RFC 3720 says
+		 * "The CmdSN of the rejected command PDU (if it is a
+		 * non-immediate command) MUST NOT be considered received
+		 * by the target (i.e., a command sequence gap must be
+		 * assumed for the CmdSN), even though the CmdSN of the
+		 * rejected command PDU may be reliably ascertained.  Upon
+		 * receiving the Reject, the initiator MUST plug the CmdSN
+		 * gap in order to continue to use the session.  The gap
+		 * may be plugged either by transmitting a command PDU
+		 * with the same CmdSN, or by aborting the task (see section
+		 * 6.9 on how an abort may plug a CmdSN gap)." (Section 6.3)
+		 */
+		iscsit_task_free(itask);
+		iscsit_send_reject(ict, rx_pdu, ISCSI_REJECT_TASK_IN_PROGRESS);
+		idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
+		return;
+	}
+
+	/* Update sequence numbers */
+	iscsit_set_cmdsn(ict, rx_pdu);
+
+	/*
+	 * Allocate STMF task
+	 */
 	itask->it_stmf_task = stmf_task_alloc(
 	    itask->it_ict->ict_sess->ist_lport,
 	    itask->it_ict->ict_sess->ist_stmf_sess, iscsi_scsi->lun,
@@ -1679,10 +1779,11 @@ iscsit_op_scsi_cmd(idm_conn_t *ic, idm_pdu_t *rx_pdu)
 		 * more likely, the LU is currently in reset.  Either way
 		 * we have no choice but to fail the request.
 		 */
+		iscsit_task_done(itask);
 		iscsit_task_free(itask);
 		iscsit_send_direct_scsi_resp(ict, rx_pdu,
 		    ISCSI_STATUS_CMD_COMPLETED, STATUS_BUSY);
-		idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
+		idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 		return;
 	}
 
@@ -2099,10 +2200,12 @@ iscsit_pdu_op_noop(iscsit_conn_t *ict, idm_pdu_t *rx_pdu)
 	int resp_datalen;
 	idm_pdu_t *resp;
 
-	/* Get iSCSI session handle */
 	/* Ignore the response from initiator */
-	if (out->ttt != ISCSI_RSVD_TASK_TAG)
+	if ((out->itt == ISCSI_RSVD_TASK_TAG) ||
+	    (out->ttt != ISCSI_RSVD_TASK_TAG)) {
+		idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 		return;
+	}
 
 	/* Allocate a PDU to respond */
 	resp_datalen = ntoh24(out->dlength);
@@ -2281,9 +2384,8 @@ iscsit_send_async_event(iscsit_conn_t *ict, uint8_t event)
 		idm_conn_event(ict->ict_ic, CE_TRANSPORT_FAIL, NULL);
 		return;
 	}
-	idm_pdu_init(abt, ict->ict_ic, NULL, NULL);
 
-	ASSERT(abt != NULL);
+	idm_pdu_init(abt, ict->ict_ic, NULL, NULL);
 	abt->isp_datalen = 0;
 
 	async_abt = (iscsi_async_evt_hdr_t *)abt->isp_hdr;
@@ -2311,6 +2413,41 @@ iscsit_send_async_event(iscsit_conn_t *ict, uint8_t event)
 	iscsit_pdu_tx(abt);
 }
 
+void
+iscsit_send_reject(iscsit_conn_t *ict, idm_pdu_t *rejected_pdu, uint8_t reason)
+{
+	idm_pdu_t		*reject_pdu;
+	iscsi_reject_rsp_hdr_t	*reject;
+
+	/*
+	 * Get a PDU to build the abort request.
+	 */
+	reject_pdu = idm_pdu_alloc(sizeof (iscsi_hdr_t),
+	    rejected_pdu->isp_hdrlen);
+	if (reject_pdu == NULL) {
+		idm_conn_event(ict->ict_ic, CE_TRANSPORT_FAIL, NULL);
+		return;
+	}
+	idm_pdu_init(reject_pdu, ict->ict_ic, NULL, NULL);
+
+	reject_pdu->isp_datalen = rejected_pdu->isp_hdrlen;
+	bcopy(rejected_pdu->isp_hdr, reject_pdu->isp_data,
+	    rejected_pdu->isp_hdrlen);
+
+	reject = (iscsi_reject_rsp_hdr_t *)reject_pdu->isp_hdr;
+	bzero(reject, sizeof (*reject));
+	reject->opcode = ISCSI_OP_REJECT_MSG;
+	reject->reason = reason;
+	reject->flags = ISCSI_FLAG_FINAL;
+	hton24(reject->dlength, rejected_pdu->isp_hdrlen);
+	reject->must_be_ff[0] = 0xff;
+	reject->must_be_ff[1] = 0xff;
+	reject->must_be_ff[2] = 0xff;
+	reject->must_be_ff[3] = 0xff;
+
+	iscsit_pdu_tx(reject_pdu);
+}
+
 
 static iscsit_task_t *
 iscsit_task_alloc(iscsit_conn_t *ict)
@@ -2400,6 +2537,47 @@ iscsit_tm_task_free(iscsit_task_t *itask)
 	kmem_free(itask, sizeof (iscsit_task_t));
 }
 
+static idm_status_t
+iscsit_task_start(iscsit_task_t *itask)
+{
+	iscsit_sess_t *ist = itask->it_ict->ict_sess;
+	avl_index_t		where;
+
+	/*
+	 * Sanity check the ITT and ensure that this task does not already
+	 * exist.  If not then add the task to the session task list.
+	 */
+	mutex_enter(&ist->ist_mutex);
+	mutex_enter(&itask->it_mutex);
+	itask->it_active = 1;
+	if (avl_find(&ist->ist_task_list, itask, &where) == NULL) {
+		/* New task, add to AVL */
+		avl_insert(&ist->ist_task_list, itask, where);
+		mutex_exit(&itask->it_mutex);
+		mutex_exit(&ist->ist_mutex);
+		return (IDM_STATUS_SUCCESS);
+	}
+	mutex_exit(&itask->it_mutex);
+	mutex_exit(&ist->ist_mutex);
+
+	return (IDM_STATUS_REJECT);
+}
+
+static void
+iscsit_task_done(iscsit_task_t *itask)
+{
+	iscsit_sess_t *ist = itask->it_ict->ict_sess;
+
+	mutex_enter(&ist->ist_mutex);
+	mutex_enter(&itask->it_mutex);
+	if (itask->it_active) {
+		avl_remove(&ist->ist_task_list, itask);
+		itask->it_active = 0;
+	}
+	mutex_exit(&itask->it_mutex);
+	mutex_exit(&ist->ist_mutex);
+}
+
 /*
  * iscsit status PDU cache
  */
diff --git a/usr/src/uts/common/io/comstar/port/iscsit/iscsit.h b/usr/src/uts/common/io/comstar/port/iscsit/iscsit.h
index 562aec035d..675bc646a2 100644
--- a/usr/src/uts/common/io/comstar/port/iscsit/iscsit.h
+++ b/usr/src/uts/common/io/comstar/port/iscsit/iscsit.h
@@ -361,6 +361,7 @@ typedef struct {
 	uint16_t		ist_tpgt_tag;
 	uint32_t		ist_expcmdsn;
 	uint32_t		ist_maxcmdsn;
+	avl_tree_t		ist_task_list;
 } iscsit_sess_t;
 
 /* Update iscsit_ils_name table whenever login states are modified */
@@ -489,6 +490,7 @@ typedef struct iscsit_conn_s {
 	iscsit_op_params_t	ict_op;
 	uint16_t		ict_cid;
 	uint32_t		ict_statsn;
+	uint32_t		ict_keepalive_ttt;
 	struct iscsit_conn_s	*ict_reinstate_conn;
 	uint32_t		ict_reinstating:1,
 				ict_lost:1,
@@ -513,11 +515,13 @@ typedef struct {
 	idm_pdu_t		*it_tm_pdu;
 	uint32_t		it_stmf_abort:1,
 				it_aborted:1,
+				it_active:1,
 				it_tm_task:1,
 				it_tm_responded:1;
 	uint32_t		it_cmdsn;
 	uint32_t		it_itt;
 	uint32_t		it_ttt;
+	avl_node_t		it_sess_ln;
 } iscsit_task_t;
 
 typedef struct iscsit_isns_cfg {
@@ -602,6 +606,7 @@ idm_rx_pdu_error_cb_t	iscsit_rx_pdu_error;
 idm_task_cb_t		iscsit_task_aborted;
 idm_client_notify_cb_t	iscsit_client_notify;
 idm_build_hdr_cb_t	iscsit_build_hdr;
+idm_keepalive_cb_t	iscsit_keepalive;
 
 /*
  * lport entry points
diff --git a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_login.c b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_login.c
index 9ac2f4ffbd..5543082350 100644
--- a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_login.c
+++ b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_login.c
@@ -1279,8 +1279,8 @@ login_sm_session_bind(iscsit_conn_t *ict)
 			    ((iscsit_global.global_svc_state != ISE_ENABLED) &&
 			    ((iscsit_global.global_svc_state != ISE_BUSY)))) {
 				SET_LOGIN_ERROR(ict,
-				    ISCSI_STATUS_CLASS_INITIATOR_ERR,
-				    ISCSI_LOGIN_STATUS_TGT_REMOVED);
+				    ISCSI_STATUS_CLASS_TARGET_ERR,
+				    ISCSI_LOGIN_STATUS_SVC_UNAVAILABLE);
 				mutex_exit(&tgt->target_mutex);
 				ISCSIT_GLOBAL_UNLOCK();
 				goto session_bind_error;
diff --git a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_sess.c b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_sess.c
index e61dda1735..59a811ffd2 100644
--- a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_sess.c
+++ b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_sess.c
@@ -83,6 +83,8 @@ static void
 sess_sm_new_state(iscsit_sess_t *ist, sess_event_ctx_t *ctx,
     iscsit_session_state_t new_state);
 
+static int
+iscsit_task_itt_compare(const void *void_task1, const void *void_task2);
 
 static uint16_t
 iscsit_tsih_alloc(void)
@@ -144,6 +146,8 @@ iscsit_sess_create(iscsit_tgt_t *tgt, iscsit_conn_t *ict,
 	    offsetof(sess_event_ctx_t, se_ctx_node));
 	list_create(&result->ist_conn_list, sizeof (iscsit_conn_t),
 	    offsetof(iscsit_conn_t, ict_sess_ln));
+	avl_create(&result->ist_task_list, iscsit_task_itt_compare,
+	    sizeof (iscsit_task_t), offsetof(iscsit_task_t, it_sess_ln));
 
 	result->ist_state = SS_Q1_FREE;
 	result->ist_last_state = SS_Q1_FREE;
@@ -247,6 +251,7 @@ iscsit_sess_destroy(iscsit_sess_t *ist)
 	if (ist->ist_target_alias)
 		kmem_free(ist->ist_target_alias,
 		    strlen(ist->ist_target_alias) + 1);
+	avl_destroy(&ist->ist_task_list);
 	list_destroy(&ist->ist_conn_list);
 	list_destroy(&ist->ist_events);
 	cv_destroy(&ist->ist_cv);
@@ -419,6 +424,19 @@ iscsit_sess_avl_compare(const void *void_sess1, const void *void_sess2)
 	return (0);
 }
 
+int
+iscsit_task_itt_compare(const void *void_task1, const void *void_task2)
+{
+	const iscsit_task_t	*task1 = void_task1;
+	const iscsit_task_t	*task2 = void_task2;
+
+	if (task1->it_itt < task2->it_itt)
+		return (-1);
+	else if (task1->it_itt > task2->it_itt)
+		return (1);
+
+	return (0);
+}
 
 /*
  * State machine
diff --git a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_tgt.c b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_tgt.c
index c875839bb3..cc857dddd7 100644
--- a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_tgt.c
+++ b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_tgt.c
@@ -1941,6 +1941,7 @@ iscsit_portal_online(iscsit_portal_t *portal)
 			sr.sr_conn_ops.icb_client_notify =
 			    &iscsit_client_notify;
 			sr.sr_conn_ops.icb_build_hdr = &iscsit_build_hdr;
+			sr.sr_conn_ops.icb_keepalive = &iscsit_keepalive;
 
 			if (idm_tgt_svc_create(&sr, &svc) !=
 			    IDM_STATUS_SUCCESS) {
diff --git a/usr/src/uts/common/io/ib/clients/iser/iser_cq.c b/usr/src/uts/common/io/ib/clients/iser/iser_cq.c
index 443226d441..9aa4d019a1 100644
--- a/usr/src/uts/common/io/ib/clients/iser/iser_cq.c
+++ b/usr/src/uts/common/io/ib/clients/iser/iser_cq.c
@@ -342,7 +342,9 @@ iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
 	iser_qp->rq_level--;
 
 	if ((iser_qp->rq_taskqpending == B_FALSE) &&
-	    (iser_qp->rq_level <= iser_qp->rq_lwm)) {
+	    (iser_qp->rq_level <= iser_qp->rq_lwm) &&
+	    (iser_chan->ic_conn->ic_stage >= ISER_CONN_STAGE_IC_CONNECTED) &&
+	    (iser_chan->ic_conn->ic_stage <= ISER_CONN_STAGE_LOGGED_IN)) {
 		/* Set the pending flag and fire off a post_recv */
 		iser_qp->rq_taskqpending = B_TRUE;
 		mutex_exit(&iser_qp->qp_lock);
@@ -360,7 +362,6 @@ iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
 	} else {
 		mutex_exit(&iser_qp->qp_lock);
 	}
-	mutex_exit(&iser_chan->ic_conn->ic_lock);
 
 	DTRACE_PROBE3(iser__recv__cqe, iser_chan_t *, iser_chan,
 	    ibt_wc_t *, &wc, ibt_wc_status_t, wc.wc_status);
@@ -369,7 +370,6 @@ iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
 		 * Tell IDM that the channel has gone down,
 		 * unless he already knows.
 		 */
-		mutex_enter(&iser_chan->ic_conn->ic_lock);
 		switch (iser_chan->ic_conn->ic_stage) {
 		case ISER_CONN_STAGE_IC_DISCONNECTED:
 		case ISER_CONN_STAGE_IC_FREED:
@@ -388,6 +388,8 @@ iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
 		iser_msg_free(msg);
 		return (DDI_SUCCESS);
 	} else {
+		mutex_exit(&iser_chan->ic_conn->ic_lock);
+
 		/*
 		 * We have an iSER message in, let's handle it.
 		 * We will free the iser_msg_t later in this path,
diff --git a/usr/src/uts/common/io/ib/clients/iser/iser_ib.c b/usr/src/uts/common/io/ib/clients/iser/iser_ib.c
index fd857fd844..0916da0db2 100644
--- a/usr/src/uts/common/io/ib/clients/iser/iser_ib.c
+++ b/usr/src/uts/common/io/ib/clients/iser/iser_ib.c
@@ -647,6 +647,14 @@ iser_ib_post_recv_async(ibt_channel_hdl_t chanhdl)
 	/* Pull our iSER channel handle from the private data */
 	chan = (iser_chan_t *)ibt_get_chan_private(chanhdl);
 
+	/*
+	 * Caller must check that chan->ic_conn->ic_stage indicates
+	 * the connection is active (not closing, not closed) and
+	 * it must hold the mutex cross the check and the call to this function
+	 */
+	ASSERT(mutex_owned(&chan->ic_conn->ic_lock));
+	ASSERT((chan->ic_conn->ic_stage >= ISER_CONN_STAGE_IC_CONNECTED) &&
+	    (chan->ic_conn->ic_stage <= ISER_CONN_STAGE_LOGGED_IN));
 	idm_conn_hold(chan->ic_conn->ic_idmc);
 	status = ddi_taskq_dispatch(iser_taskq, iser_ib_post_recv_task,
 	    (void *)chanhdl, DDI_NOSLEEP);
diff --git a/usr/src/uts/common/io/idm/idm.c b/usr/src/uts/common/io/idm/idm.c
index 4dd1700975..6ab5811a8a 100644
--- a/usr/src/uts/common/io/idm/idm.c
+++ b/usr/src/uts/common/io/idm/idm.c
@@ -833,6 +833,7 @@ idm_buf_tx_to_ini_done(idm_task_t *idt, idm_buf_t *idb, idm_status_t status)
 
 	switch (idt->idt_state) {
 	case TASK_ACTIVE:
+		idt->idt_ic->ic_timestamp = ddi_get_lbolt();
 		idm_buf_unbind_in_locked(idt, idb);
 		mutex_exit(&idt->idt_mutex);
 		(*idb->idb_buf_cb)(idb, status);
@@ -903,6 +904,7 @@ idm_buf_rx_from_ini_done(idm_task_t *idt, idm_buf_t *idb, idm_status_t status)
 
 	switch (idt->idt_state) {
 	case TASK_ACTIVE:
+		idt->idt_ic->ic_timestamp = ddi_get_lbolt();
 		idm_buf_unbind_out_locked(idt, idb);
 		mutex_exit(&idt->idt_mutex);
 		(*idb->idb_buf_cb)(idb, status);
@@ -2100,6 +2102,12 @@ idm_refcnt_init(idm_refcnt_t *refcnt, void *referenced_obj)
 void
 idm_refcnt_destroy(idm_refcnt_t *refcnt)
 {
+	/*
+	 * Grab the mutex to there are no other lingering threads holding
+	 * the mutex before we destroy it (e.g. idm_refcnt_rele just after
+	 * the refcnt goes to zero if ir_waiting == REF_WAIT_ASYNC)
+	 */
+	mutex_enter(&refcnt->ir_mutex);
 	ASSERT(refcnt->ir_refcnt == 0);
 	cv_destroy(&refcnt->ir_cv);
 	mutex_destroy(&refcnt->ir_mutex);
diff --git a/usr/src/uts/common/io/idm/idm_impl.c b/usr/src/uts/common/io/idm/idm_impl.c
index bf64226f8f..2e39caea15 100644
--- a/usr/src/uts/common/io/idm/idm_impl.c
+++ b/usr/src/uts/common/io/idm/idm_impl.c
@@ -1012,8 +1012,47 @@ idm_wd_thread(void *arg)
 			idle_time = ddi_get_lbolt() - ic->ic_timestamp;
 
 			/*
+			 * If this connection is in FFP then grab a hold
+			 * and check the various timeout thresholds.  Otherwise
+			 * the connection is closing and we should just
+			 * move on to the next one.
+			 */
+			mutex_enter(&ic->ic_state_mutex);
+			if (ic->ic_ffp) {
+				idm_conn_hold(ic);
+			} else {
+				mutex_exit(&ic->ic_state_mutex);
+				continue;
+			}
+
+			/*
 			 * If there hasn't been any activity on this
-			 * connection for the specified period then
+			 * connection for the keepalive timeout period
+			 * and if the client has provided a keepalive
+			 * callback then call the keepalive callback.
+			 * This allows the client to take action to keep
+			 * the link alive (like send a nop PDU).
+			 */
+			if ((TICK_TO_SEC(idle_time) >=
+			    IDM_TRANSPORT_KEEPALIVE_IDLE_TIMEOUT) &&
+			    !ic->ic_keepalive) {
+				ic->ic_keepalive = B_TRUE;
+				if (ic->ic_conn_ops.icb_keepalive) {
+					mutex_exit(&ic->ic_state_mutex);
+					mutex_exit(&idm.idm_global_mutex);
+					(*ic->ic_conn_ops.icb_keepalive)(ic);
+					mutex_enter(&idm.idm_global_mutex);
+					mutex_enter(&ic->ic_state_mutex);
+				}
+			} else if ((TICK_TO_SEC(idle_time) <
+			    IDM_TRANSPORT_KEEPALIVE_IDLE_TIMEOUT)) {
+				/* Reset keepalive */
+				ic->ic_keepalive = B_FALSE;
+			}
+
+			/*
+			 * If there hasn't been any activity on this
+			 * connection for the failure timeout period then
 			 * drop the connection.  We expect the initiator
 			 * to keep the connection alive if it wants the
 			 * connection to stay open.
@@ -1026,25 +1065,23 @@ idm_wd_thread(void *arg)
 			 */
 			if (TICK_TO_SEC(idle_time) >
 			    IDM_TRANSPORT_FAIL_IDLE_TIMEOUT) {
-				/*
-				 * Only send the transport fail if we're in
-				 * FFP.  State machine timers should handle
-				 * problems in non-ffp states.
-				 */
-				if (ic->ic_ffp) {
-					mutex_exit(&idm.idm_global_mutex);
-					IDM_SM_LOG(CE_WARN, "idm_wd_thread: "
-					    "conn %p idle for %d seconds, "
-					    "sending CE_TRANSPORT_FAIL",
-					    (void *)ic, (int)idle_time);
-					idm_conn_event(ic, CE_TRANSPORT_FAIL,
-					    NULL);
-					mutex_enter(&idm.idm_global_mutex);
-				}
+				mutex_exit(&ic->ic_state_mutex);
+				mutex_exit(&idm.idm_global_mutex);
+				IDM_SM_LOG(CE_WARN, "idm_wd_thread: "
+				    "conn %p idle for %d seconds, "
+				    "sending CE_TRANSPORT_FAIL",
+				    (void *)ic, (int)idle_time);
+				idm_conn_event(ic, CE_TRANSPORT_FAIL, NULL);
+				mutex_enter(&idm.idm_global_mutex);
+				mutex_enter(&ic->ic_state_mutex);
 			}
+
+			idm_conn_rele(ic);
+
+			mutex_exit(&ic->ic_state_mutex);
 		}
 
-		wake_time = lbolt + SEC_TO_TICK(IDM_WD_INTERVAL);
+		wake_time = ddi_get_lbolt() + SEC_TO_TICK(IDM_WD_INTERVAL);
 		(void) cv_timedwait(&idm.idm_wd_cv, &idm.idm_global_mutex,
 		    wake_time);
 	}