6832684 Panic in idm while running iser tests on snv_113 BAD TRAP: type=e (#pf Page fault)

6835844 iSCSI target server panic with RHEL 4.5.2 client, mutex_destroy
6755803 win2003 initiator numerous iscsi connection lost and connection retries mesgs to iscsi target
6823971 iscsit returns an overly pessimistic login error when the target is offline

11 files changed, 301 insertions, 34 deletions

diff --git a/usr/src/uts/common/io/comstar/port/iscsit/iscsit.c b/usr/src/uts/common/io/comstar/port/iscsit/iscsit.c
index cdaef9d414..77f6d19a0f 100644
--- a/usr/src/uts/common/io/comstar/port/iscsit/iscsit.c
+++ b/usr/src/uts/common/io/comstar/port/iscsit/iscsit.c
@@ -57,11 +57,6 @@ static int iscsit_drv_open(dev_t *, int, int, cred_t *);
 static int iscsit_drv_close(dev_t, int, int, cred_t *);
 static boolean_t iscsit_drv_busy(void);
 static int iscsit_drv_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
-static boolean_t iscsit_cmdsn_in_window(iscsit_conn_t *ict, uint32_t cmdsn);
-static void iscsit_send_direct_scsi_resp(iscsit_conn_t *ict, idm_pdu_t *rx_pdu,
-    uint8_t response, uint8_t cmd_status);
-static void iscsit_send_task_mgmt_resp(idm_pdu_t *tm_resp_pdu,
-    uint8_t tm_status);
 
 extern struct mod_ops mod_miscops;
 
@@ -200,6 +195,12 @@ iscsit_tm_task_alloc(iscsit_conn_t *ict);
 static void
 iscsit_tm_task_free(iscsit_task_t *itask);
 
+static idm_status_t
+iscsit_task_start(iscsit_task_t *itask);
+
+static void
+iscsit_task_done(iscsit_task_t *itask);
+
 static int
 iscsit_status_pdu_constructor(void *pdu_void, void *arg, int flags);
 
@@ -212,6 +213,14 @@ iscsit_config_merge(it_config_t *cfg);
 static idm_status_t
 iscsit_login_fail(idm_conn_t *ic);
 
+static boolean_t iscsit_cmdsn_in_window(iscsit_conn_t *ict, uint32_t cmdsn);
+static void iscsit_send_direct_scsi_resp(iscsit_conn_t *ict, idm_pdu_t *rx_pdu,
+    uint8_t response, uint8_t cmd_status);
+static void iscsit_send_task_mgmt_resp(idm_pdu_t *tm_resp_pdu,
+    uint8_t tm_status);
+static void iscsit_send_reject(iscsit_conn_t *ict, idm_pdu_t *rejected_pdu,
+    uint8_t reason);
+
 int
 _init(void)
 {
@@ -857,6 +866,11 @@ iscsit_task_aborted(idm_task_t *idt, idm_status_t status)
 		if (itask->it_stmf_abort) {
 			mutex_exit(&itask->it_mutex);
 			/*
+			 * Task is no longer active
+			 */
+			iscsit_task_done(itask);
+
+			/*
 			 * STMF has already asked for this task to be aborted
 			 *
 			 * STMF specification is wrong... says to return
@@ -967,6 +981,46 @@ iscsit_build_hdr(idm_task_t *idm_task, idm_pdu_t *pdu, uint8_t opcode)
 	 */
 }
 
+void
+iscsit_keepalive(idm_conn_t *ic)
+{
+	idm_pdu_t		*nop_in_pdu;
+	iscsi_nop_in_hdr_t	*nop_in;
+	iscsit_conn_t		*ict = ic->ic_handle;
+
+	/*
+	 * IDM noticed the connection has been idle for too long so it's
+	 * time to provoke some activity.  Build and transmit an iSCSI
+	 * nop-in PDU -- when the initiator responds it will be counted
+	 * as "activity" and keep the connection alive.
+	 *
+	 * We don't actually care about the response here at the iscsit level
+	 * so we will just throw it away without looking at it when it arrives.
+	 */
+	nop_in_pdu = idm_pdu_alloc(sizeof (*nop_in), 0);
+	idm_pdu_init(nop_in_pdu, ic, NULL, NULL);
+
+	nop_in = (iscsi_nop_in_hdr_t *)nop_in_pdu->isp_hdr;
+	bzero(nop_in, sizeof (*nop_in));
+	nop_in->opcode = ISCSI_OP_NOOP_IN;
+	nop_in->flags = ISCSI_FLAG_FINAL;
+	nop_in->itt = ISCSI_RSVD_TASK_TAG;
+	/*
+	 * This works because we don't currently allocate ttt's anywhere else
+	 * in iscsit so as long as we stay out of IDM's range we are safe.
+	 * If we need to allocate ttt's for other PDU's in the future this will
+	 * need to be improved.
+	 */
+	mutex_enter(&ict->ict_mutex);
+	nop_in->ttt = ict->ict_keepalive_ttt;
+	ict->ict_keepalive_ttt++;
+	if (ict->ict_keepalive_ttt == ISCSI_RSVD_TASK_TAG)
+		ict->ict_keepalive_ttt = IDM_TASKIDS_MAX;
+	mutex_exit(&ict->ict_mutex);
+
+	iscsit_pdu_tx(nop_in_pdu);
+}
+
 static idm_status_t
 iscsit_conn_accept(idm_conn_t *ic)
 {
@@ -994,6 +1048,7 @@ iscsit_conn_accept(idm_conn_t *ic)
 
 	ict->ict_ic = ic;
 	ict->ict_statsn = 1;
+	ict->ict_keepalive_ttt = IDM_TASKIDS_MAX; /* Avoid IDM TT range */
 	ic->ic_handle = ict;
 	mutex_init(&ict->ict_mutex, NULL, MUTEX_DRIVER, NULL);
 	idm_refcnt_init(&ict->ict_refcnt, ict);
@@ -1370,6 +1425,14 @@ iscsit_send_scsi_status(scsi_task_t *task, uint32_t ioflags)
 		return (STMF_SUCCESS);
 	}
 
+	/*
+	 * Remove the task from the session task list
+	 */
+	iscsit_task_done(itask);
+
+	/*
+	 * Send status
+	 */
 	mutex_enter(&itask->it_idm_task->idt_mutex);
 	if ((itask->it_idm_task->idt_state == TASK_ACTIVE) &&
 	    (task->task_completion_status == STMF_SUCCESS) &&
@@ -1539,6 +1602,11 @@ iscsit_abort(stmf_local_port_t *lport, int abort_cmd, void *arg, uint32_t flags)
 	if (iscsit_task->it_aborted) {
 		mutex_exit(&iscsit_task->it_mutex);
 		/*
+		 * Task is no longer active
+		 */
+		iscsit_task_done(iscsit_task);
+
+		/*
 		 * STMF specification is wrong... says to return
 		 * STMF_ABORTED, the code actually looks for
 		 * STMF_ABORT_SUCCESS.
@@ -1634,14 +1702,15 @@ iscsit_op_scsi_cmd(idm_conn_t *ic, idm_pdu_t *rx_pdu)
 
 	itask = iscsit_task_alloc(ict);
 	if (itask == NULL) {
+		/* Finish processing request */
+		iscsit_set_cmdsn(ict, rx_pdu);
+
 		iscsit_send_direct_scsi_resp(ict, rx_pdu,
 		    ISCSI_STATUS_CMD_COMPLETED, STATUS_BUSY);
 		idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 		return;
 	}
 
-	/* Finish processing request */
-	iscsit_set_cmdsn(ict, rx_pdu);
 
 	/*
 	 * Note CmdSN and ITT in task.  IDM will have already validated this
@@ -1669,6 +1738,37 @@ iscsit_op_scsi_cmd(idm_conn_t *ic, idm_pdu_t *rx_pdu)
 
 	ict = rx_pdu->isp_ic->ic_handle; /* IDM client private */
 
+	/*
+	 * Add task to session list.  This function will also check to
+	 * ensure that the task does not already exist.
+	 */
+	if (iscsit_task_start(itask) != IDM_STATUS_SUCCESS) {
+		/*
+		 * Task exists, free all resources and reject.  Don't
+		 * update expcmdsn in this case because RFC 3720 says
+		 * "The CmdSN of the rejected command PDU (if it is a
+		 * non-immediate command) MUST NOT be considered received
+		 * by the target (i.e., a command sequence gap must be
+		 * assumed for the CmdSN), even though the CmdSN of the
+		 * rejected command PDU may be reliably ascertained.  Upon
+		 * receiving the Reject, the initiator MUST plug the CmdSN
+		 * gap in order to continue to use the session.  The gap
+		 * may be plugged either by transmitting a command PDU
+		 * with the same CmdSN, or by aborting the task (see section
+		 * 6.9 on how an abort may plug a CmdSN gap)." (Section 6.3)
+		 */
+		iscsit_task_free(itask);
+		iscsit_send_reject(ict, rx_pdu, ISCSI_REJECT_TASK_IN_PROGRESS);
+		idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
+		return;
+	}
+
+	/* Update sequence numbers */
+	iscsit_set_cmdsn(ict, rx_pdu);
+
+	/*
+	 * Allocate STMF task
+	 */
 	itask->it_stmf_task = stmf_task_alloc(
 	    itask->it_ict->ict_sess->ist_lport,
 	    itask->it_ict->ict_sess->ist_stmf_sess, iscsi_scsi->lun,
@@ -1679,10 +1779,11 @@ iscsit_op_scsi_cmd(idm_conn_t *ic, idm_pdu_t *rx_pdu)
 		 * more likely, the LU is currently in reset.  Either way
 		 * we have no choice but to fail the request.
 		 */
+		iscsit_task_done(itask);
 		iscsit_task_free(itask);
 		iscsit_send_direct_scsi_resp(ict, rx_pdu,
 		    ISCSI_STATUS_CMD_COMPLETED, STATUS_BUSY);
-		idm_pdu_complete(rx_pdu, IDM_STATUS_FAIL);
+		idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 		return;
 	}
 
@@ -2099,10 +2200,12 @@ iscsit_pdu_op_noop(iscsit_conn_t *ict, idm_pdu_t *rx_pdu)
 	int resp_datalen;
 	idm_pdu_t *resp;
 
-	/* Get iSCSI session handle */
 	/* Ignore the response from initiator */
-	if (out->ttt != ISCSI_RSVD_TASK_TAG)
+	if ((out->itt == ISCSI_RSVD_TASK_TAG) ||
+	    (out->ttt != ISCSI_RSVD_TASK_TAG)) {
+		idm_pdu_complete(rx_pdu, IDM_STATUS_SUCCESS);
 		return;
+	}
 
 	/* Allocate a PDU to respond */
 	resp_datalen = ntoh24(out->dlength);
@@ -2281,9 +2384,8 @@ iscsit_send_async_event(iscsit_conn_t *ict, uint8_t event)
 		idm_conn_event(ict->ict_ic, CE_TRANSPORT_FAIL, NULL);
 		return;
 	}
-	idm_pdu_init(abt, ict->ict_ic, NULL, NULL);
 
-	ASSERT(abt != NULL);
+	idm_pdu_init(abt, ict->ict_ic, NULL, NULL);
 	abt->isp_datalen = 0;
 
 	async_abt = (iscsi_async_evt_hdr_t *)abt->isp_hdr;
@@ -2311,6 +2413,41 @@ iscsit_send_async_event(iscsit_conn_t *ict, uint8_t event)
 	iscsit_pdu_tx(abt);
 }
 
+void
+iscsit_send_reject(iscsit_conn_t *ict, idm_pdu_t *rejected_pdu, uint8_t reason)
+{
+	idm_pdu_t		*reject_pdu;
+	iscsi_reject_rsp_hdr_t	*reject;
+
+	/*
+	 * Get a PDU to build the abort request.
+	 */
+	reject_pdu = idm_pdu_alloc(sizeof (iscsi_hdr_t),
+	    rejected_pdu->isp_hdrlen);
+	if (reject_pdu == NULL) {
+		idm_conn_event(ict->ict_ic, CE_TRANSPORT_FAIL, NULL);
+		return;
+	}
+	idm_pdu_init(reject_pdu, ict->ict_ic, NULL, NULL);
+
+	reject_pdu->isp_datalen = rejected_pdu->isp_hdrlen;
+	bcopy(rejected_pdu->isp_hdr, reject_pdu->isp_data,
+	    rejected_pdu->isp_hdrlen);
+
+	reject = (iscsi_reject_rsp_hdr_t *)reject_pdu->isp_hdr;
+	bzero(reject, sizeof (*reject));
+	reject->opcode = ISCSI_OP_REJECT_MSG;
+	reject->reason = reason;
+	reject->flags = ISCSI_FLAG_FINAL;
+	hton24(reject->dlength, rejected_pdu->isp_hdrlen);
+	reject->must_be_ff[0] = 0xff;
+	reject->must_be_ff[1] = 0xff;
+	reject->must_be_ff[2] = 0xff;
+	reject->must_be_ff[3] = 0xff;
+
+	iscsit_pdu_tx(reject_pdu);
+}
+
 
 static iscsit_task_t *
 iscsit_task_alloc(iscsit_conn_t *ict)
@@ -2400,6 +2537,47 @@ iscsit_tm_task_free(iscsit_task_t *itask)
 	kmem_free(itask, sizeof (iscsit_task_t));
 }
 
+static idm_status_t
+iscsit_task_start(iscsit_task_t *itask)
+{
+	iscsit_sess_t *ist = itask->it_ict->ict_sess;
+	avl_index_t		where;
+
+	/*
+	 * Sanity check the ITT and ensure that this task does not already
+	 * exist.  If not then add the task to the session task list.
+	 */
+	mutex_enter(&ist->ist_mutex);
+	mutex_enter(&itask->it_mutex);
+	itask->it_active = 1;
+	if (avl_find(&ist->ist_task_list, itask, &where) == NULL) {
+		/* New task, add to AVL */
+		avl_insert(&ist->ist_task_list, itask, where);
+		mutex_exit(&itask->it_mutex);
+		mutex_exit(&ist->ist_mutex);
+		return (IDM_STATUS_SUCCESS);
+	}
+	mutex_exit(&itask->it_mutex);
+	mutex_exit(&ist->ist_mutex);
+
+	return (IDM_STATUS_REJECT);
+}
+
+static void
+iscsit_task_done(iscsit_task_t *itask)
+{
+	iscsit_sess_t *ist = itask->it_ict->ict_sess;
+
+	mutex_enter(&ist->ist_mutex);
+	mutex_enter(&itask->it_mutex);
+	if (itask->it_active) {
+		avl_remove(&ist->ist_task_list, itask);
+		itask->it_active = 0;
+	}
+	mutex_exit(&itask->it_mutex);
+	mutex_exit(&ist->ist_mutex);
+}
+
 /*
  * iscsit status PDU cache
  */
diff --git a/usr/src/uts/common/io/comstar/port/iscsit/iscsit.h b/usr/src/uts/common/io/comstar/port/iscsit/iscsit.h
index 562aec035d..675bc646a2 100644
--- a/usr/src/uts/common/io/comstar/port/iscsit/iscsit.h
+++ b/usr/src/uts/common/io/comstar/port/iscsit/iscsit.h
@@ -361,6 +361,7 @@ typedef struct {
 	uint16_t		ist_tpgt_tag;
 	uint32_t		ist_expcmdsn;
 	uint32_t		ist_maxcmdsn;
+	avl_tree_t		ist_task_list;
 } iscsit_sess_t;
 
 /* Update iscsit_ils_name table whenever login states are modified */
@@ -489,6 +490,7 @@ typedef struct iscsit_conn_s {
 	iscsit_op_params_t	ict_op;
 	uint16_t		ict_cid;
 	uint32_t		ict_statsn;
+	uint32_t		ict_keepalive_ttt;
 	struct iscsit_conn_s	*ict_reinstate_conn;
 	uint32_t		ict_reinstating:1,
 				ict_lost:1,
@@ -513,11 +515,13 @@ typedef struct {
 	idm_pdu_t		*it_tm_pdu;
 	uint32_t		it_stmf_abort:1,
 				it_aborted:1,
+				it_active:1,
 				it_tm_task:1,
 				it_tm_responded:1;
 	uint32_t		it_cmdsn;
 	uint32_t		it_itt;
 	uint32_t		it_ttt;
+	avl_node_t		it_sess_ln;
 } iscsit_task_t;
 
 typedef struct iscsit_isns_cfg {
@@ -602,6 +606,7 @@ idm_rx_pdu_error_cb_t	iscsit_rx_pdu_error;
 idm_task_cb_t		iscsit_task_aborted;
 idm_client_notify_cb_t	iscsit_client_notify;
 idm_build_hdr_cb_t	iscsit_build_hdr;
+idm_keepalive_cb_t	iscsit_keepalive;
 
 /*
  * lport entry points
diff --git a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_login.c b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_login.c
index 9ac2f4ffbd..5543082350 100644
--- a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_login.c
+++ b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_login.c
@@ -1279,8 +1279,8 @@ login_sm_session_bind(iscsit_conn_t *ict)
 			    ((iscsit_global.global_svc_state != ISE_ENABLED) &&
 			    ((iscsit_global.global_svc_state != ISE_BUSY)))) {
 				SET_LOGIN_ERROR(ict,
-				    ISCSI_STATUS_CLASS_INITIATOR_ERR,
-				    ISCSI_LOGIN_STATUS_TGT_REMOVED);
+				    ISCSI_STATUS_CLASS_TARGET_ERR,
+				    ISCSI_LOGIN_STATUS_SVC_UNAVAILABLE);
 				mutex_exit(&tgt->target_mutex);
 				ISCSIT_GLOBAL_UNLOCK();
 				goto session_bind_error;
diff --git a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_sess.c b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_sess.c
index e61dda1735..59a811ffd2 100644
--- a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_sess.c
+++ b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_sess.c
@@ -83,6 +83,8 @@ static void
 sess_sm_new_state(iscsit_sess_t *ist, sess_event_ctx_t *ctx,
     iscsit_session_state_t new_state);
 
+static int
+iscsit_task_itt_compare(const void *void_task1, const void *void_task2);
 
 static uint16_t
 iscsit_tsih_alloc(void)
@@ -144,6 +146,8 @@ iscsit_sess_create(iscsit_tgt_t *tgt, iscsit_conn_t *ict,
 	    offsetof(sess_event_ctx_t, se_ctx_node));
 	list_create(&result->ist_conn_list, sizeof (iscsit_conn_t),
 	    offsetof(iscsit_conn_t, ict_sess_ln));
+	avl_create(&result->ist_task_list, iscsit_task_itt_compare,
+	    sizeof (iscsit_task_t), offsetof(iscsit_task_t, it_sess_ln));
 
 	result->ist_state = SS_Q1_FREE;
 	result->ist_last_state = SS_Q1_FREE;
@@ -247,6 +251,7 @@ iscsit_sess_destroy(iscsit_sess_t *ist)
 	if (ist->ist_target_alias)
 		kmem_free(ist->ist_target_alias,
 		    strlen(ist->ist_target_alias) + 1);
+	avl_destroy(&ist->ist_task_list);
 	list_destroy(&ist->ist_conn_list);
 	list_destroy(&ist->ist_events);
 	cv_destroy(&ist->ist_cv);
@@ -419,6 +424,19 @@ iscsit_sess_avl_compare(const void *void_sess1, const void *void_sess2)
 	return (0);
 }
 
+int
+iscsit_task_itt_compare(const void *void_task1, const void *void_task2)
+{
+	const iscsit_task_t	*task1 = void_task1;
+	const iscsit_task_t	*task2 = void_task2;
+
+	if (task1->it_itt < task2->it_itt)
+		return (-1);
+	else if (task1->it_itt > task2->it_itt)
+		return (1);
+
+	return (0);
+}
 
 /*
  * State machine
diff --git a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_tgt.c b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_tgt.c
index c875839bb3..cc857dddd7 100644
--- a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_tgt.c
+++ b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_tgt.c
@@ -1941,6 +1941,7 @@ iscsit_portal_online(iscsit_portal_t *portal)
 			sr.sr_conn_ops.icb_client_notify =
 			    &iscsit_client_notify;
 			sr.sr_conn_ops.icb_build_hdr = &iscsit_build_hdr;
+			sr.sr_conn_ops.icb_keepalive = &iscsit_keepalive;
 
 			if (idm_tgt_svc_create(&sr, &svc) !=
 			    IDM_STATUS_SUCCESS) {
diff --git a/usr/src/uts/common/io/ib/clients/iser/iser_cq.c b/usr/src/uts/common/io/ib/clients/iser/iser_cq.c
index 443226d441..9aa4d019a1 100644
--- a/usr/src/uts/common/io/ib/clients/iser/iser_cq.c
+++ b/usr/src/uts/common/io/ib/clients/iser/iser_cq.c
@@ -342,7 +342,9 @@ iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
 	iser_qp->rq_level--;
 
 	if ((iser_qp->rq_taskqpending == B_FALSE) &&
-	    (iser_qp->rq_level <= iser_qp->rq_lwm)) {
+	    (iser_qp->rq_level <= iser_qp->rq_lwm) &&
+	    (iser_chan->ic_conn->ic_stage >= ISER_CONN_STAGE_IC_CONNECTED) &&
+	    (iser_chan->ic_conn->ic_stage <= ISER_CONN_STAGE_LOGGED_IN)) {
 		/* Set the pending flag and fire off a post_recv */
 		iser_qp->rq_taskqpending = B_TRUE;
 		mutex_exit(&iser_qp->qp_lock);
@@ -360,7 +362,6 @@ iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
 	} else {
 		mutex_exit(&iser_qp->qp_lock);
 	}
-	mutex_exit(&iser_chan->ic_conn->ic_lock);
 
 	DTRACE_PROBE3(iser__recv__cqe, iser_chan_t *, iser_chan,
 	    ibt_wc_t *, &wc, ibt_wc_status_t, wc.wc_status);
@@ -369,7 +370,6 @@ iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
 		 * Tell IDM that the channel has gone down,
 		 * unless he already knows.
 		 */
-		mutex_enter(&iser_chan->ic_conn->ic_lock);
 		switch (iser_chan->ic_conn->ic_stage) {
 		case ISER_CONN_STAGE_IC_DISCONNECTED:
 		case ISER_CONN_STAGE_IC_FREED:
@@ -388,6 +388,8 @@ iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
 		iser_msg_free(msg);
 		return (DDI_SUCCESS);
 	} else {
+		mutex_exit(&iser_chan->ic_conn->ic_lock);
+
 		/*
 		 * We have an iSER message in, let's handle it.
 		 * We will free the iser_msg_t later in this path,
diff --git a/usr/src/uts/common/io/ib/clients/iser/iser_ib.c b/usr/src/uts/common/io/ib/clients/iser/iser_ib.c
index fd857fd844..0916da0db2 100644
--- a/usr/src/uts/common/io/ib/clients/iser/iser_ib.c
+++ b/usr/src/uts/common/io/ib/clients/iser/iser_ib.c
@@ -647,6 +647,14 @@ iser_ib_post_recv_async(ibt_channel_hdl_t chanhdl)
 	/* Pull our iSER channel handle from the private data */
 	chan = (iser_chan_t *)ibt_get_chan_private(chanhdl);
 
+	/*
+	 * Caller must check that chan->ic_conn->ic_stage indicates
+	 * the connection is active (not closing, not closed) and
+	 * it must hold the mutex cross the check and the call to this function
+	 */
+	ASSERT(mutex_owned(&chan->ic_conn->ic_lock));
+	ASSERT((chan->ic_conn->ic_stage >= ISER_CONN_STAGE_IC_CONNECTED) &&
+	    (chan->ic_conn->ic_stage <= ISER_CONN_STAGE_LOGGED_IN));
 	idm_conn_hold(chan->ic_conn->ic_idmc);
 	status = ddi_taskq_dispatch(iser_taskq, iser_ib_post_recv_task,
 	    (void *)chanhdl, DDI_NOSLEEP);
diff --git a/usr/src/uts/common/io/idm/idm.c b/usr/src/uts/common/io/idm/idm.c
index 4dd1700975..6ab5811a8a 100644
--- a/usr/src/uts/common/io/idm/idm.c
+++ b/usr/src/uts/common/io/idm/idm.c
@@ -833,6 +833,7 @@ idm_buf_tx_to_ini_done(idm_task_t *idt, idm_buf_t *idb, idm_status_t status)
 
 	switch (idt->idt_state) {
 	case TASK_ACTIVE:
+		idt->idt_ic->ic_timestamp = ddi_get_lbolt();
 		idm_buf_unbind_in_locked(idt, idb);
 		mutex_exit(&idt->idt_mutex);
 		(*idb->idb_buf_cb)(idb, status);
@@ -903,6 +904,7 @@ idm_buf_rx_from_ini_done(idm_task_t *idt, idm_buf_t *idb, idm_status_t status)
 
 	switch (idt->idt_state) {
 	case TASK_ACTIVE:
+		idt->idt_ic->ic_timestamp = ddi_get_lbolt();
 		idm_buf_unbind_out_locked(idt, idb);
 		mutex_exit(&idt->idt_mutex);
 		(*idb->idb_buf_cb)(idb, status);
@@ -2100,6 +2102,12 @@ idm_refcnt_init(idm_refcnt_t *refcnt, void *referenced_obj)
 void
 idm_refcnt_destroy(idm_refcnt_t *refcnt)
 {
+	/*
+	 * Grab the mutex to there are no other lingering threads holding
+	 * the mutex before we destroy it (e.g. idm_refcnt_rele just after
+	 * the refcnt goes to zero if ir_waiting == REF_WAIT_ASYNC)
+	 */
+	mutex_enter(&refcnt->ir_mutex);
 	ASSERT(refcnt->ir_refcnt == 0);
 	cv_destroy(&refcnt->ir_cv);
 	mutex_destroy(&refcnt->ir_mutex);
diff --git a/usr/src/uts/common/io/idm/idm_impl.c b/usr/src/uts/common/io/idm/idm_impl.c
index bf64226f8f..2e39caea15 100644
--- a/usr/src/uts/common/io/idm/idm_impl.c
+++ b/usr/src/uts/common/io/idm/idm_impl.c
@@ -1012,8 +1012,47 @@ idm_wd_thread(void *arg)
 			idle_time = ddi_get_lbolt() - ic->ic_timestamp;
 
 			/*
+			 * If this connection is in FFP then grab a hold
+			 * and check the various timeout thresholds.  Otherwise
+			 * the connection is closing and we should just
+			 * move on to the next one.
+			 */
+			mutex_enter(&ic->ic_state_mutex);
+			if (ic->ic_ffp) {
+				idm_conn_hold(ic);
+			} else {
+				mutex_exit(&ic->ic_state_mutex);
+				continue;
+			}
+
+			/*
 			 * If there hasn't been any activity on this
-			 * connection for the specified period then
+			 * connection for the keepalive timeout period
+			 * and if the client has provided a keepalive
+			 * callback then call the keepalive callback.
+			 * This allows the client to take action to keep
+			 * the link alive (like send a nop PDU).
+			 */
+			if ((TICK_TO_SEC(idle_time) >=
+			    IDM_TRANSPORT_KEEPALIVE_IDLE_TIMEOUT) &&
+			    !ic->ic_keepalive) {
+				ic->ic_keepalive = B_TRUE;
+				if (ic->ic_conn_ops.icb_keepalive) {
+					mutex_exit(&ic->ic_state_mutex);
+					mutex_exit(&idm.idm_global_mutex);
+					(*ic->ic_conn_ops.icb_keepalive)(ic);
+					mutex_enter(&idm.idm_global_mutex);
+					mutex_enter(&ic->ic_state_mutex);
+				}
+			} else if ((TICK_TO_SEC(idle_time) <
+			    IDM_TRANSPORT_KEEPALIVE_IDLE_TIMEOUT)) {
+				/* Reset keepalive */
+				ic->ic_keepalive = B_FALSE;
+			}
+
+			/*
+			 * If there hasn't been any activity on this
+			 * connection for the failure timeout period then
 			 * drop the connection.  We expect the initiator
 			 * to keep the connection alive if it wants the
 			 * connection to stay open.
@@ -1026,25 +1065,23 @@ idm_wd_thread(void *arg)
 			 */
 			if (TICK_TO_SEC(idle_time) >
 			    IDM_TRANSPORT_FAIL_IDLE_TIMEOUT) {
-				/*
-				 * Only send the transport fail if we're in
-				 * FFP.  State machine timers should handle
-				 * problems in non-ffp states.
-				 */
-				if (ic->ic_ffp) {
-					mutex_exit(&idm.idm_global_mutex);
-					IDM_SM_LOG(CE_WARN, "idm_wd_thread: "
-					    "conn %p idle for %d seconds, "
-					    "sending CE_TRANSPORT_FAIL",
-					    (void *)ic, (int)idle_time);
-					idm_conn_event(ic, CE_TRANSPORT_FAIL,
-					    NULL);
-					mutex_enter(&idm.idm_global_mutex);
-				}
+				mutex_exit(&ic->ic_state_mutex);
+				mutex_exit(&idm.idm_global_mutex);
+				IDM_SM_LOG(CE_WARN, "idm_wd_thread: "
+				    "conn %p idle for %d seconds, "
+				    "sending CE_TRANSPORT_FAIL",
+				    (void *)ic, (int)idle_time);
+				idm_conn_event(ic, CE_TRANSPORT_FAIL, NULL);
+				mutex_enter(&idm.idm_global_mutex);
+				mutex_enter(&ic->ic_state_mutex);
 			}
+
+			idm_conn_rele(ic);
+
+			mutex_exit(&ic->ic_state_mutex);
 		}
 
-		wake_time = lbolt + SEC_TO_TICK(IDM_WD_INTERVAL);
+		wake_time = ddi_get_lbolt() + SEC_TO_TICK(IDM_WD_INTERVAL);
 		(void) cv_timedwait(&idm.idm_wd_cv, &idm.idm_global_mutex,
 		    wake_time);
 	}
diff --git a/usr/src/uts/common/sys/idm/idm.h b/usr/src/uts/common/sys/idm/idm.h
index 16e3b9fdc0..90353fa7fc 100644
--- a/usr/src/uts/common/sys/idm/idm.h
+++ b/usr/src/uts/common/sys/idm/idm.h
@@ -149,6 +149,8 @@ typedef void (idm_task_cb_t)(struct idm_task_s *task, idm_status_t status);
 typedef void (idm_build_hdr_cb_t)(
     struct idm_task_s *task, struct idm_pdu_s *pdu, uint8_t opcode);
 
+typedef void (idm_keepalive_cb_t)(struct idm_conn_s *ic);
+
 typedef union idm_sockaddr {
 	struct sockaddr		sin;
 	struct sockaddr_in	sin4;
@@ -167,6 +169,7 @@ typedef struct {
 	idm_task_cb_t		*icb_task_aborted;
 	idm_client_notify_cb_t	*icb_client_notify;
 	idm_build_hdr_cb_t	*icb_build_hdr;
+	idm_keepalive_cb_t	*icb_keepalive;
 } idm_conn_ops_t;
 
 typedef struct {
diff --git a/usr/src/uts/common/sys/idm/idm_impl.h b/usr/src/uts/common/sys/idm/idm_impl.h
index c6d8a16101..0500e0b256 100644
--- a/usr/src/uts/common/sys/idm/idm_impl.h
+++ b/usr/src/uts/common/sys/idm/idm_impl.h
@@ -53,6 +53,12 @@ typedef enum {
 #define	IDM_WD_INTERVAL			5
 
 /*
+ * Timeout period before the client "keepalive" callback is invoked in
+ * seconds if the connection is idle.
+ */
+#define	IDM_TRANSPORT_KEEPALIVE_IDLE_TIMEOUT	20
+
+/*
  * Timeout period before a TRANSPORT_FAIL event is generated in seconds
  * if the connection is idle.
  */
@@ -161,6 +167,7 @@ typedef struct idm_conn_s {
 	idm_status_t		ic_conn_sm_status;
 
 	boolean_t		ic_ffp;
+	boolean_t		ic_keepalive;
 	uint32_t		ic_internal_cid;
 
 	uint32_t		ic_conn_flags;