6844136 lun-reset can't be set correctly in iscsi initiator

6842558 pull-out cables causes reset ping-pong for a long time.
6764779 iscsi session logs out/in about every 45 minutes (object fs)

6 files changed, 385 insertions, 36 deletions

diff --git a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi.c b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi.c
index 47a0fe7ea2..23f31e796b 100644
--- a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi.c
+++ b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi.c
@@ -1068,7 +1068,7 @@ iscsi_tran_getcap(struct scsi_address *ap, char *cap, int whom)
 static int
 iscsi_tran_setcap(struct scsi_address *ap, char *cap, int value, int whom)
 {
-	return (iscsi_i_commoncap(ap, cap, 0, whom, 1));
+	return (iscsi_i_commoncap(ap, cap, value, whom, 1));
 }
 
 
diff --git a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi.h b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi.h
index 9247d92e27..bebe220c61 100644
--- a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi.h
+++ b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi.h
@@ -380,6 +380,16 @@ typedef enum iscsi_cmd_text_stage {
 #define	ISCSI_CMD_MISCFLAG_FREE		0x2
 #define	ISCSI_CMD_MISCFLAG_STUCK	0x4
 #define	ISCSI_CMD_MISCFLAG_XARQ 	0x8
+#define	ISCSI_CMD_MISCFLAG_SENT		0x10
+#define	ISCSI_CMD_MISCFLAG_FLUSH	0x20
+
+/*
+ * 1/2 of a 32 bit number, used for checking CmdSN
+ * wrapped.
+ */
+#define	ISCSI_CMD_SN_WRAP		0x80000000
+
+#define	ISCSI_CMD_PKT_STAT_INIT		0
 
 /*
  * iSCSI cmd/pkt Structure
@@ -430,6 +440,10 @@ typedef struct iscsi_cmd {
 			 * another R2T to handle.
 			 */
 			boolean_t		r2t_more;
+			/*
+			 * It is used to record pkt_statistics temporarily.
+			 */
+			uint_t			pkt_stat;
 		} scsi;
 		/* ISCSI_CMD_TYPE_ABORT */
 		struct {
@@ -439,6 +453,7 @@ typedef struct iscsi_cmd {
 		/* ISCSI_CMD_TYPE_RESET */
 		struct {
 			int			level;
+			uint8_t			response;
 		} reset;
 		/* ISCSI_CMD_TYPE_NOP */
 		struct {
@@ -497,6 +512,8 @@ typedef struct iscsi_cmd {
 	idm_pdu_t		cmd_pdu;
 
 	sm_audit_buf_t		cmd_state_audit;
+
+	uint32_t		cmd_sn;
 } iscsi_cmd_t;
 
 
@@ -528,12 +545,15 @@ typedef struct iscsi_lun {
 	uchar_t			lun_type;
 } iscsi_lun_t;
 
-#define	ISCSI_LUN_STATE_CLEAR	0		/* used to clear all states */
-#define	ISCSI_LUN_STATE_OFFLINE	1
-#define	ISCSI_LUN_STATE_ONLINE	2
-#define	ISCSI_LUN_STATE_INVALID	4		/* offline failed */
+#define	ISCSI_LUN_STATE_CLEAR	    0		/* used to clear all states */
+#define	ISCSI_LUN_STATE_OFFLINE	    1
+#define	ISCSI_LUN_STATE_ONLINE	    2
+#define	ISCSI_LUN_STATE_INVALID	    4		/* offline failed */
+#define	ISCSI_LUN_STATE_BUSY	    8		/* logic unit is in reset */
+
+#define	ISCSI_LUN_CAP_RESET	    0x01
 
-#define	ISCSI_LUN_CAP_RESET   0x01
+#define	ISCSI_SCSI_RESET_SENSE_CODE 0x29
 
 /*
  *
@@ -959,6 +979,10 @@ typedef struct iscsi_sess {
 	iscsi_thread_t		*sess_wd_thread;
 
 	sm_audit_buf_t		sess_state_audit;
+
+	kmutex_t		sess_reset_mutex;
+
+	boolean_t		sess_reset_in_progress;
 } iscsi_sess_t;
 
 /*
diff --git a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_cmd.c b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_cmd.c
index 187e6f613b..3d9f59b9e2 100644
--- a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_cmd.c
+++ b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_cmd.c
@@ -336,6 +336,9 @@ iscsi_cmd_state_free(iscsi_cmd_t *icmdp, iscsi_cmd_event_t event, void *arg)
 				    iscsi_cmd_timeout_factor);
 			else
 				icmdp->cmd_lbolt_timeout = 0;
+
+			icmdp->cmd_un.scsi.pkt_stat &=
+			    ISCSI_CMD_PKT_STAT_INIT;
 		} else {
 			icmdp->cmd_lbolt_timeout = icmdp->cmd_lbolt_pending +
 			    SEC_TO_TICK(ISCSI_INTERNAL_CMD_TIMEOUT *
@@ -636,7 +639,7 @@ iscsi_cmd_state_pending(iscsi_cmd_t *icmdp, iscsi_cmd_event_t event, void *arg)
 				    CMD_TIMEOUT, STAT_TIMEOUT);
 			} else {
 				ISCSI_CMD_SET_REASON_STAT(icmdp,
-				    CMD_TRAN_ERR, 0);
+				    CMD_TRAN_ERR, icmdp->cmd_un.scsi.pkt_stat);
 			}
 			iscsi_enqueue_completed_cmd(isp, icmdp);
 			break;
@@ -792,7 +795,7 @@ iscsi_cmd_state_active(iscsi_cmd_t *icmdp, iscsi_cmd_event_t event, void *arg)
 				 * when we complete the command.
 				 */
 				ISCSI_CMD_SET_REASON_STAT(
-				    t_icmdp, CMD_TIMEOUT, STAT_TIMEOUT);
+				    t_icmdp, CMD_TIMEOUT, STAT_ABORTED);
 				idm_task_abort(icp->conn_ic, t_icmdp->cmd_itp,
 				    AT_TASK_MGMT_ABORT);
 			} else {
@@ -811,9 +814,17 @@ iscsi_cmd_state_active(iscsi_cmd_t *icmdp, iscsi_cmd_event_t event, void *arg)
 			mutex_exit(&isp->sess_cmdsn_mutex);
 
 			/*
-			 * Complete the abort/reset successfully.
+			 * Complete the abort/reset command.
 			 */
-			ISCSI_CMD_ISSUE_CALLBACK(icmdp, ISCSI_STATUS_SUCCESS);
+			if (icmdp->cmd_un.reset.response !=
+			    SCSI_TCP_TM_RESP_COMPLETE) {
+				ISCSI_CMD_ISSUE_CALLBACK(icmdp,
+				    ISCSI_STATUS_CMD_FAILED);
+			} else {
+				ISCSI_CMD_ISSUE_CALLBACK(icmdp,
+				    ISCSI_STATUS_SUCCESS);
+			}
+
 			break;
 
 		case ISCSI_CMD_TYPE_LOGOUT:
@@ -1014,8 +1025,8 @@ iscsi_cmd_state_active(iscsi_cmd_t *icmdp, iscsi_cmd_event_t event, void *arg)
 			mutex_enter(&icp->conn_queue_idm_aborting.mutex);
 			iscsi_enqueue_idm_aborting_cmd(icmdp->cmd_conn, icmdp);
 			mutex_exit(&icp->conn_queue_idm_aborting.mutex);
-
-			ISCSI_CMD_SET_REASON_STAT(icmdp, CMD_TRAN_ERR, 0);
+			ISCSI_CMD_SET_REASON_STAT(icmdp,
+			    CMD_TRAN_ERR, icmdp->cmd_un.scsi.pkt_stat);
 			idm_task_abort(icp->conn_ic, icmdp->cmd_itp,
 			    AT_TASK_MGMT_ABORT);
 			break;
@@ -1111,7 +1122,8 @@ iscsi_cmd_state_active(iscsi_cmd_t *icmdp, iscsi_cmd_event_t event, void *arg)
 		iscsi_task_cleanup(ISCSI_OP_SCSI_RSP, icmdp);
 		iscsi_sess_release_scsi_itt(icmdp);
 
-		ISCSI_CMD_SET_REASON_STAT(icmdp, CMD_TRAN_ERR, 0);
+		ISCSI_CMD_SET_REASON_STAT(icmdp, CMD_TRAN_ERR,
+		    icmdp->cmd_un.scsi.pkt_stat);
 		iscsi_enqueue_completed_cmd(isp, icmdp);
 		break;
 
@@ -1193,7 +1205,9 @@ iscsi_cmd_state_aborting(iscsi_cmd_t *icmdp, iscsi_cmd_event_t event, void *arg)
 			    ISCSI_CMD_EVENT_E10, arg);
 		}
 
-		ISCSI_CMD_SET_REASON_STAT(icmdp, CMD_TRAN_ERR, 0);
+		ISCSI_CMD_SET_REASON_STAT(icmdp,
+		    CMD_TRAN_ERR, icmdp->cmd_un.scsi.pkt_stat);
+
 		idm_task_abort(icmdp->cmd_conn->conn_ic, icmdp->cmd_itp,
 		    AT_TASK_MGMT_ABORT);
 		break;
@@ -1205,7 +1219,8 @@ iscsi_cmd_state_aborting(iscsi_cmd_t *icmdp, iscsi_cmd_event_t event, void *arg)
 		iscsi_task_cleanup(ISCSI_OP_SCSI_RSP, icmdp);
 		iscsi_sess_release_scsi_itt(icmdp);
 
-		ISCSI_CMD_SET_REASON_STAT(icmdp, CMD_TRAN_ERR, 0);
+		ISCSI_CMD_SET_REASON_STAT(icmdp, CMD_TRAN_ERR,
+		    icmdp->cmd_un.scsi.pkt_stat);
 		iscsi_enqueue_completed_cmd(isp, icmdp);
 		break;
 
@@ -1261,8 +1276,10 @@ iscsi_cmd_state_idm_aborting(iscsi_cmd_t *icmdp, iscsi_cmd_event_t event,
 	case ISCSI_CMD_EVENT_E7:
 		/*
 		 * We have already requested IDM to stop processing this
-		 * command so ignore this request.
+		 * command so just update the pkt_statistics.
 		 */
+		ISCSI_CMD_SET_REASON_STAT(icmdp,
+		    CMD_TRAN_ERR, icmdp->cmd_un.scsi.pkt_stat);
 		break;
 
 	/* -E9: IDM is no longer processing this command */
diff --git a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_conn.c b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_conn.c
index 6a05a7fa3f..41bf1353e2 100644
--- a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_conn.c
+++ b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_conn.c
@@ -913,6 +913,13 @@ iscsi_conn_flush_active_cmds(iscsi_conn_t *icp)
 	/* Flush active queue */
 	icmdp = icp->conn_queue_active.head;
 	while (icmdp != NULL) {
+
+		mutex_enter(&icmdp->cmd_mutex);
+		if (icmdp->cmd_type == ISCSI_CMD_TYPE_SCSI) {
+			icmdp->cmd_un.scsi.pkt_stat |= STAT_ABORTED;
+		}
+		mutex_exit(&icmdp->cmd_mutex);
+
 		iscsi_cmd_state_machine(icmdp,
 		    ISCSI_CMD_EVENT_E7, isp);
 		icmdp = icp->conn_queue_active.head;
diff --git a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_io.c b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_io.c
index 71c3da8eca..d97918a692 100644
--- a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_io.c
+++ b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_io.c
@@ -33,6 +33,7 @@
 #include <netinet/tcp.h>	/* TCP_NODELAY */
 #include <sys/socketvar.h>	/* _ALLOC_SLEEP */
 #include <sys/strsun.h>		/* DB_TYPE() */
+#include <sys/scsi/generic/sense.h>
 
 #include "iscsi.h"		/* iscsi driver */
 #include <sys/iscsi_protocol.h>	/* iscsi protocol */
@@ -106,6 +107,9 @@ static void iscsi_handle_nop(iscsi_conn_t *icp, uint32_t itt, uint32_t ttt);
 
 static void iscsi_timeout_checks(iscsi_sess_t *isp);
 static void iscsi_nop_checks(iscsi_sess_t *isp);
+static boolean_t iscsi_decode_sense(uint8_t *sense_data);
+static void iscsi_flush_cmd_after_reset(uint32_t cmd_sn, uint16_t lun_num,
+    iscsi_conn_t *icp);
 
 /*
  * This file contains the main guts of the iSCSI protocol layer.
@@ -424,7 +428,7 @@ iscsi_cmd_rsp_chk(iscsi_cmd_t *icmdp, iscsi_scsi_rsp_hdr_t *issrhp)
 	}
 }
 
-static void
+static boolean_t
 iscsi_cmd_rsp_cmd_status(iscsi_cmd_t *icmdp, iscsi_scsi_rsp_hdr_t *issrhp,
     uint8_t *data)
 {
@@ -434,6 +438,7 @@ iscsi_cmd_rsp_cmd_status(iscsi_cmd_t *icmdp, iscsi_scsi_rsp_hdr_t *issrhp,
 	int32_t			statuslen	= 0;
 	int32_t			senselen_to	= 0;
 	struct scsi_pkt		*pkt;
+	boolean_t		affect		= B_FALSE;
 
 	pkt = icmdp->cmd_un.scsi.pkt;
 	dlength = n2h24(issrhp->dlength);
@@ -542,6 +547,9 @@ iscsi_cmd_rsp_cmd_status(iscsi_cmd_t *icmdp, iscsi_scsi_rsp_hdr_t *issrhp,
 			if (dlength > 0) {
 				bcopy(&data[2], (uchar_t *)&arqstat->
 				    sts_sensedata, dlength);
+
+				affect = iscsi_decode_sense(
+				    (uint8_t *)&arqstat->sts_sensedata);
 			}
 			break;
 		}
@@ -570,6 +578,8 @@ iscsi_cmd_rsp_cmd_status(iscsi_cmd_t *icmdp, iscsi_scsi_rsp_hdr_t *issrhp,
 			pkt->pkt_scbp[0] = issrhp->cmd_status;
 		}
 	}
+
+	return (affect);
 }
 
 /*
@@ -628,6 +638,9 @@ iscsi_rx_process_cmd_rsp(idm_conn_t *ic, idm_pdu_t *pdu)
 	struct scsi_pkt		*pkt	= NULL;
 	idm_status_t		rval;
 	struct buf		*bp;
+	boolean_t		flush	= B_FALSE;
+	uint32_t		cmd_sn	= 0;
+	uint16_t		lun_num = 0;
 
 	/* make sure we get status in order */
 	mutex_enter(&icp->conn_queue_active.mutex);
@@ -725,10 +738,18 @@ iscsi_rx_process_cmd_rsp(idm_conn_t *ic, idm_pdu_t *pdu)
 	} else {
 		/* success */
 		iscsi_cmd_rsp_chk(icmdp, issrhp);
-		iscsi_cmd_rsp_cmd_status(icmdp, issrhp, data);
+		flush = iscsi_cmd_rsp_cmd_status(icmdp, issrhp, data);
+		if (flush == B_TRUE) {
+			cmd_sn = icmdp->cmd_sn;
+			ASSERT(icmdp->cmd_lun != NULL);
+			lun_num = icmdp->cmd_lun->lun_num;
+		}
 	}
 
 	iscsi_cmd_state_machine(icmdp, ISCSI_CMD_EVENT_E3, isp);
+	if (flush == B_TRUE) {
+		iscsi_flush_cmd_after_reset(cmd_sn, lun_num, icp);
+	}
 	mutex_exit(&icp->conn_queue_active.mutex);
 	return (IDM_STATUS_SUCCESS);
 }
@@ -1205,11 +1226,24 @@ iscsi_rx_process_task_mgt_rsp(idm_conn_t *ic, idm_pdu_t *pdu)
 				break;
 			}
 			/* FALLTHRU */
+		case SCSI_TCP_TM_RESP_REJECTED:
+			/*
+			 * If the target rejects our reset task,
+			 * we should record the response and complete
+			 * this command with the result.
+			 */
+			if (icmdp->cmd_type == ISCSI_CMD_TYPE_RESET) {
+				icmdp->cmd_un.reset.response =
+				    istmrhp->response;
+				iscsi_cmd_state_machine(icmdp,
+				    ISCSI_CMD_EVENT_E3, isp);
+				break;
+			}
+			/* FALLTHRU */
 		case SCSI_TCP_TM_RESP_NO_LUN:
 		case SCSI_TCP_TM_RESP_TASK_ALLEGIANT:
 		case SCSI_TCP_TM_RESP_NO_FAILOVER:
 		case SCSI_TCP_TM_RESP_IN_PRGRESS:
-		case SCSI_TCP_TM_RESP_REJECTED:
 		default:
 			/*
 			 * Something is out of sync.  Flush
@@ -1801,11 +1835,12 @@ iscsi_tx_cmd(iscsi_sess_t *isp, iscsi_cmd_t *icmdp)
 
 static void
 iscsi_tx_init_hdr(iscsi_sess_t *isp, iscsi_conn_t *icp,
-    iscsi_text_hdr_t *ihp, int opcode, uint32_t cmd_itt)
+    iscsi_text_hdr_t *ihp, int opcode, iscsi_cmd_t *icmdp)
 {
 	ihp->opcode		= opcode;
-	ihp->itt		= cmd_itt;
+	ihp->itt		= icmdp->cmd_itt;
 	mutex_enter(&isp->sess_cmdsn_mutex);
+	icmdp->cmd_sn		= isp->sess_cmdsn;
 	ihp->cmdsn		= htonl(isp->sess_cmdsn);
 	isp->sess_cmdsn++;
 	mutex_exit(&isp->sess_cmdsn_mutex);
@@ -2075,7 +2110,7 @@ iscsi_tx_scsi(iscsi_sess_t *isp, iscsi_cmd_t *icmdp)
 	}
 
 	iscsi_tx_init_hdr(isp, icp, (iscsi_text_hdr_t *)ihp,
-	    ISCSI_OP_SCSI_CMD, icmdp->cmd_itt);
+	    ISCSI_OP_SCSI_CMD, icmdp);
 
 	idm_pdu_init(pdu, icp->conn_ic, (void *)icmdp, &iscsi_tx_done);
 	idm_pdu_init_hdr(pdu, (uint8_t *)ihp, len);
@@ -2099,6 +2134,8 @@ iscsi_tx_scsi(iscsi_sess_t *isp, iscsi_cmd_t *icmdp)
 
 	idm_pdu_tx(pdu);
 
+	icmdp->cmd_misc_flags |= ISCSI_CMD_MISCFLAG_SENT;
+
 	return (rval);
 }
 
@@ -2132,6 +2169,7 @@ iscsi_tx_pdu(iscsi_conn_t *icp, int opcode, void *hdr, int hdrlen,
 
 	mutex_exit(&icp->conn_sess->sess_queue_pending.mutex);
 	idm_pdu_tx(tx_pdu);
+	icmdp->cmd_misc_flags |= ISCSI_CMD_MISCFLAG_SENT;
 }
 
 
@@ -2159,6 +2197,7 @@ iscsi_tx_nop(iscsi_sess_t *isp, iscsi_cmd_t *icmdp)
 	inohp->itt	= icmdp->cmd_itt;
 	inohp->ttt	= icmdp->cmd_ttt;
 	mutex_enter(&isp->sess_cmdsn_mutex);
+	icmdp->cmd_sn	= isp->sess_cmdsn;
 	inohp->cmdsn	= htonl(isp->sess_cmdsn);
 	mutex_exit(&isp->sess_cmdsn_mutex);
 	inohp->expstatsn	= htonl(icp->conn_expstatsn);
@@ -2188,6 +2227,7 @@ iscsi_tx_abort(iscsi_sess_t *isp, iscsi_cmd_t *icmdp)
 	istmh = kmem_zalloc(sizeof (iscsi_scsi_task_mgt_hdr_t), KM_SLEEP);
 	ASSERT(istmh != NULL);
 	mutex_enter(&isp->sess_cmdsn_mutex);
+	icmdp->cmd_sn	= isp->sess_cmdsn;
 	istmh->cmdsn	= htonl(isp->sess_cmdsn);
 	mutex_exit(&isp->sess_cmdsn_mutex);
 	istmh->expstatsn = htonl(icp->conn_expstatsn);
@@ -2225,6 +2265,7 @@ iscsi_tx_reset(iscsi_sess_t *isp, iscsi_cmd_t *icmdp)
 	ASSERT(istmh != NULL);
 	istmh->opcode	= ISCSI_OP_SCSI_TASK_MGT_MSG | ISCSI_OP_IMMEDIATE;
 	mutex_enter(&isp->sess_cmdsn_mutex);
+	icmdp->cmd_sn	= isp->sess_cmdsn;
 	istmh->cmdsn	= htonl(isp->sess_cmdsn);
 	mutex_exit(&isp->sess_cmdsn_mutex);
 	istmh->expstatsn	= htonl(icp->conn_expstatsn);
@@ -2276,6 +2317,7 @@ iscsi_tx_logout(iscsi_sess_t *isp, iscsi_cmd_t *icmdp)
 	ilh->itt		= icmdp->cmd_itt;
 	ilh->cid		= icp->conn_cid;
 	mutex_enter(&isp->sess_cmdsn_mutex);
+	icmdp->cmd_sn	= isp->sess_cmdsn;
 	ilh->cmdsn	= htonl(isp->sess_cmdsn);
 	mutex_exit(&isp->sess_cmdsn_mutex);
 	ilh->expstatsn	= htonl(icp->conn_expstatsn);
@@ -2314,7 +2356,7 @@ iscsi_tx_text(iscsi_sess_t *isp, iscsi_cmd_t *icmdp)
 	hton24(ith->dlength, icmdp->cmd_un.text.data_len);
 	ith->ttt		= icmdp->cmd_un.text.ttt;
 	iscsi_tx_init_hdr(isp, icp, (iscsi_text_hdr_t *)ith,
-	    ISCSI_OP_TEXT_CMD, icmdp->cmd_itt);
+	    ISCSI_OP_TEXT_CMD, icmdp);
 	bcopy(icmdp->cmd_un.text.lun, ith->rsvd4, sizeof (ith->rsvd4));
 
 	iscsi_tx_pdu(icp, ISCSI_OP_TEXT_CMD, ith, sizeof (iscsi_text_hdr_t),
@@ -2351,11 +2393,11 @@ iscsi_handle_abort(void *arg)
 	ASSERT(icmdp->cmd_un.scsi.abort_icmdp == NULL);
 
 	new_icmdp = iscsi_cmd_alloc(icp, KM_SLEEP);
-	new_icmdp->cmd_type		= ISCSI_CMD_TYPE_ABORT;
-	new_icmdp->cmd_lun		= icmdp->cmd_lun;
-	new_icmdp->cmd_un.abort.icmdp	= icmdp;
-	new_icmdp->cmd_conn		= icmdp->cmd_conn;
-	icmdp->cmd_un.scsi.abort_icmdp	= new_icmdp;
+	new_icmdp->cmd_type		    = ISCSI_CMD_TYPE_ABORT;
+	new_icmdp->cmd_lun		    = icmdp->cmd_lun;
+	new_icmdp->cmd_un.abort.icmdp	    = icmdp;
+	new_icmdp->cmd_conn		    = icmdp->cmd_conn;
+	icmdp->cmd_un.scsi.abort_icmdp	    = new_icmdp;
 
 	/* pending queue mutex is already held by timeout_checks */
 	iscsi_cmd_state_machine(new_icmdp, ISCSI_CMD_EVENT_E1, isp);
@@ -2425,7 +2467,7 @@ iscsi_handle_nop(iscsi_conn_t *icp, uint32_t itt, uint32_t ttt)
 }
 
 /*
- * iscsi_handle_reset -
+ * iscsi_handle_reset - send reset request to the target
  *
  */
 iscsi_status_t
@@ -2437,6 +2479,29 @@ iscsi_handle_reset(iscsi_sess_t *isp, int level, iscsi_lun_t *ilp)
 
 	ASSERT(isp != NULL);
 
+	if (level == RESET_LUN) {
+		rw_enter(&isp->sess_lun_list_rwlock, RW_WRITER);
+		ASSERT(ilp != NULL);
+		if (ilp->lun_state & ISCSI_LUN_STATE_BUSY) {
+			rw_exit(&isp->sess_lun_list_rwlock);
+			return (ISCSI_STATUS_SUCCESS);
+		}
+		ilp->lun_state |= ISCSI_LUN_STATE_BUSY;
+		rw_exit(&isp->sess_lun_list_rwlock);
+	} else {
+		mutex_enter(&isp->sess_reset_mutex);
+		if (isp->sess_reset_in_progress == B_TRUE) {
+			/*
+			 * If the reset is in progress, it is unnecessary
+			 * to send reset to the target redunantly.
+			 */
+			mutex_exit(&isp->sess_reset_mutex);
+			return (ISCSI_STATUS_SUCCESS);
+		}
+		isp->sess_reset_in_progress = B_TRUE;
+		mutex_exit(&isp->sess_reset_mutex);
+	}
+
 	bzero(&icmd, sizeof (iscsi_cmd_t));
 	icmd.cmd_sig		= ISCSI_SIG_CMD;
 	icmd.cmd_state		= ISCSI_CMD_STATE_FREE;
@@ -2445,6 +2510,8 @@ iscsi_handle_reset(iscsi_sess_t *isp, int level, iscsi_lun_t *ilp)
 	icmd.cmd_un.reset.level	= level;
 	icmd.cmd_result		= ISCSI_STATUS_SUCCESS;
 	icmd.cmd_completed	= B_FALSE;
+	icmd.cmd_un.reset.response = SCSI_TCP_TM_RESP_COMPLETE;
+
 	mutex_init(&icmd.cmd_mutex, NULL, MUTEX_DRIVER, NULL);
 	cv_init(&icmd.cmd_completion, NULL, CV_DRIVER, NULL);
 	/*
@@ -2456,6 +2523,17 @@ iscsi_handle_reset(iscsi_sess_t *isp, int level, iscsi_lun_t *ilp)
 	if (!ISCSI_SESS_STATE_FULL_FEATURE(isp->sess_state)) {
 		/* We aren't connected to the target fake success */
 		mutex_exit(&isp->sess_state_mutex);
+
+		if (level == RESET_LUN) {
+			rw_enter(&isp->sess_lun_list_rwlock, RW_WRITER);
+			ilp->lun_state &= ~ISCSI_LUN_STATE_BUSY;
+			rw_exit(&isp->sess_lun_list_rwlock);
+		} else {
+			mutex_enter(&isp->sess_reset_mutex);
+			isp->sess_reset_in_progress = B_FALSE;
+			mutex_exit(&isp->sess_reset_mutex);
+		}
+
 		return (ISCSI_STATUS_SUCCESS);
 	}
 
@@ -2483,13 +2561,74 @@ iscsi_handle_reset(iscsi_sess_t *isp, int level, iscsi_lun_t *ilp)
 		icp = isp->sess_conn_list;
 		while (icp != NULL) {
 			iscsi_cmd_t *t_icmdp = NULL;
+			iscsi_cmd_t *next_icmdp = NULL;
 
 			mutex_enter(&icp->conn_queue_active.mutex);
 			t_icmdp = icp->conn_queue_active.head;
 			while (t_icmdp != NULL) {
-				iscsi_cmd_state_machine(t_icmdp,
-				    ISCSI_CMD_EVENT_E7, isp);
-				t_icmdp = icp->conn_queue_active.head;
+				next_icmdp = t_icmdp->cmd_next;
+				mutex_enter(&t_icmdp->cmd_mutex);
+				if (!(t_icmdp->cmd_misc_flags &
+				    ISCSI_CMD_MISCFLAG_SENT)) {
+					/*
+					 * Although this command is in the
+					 * active queue, it has not been sent.
+					 * Skip it.
+					 */
+					mutex_exit(&t_icmdp->cmd_mutex);
+					t_icmdp = next_icmdp;
+					continue;
+				}
+				if (level == RESET_LUN) {
+					if (icmd.cmd_lun == NULL ||
+					    t_icmdp->cmd_lun == NULL ||
+					    (icmd.cmd_lun->lun_num !=
+					    t_icmdp->cmd_lun->lun_num)) {
+						mutex_exit(&t_icmdp->cmd_mutex);
+						t_icmdp = next_icmdp;
+						continue;
+					}
+				}
+
+				if (icmd.cmd_sn == t_icmdp->cmd_sn) {
+					/*
+					 * This command may be replied with
+					 * UA sense key later. So currently
+					 * it is not a suitable time to flush
+					 * it. Mark its flag with FLUSH. There
+					 * is no harm to keep it for a while.
+					 */
+					t_icmdp->cmd_misc_flags |=
+					    ISCSI_CMD_MISCFLAG_FLUSH;
+					if (t_icmdp->cmd_type ==
+					    ISCSI_CMD_TYPE_SCSI) {
+						t_icmdp->cmd_un.scsi.pkt_stat |=
+						    STAT_BUS_RESET;
+					}
+					mutex_exit(&t_icmdp->cmd_mutex);
+				} else if ((icmd.cmd_sn > t_icmdp->cmd_sn) ||
+				    ((t_icmdp->cmd_sn - icmd.cmd_sn) >
+				    ISCSI_CMD_SN_WRAP)) {
+					/*
+					 * This reset request must act on all
+					 * the commnds from the same session
+					 * having a CmdSN lower than the task
+					 * mangement CmdSN. So flush these
+					 * commands here.
+					 */
+					if (t_icmdp->cmd_type ==
+					    ISCSI_CMD_TYPE_SCSI) {
+						t_icmdp->cmd_un.scsi.pkt_stat |=
+						    STAT_BUS_RESET;
+					}
+					mutex_exit(&t_icmdp->cmd_mutex);
+					iscsi_cmd_state_machine(t_icmdp,
+					    ISCSI_CMD_EVENT_E7, isp);
+				} else {
+					mutex_exit(&t_icmdp->cmd_mutex);
+				}
+
+				t_icmdp = next_icmdp;
 			}
 
 			mutex_exit(&icp->conn_queue_active.mutex);
@@ -2502,6 +2641,16 @@ iscsi_handle_reset(iscsi_sess_t *isp, int level, iscsi_lun_t *ilp)
 	cv_destroy(&icmd.cmd_completion);
 	mutex_destroy(&icmd.cmd_mutex);
 
+	if (level == RESET_LUN) {
+		rw_enter(&isp->sess_lun_list_rwlock, RW_WRITER);
+		ilp->lun_state &= ~ISCSI_LUN_STATE_BUSY;
+		rw_exit(&isp->sess_lun_list_rwlock);
+	} else {
+		mutex_enter(&isp->sess_reset_mutex);
+		isp->sess_reset_in_progress = B_FALSE;
+		mutex_exit(&isp->sess_reset_mutex);
+	}
+
 	return (rval);
 }
 
@@ -3227,16 +3376,31 @@ iscsi_timeout_checks(iscsi_sess_t *isp)
 			if (icmdp->cmd_lbolt_timeout == 0)
 				continue;
 
-			/* Skip if command is not active */
-			if (icmdp->cmd_state != ISCSI_CMD_STATE_ACTIVE)
+			/*
+			 * Skip if command is not active or not needed
+			 * to flush.
+			 */
+			if (icmdp->cmd_state != ISCSI_CMD_STATE_ACTIVE &&
+			    !(icmdp->cmd_misc_flags & ISCSI_CMD_MISCFLAG_FLUSH))
 				continue;
 
 			/* Skip if timeout still in the future */
 			if (now <= icmdp->cmd_lbolt_timeout)
 				continue;
 
-			/* timeout */
-			iscsi_cmd_state_machine(icmdp, ISCSI_CMD_EVENT_E6, isp);
+			if (icmdp->cmd_misc_flags & ISCSI_CMD_MISCFLAG_FLUSH) {
+				/*
+				 * This command is left during target reset,
+				 * we can flush it now.
+				 */
+				iscsi_cmd_state_machine(icmdp,
+				    ISCSI_CMD_EVENT_E7, isp);
+			} else if (icmdp->cmd_state == ISCSI_CMD_STATE_ACTIVE) {
+				/* timeout */
+				iscsi_cmd_state_machine(icmdp,
+				    ISCSI_CMD_EVENT_E6, isp);
+			}
+
 		}
 		mutex_exit(&icp->conn_queue_active.mutex);
 		mutex_exit(&isp->sess_queue_pending.mutex);
@@ -3299,3 +3463,91 @@ iscsi_nop_checks(iscsi_sess_t *isp)
  * | End of wd routines						|
  * +--------------------------------------------------------------------+
  */
+
+/*
+ * iscsi_flush_cmd_after_reset - flush commands after reset
+ *
+ * Here we will flush all the commands in the same connection whose cmdsn is
+ * less than the one received with the Unit Attention.
+ */
+static void
+iscsi_flush_cmd_after_reset(uint32_t cmd_sn, uint16_t lun_num,
+    iscsi_conn_t *icp)
+{
+	iscsi_cmd_t	*t_icmdp    = NULL;
+	iscsi_cmd_t	*next_icmdp = NULL;
+
+	ASSERT(icp != NULL);
+
+	t_icmdp = icp->conn_queue_active.head;
+	while (t_icmdp != NULL) {
+		next_icmdp = t_icmdp->cmd_next;
+		mutex_enter(&t_icmdp->cmd_mutex);
+		/*
+		 * We will flush the commands whose cmdsn is less than the one
+		 * got Unit Attention.
+		 * Here we will check for wrap by subtracting and compare to
+		 * 1/2 of a 32 bit number, if greater then we wrapped.
+		 */
+		if ((t_icmdp->cmd_misc_flags & ISCSI_CMD_MISCFLAG_SENT) &&
+		    ((cmd_sn > t_icmdp->cmd_sn) ||
+		    ((t_icmdp->cmd_sn - cmd_sn) >
+		    ISCSI_CMD_SN_WRAP))) {
+			if (t_icmdp->cmd_lun != NULL &&
+			    t_icmdp->cmd_lun->lun_num == lun_num) {
+				t_icmdp->cmd_misc_flags |=
+				    ISCSI_CMD_MISCFLAG_FLUSH;
+				if (t_icmdp->cmd_type == ISCSI_CMD_TYPE_SCSI) {
+					t_icmdp->cmd_un.scsi.pkt_stat |=
+					    STAT_BUS_RESET;
+				}
+			}
+		}
+		mutex_exit(&t_icmdp->cmd_mutex);
+		t_icmdp = next_icmdp;
+	}
+}
+
+/*
+ * iscsi_decode_sense - decode the sense data in the cmd response
+ *
+ * Here we only care about Unit Attention with 0x29.
+ */
+static boolean_t
+iscsi_decode_sense(uint8_t *sense_data)
+{
+	uint8_t	sense_key   = 0;
+	uint8_t	asc	    = 0;
+	boolean_t affect    = B_FALSE;
+
+	ASSERT(sense_data != NULL);
+
+	sense_key = scsi_sense_key(sense_data);
+	switch (sense_key) {
+		case KEY_UNIT_ATTENTION:
+			asc = scsi_sense_asc(sense_data);
+			switch (asc) {
+				case ISCSI_SCSI_RESET_SENSE_CODE:
+					/*
+					 * POWER ON, RESET, OR BUS_DEVICE RESET
+					 * OCCURRED
+					 */
+					affect = B_TRUE;
+					break;
+				default:
+					/*
+					 * Currently we don't care
+					 * about other sense key.
+					 */
+					break;
+			}
+			break;
+		default:
+			/*
+			 * Currently we don't care
+			 * about other sense key.
+			 */
+			break;
+	}
+	return (affect);
+}
diff --git a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_sess.c b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_sess.c
index 2c0d453c31..8bfee6d678 100644
--- a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_sess.c
+++ b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_sess.c
@@ -85,6 +85,7 @@ static iscsi_status_t iscsi_sess_testunitready(iscsi_sess_t *isp);
 static iscsi_status_t iscsi_sess_reportluns(iscsi_sess_t *isp);
 static void iscsi_sess_inquiry(iscsi_sess_t *isp, uint16_t lun_num,
     uint8_t lun_addr_type);
+static void iscsi_sess_update_busy_luns(iscsi_sess_t *isp, boolean_t clear);
 
 /*
  * +--------------------------------------------------------------------+
@@ -199,6 +200,7 @@ clean_failed_sess:
 	isp->sess_sig			= ISCSI_SIG_SESS;
 	isp->sess_state			= ISCSI_SESS_STATE_FREE;
 	mutex_init(&isp->sess_state_mutex, NULL, MUTEX_DRIVER, NULL);
+	mutex_init(&isp->sess_reset_mutex, NULL, MUTEX_DRIVER, NULL);
 	isp->sess_hba			= ihp;
 	isp->sess_enum_in_progress	= B_FALSE;
 
@@ -215,6 +217,7 @@ clean_failed_sess:
 	isp->sess_last_err		= NoError;
 	isp->sess_tsid			= 0;
 	isp->sess_type			= type;
+	isp->sess_reset_in_progress	= B_FALSE;
 	idm_sm_audit_init(&isp->sess_state_audit);
 
 	/* copy default driver login parameters */
@@ -291,6 +294,7 @@ iscsi_sess_cleanup2:
 	iscsi_destroy_queue(&isp->sess_queue_completion);
 	iscsi_destroy_queue(&isp->sess_queue_pending);
 	mutex_destroy(&isp->sess_state_mutex);
+	mutex_destroy(&isp->sess_reset_mutex);
 	kmem_free(isp, sizeof (iscsi_sess_t));
 
 	return (NULL);
@@ -571,6 +575,7 @@ iscsi_sess_destroy(iscsi_sess_t *isp)
 	rw_destroy(&isp->sess_conn_list_rwlock);
 	mutex_destroy(&isp->sess_cmdsn_mutex);
 	mutex_destroy(&isp->sess_state_mutex);
+	mutex_destroy(&isp->sess_reset_mutex);
 	kmem_free(isp, sizeof (iscsi_sess_t));
 
 	return (rval);
@@ -1217,6 +1222,12 @@ iscsi_sess_state_logged_in(iscsi_sess_t *isp, iscsi_sess_event_t event)
 			iscsi_thread_destroy(isp->sess_ic_thread);
 		}
 
+		mutex_enter(&isp->sess_reset_mutex);
+		isp->sess_reset_in_progress = B_FALSE;
+		mutex_exit(&isp->sess_reset_mutex);
+		/* update busy luns if needed */
+		iscsi_sess_update_busy_luns(isp, B_TRUE);
+
 		mutex_enter(&isp->sess_state_mutex);
 		break;
 
@@ -1399,6 +1410,12 @@ iscsi_sess_state_in_flush(iscsi_sess_t *isp, iscsi_sess_event_t event)
 			iscsi_thread_destroy(isp->sess_ic_thread);
 		}
 
+		mutex_enter(&isp->sess_reset_mutex);
+		isp->sess_reset_in_progress = B_FALSE;
+		mutex_exit(&isp->sess_reset_mutex);
+		/* update busy luns if needed */
+		iscsi_sess_update_busy_luns(isp, B_TRUE);
+
 		mutex_enter(&isp->sess_state_mutex);
 		break;
 
@@ -2173,6 +2190,15 @@ iscsi_sess_flush(iscsi_sess_t *isp)
 	mutex_enter(&isp->sess_queue_pending.mutex);
 	icmdp = isp->sess_queue_pending.head;
 	while (icmdp != NULL) {
+
+		if (isp->sess_state == ISCSI_SESS_STATE_FAILED) {
+			mutex_enter(&icmdp->cmd_mutex);
+			if (icmdp->cmd_type == ISCSI_CMD_TYPE_SCSI) {
+				icmdp->cmd_un.scsi.pkt_stat |= STAT_ABORTED;
+			}
+			mutex_exit(&icmdp->cmd_mutex);
+		}
+
 		iscsi_cmd_state_machine(icmdp,
 		    ISCSI_CMD_EVENT_E7, isp);
 		icmdp = isp->sess_queue_pending.head;
@@ -2242,3 +2268,26 @@ iscsi_sess_get_by_target(uint32_t target_oid, iscsi_hba_t *ihp,
 	}
 	return (rval);
 }
+
+static void
+iscsi_sess_update_busy_luns(iscsi_sess_t *isp, boolean_t clear)
+{
+	iscsi_lun_t	*ilp;
+	iscsi_hba_t	*ihp;
+
+	ASSERT(isp != NULL);
+	ihp = isp->sess_hba;
+	ASSERT(ihp != NULL);
+
+	rw_enter(&isp->sess_lun_list_rwlock, RW_WRITER);
+	ilp = isp->sess_lun_list;
+	while (ilp != NULL) {
+		if (clear == B_TRUE) {
+			ilp->lun_state &= ~ISCSI_LUN_STATE_BUSY;
+		} else {
+			ilp->lun_state |= ISCSI_LUN_STATE_BUSY;
+		}
+		ilp = ilp->lun_next;
+	}
+	rw_exit(&isp->sess_lun_list_rwlock);
+}