6748643 IDM sockets code should allocate buffers from one or more kmem caches

6818594 panic assertion failed: ist->ist_ffp_conn_count >= 1, file: ../../common/io/comstar/port/iscsit/iscs
6821084 SUNWiscsitr/SUNWiscsitu package issues
6821999 BAD TRAP: type=d (#gp General protection) rp=ffffff000fdf5460 addr=ffffff02f2564 c58

9 files changed, 94 insertions, 22 deletions

diff --git a/usr/src/pkgdefs/SUNWiscsitr/depend b/usr/src/pkgdefs/SUNWiscsitr/depend
index 719947f50e..2ea59047dd 100644
--- a/usr/src/pkgdefs/SUNWiscsitr/depend
+++ b/usr/src/pkgdefs/SUNWiscsitr/depend
@@ -17,7 +17,7 @@
 #
 # CDDL HEADER END
 #
-# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
 # This package information file defines software dependencies associated
@@ -48,3 +48,4 @@ P SUNWcsd	Core Solaris Devices
 P SUNWcsl	Core Solaris Libraries
 P SUNWiscsidmr	Sun iSCSI Data Mover (Root)
 P SUNWiscsidmu	Sun iSCSI Data Mover (Usr)
+P SUNWstmf	Sun Common Multipotocol SCSI Target
diff --git a/usr/src/pkgdefs/SUNWiscsitu/depend b/usr/src/pkgdefs/SUNWiscsitu/depend
index fd563fed32..d53aef378c 100644
--- a/usr/src/pkgdefs/SUNWiscsitu/depend
+++ b/usr/src/pkgdefs/SUNWiscsitu/depend
@@ -17,7 +17,7 @@
 #
 # CDDL HEADER END
 #
-# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
 # This package information file defines software dependencies associated
@@ -47,3 +47,4 @@ P SUNWcsu	Core Solaris, (Usr)
 P SUNWcsd	Core Solaris Devices
 P SUNWcsl	Core Solaris Libraries
 P SUNWiscsitr	Sun iSCSI COMSTAR port provider (Root)
+P SUNWstmfu	Sun Common Multipotocol SCSI Target Libraries and Tools
diff --git a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_sess.c b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_sess.c
index 0456b19b71..3db8516fbd 100644
--- a/usr/src/uts/common/io/comstar/port/iscsit/iscsit_sess.c
+++ b/usr/src/uts/common/io/comstar/port/iscsit/iscsit_sess.c
@@ -45,8 +45,6 @@
 #define	ISCSIT_SESS_SM_STRINGS
 #include <iscsit.h>
 
-
-
 typedef struct {
 	list_node_t		se_ctx_node;
 	iscsit_session_event_t	se_ctx_event;
@@ -678,6 +676,17 @@ sess_sm_q6_done(iscsit_sess_t *ist, sess_event_ctx_t *ctx)
 {
 	/* Terminal state */
 	switch (ctx->se_ctx_event) {
+	case SE_CONN_LOGGED_IN:
+		/*
+		 * It's possible to get this event if we encountered
+		 * an SE_SESSION_REINSTATE_EVENT while we were in
+		 * SS_Q2_ACTIVE state.  If so we want to update
+		 * ist->ist_ffp_conn_count because we know an
+		 * SE_CONN_FFP_FAIL or SE_CONN_FFP_DISABLE is on the
+		 * way.
+		 */
+		ist->ist_ffp_conn_count++;
+		break;
 	case SE_CONN_FFP_FAIL:
 	case SE_CONN_FFP_DISABLE:
 		ASSERT(ist->ist_ffp_conn_count >= 1);
diff --git a/usr/src/uts/common/io/ib/clients/iser/iser_cq.c b/usr/src/uts/common/io/ib/clients/iser/iser_cq.c
index fb31b77d93..443226d441 100644
--- a/usr/src/uts/common/io/ib/clients/iser/iser_cq.c
+++ b/usr/src/uts/common/io/ib/clients/iser/iser_cq.c
@@ -337,6 +337,7 @@ iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
 	 * to see if we need to fill the RQ back up (or if
 	 * we are already on the taskq).
 	 */
+	mutex_enter(&iser_chan->ic_conn->ic_lock);
 	mutex_enter(&iser_qp->qp_lock);
 	iser_qp->rq_level--;
 
@@ -346,8 +347,7 @@ iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
 		iser_qp->rq_taskqpending = B_TRUE;
 		mutex_exit(&iser_qp->qp_lock);
 
-		status = ddi_taskq_dispatch(iser_taskq, iser_ib_post_recv,
-		    (void *)iser_chan->ic_chanhdl, DDI_NOSLEEP);
+		status = iser_ib_post_recv_async(iser_chan->ic_chanhdl);
 
 		if (status != DDI_SUCCESS) {
 			ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: "
@@ -360,6 +360,7 @@ iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
 	} else {
 		mutex_exit(&iser_qp->qp_lock);
 	}
+	mutex_exit(&iser_chan->ic_conn->ic_lock);
 
 	DTRACE_PROBE3(iser__recv__cqe, iser_chan_t *, iser_chan,
 	    ibt_wc_t *, &wc, ibt_wc_status_t, wc.wc_status);
diff --git a/usr/src/uts/common/io/ib/clients/iser/iser_ib.c b/usr/src/uts/common/io/ib/clients/iser/iser_ib.c
index 29be1e3996..fd857fd844 100644
--- a/usr/src/uts/common/io/ib/clients/iser/iser_ib.c
+++ b/usr/src/uts/common/io/ib/clients/iser/iser_ib.c
@@ -76,6 +76,8 @@ static void iser_ib_handle_portdown_event(ibt_hca_hdl_t hdl,
 static void iser_ib_handle_hca_detach_event(ibt_hca_hdl_t hdl,
     ibt_async_event_t *event);
 
+static void iser_ib_post_recv_task(void *arg);
+
 static struct ibt_clnt_modinfo_s iser_ib_modinfo = {
 	IBTI_V_CURR,
 	IBT_STORAGE_DEV,
@@ -635,10 +637,42 @@ iser_ib_free_rc_channel(iser_chan_t *chan)
  * current fill level of the RQ, and post as many WRs as necessary
  * to fill it again.
  */
+
+int
+iser_ib_post_recv_async(ibt_channel_hdl_t chanhdl)
+{
+	iser_chan_t	*chan;
+	int		status;
+
+	/* Pull our iSER channel handle from the private data */
+	chan = (iser_chan_t *)ibt_get_chan_private(chanhdl);
+
+	idm_conn_hold(chan->ic_conn->ic_idmc);
+	status = ddi_taskq_dispatch(iser_taskq, iser_ib_post_recv_task,
+	    (void *)chanhdl, DDI_NOSLEEP);
+	if (status != DDI_SUCCESS) {
+		idm_conn_rele(chan->ic_conn->ic_idmc);
+	}
+
+	return (status);
+}
+
+static void
+iser_ib_post_recv_task(void *arg)
+{
+	ibt_channel_hdl_t	chanhdl = arg;
+	iser_chan_t		*chan;
+
+	/* Pull our iSER channel handle from the private data */
+	chan = (iser_chan_t *)ibt_get_chan_private(chanhdl);
+
+	iser_ib_post_recv(chanhdl);
+	idm_conn_rele(chan->ic_conn->ic_idmc);
+}
+
 void
-iser_ib_post_recv(void *arg)
+iser_ib_post_recv(ibt_channel_hdl_t chanhdl)
 {
-	ibt_channel_hdl_t chanhdl;
 	iser_chan_t	*chan;
 	iser_hca_t	*hca;
 	iser_msg_t	*msg;
@@ -650,15 +684,11 @@ iser_ib_post_recv(void *arg)
 	iser_qp_t	*iser_qp;
 	ib_gid_t	lgid;
 
-	chanhdl = (ibt_channel_hdl_t)arg;
-
 	/* Pull our iSER channel handle from the private data */
 	chan = (iser_chan_t *)ibt_get_chan_private(chanhdl);
 
-	/* It is possible to run after the channel has been freed */
-	if (chan == NULL) {
-		return;
-	}
+	ASSERT(chan != NULL);
+
 	mutex_enter(&chan->ic_conn->ic_lock);
 
 	/* Bail out if the connection is closed; no need for more recv WRs */
@@ -710,8 +740,7 @@ iser_ib_post_recv(void *arg)
 		 * second, then try again.
 		 */
 		delay(drv_usectohz(ISER_DELAY_HALF_SECOND));
-		status = ddi_taskq_dispatch(iser_taskq, iser_ib_post_recv,
-		    (void *)chanhdl, DDI_NOSLEEP);
+		status = iser_ib_post_recv_async(chanhdl);
 		if (status != DDI_SUCCESS) {
 			ISER_LOG(CE_NOTE, "iser_ib_post_recv: failed to "
 			    "redispatch routine");
@@ -784,8 +813,7 @@ iser_ib_post_recv(void *arg)
 	 */
 	if (iser_qp->rq_level == 0) {
 		mutex_exit(&iser_qp->qp_lock);
-		status = ddi_taskq_dispatch(iser_taskq, iser_ib_post_recv,
-		    (void *)chanhdl, DDI_NOSLEEP);
+		status = iser_ib_post_recv_async(chanhdl);
 		if (status != DDI_SUCCESS) {
 			ISER_LOG(CE_NOTE, "iser_ib_post_recv: failed to "
 			    "dispatch followup routine");
diff --git a/usr/src/uts/common/io/idm/idm_so.c b/usr/src/uts/common/io/idm/idm_so.c
index 376d2162bf..04353496b2 100644
--- a/usr/src/uts/common/io/idm/idm_so.c
+++ b/usr/src/uts/common/io/idm/idm_so.c
@@ -167,6 +167,10 @@ idm_so_init(idm_transport_t *it)
 	    sizeof (idm_pdu_t) + IDM_SORX_CACHE_HDRLEN, 8,
 	    &idm_sorx_pdu_constructor, NULL, NULL, NULL, NULL, KM_SLEEP);
 
+	/* 128k buffer cache */
+	idm.idm_so_128k_buf_cache = kmem_cache_create("idm_128k_buf_cache",
+	    IDM_SO_BUF_CACHE_UB, 8, NULL, NULL, NULL, NULL, NULL, KM_SLEEP);
+
 	/* Set the sockets transport ops */
 	it->it_ops = &idm_so_transport_ops;
 }
@@ -178,6 +182,7 @@ idm_so_init(idm_transport_t *it)
 void
 idm_so_fini(void)
 {
+	kmem_cache_destroy(idm.idm_so_128k_buf_cache);
 	kmem_cache_destroy(idm.idm_sotx_pdu_cache);
 	kmem_cache_destroy(idm.idm_sorx_pdu_cache);
 }
@@ -2146,12 +2151,21 @@ idm_so_buf_rx_from_ini(idm_task_t *idt, idm_buf_t *idb)
 static idm_status_t
 idm_so_buf_alloc(idm_buf_t *idb, uint64_t buflen)
 {
-	idb->idb_buf = kmem_alloc(buflen, KM_NOSLEEP);
+	if ((buflen > IDM_SO_BUF_CACHE_LB) && (buflen <= IDM_SO_BUF_CACHE_UB)) {
+		idb->idb_buf = kmem_cache_alloc(idm.idm_so_128k_buf_cache,
+		    KM_NOSLEEP);
+		idb->idb_buf_private = idm.idm_so_128k_buf_cache;
+	} else {
+		idb->idb_buf = kmem_alloc(buflen, KM_NOSLEEP);
+		idb->idb_buf_private = NULL;
+	}
+
 	if (idb->idb_buf == NULL) {
 		IDM_CONN_LOG(CE_NOTE,
 		    "idm_so_buf_alloc: failed buffer allocation");
 		return (IDM_STATUS_FAIL);
 	}
+
 	return (IDM_STATUS_SUCCESS);
 }
 
@@ -2175,7 +2189,11 @@ idm_so_buf_teardown(idm_buf_t *idb)
 static void
 idm_so_buf_free(idm_buf_t *idb)
 {
-	kmem_free(idb->idb_buf, idb->idb_buflen);
+	if (idb->idb_buf_private == NULL) {
+		kmem_free(idb->idb_buf, idb->idb_buflen);
+	} else {
+		kmem_cache_free(idb->idb_buf_private, idb->idb_buf);
+	}
 }
 
 static void
diff --git a/usr/src/uts/common/sys/ib/clients/iser/iser_ib.h b/usr/src/uts/common/sys/ib/clients/iser/iser_ib.h
index 9fa00ebae5..bce5e06ca1 100644
--- a/usr/src/uts/common/sys/ib/clients/iser/iser_ib.h
+++ b/usr/src/uts/common/sys/ib/clients/iser/iser_ib.h
@@ -187,7 +187,9 @@ void iser_ib_close_rc_channel(iser_chan_t *chan);
 
 void iser_ib_free_rc_channel(iser_chan_t *chan);
 
-void iser_ib_post_recv(void *arg);
+int iser_ib_post_recv_async(ibt_channel_hdl_t chanhdl);
+
+void iser_ib_post_recv(ibt_channel_hdl_t chanhdl);
 
 void iser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl, void *arg);
 
diff --git a/usr/src/uts/common/sys/idm/idm_impl.h b/usr/src/uts/common/sys/idm/idm_impl.h
index 9a407fcc35..c6d8a16101 100644
--- a/usr/src/uts/common/sys/idm/idm_impl.h
+++ b/usr/src/uts/common/sys/idm/idm_impl.h
@@ -421,6 +421,7 @@ typedef struct {
 	idm_idpool_t	idm_conn_id_pool;
 	kmem_cache_t	*idm_sotx_pdu_cache;
 	kmem_cache_t	*idm_sorx_pdu_cache;
+	kmem_cache_t	*idm_so_128k_buf_cache;
 } idm_global_t;
 
 idm_global_t	idm; /* Global state */
diff --git a/usr/src/uts/common/sys/idm/idm_so.h b/usr/src/uts/common/sys/idm/idm_so.h
index 42c39c6461..c5d9a8b0b0 100644
--- a/usr/src/uts/common/sys/idm/idm_so.h
+++ b/usr/src/uts/common/sys/idm/idm_so.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -32,6 +32,7 @@ extern "C" {
 
 #include <sys/idm/idm_transport.h>
 #include <sys/ksocket.h>
+
 /*
  * Define TCP window size (send and receive buffer sizes)
  */
@@ -39,6 +40,16 @@ extern "C" {
 #define	IDM_RCVBUF_SIZE		(256 * 1024)
 #define	IDM_SNDBUF_SIZE		(256 * 1024)
 
+/*
+ * Lower and upper bounds to use the 128k buffer cache.  Below the lower bound
+ * allocations will use the built-in Solaris buffer caches.  We don't expect
+ * to see allocations above the upper bound because SBD currently allocates
+ * 128k buffers.
+ */
+
+#define	IDM_SO_BUF_CACHE_LB	(32 * 1024)
+#define	IDM_SO_BUF_CACHE_UB	(128 * 1024)
+
 /* sockets-specific portion of idm_svc_t */
 typedef struct idm_so_svc_s {
 	ksocket_t		is_so;