6306925 ibt_get_paths() should cache recently found path records.

6 files changed, 515 insertions, 178 deletions

diff --git a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_impl.c b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_impl.c
index f26e144c44..b1ce8f4361 100644
--- a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_impl.c
+++ b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_impl.c
@@ -64,6 +64,9 @@ static ibt_status_t	ibcm_hca_init_port(ibcm_hca_info_t *hcap,
 static ibcm_status_t	ibcm_hca_fini_port(ibcm_hca_info_t *hcap,
 			    uint8_t port_index);
 
+static void ibcm_rc_flow_control_init(void);
+static void ibcm_rc_flow_control_fini(void);
+
 /*
  * Routines that check if hca's avl trees and sidr lists are free of any
  * active client resources ie., RC or UD state structures in certain states
@@ -120,28 +123,7 @@ kcondvar_t		ibcm_global_hca_cv;
 /* mutex and cv to sa session open */
 kmutex_t		ibcm_sa_open_lock;
 kcondvar_t		ibcm_sa_open_cv;
-
-/* Deal with poor SA timeout behavior during stress */
-kmutex_t		ibcm_sa_timeout_lock;
-kcondvar_t		ibcm_sa_timeout_cv;
 int			ibcm_sa_timeout_delay = 1;		/* in ticks */
-int			ibcm_sa_timeout_simul;
-int			ibcm_sa_timeout_simul_max;
-int			ibcm_sa_timeout_simul_init = 8;		/* tunable */
-
-/* Control the number of RC connection requests get started simultaneously */
-kcondvar_t		ibcm_rc_flow_control_cv;
-int			ibcm_rc_flow_control_simul;
-int			ibcm_rc_flow_control_simul_max;
-int			ibcm_rc_flow_control_simul_init = 8;	/* tunable */
-int			ibcm_rc_flow_control_simul_stalls;
-
-/* Control the number of disconnection requests get started simultaneously */
-kcondvar_t		ibcm_close_flow_control_cv;
-int			ibcm_close_flow_control_simul;
-int			ibcm_close_flow_control_simul_max;
-int			ibcm_close_flow_control_simul_init = 8;	/* tunable */
-
 _NOTE(MUTEX_PROTECTS_DATA(ibcm_sa_open_lock,
     ibcm_port_info_s::{port_ibmf_saa_hdl port_saa_open_in_progress}))
 
@@ -580,21 +562,12 @@ ibcm_init(void)
 	ibcm_init_hcas();
 	ibcm_finit_state = IBCM_FINIT_IDLE;
 
+	ibcm_path_cache_init();
+
 	/* Unblock any waiting HCA DR asyncs in CM */
 	mutex_exit(&ibcm_global_hca_lock);
 
-	mutex_init(&ibcm_sa_timeout_lock, NULL, MUTEX_DEFAULT, NULL);
-	cv_init(&ibcm_sa_timeout_cv, NULL, CV_DRIVER, NULL);
-	cv_init(&ibcm_rc_flow_control_cv, NULL, CV_DRIVER, NULL);
-	cv_init(&ibcm_close_flow_control_cv, NULL, CV_DRIVER, NULL);
-	mutex_enter(&ibcm_sa_timeout_lock);
-	ibcm_sa_timeout_simul_max = ibcm_sa_timeout_simul_init;
-	ibcm_sa_timeout_simul = 0;
-	ibcm_rc_flow_control_simul_max = ibcm_rc_flow_control_simul_init;
-	ibcm_rc_flow_control_simul = 0;
-	ibcm_close_flow_control_simul_max = ibcm_close_flow_control_simul_init;
-	ibcm_close_flow_control_simul = 0;
-	mutex_exit(&ibcm_sa_timeout_lock);
+	ibcm_rc_flow_control_init();
 
 	IBTF_DPRINTF_L4(cmlog, "ibcm_init: done");
 	return (IBCM_SUCCESS);
@@ -711,10 +684,9 @@ ibcm_fini(void)
 	if (status != IBT_SUCCESS)
 	    IBTF_DPRINTF_L1(cmlog, "ibcm_fini: ibt_detach failed %d", status);
 
-	mutex_destroy(&ibcm_sa_timeout_lock);
-	cv_destroy(&ibcm_sa_timeout_cv);
-	cv_destroy(&ibcm_rc_flow_control_cv);
-	cv_destroy(&ibcm_close_flow_control_cv);
+	ibcm_rc_flow_control_fini();
+
+	ibcm_path_cache_fini();
 
 	ibcm_fini_ids();
 	ibcm_fini_locks();
@@ -1376,136 +1348,205 @@ ibcm_dec_hca_svc_cnt(ibcm_hca_info_t *hcap)
 	mutex_exit(&ibcm_global_hca_lock);
 }
 
-void
-ibcm_rc_flow_control_enter()
+/*
+ * Flow control logic for open_rc_channel and close_rc_channel follows.
+ * We use one instance of the same data structure to control each of
+ * "open" and "close".  We allow up to IBCM_FLOW_SIMUL_MAX requests to be
+ * initiated at one time.  We initiate the next group any time there are
+ * less than IBCM_FLOW_LOW_WATER.
+ */
+
+/* These variables are for both open_rc_channel and close_rc_channel */
+#define	IBCM_FLOW_SIMUL_MAX	32
+int ibcm_flow_simul_max = IBCM_FLOW_SIMUL_MAX;
+#define	IBCM_SAA_SIMUL_MAX	8
+int ibcm_saa_simul_max = IBCM_SAA_SIMUL_MAX;
+
+typedef struct ibcm_flow1_s {
+	struct ibcm_flow1_s	*link;
+	kcondvar_t		cv;
+	uint8_t			waiters;	/* 1 to IBCM_FLOW_SIMUL_MAX */
+} ibcm_flow1_t;
+
+typedef struct ibcm_flow_s {
+	ibcm_flow1_t	*list;
+	uint_t		simul;		/* #requests currently outstanding */
+	uint_t		simul_max;
+	uint_t		waiters_per_chunk;
+	uint_t		lowat;
+	uint_t		lowat_default;
+	/* statistics */
+	uint_t		total;
+	uint_t		enable_queuing;
+} ibcm_flow_t;
+
+kmutex_t ibcm_rc_flow_mutex;
+ibcm_flow_t ibcm_open_flow;
+ibcm_flow_t ibcm_close_flow;
+ibcm_flow_t ibcm_saa_flow;
+
+static void
+ibcm_flow_init(ibcm_flow_t *flow, uint_t simul_max)
 {
-	mutex_enter(&ibcm_sa_timeout_lock);
-	if (ibcm_rc_flow_control_simul_max != ibcm_rc_flow_control_simul_init) {
-		if (ibcm_rc_flow_control_simul_max <
-		    ibcm_rc_flow_control_simul_init) {
-			/* letting more flow should be gradual */
-			ibcm_rc_flow_control_simul_max++;
-			if (ibcm_rc_flow_control_simul <
-			    ibcm_rc_flow_control_simul_max - 1)
-				cv_signal(&ibcm_rc_flow_control_cv);
-		} else {
-			ibcm_rc_flow_control_simul_max =
-			    ibcm_rc_flow_control_simul_init;
-		}
+	flow->list = NULL;
+	flow->simul = 0;
+	flow->waiters_per_chunk = 4;
+	flow->simul_max = simul_max;
+	flow->lowat = flow->simul_max - flow->waiters_per_chunk;
+	flow->lowat_default = flow->lowat;
+	/* stats */
+	flow->total = 0;
+	flow->enable_queuing = 0;
+}
+
+static void
+ibcm_rc_flow_control_init(void)
+{
+	mutex_init(&ibcm_rc_flow_mutex, NULL, MUTEX_DEFAULT, NULL);
+	mutex_enter(&ibcm_rc_flow_mutex);
+	ibcm_flow_init(&ibcm_open_flow, ibcm_flow_simul_max);
+	ibcm_flow_init(&ibcm_close_flow, ibcm_flow_simul_max);
+	ibcm_flow_init(&ibcm_saa_flow, ibcm_saa_simul_max);
+	mutex_exit(&ibcm_rc_flow_mutex);
+}
+
+static void
+ibcm_rc_flow_control_fini(void)
+{
+	mutex_destroy(&ibcm_rc_flow_mutex);
+}
+
+static ibcm_flow1_t *
+ibcm_flow_find(ibcm_flow_t *flow)
+{
+	ibcm_flow1_t *flow1;
+	ibcm_flow1_t *f;
+
+	f = flow->list;
+	if (f) {	/* most likely code path */
+		while (f->link != NULL)
+			f = f->link;
+		if (f->waiters < flow->waiters_per_chunk)
+			return (f);
 	}
-	while (ibcm_rc_flow_control_simul >= ibcm_rc_flow_control_simul_max) {
-		cv_wait(&ibcm_rc_flow_control_cv, &ibcm_sa_timeout_lock);
+
+	/* There was no flow1 list element ready for another waiter */
+	mutex_exit(&ibcm_rc_flow_mutex);
+	flow1 = kmem_alloc(sizeof (*flow1), KM_SLEEP);
+	mutex_enter(&ibcm_rc_flow_mutex);
+
+	f = flow->list;
+	if (f) {
+		while (f->link != NULL)
+			f = f->link;
+		if (f->waiters < flow->waiters_per_chunk) {
+			kmem_free(flow1, sizeof (*flow1));
+			return (f);
+		}
+		f->link = flow1;
+	} else {
+		flow->list = flow1;
 	}
-	ibcm_rc_flow_control_simul++;
-	mutex_exit(&ibcm_sa_timeout_lock);
+	cv_init(&flow1->cv, NULL, CV_DRIVER, NULL);
+	flow1->waiters = 0;
+	flow1->link = NULL;
+	return (flow1);
 }
 
-void
-ibcm_rc_flow_control_exit()
+static void
+ibcm_flow_enter(ibcm_flow_t *flow)
 {
-	mutex_enter(&ibcm_sa_timeout_lock);
-	if (ibcm_rc_flow_control_simul_max != ibcm_rc_flow_control_simul_init) {
-		if (ibcm_rc_flow_control_simul_max <
-		    ibcm_rc_flow_control_simul_init) {
-			/* letting more flow should be gradual */
-			ibcm_rc_flow_control_simul_max++;
-			if (ibcm_rc_flow_control_simul <
-			    ibcm_rc_flow_control_simul_max)
-				cv_signal(&ibcm_rc_flow_control_cv);
-		} else {
-			ibcm_rc_flow_control_simul_max =
-			    ibcm_rc_flow_control_simul_init;
+	mutex_enter(&ibcm_rc_flow_mutex);
+	if (flow->list == NULL && flow->simul < flow->simul_max) {
+		flow->simul++;
+		flow->total++;
+	} else {
+		ibcm_flow1_t *flow1;
+
+		flow1 = ibcm_flow_find(flow);
+		flow1->waiters++;
+		cv_wait(&flow1->cv, &ibcm_rc_flow_mutex);
+		if (--flow1->waiters == 0) {
+			cv_destroy(&flow1->cv);
+			kmem_free(flow1, sizeof (*flow1));
 		}
 	}
-	if (--ibcm_rc_flow_control_simul < ibcm_rc_flow_control_simul_max)
-		cv_signal(&ibcm_rc_flow_control_cv);
-	mutex_exit(&ibcm_sa_timeout_lock);
+	mutex_exit(&ibcm_rc_flow_mutex);
 }
 
-void
-ibcm_close_flow_control_enter()
+static void
+ibcm_flow_exit(ibcm_flow_t *flow)
 {
-	mutex_enter(&ibcm_sa_timeout_lock);
-	if (ibcm_close_flow_control_simul_max !=
-	    ibcm_close_flow_control_simul_init) {
-		if (ibcm_close_flow_control_simul_max <
-		    ibcm_close_flow_control_simul_init) {
-			/* letting more flow should be gradual */
-			ibcm_close_flow_control_simul_max++;
-			if (ibcm_close_flow_control_simul <
-			    ibcm_close_flow_control_simul_max - 1)
-				cv_signal(&ibcm_close_flow_control_cv);
-		} else {
-			ibcm_close_flow_control_simul_max =
-			    ibcm_close_flow_control_simul_init;
+	mutex_enter(&ibcm_rc_flow_mutex);
+	if (--flow->simul < flow->lowat) {
+		flow->lowat += flow->waiters_per_chunk;
+		if (flow->lowat > flow->lowat_default)
+			flow->lowat = flow->lowat_default;
+		if (flow->list) {
+			ibcm_flow1_t *flow1;
+
+			flow1 = flow->list;
+			flow->list = flow1->link;	/* unlink */
+			flow1->link = NULL;		/* be clean */
+			flow->total += flow1->waiters;
+			flow->simul += flow1->waiters;
+			cv_broadcast(&flow1->cv);
 		}
 	}
-	while (ibcm_close_flow_control_simul >=
-	    ibcm_close_flow_control_simul_max) {
-		cv_wait(&ibcm_close_flow_control_cv, &ibcm_sa_timeout_lock);
+	mutex_exit(&ibcm_rc_flow_mutex);
+}
+
+static void
+ibcm_flow_stall(ibcm_flow_t *flow)
+{
+	mutex_enter(&ibcm_rc_flow_mutex);
+	if (flow->lowat > 1) {
+		flow->lowat >>= 1;
+		IBTF_DPRINTF_L2(cmlog, "stall - lowat = %d", flow->lowat);
 	}
-	ibcm_close_flow_control_simul++;
-	mutex_exit(&ibcm_sa_timeout_lock);
+	mutex_exit(&ibcm_rc_flow_mutex);
+}
+
+void
+ibcm_rc_flow_control_enter(void)
+{
+	ibcm_flow_enter(&ibcm_open_flow);
+}
+
+void
+ibcm_rc_flow_control_exit(void)
+{
+	ibcm_flow_exit(&ibcm_open_flow);
+}
+
+void
+ibcm_close_flow_control_enter()
+{
+	ibcm_flow_enter(&ibcm_close_flow);
 }
 
 void
 ibcm_close_flow_control_exit()
 {
-	mutex_enter(&ibcm_sa_timeout_lock);
-	if (ibcm_close_flow_control_simul_max !=
-	    ibcm_close_flow_control_simul_init) {
-		if (ibcm_close_flow_control_simul_max <
-		    ibcm_close_flow_control_simul_init) {
-			/* letting more flow should be gradual */
-			ibcm_close_flow_control_simul_max++;
-			if (ibcm_close_flow_control_simul <
-			    ibcm_close_flow_control_simul_max)
-				cv_signal(&ibcm_close_flow_control_cv);
-		} else {
-			ibcm_close_flow_control_simul_max =
-			    ibcm_close_flow_control_simul_init;
-		}
-	}
-	if (--ibcm_close_flow_control_simul < ibcm_close_flow_control_simul_max)
-		cv_signal(&ibcm_close_flow_control_cv);
-	mutex_exit(&ibcm_sa_timeout_lock);
+	ibcm_flow_exit(&ibcm_close_flow);
 }
 
-/*
- * This function is called when we hit any timeout, and have to retry.
- * The logic here is that the timeout may have been because the other
- * side of this connection is too busy to respond in time.  We provide
- * relief here by temporarily reducing the number of connections we
- * allow to be initiated.  As pending ones complete, we let new ones start.
- */
 void
 ibcm_rc_flow_control_stall()
 {
-	mutex_enter(&ibcm_sa_timeout_lock);
-	ibcm_rc_flow_control_simul_stalls++;
-	if (ibcm_rc_flow_control_simul_max)
-		ibcm_rc_flow_control_simul_max--;
-	mutex_exit(&ibcm_sa_timeout_lock);
+	ibcm_flow_stall(&ibcm_open_flow);
 }
 
 void
 ibcm_sa_access_enter()
 {
-	mutex_enter(&ibcm_sa_timeout_lock);
-	while (ibcm_sa_timeout_simul == ibcm_sa_timeout_simul_max) {
-		cv_wait(&ibcm_sa_timeout_cv, &ibcm_sa_timeout_lock);
-	}
-	ibcm_sa_timeout_simul++;
-	mutex_exit(&ibcm_sa_timeout_lock);
+	ibcm_flow_enter(&ibcm_saa_flow);
 }
 
 void
 ibcm_sa_access_exit()
 {
-	mutex_enter(&ibcm_sa_timeout_lock);
-	cv_signal(&ibcm_sa_timeout_cv);
-	ibcm_sa_timeout_simul--;
-	mutex_exit(&ibcm_sa_timeout_lock);
+	ibcm_flow_exit(&ibcm_saa_flow);
 }
 
 static void
@@ -1533,9 +1574,11 @@ ibcm_sm_notice_handler(ibmf_saa_handle_t saa_handle,
 		break;
 	case IBMF_SAA_EVENT_GID_AVAILABLE:
 		code = IBT_SM_EVENT_GID_AVAIL;
+		ibcm_path_cache_purge();
 		break;
 	case IBMF_SAA_EVENT_GID_UNAVAILABLE:
 		code = IBT_SM_EVENT_GID_UNAVAIL;
+		ibcm_path_cache_purge();
 		break;
 	case IBMF_SAA_EVENT_SUBSCRIBER_STATUS_CHG:
 		event_status =
@@ -2010,7 +2053,8 @@ ibcm_comm_est_handler(ibt_async_event_t *eventp)
 		ibcm_cep_state_rtu(statep, NULL);
 
 	} else {
-		if (statep->state == IBCM_STATE_ESTABLISHED) {
+		if (statep->state == IBCM_STATE_ESTABLISHED ||
+		    statep->state == IBCM_STATE_TRANSIENT_ESTABLISHED) {
 			IBTF_DPRINTF_L4(cmlog, "ibcm_comm_est_handler: "
 			    "Channel already in ESTABLISHED state");
 		} else {
@@ -2085,6 +2129,7 @@ ibcm_async_handler(void *clnt_hdl, ibt_hca_hdl_t hca_hdl,
 		_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*pup))
 		(void) taskq_dispatch(ibcm_taskq,
 		    ibcm_service_record_rewrite_task, pup, TQ_SLEEP);
+		ibcm_path_cache_purge();
 		return;
 
 	case IBT_HCA_ATTACH_EVENT:
diff --git a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_path.c b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_path.c
index a13f57bf60..3af18a9993 100644
--- a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_path.c
+++ b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_path.c
@@ -150,6 +150,226 @@ ibt_aget_paths(ibt_clnt_hdl_t ibt_hdl, ibt_path_flags_t flags,
 
 
 /*
+ * ibt_get_paths() cache consists of one or more of:
+ *
+ *	ib_gid_t dgid (attrp->pa_dgids[0])
+ *	ibt_path_attr_t attr
+ *	ibt_path_flags_t flags
+ *	ibt_path_info_t path
+ *
+ * If the first 3 match, max_paths is 1, sname is NULL, and sid is 0,
+ * then the path is returned immediately.
+ *
+ * Note that a compare of "attr" is non-trivial.  Only accept ones
+ * that memcmp() succeeds, i.e., basically assume a bzero was done.
+ *
+ * Cache must be invalidated if PORT_DOWN event or GID_UNAVAIL occurs.
+ * Cache must be freed as part of _fini.
+ */
+
+#define	IBCM_PATH_CACHE_SIZE	16	/* keep small for linear search */
+#define	IBCM_PATH_CACHE_TIMEOUT	60	/* purge cache after 60 seconds */
+
+typedef struct ibcm_path_cache_s {
+	ib_gid_t		dgid;
+	ibt_path_attr_t		attr;
+	ibt_path_flags_t	flags;
+	ibt_path_info_t		path;
+} ibcm_path_cache_t;
+
+kmutex_t ibcm_path_cache_mutex;
+int ibcm_path_cache_invalidate;	/* invalidate cache on next ibt_get_paths */
+clock_t ibcm_path_cache_timeout = IBCM_PATH_CACHE_TIMEOUT; /* tunable */
+timeout_id_t ibcm_path_cache_timeout_id;
+int ibcm_path_cache_size_init = IBCM_PATH_CACHE_SIZE;	/* tunable */
+int ibcm_path_cache_size;
+ibcm_path_cache_t *ibcm_path_cachep;
+
+struct ibcm_path_cache_stat_s {
+	int hits;
+	int misses;
+	int adds;
+	int already_in_cache;
+	int bad_path_for_cache;
+	int purges;
+	int timeouts;
+} ibcm_path_cache_stats;
+
+/*ARGSUSED*/
+static void
+ibcm_path_cache_timeout_cb(void *arg)
+{
+	clock_t timeout_in_hz;
+
+	timeout_in_hz = drv_usectohz(ibcm_path_cache_timeout * 1000000);
+	mutex_enter(&ibcm_path_cache_mutex);
+	ibcm_path_cache_invalidate = 1;	/* invalidate cache on next check */
+	if (ibcm_path_cache_timeout_id)
+		ibcm_path_cache_timeout_id = timeout(ibcm_path_cache_timeout_cb,
+		    NULL, timeout_in_hz);
+	/* else we're in _fini */
+	mutex_exit(&ibcm_path_cache_mutex);
+}
+
+void
+ibcm_path_cache_init(void)
+{
+	clock_t timeout_in_hz;
+	int cache_size = ibcm_path_cache_size_init;
+	ibcm_path_cache_t *path_cachep;
+
+	timeout_in_hz = drv_usectohz(ibcm_path_cache_timeout * 1000000);
+	path_cachep = kmem_zalloc(cache_size * sizeof (*path_cachep), KM_SLEEP);
+	mutex_init(&ibcm_path_cache_mutex, NULL, MUTEX_DRIVER, NULL);
+	mutex_enter(&ibcm_path_cache_mutex);
+	ibcm_path_cache_size = cache_size;
+	ibcm_path_cachep = path_cachep;
+	ibcm_path_cache_timeout_id = timeout(ibcm_path_cache_timeout_cb,
+	    NULL, timeout_in_hz);
+	mutex_exit(&ibcm_path_cache_mutex);
+}
+
+void
+ibcm_path_cache_fini(void)
+{
+	timeout_id_t tmp_timeout_id;
+	int cache_size;
+	ibcm_path_cache_t *path_cachep;
+
+	mutex_enter(&ibcm_path_cache_mutex);
+	if (ibcm_path_cache_timeout_id) {
+		tmp_timeout_id = ibcm_path_cache_timeout_id;
+		ibcm_path_cache_timeout_id = 0;	/* no more timeouts */
+	}
+	cache_size = ibcm_path_cache_size;
+	path_cachep = ibcm_path_cachep;
+	mutex_exit(&ibcm_path_cache_mutex);
+	if (tmp_timeout_id)
+		(void) untimeout(tmp_timeout_id);
+	mutex_destroy(&ibcm_path_cache_mutex);
+	kmem_free(path_cachep, cache_size * sizeof (*path_cachep));
+}
+
+static ibcm_status_t
+ibcm_path_cache_check(ibt_path_flags_t flags, ibt_path_attr_t *attrp,
+    uint8_t max_paths, ibt_path_info_t *path, uint8_t *num_paths_p)
+{
+	int i;
+	ib_gid_t dgid;
+	ibcm_path_cache_t *path_cachep;
+
+	if (max_paths != 1 || attrp->pa_num_dgids != 1 ||
+	    attrp->pa_sname != NULL || attrp->pa_sid != 0) {
+		mutex_enter(&ibcm_path_cache_mutex);
+		ibcm_path_cache_stats.bad_path_for_cache++;
+		mutex_exit(&ibcm_path_cache_mutex);
+		return (IBCM_FAILURE);
+	}
+
+	dgid = attrp->pa_dgids[0];
+	if ((dgid.gid_guid | dgid.gid_prefix) == 0ULL)
+		return (IBCM_FAILURE);
+
+	mutex_enter(&ibcm_path_cache_mutex);
+	if (ibcm_path_cache_invalidate) {	/* invalidate all entries */
+		ibcm_path_cache_stats.timeouts++;
+		ibcm_path_cache_invalidate = 0;
+		path_cachep = ibcm_path_cachep;
+		for (i = 0; i < ibcm_path_cache_size; i++, path_cachep++) {
+			path_cachep->dgid.gid_guid = 0ULL;
+			path_cachep->dgid.gid_prefix = 0ULL;
+		}
+		mutex_exit(&ibcm_path_cache_mutex);
+		return (IBCM_FAILURE);
+	}
+
+	path_cachep = ibcm_path_cachep;
+	for (i = 0; i < ibcm_path_cache_size; i++, path_cachep++) {
+		if (path_cachep->dgid.gid_guid == 0ULL)
+			break;	/* end of search, no more valid cache entries */
+
+		/* make pa_dgids pointers match, so we can use memcmp */
+		path_cachep->attr.pa_dgids = attrp->pa_dgids;
+		if (path_cachep->flags != flags ||
+		    path_cachep->dgid.gid_guid != dgid.gid_guid ||
+		    path_cachep->dgid.gid_prefix != dgid.gid_prefix ||
+		    memcmp(&(path_cachep->attr), attrp, sizeof (*attrp)) != 0) {
+			/* make pa_dgids NULL again */
+			path_cachep->attr.pa_dgids = NULL;
+			continue;
+		}
+		/* else we have a match */
+		/* make pa_dgids NULL again */
+		path_cachep->attr.pa_dgids = NULL;
+		*path = path_cachep->path;	/* retval */
+		if (num_paths_p)
+			*num_paths_p = 1;	/* retval */
+		ibcm_path_cache_stats.hits++;
+		mutex_exit(&ibcm_path_cache_mutex);
+		return (IBCM_SUCCESS);
+	}
+	ibcm_path_cache_stats.misses++;
+	mutex_exit(&ibcm_path_cache_mutex);
+	return (IBCM_FAILURE);
+}
+
+static void
+ibcm_path_cache_add(ibt_path_flags_t flags,
+    ibt_path_attr_t *attrp, uint8_t max_paths, ibt_path_info_t *path)
+{
+	int i;
+	ib_gid_t dgid;
+	ibcm_path_cache_t *path_cachep;
+
+	if (max_paths != 1 || attrp->pa_num_dgids != 1 ||
+	    attrp->pa_sname != NULL || attrp->pa_sid != 0)
+		return;
+
+	dgid = attrp->pa_dgids[0];
+	if ((dgid.gid_guid | dgid.gid_prefix) == 0ULL)
+		return;
+
+	mutex_enter(&ibcm_path_cache_mutex);
+	path_cachep = ibcm_path_cachep;
+	for (i = 0; i < ibcm_path_cache_size; i++, path_cachep++) {
+		path_cachep->attr.pa_dgids = attrp->pa_dgids;
+		if (path_cachep->flags == flags &&
+		    path_cachep->dgid.gid_guid == dgid.gid_guid &&
+		    path_cachep->dgid.gid_prefix == dgid.gid_prefix &&
+		    memcmp(&(path_cachep->attr), attrp, sizeof (*attrp)) == 0) {
+			/* already in cache */
+			ibcm_path_cache_stats.already_in_cache++;
+			path_cachep->attr.pa_dgids = NULL;
+			mutex_exit(&ibcm_path_cache_mutex);
+			return;
+		}
+		if (path_cachep->dgid.gid_guid != 0ULL) {
+			path_cachep->attr.pa_dgids = NULL;
+			continue;
+		}
+		/* else the rest of the entries are free, so use this one */
+		ibcm_path_cache_stats.adds++;
+		path_cachep->flags = flags;
+		path_cachep->attr = *attrp;
+		path_cachep->attr.pa_dgids = NULL;
+		path_cachep->dgid = attrp->pa_dgids[0];
+		path_cachep->path = *path;
+		mutex_exit(&ibcm_path_cache_mutex);
+		return;
+	}
+	mutex_exit(&ibcm_path_cache_mutex);
+}
+
+void
+ibcm_path_cache_purge(void)
+{
+	mutex_enter(&ibcm_path_cache_mutex);
+	ibcm_path_cache_invalidate = 1;	/* invalidate cache on next check */
+	ibcm_path_cache_stats.purges++;
+	mutex_exit(&ibcm_path_cache_mutex);
+}
+
+/*
  * Function:
  *	ibt_get_paths
  * Input:
@@ -183,6 +403,8 @@ ibt_get_paths(ibt_clnt_hdl_t ibt_hdl, ibt_path_flags_t flags,
     ibt_path_attr_t *attrp, uint8_t max_paths, ibt_path_info_t *paths,
     uint8_t *num_paths_p)
 {
+	ibt_status_t	retval;
+
 	ASSERT(paths != NULL);
 
 	IBTF_DPRINTF_L3(cmlog, "ibt_get_paths(%p, 0x%lX, %p, %d)",
@@ -197,8 +419,16 @@ ibt_get_paths(ibt_clnt_hdl_t ibt_hdl, ibt_path_flags_t flags,
 	if (num_paths_p != NULL)
 		*num_paths_p = 0;
 
-	return (ibcm_handle_get_path(attrp, flags, max_paths, paths,
-	    num_paths_p, NULL, NULL));
+	if (ibcm_path_cache_check(flags, attrp, max_paths, paths,
+	    num_paths_p) == IBCM_SUCCESS)
+		return (IBT_SUCCESS);
+
+	retval = ibcm_handle_get_path(attrp, flags, max_paths, paths,
+	    num_paths_p, NULL, NULL);
+
+	if (retval == IBT_SUCCESS)
+		ibcm_path_cache_add(flags, attrp, max_paths, paths);
+	return (retval);
 }
 
 
diff --git a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_sm.c b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_sm.c
index cad9634929..c9327092eb 100644
--- a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_sm.c
+++ b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_sm.c
@@ -537,7 +537,7 @@ ibcm_process_incoming_mad(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp,
 	/* Get the HCA entry pointer */
 	cm_qp_entry = (ibcm_qp_list_t *)args;
 
-	IBTF_DPRINTF_L5(cmlog, "ibcm_process_incoming_mad: ibmf_hdl %p"
+	IBTF_DPRINTF_L5(cmlog, "ibcm_process_incoming_mad: ibmf_hdl %p "
 	    "msg %p args %p", ibmf_handle, msgp, args);
 
 #ifdef	DEBUG
@@ -701,7 +701,7 @@ typedef struct ibcm_taskq_args_s {
 	void		*tq_args;
 } ibcm_taskq_args_t;
 
-#define	IBCM_RECV_MAX	64
+#define	IBCM_RECV_MAX	128
 ibcm_taskq_args_t ibcm_recv_array[IBCM_RECV_MAX + 1];
 int ibcm_get, ibcm_put;
 int ibcm_recv_total;
@@ -1075,8 +1075,9 @@ new_req:
 		 * arriving  just after expected retry clock
 		 */
 		statep->stale_clock = gethrtime() +
-		    (ibcm_adj_btime  * 1000000000) + statep->max_cm_retries *
-		    (statep->remote_ack_delay/2) * 1000;
+		    (hrtime_t)(ibcm_adj_btime  * 1000000000) +
+		    (hrtime_t)statep->remote_ack_delay *
+		    (statep->max_cm_retries * (1000 / 2));
 
 		mutex_exit(&statep->state_mutex);
 
@@ -1478,7 +1479,8 @@ ibcm_process_rep_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 	 * First, handle the re-send cases
 	 * The resend routines below release the state mutex
 	 */
-	if (statep->state == IBCM_STATE_ESTABLISHED)
+	if (statep->state == IBCM_STATE_ESTABLISHED ||
+	    statep->state == IBCM_STATE_DREQ_SENT)
 		ibcm_resend_rtu_mad(statep);
 	else if (statep->state == IBCM_STATE_REJ_SENT)
 		ibcm_resend_rej_mad(statep);
@@ -1792,7 +1794,7 @@ ibcm_handle_cep_rep_response(ibcm_state_data_t *statep, ibcm_status_t response,
 		 */
 		ibcm_cep_send_rtu(statep);
 	} else {
-		IBTF_DPRINTF_L4(cmlog, "ibcm_process_rep_msg: statep 0x%p"
+		IBTF_DPRINTF_L4(cmlog, "ibcm_handle_cep_rep_response: statep %p"
 		    " posting REJ reject_reason = %d", statep, reject_reason);
 
 		ASSERT(response == IBCM_SEND_REJ);
@@ -1884,7 +1886,7 @@ ibcm_process_mra_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 		mutex_enter(&statep->state_mutex);
 		IBCM_REF_CNT_DECR(statep);
 		mutex_exit(&statep->state_mutex);
-		IBTF_DPRINTF_L3(cmlog, "ibcm_process_mra_msg: statep 0x%p"
+		IBTF_DPRINTF_L3(cmlog, "ibcm_process_mra_msg: statep 0x%p "
 		    "MRA MAD with tid expected 0x%llX tid found 0x%llX "
 		    "com id 0x%x arrived", statep,
 		    b2h64(IBCM_OUT_HDRP(statep->stored_msg)->TransactionID),
@@ -2050,7 +2052,7 @@ ibcm_process_rtu_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 		mutex_enter(&statep->state_mutex);
 		IBCM_REF_CNT_DECR(statep);
 		mutex_exit(&statep->state_mutex);
-		IBTF_DPRINTF_L3(cmlog, "ibcm_process_rtu_msg: statep 0x%p"
+		IBTF_DPRINTF_L3(cmlog, "ibcm_process_rtu_msg: statep 0x%p "
 		    "An RTU MAD with tid expected 0x%llX tid found 0x%llX "
 		    "com id 0x%x arrived", statep,
 		    b2h64(IBCM_OUT_HDRP(statep->stored_msg)->TransactionID),
@@ -2224,7 +2226,7 @@ ibcm_process_rej_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 	} else if ((statep->state == IBCM_STATE_ESTABLISHED) &&
 	    (statep->mode == IBCM_ACTIVE_MODE)) {
 
-		IBTF_DPRINTF_L4(cmlog, "ibcm_process_rej_msg: statep 0x%p"
+		IBTF_DPRINTF_L4(cmlog, "ibcm_process_rej_msg: statep 0x%p "
 		    "REJ in established state", statep);
 
 		statep->state = IBCM_STATE_TIMEWAIT;
@@ -2329,7 +2331,7 @@ ibcm_process_dreq_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 	local_qpn = b2h32(dreq_msgp->dreq_remote_qpn_eecn_plus) >> 8;
 
 	if (state_lookup_status != IBCM_LOOKUP_EXISTS) {
-		IBTF_DPRINTF_L3(cmlog, "ibcm_process_drep_msg: no statep with"
+		IBTF_DPRINTF_L3(cmlog, "ibcm_process_dreq_msg: no statep with"
 		    "com id %x", b2h32(dreq_msgp->dreq_remote_comm_id));
 		/* implies a bogus message */
 		return;
@@ -2344,6 +2346,7 @@ ibcm_process_dreq_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 	 * a remote stale connection processing with the same com id, but
 	 * not intended for this statep
 	 */
+	mutex_enter(&statep->state_mutex);
 	if ((statep->local_qpn != local_qpn) ||
 	    (statep->remote_comid != b2h32(dreq_msgp->dreq_local_comm_id))) {
 
@@ -2353,29 +2356,52 @@ ibcm_process_dreq_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 		    statep->local_qpn, local_qpn, statep->remote_comid,
 		    b2h32(dreq_msgp->dreq_local_comm_id));
 
-		mutex_enter(&statep->state_mutex);
 		IBCM_REF_CNT_DECR(statep);
 		mutex_exit(&statep->state_mutex);
 		return;
 	}
+	/*
+	 * If another thread is processing a copy of this same DREQ,
+	 * bail out here.
+	 */
+	if (statep->drep_in_progress) {
+		IBCM_REF_CNT_DECR(statep);
+		mutex_exit(&statep->state_mutex);
+		return;
+	}
+	switch (statep->state) {
+	case IBCM_STATE_ESTABLISHED:
+	case IBCM_STATE_DREQ_SENT:
+	case IBCM_STATE_TRANSIENT_DREQ_SENT:
+	case IBCM_STATE_TIMEWAIT:
+		break;
+	default:
+		/* All other states ignore DREQ */
+		IBCM_REF_CNT_DECR(statep);
+		mutex_exit(&statep->state_mutex);
+		return;
+	}
+	statep->drep_in_progress = 1;
 
 	/*
 	 * If drep msg wasn't really required, it shall be deleted finally
 	 * when statep goes away
 	 */
 	if (statep->drep_msg == NULL) {
+		mutex_exit(&statep->state_mutex);
 		if (ibcm_alloc_out_msg(statep->stored_reply_addr.ibmf_hdl,
 		    &statep->drep_msg, MAD_METHOD_SEND) != IBT_SUCCESS) {
 			IBTF_DPRINTF_L2(cmlog, "ibcm_process_dreq_msg: "
 			    "statep 0x%p ibcm_alloc_out_msg failed", statep);
 			mutex_enter(&statep->state_mutex);
+			statep->drep_in_progress = 0;
 			IBCM_REF_CNT_DECR(statep);
 			mutex_exit(&statep->state_mutex);
 			return;
 		}
+		mutex_enter(&statep->state_mutex);
 	}
 
-	mutex_enter(&statep->state_mutex);
 
 	while (statep->state == IBCM_STATE_TRANSIENT_DREQ_SENT)
 		cv_wait(&statep->block_mad_cv, &statep->state_mutex);
@@ -2416,6 +2442,8 @@ ibcm_process_dreq_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 			/* Move CEP to error state */
 			(void) ibcm_cep_to_error_state(statep);
 		}
+		mutex_enter(&statep->state_mutex);
+		statep->drep_in_progress = 0;
 
 		IBCM_OUT_HDRP(statep->drep_msg)->TransactionID =
 		    ((ib_mad_hdr_t *)(input_madp))->TransactionID;
@@ -2426,7 +2454,6 @@ ibcm_process_dreq_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 		if (statep->close_ret_status)
 			*statep->close_ret_status = close_event_type;
 
-		mutex_enter(&statep->state_mutex);
 		if (statep->close_nocb_state != IBCM_FAIL) {
 			ibtl_cm_chan_is_closing(statep->channel);
 			statep->close_nocb_state = IBCM_BLOCK;
@@ -2493,6 +2520,7 @@ ibcm_process_dreq_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 		ibcm_handle_cep_dreq_response(statep, NULL, 0);
 
 	} else if (statep->state == IBCM_STATE_TIMEWAIT) {
+		statep->drep_in_progress = 0;
 		if (statep->send_mad_flags & IBCM_DREP_POST_BUSY) {
 			IBCM_REF_CNT_DECR(statep);
 			mutex_exit(&statep->state_mutex);
@@ -2517,6 +2545,7 @@ ibcm_process_dreq_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 			    statep, statep->state);
 #endif
 		IBCM_REF_CNT_DECR(statep);
+		statep->drep_in_progress = 0;
 		mutex_exit(&statep->state_mutex);
 	}
 }
@@ -2593,6 +2622,10 @@ ibcm_post_dreq_mad(void *vstatep)
 
 	statep->timer_stored_state = statep->state;
 	/* client cannot specify more than 16 retries */
+	statep->timer_value = statep->remote_ack_delay;
+	if (statep->mode == IBCM_ACTIVE_MODE) {
+		statep->timer_value += (2 * statep->pkt_life_time);
+	}
 	statep->remaining_retry_cnt = statep->max_cm_retries + 1;
 	statep->timerid = IBCM_TIMEOUT(statep, 0);
 	mutex_exit(&statep->state_mutex);
@@ -2618,7 +2651,7 @@ ibcm_post_drep_mad(ibcm_state_data_t *statep)
 
 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*drep_msgp))
 
-	IBTF_DPRINTF_L4(cmlog, "ibcm_post_dreq_mad:");
+	IBTF_DPRINTF_L4(cmlog, "ibcm_post_drep_mad:");
 
 	/* Fill up DREP fields */
 	drep_msgp->drep_local_comm_id = h2b32(statep->local_comid);
@@ -2677,9 +2710,8 @@ ibcm_process_drep_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 		mutex_enter(&statep->state_mutex);
 		IBCM_REF_CNT_DECR(statep);
 		mutex_exit(&statep->state_mutex);
-		IBTF_DPRINTF_L3(cmlog, "ibcm_process_drep_msg: statep 0x%p"
-		    "DREP with tid expected 0x%llX tid found 0x%llX",
-		    statep,
+		IBTF_DPRINTF_L3(cmlog, "ibcm_process_drep_msg: statep 0x%p "
+		    "DREP with tid expected 0x%llX tid found 0x%llX", statep,
 		    b2h64(IBCM_OUT_HDRP(statep->dreq_msg)->TransactionID),
 		    b2h64(((ib_mad_hdr_t *)(input_madp))->TransactionID));
 		return;
@@ -3022,7 +3054,7 @@ ibcm_build_n_post_rej_mad(uint8_t *input_madp, ib_com_id_t remote_comid,
 	rej_msg = (ibcm_rej_msg_t *)IBCM_OUT_MSGP(cm_rej_msg);
 	rej_msg->rej_local_comm_id = 0;
 	rej_msg->rej_remote_comm_id = h2b32(remote_comid);
-	rej_msg->rej_msg_type_plus = (which_msg & 0x2) << 6;
+	rej_msg->rej_msg_type_plus = (which_msg & 0x3) << 6;
 	rej_msg->rej_reject_info_len_plus = 0;
 	rej_msg->rej_rejection_reason = h2b16(reject_reason);
 
@@ -3291,6 +3323,8 @@ ibcm_process_abort(ibcm_state_data_t *statep)
 			ibcm_handler_conn_fail(statep, IBT_CM_FAILURE_REJ_RCV,
 			    IBT_CM_FAILURE_UNKNOWN, IBT_CM_TIMEOUT, NULL, 0);
 		else {
+			ibcm_path_cache_purge();
+
 			event.cm_type = IBT_CM_EVENT_CONN_CLOSED;
 			event.cm_channel = statep->channel;
 			event.cm_event.closed = IBT_CM_CLOSED_ABORT;
@@ -3412,7 +3446,7 @@ ibcm_timeout_cb(void *arg)
 		ibcm_ap_state_t		stored_ap_state;
 
 		statep->remaining_retry_cnt--;
-		IBTF_DPRINTF_L3(cmlog, "ibcm_timeout_cb: statep 0x%p"
+		IBTF_DPRINTF_L3(cmlog, "ibcm_timeout_cb: statep 0x%p "
 		    "attr-id= 0x%x, retries remaining = 0x%x", statep,
 		    b2h16(IBCM_OUT_HDRP(statep->stored_msg)->AttributeID),
 		    statep->remaining_retry_cnt);
@@ -3561,7 +3595,7 @@ ibcm_timeout_cb(void *arg)
 	} else if ((statep->ap_state == IBCM_AP_STATE_LAP_SENT) ||
 	    (statep->ap_state == IBCM_AP_STATE_MRA_LAP_RCVD)) {
 
-		IBTF_DPRINTF_L4(cmlog, "ibcm_timeout_cb: statep 0x%p"
+		IBTF_DPRINTF_L4(cmlog, "ibcm_timeout_cb: statep 0x%p "
 		    "LAP timed out",  statep);
 		statep->timedout_state = statep->state;
 		/*
@@ -3699,7 +3733,7 @@ ibcm_resend_post_rep_complete(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp,
 {
 	ibcm_state_data_t	*statep = (ibcm_state_data_t *)args;
 
-	IBTF_DPRINTF_L4(cmlog, "ibcm_post_rep_complete statep %p", statep);
+	IBTF_DPRINTF_L4(cmlog, "ibcm_resend_post_rep_complete(%p)", statep);
 
 	ibcm_insert_trace(statep, IBCM_TRACE_REP_POST_COMPLETE);
 	mutex_enter(&statep->state_mutex);
@@ -3767,6 +3801,9 @@ ibcm_post_dreq_complete(ibmf_handle_t ibmf_handle, ibmf_msg_t *msgp, void *args)
 	mutex_enter(&statep->state_mutex);
 	if (statep->state == IBCM_STATE_DREQ_SENT)
 		statep->timerid = IBCM_TIMEOUT(statep, statep->timer_value);
+	if (statep->close_flow == 1)
+		ibcm_close_flow_control_exit();
+	statep->close_flow = 0;
 	IBCM_REF_CNT_DECR(statep);
 	mutex_exit(&statep->state_mutex);
 }
@@ -4597,7 +4634,7 @@ ibcm_process_sidr_rep_msg(ibcm_hca_info_t *hcap, uint8_t *input_madp,
 	    IBCM_FLAG_LOOKUP);
 	rw_exit(&hcap->hca_sidr_list_lock);
 
-	IBTF_DPRINTF_L4(cmlog, "ibcm_process_sidr_rep_msg: ud_statep 0x%p"
+	IBTF_DPRINTF_L4(cmlog, "ibcm_process_sidr_rep_msg: ud_statep 0x%p "
 	    "find sidr entry status = %x", ud_statep, status);
 
 	if (status != IBCM_LOOKUP_EXISTS) {
@@ -5041,7 +5078,8 @@ ibcm_post_mad(ibmf_msg_t *msgp, ibcm_mad_addr_t *cm_mad_addr,
 	    cm_mad_addr->ibmf_hdl, cm_mad_addr->cm_qp_entry->qp_cm, msgp,
 	    NULL, post_cb, args, 0);
 
-	IBTF_DPRINTF_L4(cmlog, "ibmf_send_mad returned %d", post_status);
+	IBTF_DPRINTF_L4(cmlog, "ibcm_post_mad: ibmf_msg_transport returned %d",
+	    post_status);
 
 	if (post_status != IBMF_SUCCESS) {
 		/* Analyze the reason for failure */
@@ -5143,6 +5181,8 @@ ibcm_handler_conn_fail(ibcm_state_data_t *statep, uint8_t cf_code,
 {
 	ibt_cm_event_t	event;
 
+	ibcm_path_cache_purge();
+
 	/* Invoke CM handler w/ event passed as arg */
 	if (statep->cm_handler != NULL) {
 		bzero(&event, sizeof (ibt_cm_event_t));
@@ -6392,7 +6432,7 @@ ibcm_cep_state_rep(ibcm_state_data_t *statep, ibcm_rep_msg_t *cm_rep_msgp,
 		if ((cm_rep_msgp->rep_rnr_retry_cnt_plus >> 4) & 0x1)
 			IBCM_EVT_REP(event).rep_flags |= IBT_CM_SRQ_EXISTS;
 
-		IBTF_DPRINTF_L4(cmlog, "ibcm_cep_state_rep: statep 0x%p"
+		IBTF_DPRINTF_L4(cmlog, "ibcm_cep_state_rep: statep 0x%p "
 		    "rep_service_time %d", statep,
 		    IBCM_EVT_REP(event).rep_service_time);
 
@@ -6414,7 +6454,7 @@ ibcm_cep_state_rep(ibcm_state_data_t *statep, ibcm_rep_msg_t *cm_rep_msgp,
 
 		ibcm_insert_trace(statep, IBCM_TRACE_RET_REP_RCVD_EVENT);
 
-		IBTF_DPRINTF_L4(cmlog, "ibcm_cep_state_rep: statep 0x%p"
+		IBTF_DPRINTF_L4(cmlog, "ibcm_cep_state_rep: statep 0x%p "
 		    "Client handler returned %x", statep, cb_status);
 
 		if (cb_status == IBT_CM_DEFER) {
@@ -6479,8 +6519,8 @@ ibcm_process_cep_rep_cm_hdlr(ibcm_state_data_t *statep,
 	} else if (cb_status == IBT_CM_NO_RESOURCE) {
 		*reject_reason = IBT_CM_NO_RESC;
 	} else if (cb_status != IBT_CM_ACCEPT) {
-		IBTF_DPRINTF_L2(cmlog, "ibcm_cep_state_rep: statep 0x%p "
-		    "Client handler returned unexpected value %x",
+		IBTF_DPRINTF_L2(cmlog, "ibcm_process_cep_rep_cm_hdlr: statep "
+		    "0x%p, Client handler returned unexpected value %d",
 		    statep, cb_status);
 		*reject_reason = IBT_CM_CONSUMER;
 	} else
@@ -6494,14 +6534,14 @@ ibcm_process_cep_rep_cm_hdlr(ibcm_state_data_t *statep,
 		time = ibt_usec2ib(statep->pkt_life_time * 2 +
 		    ibt_ib2usec(cm_rep_msgp->rep_target_delay_plus >> 3));
 
-		IBTF_DPRINTF_L5(cmlog, "ibcm_cep_state_rep: statep 0x%p"
+		IBTF_DPRINTF_L5(cmlog, "ibcm_process_cep_rep_cm_hdlr: statep %p"
 		    " active cep_timeout(usec) 0x%x ", statep, time);
 
-		IBTF_DPRINTF_L4(cmlog, "ibcm_cep_state_rep: statep 0x%p"
+		IBTF_DPRINTF_L4(cmlog, "ibcm_process_cep_rep_cm_hdlr: statep %p"
 		    " passive hca_ack_delay(ib_time) = 0x%x, ", statep,
 		    cm_rep_msgp->rep_target_delay_plus >> 3);
 
-		IBTF_DPRINTF_L5(cmlog, "ibcm_cep_state_rep: statep 0x%p"
+		IBTF_DPRINTF_L5(cmlog, "ibcm_process_cep_rep_cm_hdlr: statep %p"
 		    " rnr_retry_cnt = 0x%x", statep,
 		    cm_rep_msgp->rep_rnr_retry_cnt_plus >> 5);
 
@@ -6516,8 +6556,9 @@ ibcm_process_cep_rep_cm_hdlr(ibcm_state_data_t *statep,
 		    (ibcm_req_msg_t *)IBCM_OUT_MSGP(statep->stored_msg),
 		    cm_rep_msgp) != IBT_SUCCESS) {
 
-			IBTF_DPRINTF_L2(cmlog, "ibcm_cep_state_rep: statep %p "
-			    "ibcm_invoke_qp_modify to RTR failed", statep);
+			IBTF_DPRINTF_L2(cmlog, "ibcm_process_cep_rep_cm_hdlr: "
+			    "statep %p, ibcm_invoke_qp_modify to RTR failed",
+			    statep);
 			*reject_reason = IBT_CM_NO_RESC;
 		/*
 		 * Call modify qp function from RTR to RTS
@@ -6527,8 +6568,9 @@ ibcm_process_cep_rep_cm_hdlr(ibcm_state_data_t *statep,
 		} else if (ibcm_invoke_rtu_qp_modify(statep, time, cm_rep_msgp)
 		    != IBT_SUCCESS) {
 
-			IBTF_DPRINTF_L2(cmlog, "ibcm_cep_state_rep: statep %p "
-			    "ibcm_invoke_rtu_qp_modify to RTS failed", statep);
+			IBTF_DPRINTF_L2(cmlog, "ibcm_process_cep_rep_cm_hdlr: "
+			    "statep %p ibcm_invoke_rtu_qp_modify to RTS failed",
+			    statep);
 			(void) ibcm_cep_to_error_state(statep);
 			*reject_reason = IBT_CM_NO_RESC;
 		}
@@ -6887,10 +6929,12 @@ ibcm_cep_state_rej(ibcm_state_data_t *statep, ibcm_rej_msg_t *rej_msgp,
 
 	IBTF_DPRINTF_L4(cmlog, "ibcm_cep_state_rej: statep 0x%p", statep);
 
+	ibcm_path_cache_purge();
+
 	if ((rej_state == IBCM_STATE_REP_SENT) ||
 	    (rej_state == IBCM_STATE_MRA_REP_RCVD)) {
 		status = ibcm_cep_to_error_state(statep);
-		IBTF_DPRINTF_L5(cmlog, "ibcm_cep_state_rej: statep 0x%p"
+		IBTF_DPRINTF_L5(cmlog, "ibcm_cep_state_rej: statep 0x%p "
 		    "ibcm_cep_to_error_state returned %d", statep,
 		    status);
 	}
@@ -7152,7 +7196,7 @@ ibcm_cep_state_rej_est(ibcm_state_data_t *statep)
 		event.cm_event.closed = IBT_CM_CLOSED_REJ_RCVD;
 
 		IBTF_DPRINTF_L4(cmlog, "ibcm_cep_state_rej_est: "
-		    "rej_reason = %x", event.cm_event.failed.cf_reason);
+		    "rej_reason = %d", event.cm_event.failed.cf_reason);
 
 		ibcm_insert_trace(statep, IBCM_TRACE_CALLED_CONN_CLOSE_EVENT);
 
@@ -7285,7 +7329,7 @@ ibcm_process_sidr_req_cm_hdlr(ibcm_ud_state_data_t *ud_statep,
 	else *sidr_status = IBT_CM_SREP_REJ;
 
 	if (*sidr_status != IBT_CM_SREP_CHAN_VALID) {
-		IBTF_DPRINTF_L2(cmlog, "ibcm_sidr_req_ud_handler: "
+		IBTF_DPRINTF_L2(cmlog, "ibcm_process_sidr_req_cm_hdlr: "
 		    "ud_handler return a failure: %d", cb_status);
 		if (*sidr_status == IBT_CM_SREP_REDIRECT) {
 		/*
@@ -7654,10 +7698,10 @@ ibcm_fill_adds_from_lap(ibt_adds_vect_t *adds, ibcm_lap_msg_t *lap_msg,
 		adds->av_dlid = b2h16(lap_msg->lap_alt_r_port_lid);
 	}
 
-	IBTF_DPRINTF_L4(cmlog, "ibcm_cep_state_lap: SGID=(%llX:%llX)",
+	IBTF_DPRINTF_L4(cmlog, "ibcm_fill_adds_from_lap: SGID=(%llX:%llX)",
 	    adds->av_sgid.gid_prefix, adds->av_sgid.gid_guid);
 
-	IBTF_DPRINTF_L4(cmlog, "ibcm_cep_state_lap: DGID=(%llX:%llX)",
+	IBTF_DPRINTF_L4(cmlog, "ibcm_fill_adds_from_lap: DGID=(%llX:%llX)",
 	    adds->av_dgid.gid_prefix, adds->av_dgid.gid_guid);
 
 	adds->av_srate = lap_msg->lap_alt_srate_plus & 0x3f;
@@ -7757,7 +7801,7 @@ ibcm_process_cep_lap_cm_hdlr(ibcm_state_data_t *statep,
 
 	IBCM_QP_RC(qp_attrs).rc_alt_path.cep_hca_port_num = port.hp_port;
 
-	IBTF_DPRINTF_L4(cmlog, "ibcm_process_cep_lap_cm_hdlr: statep 0x%p"
+	IBTF_DPRINTF_L4(cmlog, "ibcm_process_cep_lap_cm_hdlr: statep 0x%p "
 	    "gid = (%llx, %llx), port_num = %d", statep,
 	    IBCM_QP_RC(qp_attrs).rc_alt_path.cep_adds_vect.av_dgid.
 	    gid_prefix,
@@ -7781,7 +7825,7 @@ ibcm_process_cep_lap_cm_hdlr(ibcm_state_data_t *statep,
 
 	qp_attrs.qp_info.qp_trans = IBT_RC_SRV;
 	if (IBCM_QP_RC(qp_attrs).rc_mig_state == IBT_STATE_MIGRATED) {
-		IBTF_DPRINTF_L3(cmlog, "ibcm_cep_state_lap: statep 0x%p: "
+		IBTF_DPRINTF_L3(cmlog, "ibcm_cep_state_lap_cm_hdlr: statep %p: "
 		    "rearming APM", statep);
 		cep_flags |= IBT_CEP_SET_MIG;
 		IBCM_QP_RC(qp_attrs).rc_mig_state = IBT_STATE_REARMED;
@@ -7944,6 +7988,9 @@ ibcm_set_apr_arej(int ap_status, ibcm_apr_msg_t *apr_msgp,
 
 	*ari_valid = B_FALSE;
 
+	IBTF_DPRINTF_L3(cmlog, "ibcm_set_apr_arej: apr_status = %d "
+	    "ari_len = %d", ap_status, ari_len);
+
 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ari))
 
 	switch (ap_status) {
@@ -7970,6 +8017,9 @@ ibcm_set_apr_arej(int ap_status, ibcm_apr_msg_t *apr_msgp,
 		    sizeof (ib_gid_t));
 		ari->ari_gid.gid_guid = b2h64(ari->ari_gid.gid_guid);
 		ari->ari_gid.gid_prefix = b2h64(ari->ari_gid.gid_prefix);
+
+		IBTF_DPRINTF_L4(cmlog, "ibcm_set_apr_arej: ari_gid= %llX:%llX",
+		    ari->ari_gid.gid_prefix, ari->ari_gid.gid_guid);
 		break;
 	case IBT_CM_AP_FLOW_REJECTED:
 		if (ari_len < 3)	/* 3 bytes needed for 20 bits */
diff --git a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_ti.c b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_ti.c
index 7c1565e923..5a95d1201f 100644
--- a/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_ti.c
+++ b/usr/src/uts/common/io/ib/mgt/ibcm/ibcm_ti.c
@@ -1520,6 +1520,8 @@ ibt_close_rc_channel(ibt_channel_hdl_t channel, ibt_execution_mode_t mode,
 			statep->state = IBCM_STATE_ESTABLISHED;
 			IBCM_REF_CNT_DECR(statep);
 			cv_broadcast(&statep->block_mad_cv);
+			statep->close_flow = 0;
+			ibcm_close_flow_control_exit();
 			mutex_exit(&statep->state_mutex);
 			return (status);
 		}
@@ -1549,6 +1551,8 @@ ibt_close_rc_channel(ibt_channel_hdl_t channel, ibt_execution_mode_t mode,
 			statep->state = IBCM_STATE_ESTABLISHED;
 			IBCM_REF_CNT_DECR(statep);
 			cv_broadcast(&statep->block_mad_cv);
+			statep->close_flow = 0;
+			ibcm_close_flow_control_exit();
 			mutex_exit(&statep->state_mutex);
 			return (IBT_INSUFF_KERNEL_RESOURCE);
 		}
@@ -2353,7 +2357,8 @@ ibt_cm_delay(ibt_cmdelay_flags_t flags, void *cm_session_id,
 		return (IBT_CHAN_STATE_INVALID);
 	}
 	/* service time is usecs, stale_clock is nsecs */
-	statep->stale_clock += (hrtime_t)service_time * (1000 *
+	statep->stale_clock = gethrtime() +
+	    (hrtime_t)ibt_ib2usec(ibt_usec2ib(service_time)) * (1000 *
 	    statep->max_cm_retries);
 
 	statep->send_mad_flags |= IBCM_MRA_POST_BUSY;
diff --git a/usr/src/uts/common/sys/ib/mgt/ibcm/ibcm_impl.h b/usr/src/uts/common/sys/ib/mgt/ibcm/ibcm_impl.h
index 328520d762..2b53b66afb 100644
--- a/usr/src/uts/common/sys/ib/mgt/ibcm/ibcm_impl.h
+++ b/usr/src/uts/common/sys/ib/mgt/ibcm/ibcm_impl.h
@@ -403,6 +403,8 @@ typedef struct ibcm_state_data_s {
 	uint8_t			remaining_retry_cnt;
 	uint8_t			max_cm_retries;
 
+	uint8_t			drep_in_progress;
+
 	/* some cep stuff, stored here temporarily during connection est  */
 	uint8_t			cep_retry_cnt:3;
 	ibt_srate_t		local_srate;
@@ -2111,6 +2113,10 @@ ibt_status_t ibcm_ibmf_analyze_error(int ibmf_status);
 ibt_status_t ibcm_contact_sa_access(ibmf_saa_handle_t saa_handle,
     ibmf_saa_access_args_t *access_args, size_t *length, void **results_p);
 
+void ibcm_path_cache_init(void);
+void ibcm_path_cache_fini(void);
+void ibcm_path_cache_purge(void);
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/usr/src/uts/sparc/ibcm/ibcm.wlcmd b/usr/src/uts/sparc/ibcm/ibcm.wlcmd
index fdda354132..9df97bec27 100644
--- a/usr/src/uts/sparc/ibcm/ibcm.wlcmd
+++ b/usr/src/uts/sparc/ibcm/ibcm.wlcmd
@@ -104,6 +104,7 @@ root	ibcm_service_record_rewrite_task
 
 # kernel callbacks to ibcm
 
+root	ibcm_path_cache_timeout_cb
 root	ibcm_timeout_cb
 root	ibcm_recv_timeout_cb
 root	ibcm_sidr_timeout_cb