OS-2634 mci_rw_lock white-hot under load

Reviewed by: Robert Mustacchi <rm@joyent.com>

2 files changed, 42 insertions, 1 deletions

diff --git a/usr/src/uts/common/io/mac/mac_client.c b/usr/src/uts/common/io/mac/mac_client.c
index 766c5f8f41..0f0d44cda5 100644
--- a/usr/src/uts/common/io/mac/mac_client.c
+++ b/usr/src/uts/common/io/mac/mac_client.c
@@ -1344,6 +1344,7 @@ mac_client_open(mac_handle_t mh, mac_client_handle_t *mchp, char *name,
 	mcip->mci_p_unicast_list = NULL;
 	mcip->mci_direct_rx_fn = NULL;
 	mcip->mci_direct_rx_arg = NULL;
+	mcip->mci_vidcache = MCIP_VIDCACHE_INVALID;
 
 	mcip->mci_unicast_list = NULL;
 
@@ -4800,6 +4801,8 @@ mac_client_add_to_flow_list(mac_client_impl_t *mcip, flow_entry_t *flent)
 	 */
 	rw_enter(&mcip->mci_rw_lock, RW_WRITER);
 
+	mcip->mci_vidcache = MCIP_VIDCACHE_INVALID;
+
 	/* Add it to the head */
 	flent->fe_client_next = mcip->mci_flent_list;
 	mcip->mci_flent_list = flent;
@@ -4830,6 +4833,8 @@ mac_client_remove_flow_from_list(mac_client_impl_t *mcip, flow_entry_t *flent)
 	 * using mci_rw_lock
 	 */
 	rw_enter(&mcip->mci_rw_lock, RW_WRITER);
+	mcip->mci_vidcache = MCIP_VIDCACHE_INVALID;
+
 	while ((fe != NULL) && (fe != flent)) {
 		prev_fe = fe;
 		fe = fe->fe_client_next;
@@ -4858,6 +4863,14 @@ mac_client_check_flow_vid(mac_client_impl_t *mcip, uint16_t vid)
 {
 	flow_entry_t	*flent;
 	uint16_t	mci_vid;
+	uint32_t	cache = mcip->mci_vidcache;
+
+	/*
+	 * In hopes of not having to touch the mci_rw_lock, check to see if
+	 * this vid matches our cached result.
+	 */
+	if (MCIP_VIDCACHE_ISVALID(cache) && MCIP_VIDCACHE_VID(cache) == vid)
+		return (MCIP_VIDCACHE_BOOL(cache) ? B_TRUE : B_FALSE);
 
 	/* The mci_flent_list is protected by mci_rw_lock */
 	rw_enter(&mcip->mci_rw_lock, RW_WRITER);
@@ -4865,10 +4878,13 @@ mac_client_check_flow_vid(mac_client_impl_t *mcip, uint16_t vid)
 	    flent = flent->fe_client_next) {
 		mci_vid = i_mac_flow_vid(flent);
 		if (vid == mci_vid) {
+			mcip->mci_vidcache = MCIP_VIDCACHE_CACHE(vid, B_TRUE);
 			rw_exit(&mcip->mci_rw_lock);
 			return (B_TRUE);
 		}
 	}
+
+	mcip->mci_vidcache = MCIP_VIDCACHE_CACHE(vid, B_FALSE);
 	rw_exit(&mcip->mci_rw_lock);
 	return (B_FALSE);
 }
diff --git a/usr/src/uts/common/sys/mac_client_impl.h b/usr/src/uts/common/sys/mac_client_impl.h
index ec49527300..ce1a6f1d7c 100644
--- a/usr/src/uts/common/sys/mac_client_impl.h
+++ b/usr/src/uts/common/sys/mac_client_impl.h
@@ -21,7 +21,9 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
- * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
+ */
+/*
+ * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
  */
 
 #ifndef	_SYS_MAC_CLIENT_IMPL_H
@@ -138,6 +140,7 @@ struct mac_client_impl_s {			/* Protected by */
 	flow_entry_t		*mci_flent_list;	/* mci_rw_lock */
 	uint_t			mci_nflents;		/* mci_rw_lock */
 	uint_t			mci_nvids;		/* mci_rw_lock */
+	volatile uint32_t	mci_vidcache;		/* VID cache */
 
 	/* Resource Management Functions */
 	mac_resource_add_t	mci_resource_add;	/* SL */
@@ -281,6 +284,28 @@ extern	int	mac_tx_percpu_cnt;
 	}								\
 }
 
+/*
+ * To allow the hot path to not grab any additional locks, we keep a single
+ * entry VLAD ID cache that caches whether or not a given VID belongs to a
+ * MAC client.
+ */
+#define	MCIP_VIDCACHE_VALIDSHIFT	31
+#define	MCIP_VIDCACHE_VIDSHIFT		1
+#define	MCIP_VIDCACHE_VIDMASK		(UINT16_MAX << MCIP_VIDCACHE_VIDSHIFT)
+#define	MCIP_VIDCACHE_BOOLSHIFT		0
+
+#define	MCIP_VIDCACHE_INVALID		0
+
+#define	MCIP_VIDCACHE_CACHE(vid, bool)	\
+	((1U << MCIP_VIDCACHE_VALIDSHIFT) | \
+	((vid) << MCIP_VIDCACHE_VIDSHIFT) | \
+	((bool) ? (1U << MCIP_VIDCACHE_BOOLSHIFT) : 0))
+
+#define	MCIP_VIDCACHE_ISVALID(v)	((v) & (1U << MCIP_VIDCACHE_VALIDSHIFT))
+#define	MCIP_VIDCACHE_VID(v)		\
+	(((v) & MCIP_VIDCACHE_VIDMASK) >> MCIP_VIDCACHE_VIDSHIFT)
+#define	MCIP_VIDCACHE_BOOL(v)		((v) & (1U << MCIP_VIDCACHE_BOOLSHIFT))
+
 #define	MAC_TAG_NEEDED(mcip)						\
 	(((mcip)->mci_state_flags & MCIS_TAG_DISABLE) == 0 &&		\
 	(mcip)->mci_nvids == 1)						\