12674 want mac rx barrier function

Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Ryan Zezeski <rpz@joyent.com>
Approved by: Dan McDonald <danmcd@joyent.com>

4 files changed, 167 insertions, 79 deletions

diff --git a/usr/src/uts/common/io/mac/mac.c b/usr/src/uts/common/io/mac/mac.c
index 79a518a164..76b4765de6 100644
--- a/usr/src/uts/common/io/mac/mac.c
+++ b/usr/src/uts/common/io/mac/mac.c
@@ -707,12 +707,45 @@ mac_callback_remove_wait(mac_cb_info_t *mcbi)
 	}
 }
 
+void
+mac_callback_barrier(mac_cb_info_t *mcbi)
+{
+	ASSERT(MUTEX_HELD(mcbi->mcbi_lockp));
+	ASSERT3U(mcbi->mcbi_barrier_cnt, <, UINT_MAX);
+
+	if (mcbi->mcbi_walker_cnt == 0) {
+		return;
+	}
+
+	mcbi->mcbi_barrier_cnt++;
+	do {
+		cv_wait(&mcbi->mcbi_cv, mcbi->mcbi_lockp);
+	} while (mcbi->mcbi_walker_cnt > 0);
+	mcbi->mcbi_barrier_cnt--;
+	cv_broadcast(&mcbi->mcbi_cv);
+}
+
+void
+mac_callback_walker_enter(mac_cb_info_t *mcbi)
+{
+	mutex_enter(mcbi->mcbi_lockp);
+	/*
+	 * Incoming walkers should give precedence to timely clean-up of
+	 * deleted callback entries and requested barriers.
+	 */
+	while (mcbi->mcbi_del_cnt > 0 || mcbi->mcbi_barrier_cnt > 0) {
+		cv_wait(&mcbi->mcbi_cv, mcbi->mcbi_lockp);
+	}
+	mcbi->mcbi_walker_cnt++;
+	mutex_exit(mcbi->mcbi_lockp);
+}
+
 /*
  * The last mac callback walker does the cleanup. Walk the list and unlik
  * all the logically deleted entries and construct a temporary list of
  * removed entries. Return the list of removed entries to the caller.
  */
-mac_cb_t *
+static mac_cb_t *
 mac_callback_walker_cleanup(mac_cb_info_t *mcbi, mac_cb_t **mcb_head)
 {
 	mac_cb_t	*p;
@@ -741,7 +774,90 @@ mac_callback_walker_cleanup(mac_cb_info_t *mcbi, mac_cb_t **mcb_head)
 	return (rmlist);
 }
 
-boolean_t
+void
+mac_callback_walker_exit(mac_cb_info_t *mcbi, mac_cb_t **headp,
+    boolean_t is_promisc)
+{
+	boolean_t do_wake = B_FALSE;
+
+	mutex_enter(mcbi->mcbi_lockp);
+
+	/* If walkers remain, nothing more can be done for now */
+	if (--mcbi->mcbi_walker_cnt != 0) {
+		mutex_exit(mcbi->mcbi_lockp);
+		return;
+	}
+
+	if (mcbi->mcbi_del_cnt != 0) {
+		mac_cb_t *rmlist;
+
+		rmlist = mac_callback_walker_cleanup(mcbi, headp);
+
+		if (!is_promisc) {
+			/* The "normal" non-promisc callback clean-up */
+			mac_callback_free(rmlist);
+		} else {
+			mac_cb_t *mcb, *mcb_next;
+
+			/*
+			 * The promisc callbacks are in 2 lists, one off the
+			 * 'mip' and another off the 'mcip' threaded by
+			 * mpi_mi_link and mpi_mci_link respectively.  There
+			 * is, however, only a single shared total walker
+			 * count, and an entry cannot be physically unlinked if
+			 * a walker is active on either list. The last walker
+			 * does this cleanup of logically deleted entries.
+			 *
+			 * With a list of callbacks deleted from above from
+			 * mi_promisc_list (headp), remove the corresponding
+			 * entry from mci_promisc_list (headp_pair) and free
+			 * the structure.
+			 */
+			for (mcb = rmlist; mcb != NULL; mcb = mcb_next) {
+				mac_promisc_impl_t *mpip;
+				mac_client_impl_t *mcip;
+
+				mcb_next = mcb->mcb_nextp;
+				mpip = (mac_promisc_impl_t *)mcb->mcb_objp;
+				mcip = mpip->mpi_mcip;
+
+				ASSERT3P(&mcip->mci_mip->mi_promisc_cb_info,
+				    ==, mcbi);
+				ASSERT3P(&mcip->mci_mip->mi_promisc_list,
+				    ==, headp);
+
+				VERIFY(mac_callback_remove(mcbi,
+				    &mcip->mci_promisc_list,
+				    &mpip->mpi_mci_link));
+				mcb->mcb_flags = 0;
+				mcb->mcb_nextp = NULL;
+				kmem_cache_free(mac_promisc_impl_cache, mpip);
+			}
+		}
+
+		/*
+		 * Wake any walker threads that could be waiting in
+		 * mac_callback_walker_enter() until deleted items have been
+		 * cleaned from the list.
+		 */
+		do_wake = B_TRUE;
+	}
+
+	if (mcbi->mcbi_barrier_cnt != 0) {
+		/*
+		 * One or more threads are waiting for all walkers to exit the
+		 * callback list.  Notify them, now that the list is clear.
+		 */
+		do_wake = B_TRUE;
+	}
+
+	if (do_wake) {
+		cv_broadcast(&mcbi->mcbi_cv);
+	}
+	mutex_exit(mcbi->mcbi_lockp);
+}
+
+static boolean_t
 mac_callback_lookup(mac_cb_t **mcb_headp, mac_cb_t *mcb_elem)
 {
 	mac_cb_t	*mcb;
@@ -755,7 +871,7 @@ mac_callback_lookup(mac_cb_t **mcb_headp, mac_cb_t *mcb_elem)
 	return (B_FALSE);
 }
 
-boolean_t
+static boolean_t
 mac_callback_find(mac_cb_info_t *mcbi, mac_cb_t **mcb_headp, mac_cb_t *mcb_elem)
 {
 	boolean_t	found;
@@ -780,40 +896,6 @@ mac_callback_free(mac_cb_t *rmlist)
 	}
 }
 
-/*
- * The promisc callbacks are in 2 lists, one off the 'mip' and another off the
- * 'mcip' threaded by mpi_mi_link and mpi_mci_link respectively. However there
- * is only a single shared total walker count, and an entry can't be physically
- * unlinked if a walker is active on either list. The last walker does this
- * cleanup of logically deleted entries.
- */
-void
-i_mac_promisc_walker_cleanup(mac_impl_t *mip)
-{
-	mac_cb_t	*rmlist;
-	mac_cb_t	*mcb;
-	mac_cb_t	*mcb_next;
-	mac_promisc_impl_t	*mpip;
-
-	/*
-	 * Construct a temporary list of deleted callbacks by walking the
-	 * the mi_promisc_list. Then for each entry in the temporary list,
-	 * remove it from the mci_promisc_list and free the entry.
-	 */
-	rmlist = mac_callback_walker_cleanup(&mip->mi_promisc_cb_info,
-	    &mip->mi_promisc_list);
-
-	for (mcb = rmlist; mcb != NULL; mcb = mcb_next) {
-		mcb_next = mcb->mcb_nextp;
-		mpip = (mac_promisc_impl_t *)mcb->mcb_objp;
-		VERIFY(mac_callback_remove(&mip->mi_promisc_cb_info,
-		    &mpip->mpi_mcip->mci_promisc_list, &mpip->mpi_mci_link));
-		mcb->mcb_flags = 0;
-		mcb->mcb_nextp = NULL;
-		kmem_cache_free(mac_promisc_impl_cache, mpip);
-	}
-}
-
 void
 i_mac_notify(mac_impl_t *mip, mac_notify_type_t type)
 {
diff --git a/usr/src/uts/common/io/mac/mac_client.c b/usr/src/uts/common/io/mac/mac_client.c
index 779bdf204a..6563d51069 100644
--- a/usr/src/uts/common/io/mac/mac_client.c
+++ b/usr/src/uts/common/io/mac/mac_client.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2018 Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  * Copyright 2017 RackTop Systems.
  * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
  */
@@ -1633,6 +1633,32 @@ mac_rx_clear(mac_client_handle_t mch)
 }
 
 void
+mac_rx_barrier(mac_client_handle_t mch)
+{
+	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
+	mac_impl_t *mip = mcip->mci_mip;
+
+	i_mac_perim_enter(mip);
+
+	/* If a RX callback is set, quiesce and restart that datapath */
+	if (mcip->mci_rx_fn != mac_pkt_drop) {
+		mac_rx_client_quiesce(mch);
+		mac_rx_client_restart(mch);
+	}
+
+	/* If any promisc callbacks are registered, perform a barrier there */
+	if (mcip->mci_promisc_list != NULL || mip->mi_promisc_list != NULL) {
+		mac_cb_info_t *mcbi =  &mip->mi_promisc_cb_info;
+
+		mutex_enter(mcbi->mcbi_lockp);
+		mac_callback_barrier(mcbi);
+		mutex_exit(mcbi->mcbi_lockp);
+	}
+
+	i_mac_perim_exit(mip);
+}
+
+void
 mac_secondary_dup(mac_client_handle_t smch, mac_client_handle_t dmch)
 {
 	mac_client_impl_t *smcip = (mac_client_impl_t *)smch;
diff --git a/usr/src/uts/common/sys/mac_client.h b/usr/src/uts/common/sys/mac_client.h
index 74f4cbb310..88ab5f4756 100644
--- a/usr/src/uts/common/sys/mac_client.h
+++ b/usr/src/uts/common/sys/mac_client.h
@@ -22,7 +22,7 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
- * Copyright 2018 Joyent, Inc.  All rights reserved.
+ * Copyright 2019 Joyent, Inc.
  */
 
 /*
@@ -137,6 +137,7 @@ extern void mac_multicast_remove(mac_client_handle_t, const uint8_t *);
 
 extern void mac_rx_set(mac_client_handle_t, mac_rx_t, void *);
 extern void mac_rx_clear(mac_client_handle_t);
+extern void mac_rx_barrier(mac_client_handle_t);
 extern void mac_secondary_dup(mac_client_handle_t, mac_client_handle_t);
 extern void mac_secondary_cleanup(mac_client_handle_t);
 extern mac_tx_cookie_t mac_tx(mac_client_handle_t, mblk_t *,
diff --git a/usr/src/uts/common/sys/mac_impl.h b/usr/src/uts/common/sys/mac_impl.h
index df03a76715..4625417828 100644
--- a/usr/src/uts/common/sys/mac_impl.h
+++ b/usr/src/uts/common/sys/mac_impl.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2018, Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  */
 
 #ifndef	_SYS_MAC_IMPL_H
@@ -108,6 +108,7 @@ typedef struct mac_cb_info_s {
 	kcondvar_t	mcbi_cv;
 	uint_t		mcbi_del_cnt;		/* Deleted callback cnt */
 	uint_t		mcbi_walker_cnt;	/* List walker count */
+	uint_t		mcbi_barrier_cnt;	/* Barrier waiter count */
 } mac_cb_info_t;
 
 typedef struct mac_notify_cb_s {
@@ -123,40 +124,18 @@ typedef struct mac_notify_cb_s {
  */
 typedef boolean_t (*mcb_func_t)(mac_cb_info_t *, mac_cb_t **, mac_cb_t *);
 
-#define	MAC_CALLBACK_WALKER_INC(mcbi) {				\
-	mutex_enter((mcbi)->mcbi_lockp);			\
-	(mcbi)->mcbi_walker_cnt++;				\
-	mutex_exit((mcbi)->mcbi_lockp);				\
-}
+#define	MAC_CALLBACK_WALKER_INC(mcbi) \
+	mac_callback_walker_enter(mcbi)
 
-#define	MAC_CALLBACK_WALKER_INC_HELD(mcbi)	(mcbi)->mcbi_walker_cnt++;
-
-#define	MAC_CALLBACK_WALKER_DCR(mcbi, headp) {			\
-	mac_cb_t	*rmlist;				\
-								\
-	mutex_enter((mcbi)->mcbi_lockp);			\
-	if (--(mcbi)->mcbi_walker_cnt == 0 && (mcbi)->mcbi_del_cnt != 0) { \
-		rmlist = mac_callback_walker_cleanup((mcbi), headp);	\
-		mac_callback_free(rmlist);			\
-		cv_broadcast(&(mcbi)->mcbi_cv);			\
-	}							\
-	mutex_exit((mcbi)->mcbi_lockp);				\
-}
+#define	MAC_CALLBACK_WALKER_DCR(mcbi, headp) \
+	mac_callback_walker_exit(mcbi, headp, B_FALSE)
 
-#define	MAC_PROMISC_WALKER_INC(mip)				\
-	MAC_CALLBACK_WALKER_INC(&(mip)->mi_promisc_cb_info)
-
-#define	MAC_PROMISC_WALKER_DCR(mip) {				\
-	mac_cb_info_t	*mcbi;					\
-								\
-	mcbi = &(mip)->mi_promisc_cb_info;			\
-	mutex_enter(mcbi->mcbi_lockp);				\
-	if (--mcbi->mcbi_walker_cnt == 0 && mcbi->mcbi_del_cnt != 0) { \
-		i_mac_promisc_walker_cleanup(mip);		\
-		cv_broadcast(&mcbi->mcbi_cv);			\
-	}							\
-	mutex_exit(mcbi->mcbi_lockp);				\
-}
+#define	MAC_PROMISC_WALKER_INC(mip) \
+	mac_callback_walker_enter(&(mip)->mi_promisc_cb_info)
+
+#define	MAC_PROMISC_WALKER_DCR(mip) \
+	mac_callback_walker_exit(&(mip)->mi_promisc_cb_info, \
+	    &(mip)->mi_promisc_list, B_TRUE)
 
 typedef struct mactype_s {
 	const char	*mt_ident;
@@ -753,12 +732,13 @@ extern void mac_rx_deliver(void *, mac_resource_handle_t, mblk_t *,
     mac_header_info_t *);
 extern void mac_tx_notify(mac_impl_t *);
 
-extern	boolean_t mac_callback_find(mac_cb_info_t *, mac_cb_t **, mac_cb_t *);
-extern	void	mac_callback_add(mac_cb_info_t *, mac_cb_t **, mac_cb_t *);
-extern	boolean_t mac_callback_remove(mac_cb_info_t *, mac_cb_t **, mac_cb_t *);
-extern	void	mac_callback_remove_wait(mac_cb_info_t *);
-extern	void	mac_callback_free(mac_cb_t *);
-extern	mac_cb_t *mac_callback_walker_cleanup(mac_cb_info_t *, mac_cb_t **);
+extern void mac_callback_add(mac_cb_info_t *, mac_cb_t **, mac_cb_t *);
+extern boolean_t mac_callback_remove(mac_cb_info_t *, mac_cb_t **, mac_cb_t *);
+extern void mac_callback_remove_wait(mac_cb_info_t *);
+extern void mac_callback_barrier(mac_cb_info_t *);
+extern void mac_callback_free(mac_cb_t *);
+extern void mac_callback_walker_enter(mac_cb_info_t *);
+extern void mac_callback_walker_exit(mac_cb_info_t *, mac_cb_t **, boolean_t);
 
 /* in mac_bcast.c */
 extern void mac_bcast_init(void);
@@ -872,7 +852,6 @@ extern void mac_tx_client_block(mac_client_impl_t *);
 extern void mac_tx_client_unblock(mac_client_impl_t *);
 extern void mac_tx_invoke_callbacks(mac_client_impl_t *, mac_tx_cookie_t);
 extern int i_mac_promisc_set(mac_impl_t *, boolean_t);
-extern void i_mac_promisc_walker_cleanup(mac_impl_t *);
 extern mactype_t *mactype_getplugin(const char *);
 extern void mac_addr_factory_init(mac_impl_t *);
 extern void mac_addr_factory_fini(mac_impl_t *);