OS-4692 MAC tries to use aggr rings from downed links

Reviewed by: Bryan Cantrill <bryan@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>

5 files changed, 74 insertions, 1 deletions

diff --git a/usr/src/uts/common/io/aggr/aggr_grp.c b/usr/src/uts/common/io/aggr/aggr_grp.c
index 6863b7c5cb..3554e111c1 100644
--- a/usr/src/uts/common/io/aggr/aggr_grp.c
+++ b/usr/src/uts/common/io/aggr/aggr_grp.c
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 Joyent, Inc.
  */
 
 /*
@@ -572,6 +573,38 @@ aggr_grp_add_port(aggr_grp_t *grp, datalink_id_t port_linkid, boolean_t force,
 }
 
 /*
+ * This is called in response to either our LACP state machine or a MAC
+ * notification that the link has gone down via aggr_send_port_disable(). At
+ * this point, we may need to update our default ring. To that end, we go
+ * through the set of ports (underlying datalinks in an aggregation) that are
+ * currently enabled to transmit data. If all our links have been disabled for
+ * transmit, then we don't do anything.
+ *
+ * Note, because we only have a single TX group, we don't have to worry about
+ * the rings moving between groups and the chance that mac will reassign it
+ * unless someone removes a port, at which point, we play it safe and call this
+ * again.
+ */
+void
+aggr_grp_update_default(aggr_grp_t *grp)
+{
+	aggr_port_t *port;
+	ASSERT(MAC_PERIM_HELD(grp->lg_mh));
+
+	rw_enter(&grp->lg_tx_lock, RW_WRITER);
+
+	if (grp->lg_ntx_ports == 0) {
+		rw_exit(&grp->lg_tx_lock);
+		return;
+	}
+
+	port = grp->lg_tx_ports[0];
+	ASSERT(port->lp_tx_ring_cnt > 0);
+	mac_hwring_set_default(grp->lg_mh, port->lp_pseudo_tx_rings[0]);
+	rw_exit(&grp->lg_tx_lock);
+}
+
+/*
  * Add a pseudo RX ring for the given HW ring handle.
  */
 static int
@@ -817,6 +850,7 @@ aggr_add_pseudo_tx_ring(aggr_port_t *port,
 			    mac_find_ring(tx_grp->atg_gh, i));
 		}
 	}
+
 	return (err);
 }
 
@@ -919,6 +953,7 @@ aggr_add_pseudo_tx_group(aggr_port_t *port, aggr_pseudo_tx_group_t *tx_grp)
 		    aggr_tx_ring_update, port);
 	}
 	mac_perim_exit(pmph);
+	aggr_grp_update_default(grp);
 	return (err);
 }
 
@@ -952,6 +987,7 @@ aggr_rem_pseudo_tx_group(aggr_port_t *port, aggr_pseudo_tx_group_t *tx_grp)
 	port->lp_tx_ring_cnt = 0;
 	(void) mac_client_tx_notify(port->lp_mch, NULL, port->lp_tx_notify_mh);
 	port->lp_tx_grp_added = B_FALSE;
+	aggr_grp_update_default(grp);
 done:
 	mac_perim_exit(pmph);
 }
diff --git a/usr/src/uts/common/io/aggr/aggr_send.c b/usr/src/uts/common/io/aggr/aggr_send.c
index 7d423f267e..4095e8fda5 100644
--- a/usr/src/uts/common/io/aggr/aggr_send.c
+++ b/usr/src/uts/common/io/aggr/aggr_send.c
@@ -274,6 +274,8 @@ aggr_send_port_enable(aggr_port_t *port)
 	rw_exit(&grp->lg_tx_lock);
 
 	port->lp_tx_enabled = B_TRUE;
+
+	aggr_grp_update_default(grp);
 }
 
 /*
@@ -316,4 +318,6 @@ aggr_send_port_disable(aggr_port_t *port)
 	rw_exit(&grp->lg_tx_lock);
 
 	port->lp_tx_enabled = B_FALSE;
+
+	aggr_grp_update_default(grp);
 }
diff --git a/usr/src/uts/common/io/mac/mac.c b/usr/src/uts/common/io/mac/mac.c
index a36f220ac2..c46566f012 100644
--- a/usr/src/uts/common/io/mac/mac.c
+++ b/usr/src/uts/common/io/mac/mac.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, Joyent, Inc.  All rights reserved.
+ * Copyright 2015 Joyent, Inc.
  */
 
 /*
@@ -1684,6 +1684,37 @@ mac_hwring_send_priv(mac_client_handle_t mch, mac_ring_handle_t rh, mblk_t *mp)
 	return (mp);
 }
 
+/*
+ * Private function that is only used by aggr to update the default transmission
+ * ring. Because aggr exposes a pseudo Tx ring even for ports that may
+ * temporarily be down, it may need to update the default ring that is used by
+ * MAC such that it refers to a link that can actively be used to send traffic.
+ * Note that this is different from the case where the port has been removed
+ * from the group. In those cases, all of the rings will be torn down because
+ * the ring will no longer exist. It's important to give aggr a case where the
+ * rings can still exist such that it may be able to continue to send LACP PDUs
+ * to potentially restore the link.
+ *
+ * Finally, we explicitly don't do anything if the ring hasn't been enabled yet.
+ * This is to help out aggr which doesn't really know the internal state that
+ * MAC does about the rings and can't know that it's not quite ready for use
+ * yet.
+ */
+void
+mac_hwring_set_default(mac_handle_t mh, mac_ring_handle_t rh)
+{
+	mac_impl_t *mip = (mac_impl_t *)mh;
+	mac_ring_t *ring = (mac_ring_t *)rh;
+
+	ASSERT(MAC_PERIM_HELD(mh));
+	VERIFY(mip->mi_state_flags & MIS_IS_AGGR);
+
+	if (ring->mr_state != MR_INUSE)
+		return;
+
+	mip->mi_default_tx_ring = rh;
+}
+
 int
 mac_hwgroup_addmac(mac_group_handle_t gh, const uint8_t *addr)
 {
diff --git a/usr/src/uts/common/sys/aggr_impl.h b/usr/src/uts/common/sys/aggr_impl.h
index a02240ecea..a4c0409304 100644
--- a/usr/src/uts/common/sys/aggr_impl.h
+++ b/usr/src/uts/common/sys/aggr_impl.h
@@ -291,6 +291,7 @@ extern int aggr_grp_modify(datalink_id_t, uint8_t, uint32_t, boolean_t,
     const uchar_t *, aggr_lacp_mode_t, aggr_lacp_timer_t);
 extern void aggr_grp_multicst_port(aggr_port_t *, boolean_t);
 extern uint_t aggr_grp_count(void);
+extern void aggr_grp_update_default(aggr_grp_t *);
 
 extern void aggr_port_init(void);
 extern void aggr_port_fini(void);
diff --git a/usr/src/uts/common/sys/mac_client_priv.h b/usr/src/uts/common/sys/mac_client_priv.h
index fa3a1ebfad..a5848625c2 100644
--- a/usr/src/uts/common/sys/mac_client_priv.h
+++ b/usr/src/uts/common/sys/mac_client_priv.h
@@ -136,6 +136,7 @@ extern mblk_t *mac_hwring_tx(mac_ring_handle_t, mblk_t *);
 extern int mac_hwring_getstat(mac_ring_handle_t, uint_t, uint64_t *);
 extern mblk_t *mac_hwring_send_priv(mac_client_handle_t,
     mac_ring_handle_t, mblk_t *);
+extern void mac_hwring_set_default(mac_handle_t, mac_ring_handle_t);
 
 #define	MAC_HWRING_POLL(ring, bytes)			\
 	(((ring)->mr_info.mri_poll)			\