5013 add support for multiple mac addresses per client

Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: Dan McDonald <danmcd@omniti.com>
Approved by: Garrett D'Amore <garrett@damore.org>

6 files changed, 328 insertions, 28 deletions

diff --git a/usr/src/uts/common/io/mac/mac_client.c b/usr/src/uts/common/io/mac/mac_client.c
index dc1132941b..ae2a39ff49 100644
--- a/usr/src/uts/common/io/mac/mac_client.c
+++ b/usr/src/uts/common/io/mac/mac_client.c
@@ -21,6 +21,7 @@
 
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
  */
 
 /*
@@ -84,13 +85,23 @@
  * client opens a VNIC (upper MAC), the MAC layer detects that
  * the MAC being opened is a VNIC, and gets the MAC client handle
  * that the VNIC driver obtained from the lower MAC. This exchange
- * is doing through a private capability between the MAC layer
+ * is done through a private capability between the MAC layer
  * and the VNIC driver. The upper MAC then returns that handle
  * directly to its MAC client. Any operation done by the upper
  * MAC client is now done on the lower MAC client handle, which
  * allows the VNIC driver to be completely bypassed for the
  * performance sensitive data-path.
  *
+ * - Secondary MACs for VNICs:
+ *
+ * VNICs support multiple upper mac clients to enable support for
+ * multiple MAC addresses on the VNIC. When the VNIC is created the
+ * initial mac client is the primary upper mac. Any additional mac
+ * clients are secondary macs. These are kept in sync with the primary
+ * (for things such as the rx function and resource control settings)
+ * using the same private capability interface between the MAC layer
+ * and the VNIC layer.
+ *
  */
 
 #include <sys/types.h>
@@ -148,6 +159,7 @@ static int mac_client_datapath_setup(mac_client_impl_t *, uint16_t,
     uint8_t *, mac_resource_props_t *, boolean_t, mac_unicast_impl_t *);
 static void mac_client_datapath_teardown(mac_client_handle_t,
     mac_unicast_impl_t *, flow_entry_t *);
+static int mac_resource_ctl_set(mac_client_handle_t, mac_resource_props_t *);
 
 /* ARGSUSED */
 static int
@@ -266,6 +278,18 @@ mac_vnic_lower(mac_impl_t *mip)
 }
 
 /*
+ * Update the secondary macs
+ */
+void
+mac_vnic_secondary_update(mac_impl_t *mip)
+{
+	mac_capab_vnic_t cap;
+
+	VERIFY(i_mac_capab_get((mac_handle_t)mip, MAC_CAPAB_VNIC, &cap));
+	cap.mcv_mac_secondary_update(cap.mcv_arg);
+}
+
+/*
  * Return the MAC client handle of the primary MAC client for the
  * specified MAC instance, or NULL otherwise.
  */
@@ -1048,6 +1072,18 @@ mac_unicast_primary_get(mac_handle_t mh, uint8_t *addr)
 }
 
 /*
+ * Return the secondary MAC address for the specified handle
+ */
+void
+mac_unicast_secondary_get(mac_client_handle_t mh, uint8_t *addr)
+{
+	mac_client_impl_t *mcip = (mac_client_impl_t *)mh;
+
+	ASSERT(mcip->mci_unicast != NULL);
+	bcopy(mcip->mci_unicast->ma_addr, addr, mcip->mci_unicast->ma_len);
+}
+
+/*
  * Return information about the use of the primary MAC address of the
  * specified MAC instance:
  *
@@ -1290,6 +1326,10 @@ mac_client_open(mac_handle_t mh, mac_client_handle_t *mchp, char *name,
 		mip->mi_clients_list = mcip;
 		i_mac_perim_exit(mip);
 		*mchp = (mac_client_handle_t)mcip;
+
+		DTRACE_PROBE2(mac__client__open__nonallocated, mac_impl_t *,
+		    mcip->mci_mip, mac_client_impl_t *, mcip);
+
 		return (err);
 	}
 
@@ -1394,10 +1434,6 @@ mac_client_open(mac_handle_t mh, mac_client_handle_t *mchp, char *name,
 	if (share_desired)
 		i_mac_share_alloc(mcip);
 
-	DTRACE_PROBE2(mac__client__open__allocated, mac_impl_t *,
-	    mcip->mci_mip, mac_client_impl_t *, mcip);
-	*mchp = (mac_client_handle_t)mcip;
-
 	/*
 	 * We will do mimimal datapath setup to allow a MAC client to
 	 * transmit or receive non-unicast packets without waiting
@@ -1409,6 +1445,11 @@ mac_client_open(mac_handle_t mh, mac_client_handle_t *mchp, char *name,
 			goto done;
 		}
 	}
+
+	DTRACE_PROBE2(mac__client__open__allocated, mac_impl_t *,
+	    mcip->mci_mip, mac_client_impl_t *, mcip);
+
+	*mchp = (mac_client_handle_t)mcip;
 	i_mac_perim_exit(mip);
 	return (0);
 
@@ -1532,6 +1573,7 @@ mac_rx_set(mac_client_handle_t mch, mac_rx_t rx_fn, void *arg)
 {
 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
 	mac_impl_t	*mip = mcip->mci_mip;
+	mac_impl_t	*umip = mcip->mci_upper_mip;
 
 	/*
 	 * Instead of adding an extra set of locks and refcnts in
@@ -1547,6 +1589,15 @@ mac_rx_set(mac_client_handle_t mch, mac_rx_t rx_fn, void *arg)
 	mcip->mci_rx_arg = arg;
 	mac_rx_client_restart(mch);
 	i_mac_perim_exit(mip);
+
+	/*
+	 * If we're changing the rx function on the primary mac of a vnic,
+	 * make sure any secondary macs on the vnic are updated as well.
+	 */
+	if (umip != NULL) {
+		ASSERT((umip->mi_state_flags & MIS_IS_VNIC) != 0);
+		mac_vnic_secondary_update(umip);
+	}
 }
 
 /*
@@ -1558,6 +1609,42 @@ mac_rx_clear(mac_client_handle_t mch)
 	mac_rx_set(mch, mac_pkt_drop, NULL);
 }
 
+void
+mac_secondary_dup(mac_client_handle_t smch, mac_client_handle_t dmch)
+{
+	mac_client_impl_t *smcip = (mac_client_impl_t *)smch;
+	mac_client_impl_t *dmcip = (mac_client_impl_t *)dmch;
+	flow_entry_t *flent = dmcip->mci_flent;
+
+	/* This should only be called to setup secondary macs */
+	ASSERT((flent->fe_type & FLOW_PRIMARY_MAC) == 0);
+
+	mac_rx_set(dmch, smcip->mci_rx_fn, smcip->mci_rx_arg);
+	dmcip->mci_promisc_list = smcip->mci_promisc_list;
+
+	/*
+	 * Duplicate the primary mac resources to the secondary.
+	 * Since we already validated the resource controls when setting
+	 * them on the primary, we can ignore errors here.
+	 */
+	(void) mac_resource_ctl_set(dmch, MCIP_RESOURCE_PROPS(smcip));
+}
+
+/*
+ * Called when removing a secondary MAC. Currently only clears the promisc_list
+ * since we share the primary mac's promisc_list.
+ */
+void
+mac_secondary_cleanup(mac_client_handle_t mch)
+{
+	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
+	flow_entry_t *flent = mcip->mci_flent;
+
+	/* This should only be called for secondary macs */
+	ASSERT((flent->fe_type & FLOW_PRIMARY_MAC) == 0);
+	mcip->mci_promisc_list = NULL;
+}
+
 /*
  * Walk the MAC client subflow table and updates their priority values.
  */
@@ -1910,11 +1997,12 @@ mac_client_set_rings_prop(mac_client_impl_t *mcip, mac_resource_props_t *mrp,
  * mac_client_impl_t from the mac_impl_t (i.e if there are any cached
  * properties before the flow entry for the unicast address was created).
  */
-int
+static int
 mac_resource_ctl_set(mac_client_handle_t mch, mac_resource_props_t *mrp)
 {
 	mac_client_impl_t 	*mcip = (mac_client_impl_t *)mch;
 	mac_impl_t		*mip = (mac_impl_t *)mcip->mci_mip;
+	mac_impl_t		*umip = mcip->mci_upper_mip;
 	int			err = 0;
 	flow_entry_t		*flent = mcip->mci_flent;
 	mac_resource_props_t	*omrp, *nmrp = MCIP_RESOURCE_PROPS(mcip);
@@ -1998,20 +2086,17 @@ mac_resource_ctl_set(mac_client_handle_t mch, mac_resource_props_t *mrp)
 		mac_flow_modify(mip->mi_flow_tab, flent, mrp);
 		if (mrp->mrp_mask & MRP_PRIORITY)
 			mac_update_subflow_priority(mcip);
+
+		/* Apply these resource settings to any secondary macs */
+		if (umip != NULL) {
+			ASSERT((umip->mi_state_flags & MIS_IS_VNIC) != 0);
+			mac_vnic_secondary_update(umip);
+		}
 	}
 	kmem_free(omrp, sizeof (*omrp));
 	return (0);
 }
 
-void
-mac_resource_ctl_get(mac_client_handle_t mch, mac_resource_props_t *mrp)
-{
-	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
-	mac_resource_props_t	*mcip_mrp = MCIP_RESOURCE_PROPS(mcip);
-
-	bcopy(mcip_mrp, mrp, sizeof (mac_resource_props_t));
-}
-
 static int
 mac_unicast_flow_create(mac_client_impl_t *mcip, uint8_t *mac_addr,
     uint16_t vid, boolean_t is_primary, boolean_t first_flow,
@@ -3219,7 +3304,16 @@ mac_promisc_add(mac_client_handle_t mch, mac_client_promisc_type_t type,
 	mutex_exit(mcbi->mcbi_lockp);
 
 	*mphp = (mac_promisc_handle_t)mpip;
+
+	if (mcip->mci_state_flags & MCIS_IS_VNIC) {
+		mac_impl_t *umip = mcip->mci_upper_mip;
+
+		ASSERT(umip != NULL);
+		mac_vnic_secondary_update(umip);
+	}
+
 	i_mac_perim_exit(mip);
+
 	return (0);
 }
 
@@ -3258,6 +3352,14 @@ mac_promisc_remove(mac_promisc_handle_t mph)
 	} else {
 		mac_callback_remove_wait(&mip->mi_promisc_cb_info);
 	}
+
+	if (mcip->mci_state_flags & MCIS_IS_VNIC) {
+		mac_impl_t *umip = mcip->mci_upper_mip;
+
+		ASSERT(umip != NULL);
+		mac_vnic_secondary_update(umip);
+	}
+
 	mutex_exit(mcbi->mcbi_lockp);
 	mac_stop((mac_handle_t)mip);
 
diff --git a/usr/src/uts/common/io/vnic/vnic_dev.c b/usr/src/uts/common/io/vnic/vnic_dev.c
index 5c46901f36..2af812547e 100644
--- a/usr/src/uts/common/io/vnic/vnic_dev.c
+++ b/usr/src/uts/common/io/vnic/vnic_dev.c
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, Joyent, Inc.  All rights reserved.
  */
 
 #include <sys/types.h>
@@ -70,6 +71,11 @@
  * Due to this passthrough, some of the entry points exported by the
  * VNIC driver are never directly invoked. These entry points include
  * vnic_m_start, vnic_m_stop, vnic_m_promisc, vnic_m_multicst, etc.
+ *
+ * VNICs support multiple upper mac clients to enable support for
+ * multiple MAC addresses on the VNIC. When the VNIC is created the
+ * initial mac client is the primary upper mac. Any additional mac
+ * clients are secondary macs.
  */
 
 static int vnic_m_start(void *);
@@ -81,11 +87,13 @@ static int vnic_m_stat(void *, uint_t, uint64_t *);
 static void vnic_m_ioctl(void *, queue_t *, mblk_t *);
 static int vnic_m_setprop(void *, const char *, mac_prop_id_t, uint_t,
     const void *);
+static int vnic_m_getprop(void *, const char *, mac_prop_id_t, uint_t, void *);
 static void vnic_m_propinfo(void *, const char *, mac_prop_id_t,
     mac_prop_info_handle_t);
 static mblk_t *vnic_m_tx(void *, mblk_t *);
 static boolean_t vnic_m_capab_get(void *, mac_capab_t, void *);
 static void vnic_notify_cb(void *, mac_notify_type_t);
+static void vnic_cleanup_secondary_macs(vnic_t *, int);
 
 static kmem_cache_t	*vnic_cache;
 static krwlock_t	vnic_lock;
@@ -100,7 +108,7 @@ static mod_hash_t	*vnic_hash;
 #define	VNIC_HASH_KEY(vnic_id)	((mod_hash_key_t)(uintptr_t)vnic_id)
 
 #define	VNIC_M_CALLBACK_FLAGS	\
-	(MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_PROPINFO)
+	(MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO)
 
 static mac_callbacks_t vnic_m_callbacks = {
 	VNIC_M_CALLBACK_FLAGS,
@@ -117,7 +125,7 @@ static mac_callbacks_t vnic_m_callbacks = {
 	NULL,
 	NULL,
 	vnic_m_setprop,
-	NULL,
+	vnic_m_getprop,
 	vnic_m_propinfo
 };
 
@@ -621,6 +629,8 @@ vnic_dev_delete(datalink_id_t vnic_id, uint32_t flags, cred_t *credp)
 		return (rc);
 	}
 
+	vnic_cleanup_secondary_macs(vnic, vnic->vn_nhandles);
+
 	vnic->vn_enabled = B_FALSE;
 	(void) mod_hash_remove(vnic_hash, VNIC_HASH_KEY(vnic_id), &val);
 	ASSERT(vnic == (vnic_t *)val);
@@ -741,6 +751,20 @@ vnic_mac_client_handle(void *vnic_arg)
 	return (vnic->vn_mch);
 }
 
+/*
+ * Invoked when updating the primary MAC so that the secondary MACs are
+ * kept in sync.
+ */
+static void
+vnic_mac_secondary_update(void *vnic_arg)
+{
+	vnic_t *vn = vnic_arg;
+	int i;
+
+	for (i = 1; i <= vn->vn_nhandles; i++) {
+		mac_secondary_dup(vn->vn_mc_handles[0], vn->vn_mc_handles[i]);
+	}
+}
 
 /*
  * Return information about the specified capability.
@@ -775,6 +799,8 @@ vnic_m_capab_get(void *arg, mac_capab_t cap, void *cap_data)
 			vnic_capab->mcv_arg = vnic;
 			vnic_capab->mcv_mac_client_handle =
 			    vnic_mac_client_handle;
+			vnic_capab->mcv_mac_secondary_update =
+			    vnic_mac_secondary_update;
 		}
 		break;
 	}
@@ -841,6 +867,126 @@ vnic_m_unicst(void *arg, const uint8_t *macaddr)
 	return (mac_vnic_unicast_set(vnic->vn_mch, macaddr));
 }
 
+static void
+vnic_cleanup_secondary_macs(vnic_t *vn, int cnt)
+{
+	int i;
+
+	/* Remove existing secondaries (primary is at 0) */
+	for (i = 1; i <= cnt; i++) {
+		mac_rx_clear(vn->vn_mc_handles[i]);
+
+		/* unicast handle might not have been set yet */
+		if (vn->vn_mu_handles[i] != NULL)
+			(void) mac_unicast_remove(vn->vn_mc_handles[i],
+			    vn->vn_mu_handles[i]);
+
+		mac_secondary_cleanup(vn->vn_mc_handles[i]);
+
+		mac_client_close(vn->vn_mc_handles[i], MAC_CLOSE_FLAGS_IS_VNIC);
+
+		vn->vn_mu_handles[i] = NULL;
+		vn->vn_mc_handles[i] = NULL;
+	}
+
+	vn->vn_nhandles = 0;
+}
+
+/*
+ * Setup secondary MAC addresses on the vnic. Due to limitations in the mac
+ * code, each mac address must be associated with a mac_client (and the
+ * flow that goes along with the client) so we need to create those clients
+ * here.
+ */
+static int
+vnic_set_secondary_macs(vnic_t *vn, mac_secondary_addr_t *msa)
+{
+	int i, err;
+	char primary_name[MAXNAMELEN];
+
+	/* First, remove pre-existing secondaries */
+	ASSERT(vn->vn_nhandles < MPT_MAXMACADDR);
+	vnic_cleanup_secondary_macs(vn, vn->vn_nhandles);
+
+	if (msa->ms_addrcnt == (uint32_t)-1)
+		msa->ms_addrcnt = 0;
+
+	vn->vn_nhandles = msa->ms_addrcnt;
+
+	(void) dls_mgmt_get_linkinfo(vn->vn_id, primary_name, NULL, NULL, NULL);
+
+	/*
+	 * Now add the new secondary MACs
+	 * Recall that the primary MAC address is the first element.
+	 * The secondary clients are named after the primary with their
+	 * index to distinguish them.
+	 */
+	for (i = 1; i <= vn->vn_nhandles; i++) {
+		uint8_t *addr;
+		mac_diag_t mac_diag;
+		char secondary_name[MAXNAMELEN];
+
+		(void) snprintf(secondary_name, sizeof (secondary_name),
+		    "%s%02d", primary_name, i);
+
+		err = mac_client_open(vn->vn_lower_mh, &vn->vn_mc_handles[i],
+		    secondary_name, MAC_OPEN_FLAGS_IS_VNIC);
+		if (err != 0) {
+			/* Remove any that we successfully added */
+			vnic_cleanup_secondary_macs(vn, --i);
+			return (err);
+		}
+
+		/*
+		 * Assign a MAC address to the VNIC
+		 *
+		 * Normally this would be done with vnic_unicast_add but since
+		 * we know these are fixed adddresses, and since we need to
+		 * save this in the proper array slot, we bypass that function
+		 * and go direct.
+		 */
+		addr = msa->ms_addrs[i - 1];
+		err = mac_unicast_add(vn->vn_mc_handles[i], addr, 0,
+		    &vn->vn_mu_handles[i], vn->vn_vid, &mac_diag);
+		if (err != 0) {
+			/* Remove any that we successfully added */
+			vnic_cleanup_secondary_macs(vn, i);
+			return (err);
+		}
+
+		/*
+		 * Setup the secondary the same way as the primary (i.e.
+		 * receiver function/argument (e.g. i_dls_link_rx, mac_pkt_drop,
+		 * etc.), the promisc list, and the resource controls).
+		 */
+		mac_secondary_dup(vn->vn_mc_handles[0], vn->vn_mc_handles[i]);
+	}
+
+	return (0);
+}
+
+static int
+vnic_get_secondary_macs(vnic_t *vn, uint_t pr_valsize, void *pr_val)
+{
+	int i;
+	mac_secondary_addr_t msa;
+
+	if (pr_valsize < sizeof (msa))
+		return (EINVAL);
+
+	/* Get existing addresses (primary is at 0) */
+	ASSERT(vn->vn_nhandles < MPT_MAXMACADDR);
+	for (i = 1; i <= vn->vn_nhandles; i++) {
+		ASSERT(vn->vn_mc_handles[i] != NULL);
+		mac_unicast_secondary_get(vn->vn_mc_handles[i],
+		    msa.ms_addrs[i - 1]);
+	}
+	msa.ms_addrcnt = vn->vn_nhandles;
+
+	bcopy(&msa, pr_val, sizeof (msa));
+	return (0);
+}
+
 /*
  * Callback functions for set/get of properties
  */
@@ -849,17 +995,17 @@ static int
 vnic_m_setprop(void *m_driver, const char *pr_name, mac_prop_id_t pr_num,
     uint_t pr_valsize, const void *pr_val)
 {
-	int 		err = ENOTSUP;
+	int 		err = 0;
 	vnic_t		*vn = m_driver;
 
-	/* allow setting MTU only on an etherstub */
-	if (vn->vn_link_id != DATALINK_INVALID_LINKID)
-		return (err);
-
 	switch (pr_num) {
 	case MAC_PROP_MTU: {
 		uint32_t	mtu;
 
+		/* allow setting MTU only on an etherstub */
+		if (vn->vn_link_id != DATALINK_INVALID_LINKID)
+			return (err);
+
 		if (pr_valsize < sizeof (mtu)) {
 			err = EINVAL;
 			break;
@@ -872,13 +1018,41 @@ vnic_m_setprop(void *m_driver, const char *pr_name, mac_prop_id_t pr_num,
 		err = mac_maxsdu_update(vn->vn_mh, mtu);
 		break;
 	}
+	case MAC_PROP_SECONDARY_ADDRS: {
+		mac_secondary_addr_t msa;
+
+		bcopy(pr_val, &msa, sizeof (msa));
+		err = vnic_set_secondary_macs(vn, &msa);
+		break;
+	}
 	default:
+		err = ENOTSUP;
 		break;
 	}
 	return (err);
 }
 
 /* ARGSUSED */
+static int
+vnic_m_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num,
+    uint_t pr_valsize, void *pr_val)
+{
+	vnic_t		*vn = arg;
+	int 		ret = 0;
+
+	switch (pr_num) {
+	case MAC_PROP_SECONDARY_ADDRS:
+		ret = vnic_get_secondary_macs(vn, pr_valsize, pr_val);
+		break;
+	default:
+		ret = EINVAL;
+		break;
+	}
+
+	return (ret);
+}
+
+/* ARGSUSED */
 static void vnic_m_propinfo(void *m_driver, const char *pr_name,
     mac_prop_id_t pr_num, mac_prop_info_handle_t prh)
 {
@@ -929,7 +1103,8 @@ vnic_info(vnic_info_t *info, cred_t *credp)
 
 	bzero(&info->vn_resource_props, sizeof (mac_resource_props_t));
 	if (vnic->vn_mch != NULL)
-		mac_resource_ctl_get(vnic->vn_mch, &info->vn_resource_props);
+		mac_client_get_resources(vnic->vn_mch,
+		    &info->vn_resource_props);
 
 	rw_exit(&vnic_lock);
 	return (0);
diff --git a/usr/src/uts/common/sys/mac.h b/usr/src/uts/common/sys/mac.h
index 6876fccb1a..f1bfa1aec2 100644
--- a/usr/src/uts/common/sys/mac.h
+++ b/usr/src/uts/common/sys/mac.h
@@ -21,6 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
  */
 
 #ifndef	_SYS_MAC_H
@@ -128,6 +129,13 @@ typedef struct mac_propval_range_s {
  */
 #define	MAXMACADDRLEN		20
 
+#define	MPT_MAXMACADDR		32
+
+typedef struct mac_secondary_addr_s {
+	uint32_t	ms_addrcnt;
+	uint8_t		ms_addrs[MPT_MAXMACADDR][MAXMACADDRLEN];
+} mac_secondary_addr_t;
+
 typedef enum {
 	MAC_LOGTYPE_LINK = 1,
 	MAC_LOGTYPE_FLOW
@@ -205,6 +213,7 @@ typedef enum {
 	MAC_PROP_MAX_RXHWCLNT_AVAIL,
 	MAC_PROP_MAX_TXHWCLNT_AVAIL,
 	MAC_PROP_IB_LINKMODE,
+	MAC_PROP_SECONDARY_ADDRS,
 	MAC_PROP_PRIVATE = -1
 } mac_prop_id_t;
 
@@ -312,13 +321,18 @@ typedef struct mac_info_s {
  *
  * This capability allows the MAC layer to detect when a VNIC is being
  * access, and implement the required shortcuts.
+ *
+ * In addition, this capability is used to keep the VNIC's secondary
+ * mac_clients in sync when the primary MAC is updated.
  */
 
 typedef void *(*mac_client_handle_fn_t)(void *);
+typedef void (*mac_client_update_fn_t)(void *);
 
 typedef struct mac_capab_vnic_s {
 	void			*mcv_arg;
 	mac_client_handle_fn_t	mcv_mac_client_handle;
+	mac_client_update_fn_t	mcv_mac_secondary_update;
 } mac_capab_vnic_t;
 
 typedef void (*mac_rename_fn_t)(const char *, void *);
diff --git a/usr/src/uts/common/sys/mac_client.h b/usr/src/uts/common/sys/mac_client.h
index 40cd15a1b8..0fc4939503 100644
--- a/usr/src/uts/common/sys/mac_client.h
+++ b/usr/src/uts/common/sys/mac_client.h
@@ -22,6 +22,7 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2013 Joyent, Inc.  All rights reserved.
  */
 
 /*
@@ -135,6 +136,8 @@ extern void mac_multicast_remove(mac_client_handle_t, const uint8_t *);
 
 extern void mac_rx_set(mac_client_handle_t, mac_rx_t, void *);
 extern void mac_rx_clear(mac_client_handle_t);
+extern void mac_secondary_dup(mac_client_handle_t, mac_client_handle_t);
+extern void mac_secondary_cleanup(mac_client_handle_t);
 extern mac_tx_cookie_t mac_tx(mac_client_handle_t, mblk_t *,
     uintptr_t, uint16_t, mblk_t **);
 extern boolean_t mac_tx_is_flow_blocked(mac_client_handle_t, mac_tx_cookie_t);
@@ -158,6 +161,8 @@ extern int mac_unicast_primary_set(mac_handle_t, const uint8_t *);
 extern void mac_unicast_primary_get(mac_handle_t, uint8_t *);
 extern void mac_unicast_primary_info(mac_handle_t, char *, boolean_t *);
 
+extern void mac_unicast_secondary_get(mac_client_handle_t, uint8_t *);
+
 extern boolean_t mac_dst_get(mac_handle_t, uint8_t *);
 
 extern int mac_addr_random(mac_client_handle_t, uint_t, uint8_t *,
diff --git a/usr/src/uts/common/sys/mac_client_priv.h b/usr/src/uts/common/sys/mac_client_priv.h
index 0ddc1f074d..bed2e293b3 100644
--- a/usr/src/uts/common/sys/mac_client_priv.h
+++ b/usr/src/uts/common/sys/mac_client_priv.h
@@ -22,6 +22,7 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2013 Joyent, Inc.  All rights reserved.
  */
 
 /*
@@ -91,9 +92,6 @@ extern boolean_t mac_client_is_vlan_vnic(mac_client_handle_t);
 extern void mac_client_poll_enable(mac_client_handle_t);
 extern void mac_client_poll_disable(mac_client_handle_t);
 
-extern int mac_resource_ctl_set(mac_client_handle_t, mac_resource_props_t *);
-extern void mac_resource_ctl_get(mac_client_handle_t, mac_resource_props_t *);
-
 /*
  * Flow-related APIs for MAC clients.
  */
diff --git a/usr/src/uts/common/sys/vnic_impl.h b/usr/src/uts/common/sys/vnic_impl.h
index 2bb48a60c6..ffaa2939f5 100644
--- a/usr/src/uts/common/sys/vnic_impl.h
+++ b/usr/src/uts/common/sys/vnic_impl.h
@@ -21,6 +21,7 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2013 Joyent, Inc.  All rights reserved.
  */
 
 #ifndef	_SYS_VNIC_IMPL_H
@@ -46,8 +47,10 @@ typedef struct vnic_s {
 
 	mac_handle_t		vn_mh;
 	mac_handle_t		vn_lower_mh;
-	mac_client_handle_t	vn_mch;
-	mac_unicast_handle_t	vn_muh;
+	uint_t			vn_nhandles; /* # of secondary mac handles */
+	/* The primary handle is always the first element in the array */
+	mac_client_handle_t	vn_mc_handles[MPT_MAXMACADDR];
+	mac_unicast_handle_t	vn_mu_handles[MPT_MAXMACADDR];
 	uint32_t		vn_margin;
 	int			vn_slot_id;
 	vnic_mac_addr_type_t	vn_addr_type;
@@ -63,6 +66,9 @@ typedef struct vnic_s {
 	uint32_t		vn_hcksum_txflags;
 } vnic_t;
 
+#define	vn_mch	vn_mc_handles[0]
+#define	vn_muh	vn_mu_handles[0]
+
 extern int vnic_dev_create(datalink_id_t, datalink_id_t, vnic_mac_addr_type_t *,
     int *, uchar_t *, int *, uint_t, uint16_t, vrid_t, int,
     mac_resource_props_t *, uint32_t, vnic_ioc_diag_t *, cred_t *);