OS-2495 add support for multiple mac addresses per client

7 files changed, 426 insertions, 30 deletions

diff --git a/usr/src/lib/libdladm/common/linkprop.c b/usr/src/lib/libdladm/common/linkprop.c
index 8c0a5daf9f..b89860e1c1 100644
--- a/usr/src/lib/libdladm/common/linkprop.c
+++ b/usr/src/lib/libdladm/common/linkprop.c
@@ -66,6 +66,7 @@
 #include <net/if_types.h>
 #include <libinetutil.h>
 #include <pool.h>
+#include <libdlaggr.h>
 
 /*
  * The linkprop get() callback.
@@ -150,20 +151,20 @@ static pd_getf_t	get_zone, get_autopush, get_rate_mod, get_rate,
 			get_flowctl, get_maxbw, get_cpus, get_priority,
 			get_tagmode, get_range, get_stp, get_bridge_forward,
 			get_bridge_pvid, get_protection, get_rxrings,
-			get_txrings, get_cntavail,
+			get_txrings, get_cntavail, get_secondary_macs,
 			get_allowedips, get_allowedcids, get_pool,
 			get_rings_range, get_linkmode_prop,
 			get_promisc_filtered;
 
 static pd_setf_t	set_zone, set_rate, set_powermode, set_radio,
 			set_public_prop, set_resource, set_stp_prop,
-			set_bridge_forward, set_bridge_pvid,
+			set_bridge_forward, set_bridge_pvid, set_secondary_macs,
 			set_promisc_filtered;
 
 static pd_checkf_t	check_zone, check_autopush, check_rate, check_hoplimit,
 			check_encaplim, check_uint32, check_maxbw, check_cpus,
 			check_stp_prop, check_bridge_pvid, check_allowedips,
-			check_allowedcids, check_rings,
+			check_allowedcids, check_secondary_macs, check_rings,
 			check_pool, check_prop;
 
 struct prop_desc {
@@ -368,6 +369,9 @@ static link_attr_t link_attr[] = {
 
 	{ MAC_PROP_VN_PROMISC_FILTERED,	sizeof (boolean_t), "promisc-filtered"},
 
+	{ MAC_PROP_SECONDARY_ADDRS, sizeof (mac_secondary_addr_t),
+	    "secondary-macs"},
+
 	{ MAC_PROP_PRIVATE,	0,			"driver-private"}
 };
 
@@ -453,6 +457,12 @@ static  val_desc_t	dladm_part_linkmode_vals[] = {
 #define	RESET_VAL	((uintptr_t)-1)
 #define	UNSPEC_VAL	((uintptr_t)-2)
 
+/*
+ * For the default, if defaults are not defined for the property,
+ * pd_defval.vd_name should be null. If the driver has to be contacted for the
+ * value, vd_name should be the empty string (""). Otherwise, dladm will
+ * just print whatever is in the table.
+ */
 static prop_desc_t	prop_table[] = {
 	{ "channel",	{ NULL, 0 },
 	    NULL, 0, NULL, NULL,
@@ -516,6 +526,11 @@ static prop_desc_t	prop_table[] = {
 	    set_public_prop, NULL, get_flowctl, NULL,
 	    0, DATALINK_CLASS_PHYS, DL_ETHER },
 
+	{ "secondary-macs", { "--", 0 }, NULL, 0,
+	    set_secondary_macs, NULL,
+	    get_secondary_macs, check_secondary_macs, PD_CHECK_ALLOC,
+	    DATALINK_CLASS_VNIC, DL_ETHER },
+
 	{ "adv_10gfdx_cap", { "", 0 },
 	    link_01_vals, VALCNT(link_01_vals),
 	    NULL, NULL, get_binary, NULL,
@@ -2394,7 +2409,7 @@ get_allowedips(dladm_handle_t handle, prop_desc_t *pdp,
 			(void) dladm_ipv4addr2str(&v4addr, prop_val[i]);
 			if (mask != 0) {
 				int len = strlen(prop_val[i]);
-				(void)sprintf(prop_val[i] + len, "/%d",
+				(void) sprintf(prop_val[i] + len, "/%d",
 				    mask_to_nbits(mask));
 			}
 		} else {
@@ -2884,6 +2899,106 @@ fail:
 
 /* ARGSUSED */
 static dladm_status_t
+get_secondary_macs(dladm_handle_t handle, prop_desc_t *pdp,
+    datalink_id_t linkid, char **prop_val, uint_t *val_cnt,
+    datalink_media_t media, uint_t flags, uint_t *perm_flags)
+{
+	mac_secondary_addr_t	sa;
+	dladm_status_t		status;
+	int			i;
+
+	status = i_dladm_get_public_prop(handle, linkid, pdp->pd_name, flags,
+	    perm_flags, &sa, sizeof (sa));
+	if (status != DLADM_STATUS_OK)
+		return (status);
+
+	if (sa.ms_addrcnt == 0) {
+		*val_cnt = 0;
+		return (DLADM_STATUS_OK);
+	}
+	if (sa.ms_addrcnt > *val_cnt)
+		return (DLADM_STATUS_BADVALCNT);
+
+	for (i = 0; i < sa.ms_addrcnt; i++) {
+		(void) dladm_aggr_macaddr2str(
+		    (const unsigned char *)&sa.ms_addrs[i], prop_val[i]);
+	}
+	*val_cnt = sa.ms_addrcnt;
+	return (DLADM_STATUS_OK);
+}
+
+/* ARGSUSED */
+static dladm_status_t
+check_secondary_macs(dladm_handle_t handle, prop_desc_t *pdp,
+    datalink_id_t linkid, char **prop_val, uint_t *val_cntp, uint_t flags,
+    val_desc_t **vdpp, datalink_media_t media)
+{
+	dladm_status_t	status;
+	uchar_t		*addr;
+	uint_t		len = 0;
+	int		i;
+	uint_t		val_cnt = *val_cntp;
+	val_desc_t	*vdp = *vdpp;
+
+	if (val_cnt >= MPT_MAXMACADDR)
+		return (DLADM_STATUS_BADVALCNT);
+
+	for (i = 0; i < val_cnt; i++) {
+		addr = _link_aton(prop_val[i], (int *)&len);
+		if (addr == NULL) {
+			if (len == (uint_t)-1)
+				status = DLADM_STATUS_MACADDRINVAL;
+			else
+				status = DLADM_STATUS_NOMEM;
+			goto fail;
+		}
+
+		vdp[i].vd_val = (uintptr_t)addr;
+	}
+	return (DLADM_STATUS_OK);
+
+fail:
+	for (i = 0; i < val_cnt; i++) {
+		free((void *)vdp[i].vd_val);
+		vdp[i].vd_val = NULL;
+	}
+	return (status);
+}
+
+/* ARGSUSED */
+static dladm_status_t
+set_secondary_macs(dladm_handle_t handle, prop_desc_t *pd, datalink_id_t linkid,
+    val_desc_t *vdp, uint_t val_cnt, uint_t flags, datalink_media_t media)
+{
+	dladm_status_t status;
+	dld_ioc_macprop_t *dip;
+	int i;
+	mac_secondary_addr_t msa;
+
+	dip = i_dladm_buf_alloc_by_name(0, linkid, "secondary-macs", 0,
+	    &status);
+	if (dip == NULL)
+		return (status);
+
+	if (vdp->vd_val == 0) {
+		val_cnt = (uint_t)-1;
+	} else {
+		for (i = 0; i < val_cnt; i++) {
+			bcopy((void *)vdp[i].vd_val, msa.ms_addrs[i],
+			    MAXMACADDRLEN);
+		}
+	}
+	msa.ms_addrcnt = val_cnt;
+	(void) memcpy(dip->pr_val, &msa, dip->pr_valsize);
+
+	status = i_dladm_macprop(handle, dip, B_TRUE);
+
+	free(dip);
+	return (status);
+}
+
+/* ARGSUSED */
+static dladm_status_t
 get_autopush(dladm_handle_t handle, prop_desc_t *pdp, datalink_id_t linkid,
     char **prop_val, uint_t *val_cnt, datalink_media_t media,
     uint_t flags, uint_t *perm_flags)
@@ -4618,11 +4733,13 @@ dladm_linkprop_is_set(dladm_handle_t handle, datalink_id_t linkid,
 
 	/*
 	 * valcnt is always set to 1 by get_pool(), hence we need to check
-	 * for a non-null string to see if it is set. For protection and
-	 * allowed-ips, we can check either the *propval or the valcnt.
+	 * for a non-null string to see if it is set. For protection,
+	 * secondary-macs and allowed-ips, we can check either the *propval
+	 * or the valcnt.
 	 */
 	if ((strcmp(prop_name, "pool") == 0 ||
 	    strcmp(prop_name, "protection") == 0 ||
+	    strcmp(prop_name, "secondary-macs") == 0 ||
 	    strcmp(prop_name, "allowed-ips") == 0) &&
 	    (strlen(*propvals) != 0)) {
 		*is_set = B_TRUE;
diff --git a/usr/src/uts/common/io/mac/mac_client.c b/usr/src/uts/common/io/mac/mac_client.c
index dc1e40b424..766c5f8f41 100644
--- a/usr/src/uts/common/io/mac/mac_client.c
+++ b/usr/src/uts/common/io/mac/mac_client.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
+ * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
  */
 
 /*
@@ -85,13 +85,23 @@
  * client opens a VNIC (upper MAC), the MAC layer detects that
  * the MAC being opened is a VNIC, and gets the MAC client handle
  * that the VNIC driver obtained from the lower MAC. This exchange
- * is doing through a private capability between the MAC layer
+ * is done through a private capability between the MAC layer
  * and the VNIC driver. The upper MAC then returns that handle
  * directly to its MAC client. Any operation done by the upper
  * MAC client is now done on the lower MAC client handle, which
  * allows the VNIC driver to be completely bypassed for the
  * performance sensitive data-path.
  *
+ * - Secondary MACs for VNICs:
+ *
+ * VNICs support multiple upper mac clients to enable support for
+ * multiple MAC addresses on the VNIC. When the VNIC is created the
+ * initial mac client is the primary upper mac. Any additional mac
+ * clients are secondary macs. These are kept in sync with the primary
+ * (for things such as the rx function and resource control settings)
+ * using the same private capability interface between the MAC layer
+ * and the VNIC layer.
+ *
  */
 
 #include <sys/types.h>
@@ -149,6 +159,7 @@ static int mac_client_datapath_setup(mac_client_impl_t *, uint16_t,
     uint8_t *, mac_resource_props_t *, boolean_t, mac_unicast_impl_t *);
 static void mac_client_datapath_teardown(mac_client_handle_t,
     mac_unicast_impl_t *, flow_entry_t *);
+static int mac_resource_ctl_set(mac_client_handle_t, mac_resource_props_t *);
 
 /* ARGSUSED */
 static int
@@ -267,6 +278,18 @@ mac_vnic_lower(mac_impl_t *mip)
 }
 
 /*
+ * Update the secondary macs
+ */
+void
+mac_vnic_secondary_update(mac_impl_t *mip)
+{
+	mac_capab_vnic_t cap;
+
+	VERIFY(i_mac_capab_get((mac_handle_t)mip, MAC_CAPAB_VNIC, &cap));
+	cap.mcv_mac_secondary_update(cap.mcv_arg);
+}
+
+/*
  * Return the MAC client handle of the primary MAC client for the
  * specified MAC instance, or NULL otherwise.
  */
@@ -1049,6 +1072,18 @@ mac_unicast_primary_get(mac_handle_t mh, uint8_t *addr)
 }
 
 /*
+ * Return the secondary MAC address for the specified handle
+ */
+void
+mac_unicast_secondary_get(mac_client_handle_t mh, uint8_t *addr)
+{
+	mac_client_impl_t *mcip = (mac_client_impl_t *)mh;
+
+	ASSERT(mcip->mci_unicast != NULL);
+	bcopy(mcip->mci_unicast->ma_addr, addr, mcip->mci_unicast->ma_len);
+}
+
+/*
  * Return information about the use of the primary MAC address of the
  * specified MAC instance:
  *
@@ -1291,6 +1326,10 @@ mac_client_open(mac_handle_t mh, mac_client_handle_t *mchp, char *name,
 		mip->mi_clients_list = mcip;
 		i_mac_perim_exit(mip);
 		*mchp = (mac_client_handle_t)mcip;
+
+		DTRACE_PROBE2(mac__client__open__nonallocated, mac_impl_t *,
+		    mcip->mci_mip, mac_client_impl_t *, mcip);
+
 		return (err);
 	}
 
@@ -1395,10 +1434,6 @@ mac_client_open(mac_handle_t mh, mac_client_handle_t *mchp, char *name,
 	if (share_desired)
 		i_mac_share_alloc(mcip);
 
-	DTRACE_PROBE2(mac__client__open__allocated, mac_impl_t *,
-	    mcip->mci_mip, mac_client_impl_t *, mcip);
-	*mchp = (mac_client_handle_t)mcip;
-
 	/*
 	 * We will do mimimal datapath setup to allow a MAC client to
 	 * transmit or receive non-unicast packets without waiting
@@ -1410,6 +1445,11 @@ mac_client_open(mac_handle_t mh, mac_client_handle_t *mchp, char *name,
 			goto done;
 		}
 	}
+
+	DTRACE_PROBE2(mac__client__open__allocated, mac_impl_t *,
+	    mcip->mci_mip, mac_client_impl_t *, mcip);
+
+	*mchp = (mac_client_handle_t)mcip;
 	i_mac_perim_exit(mip);
 	return (0);
 
@@ -1533,6 +1573,7 @@ mac_rx_set(mac_client_handle_t mch, mac_rx_t rx_fn, void *arg)
 {
 	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
 	mac_impl_t	*mip = mcip->mci_mip;
+	mac_impl_t	*umip = mcip->mci_upper_mip;
 
 	/*
 	 * Instead of adding an extra set of locks and refcnts in
@@ -1548,6 +1589,15 @@ mac_rx_set(mac_client_handle_t mch, mac_rx_t rx_fn, void *arg)
 	mcip->mci_rx_arg = arg;
 	mac_rx_client_restart(mch);
 	i_mac_perim_exit(mip);
+
+	/*
+	 * If we're changing the rx function on the primary mac of a vnic,
+	 * make sure any secondary macs on the vnic are updated as well.
+	 */
+	if (umip != NULL) {
+		ASSERT((umip->mi_state_flags & MIS_IS_VNIC) != 0);
+		mac_vnic_secondary_update(umip);
+	}
 }
 
 /*
@@ -1559,6 +1609,42 @@ mac_rx_clear(mac_client_handle_t mch)
 	mac_rx_set(mch, mac_pkt_drop, NULL);
 }
 
+void
+mac_secondary_dup(mac_client_handle_t smch, mac_client_handle_t dmch)
+{
+	mac_client_impl_t *smcip = (mac_client_impl_t *)smch;
+	mac_client_impl_t *dmcip = (mac_client_impl_t *)dmch;
+	flow_entry_t *flent = dmcip->mci_flent;
+
+	/* This should only be called to setup secondary macs */
+	ASSERT((flent->fe_type & FLOW_PRIMARY_MAC) == 0);
+
+	mac_rx_set(dmch, smcip->mci_rx_fn, smcip->mci_rx_arg);
+	dmcip->mci_promisc_list = smcip->mci_promisc_list;
+
+	/*
+	 * Duplicate the primary mac resources to the secondary.
+	 * Since we already validated the resource controls when setting
+	 * them on the primary, we can ignore errors here.
+	 */
+	(void) mac_resource_ctl_set(dmch, MCIP_RESOURCE_PROPS(smcip));
+}
+
+/*
+ * Called when removing a secondary MAC. Currently only clears the promisc_list
+ * since we share the primary mac's promisc_list.
+ */
+void
+mac_secondary_cleanup(mac_client_handle_t mch)
+{
+	mac_client_impl_t *mcip = (mac_client_impl_t *)mch;
+	flow_entry_t *flent = mcip->mci_flent;
+
+	/* This should only be called for secondary macs */
+	ASSERT((flent->fe_type & FLOW_PRIMARY_MAC) == 0);
+	mcip->mci_promisc_list = NULL;
+}
+
 /*
  * Walk the MAC client subflow table and updates their priority values.
  */
@@ -1911,11 +1997,12 @@ mac_client_set_rings_prop(mac_client_impl_t *mcip, mac_resource_props_t *mrp,
  * mac_client_impl_t from the mac_impl_t (i.e if there are any cached
  * properties before the flow entry for the unicast address was created).
  */
-int
+static int
 mac_resource_ctl_set(mac_client_handle_t mch, mac_resource_props_t *mrp)
 {
 	mac_client_impl_t 	*mcip = (mac_client_impl_t *)mch;
 	mac_impl_t		*mip = (mac_impl_t *)mcip->mci_mip;
+	mac_impl_t		*umip = mcip->mci_upper_mip;
 	int			err = 0;
 	flow_entry_t		*flent = mcip->mci_flent;
 	mac_resource_props_t	*omrp, *nmrp = MCIP_RESOURCE_PROPS(mcip);
@@ -1999,20 +2086,17 @@ mac_resource_ctl_set(mac_client_handle_t mch, mac_resource_props_t *mrp)
 		mac_flow_modify(mip->mi_flow_tab, flent, mrp);
 		if (mrp->mrp_mask & MRP_PRIORITY)
 			mac_update_subflow_priority(mcip);
+
+		/* Apply these resource settings to any secondary macs */
+		if (umip != NULL) {
+			ASSERT((umip->mi_state_flags & MIS_IS_VNIC) != 0);
+			mac_vnic_secondary_update(umip);
+		}
 	}
 	kmem_free(omrp, sizeof (*omrp));
 	return (0);
 }
 
-void
-mac_resource_ctl_get(mac_client_handle_t mch, mac_resource_props_t *mrp)
-{
-	mac_client_impl_t	*mcip = (mac_client_impl_t *)mch;
-	mac_resource_props_t	*mcip_mrp = MCIP_RESOURCE_PROPS(mcip);
-
-	bcopy(mcip_mrp, mrp, sizeof (mac_resource_props_t));
-}
-
 static int
 mac_unicast_flow_create(mac_client_impl_t *mcip, uint8_t *mac_addr,
     uint16_t vid, boolean_t is_primary, boolean_t first_flow,
@@ -3221,7 +3305,16 @@ mac_promisc_add(mac_client_handle_t mch, mac_client_promisc_type_t type,
 	mutex_exit(mcbi->mcbi_lockp);
 
 	*mphp = (mac_promisc_handle_t)mpip;
+
+	if (mcip->mci_state_flags & MCIS_IS_VNIC) {
+		mac_impl_t *umip = mcip->mci_upper_mip;
+
+		ASSERT(umip != NULL);
+		mac_vnic_secondary_update(umip);
+	}
+
 	i_mac_perim_exit(mip);
+
 	return (0);
 }
 
@@ -3260,6 +3353,14 @@ mac_promisc_remove(mac_promisc_handle_t mph)
 	} else {
 		mac_callback_remove_wait(&mip->mi_promisc_cb_info);
 	}
+
+	if (mcip->mci_state_flags & MCIS_IS_VNIC) {
+		mac_impl_t *umip = mcip->mci_upper_mip;
+
+		ASSERT(umip != NULL);
+		mac_vnic_secondary_update(umip);
+	}
+
 	mutex_exit(mcbi->mcbi_lockp);
 	mac_stop((mac_handle_t)mip);
 
diff --git a/usr/src/uts/common/io/vnic/vnic_dev.c b/usr/src/uts/common/io/vnic/vnic_dev.c
index 065d7f2cbc..dd2a359206 100644
--- a/usr/src/uts/common/io/vnic/vnic_dev.c
+++ b/usr/src/uts/common/io/vnic/vnic_dev.c
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
+ * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
  */
 
 #include <sys/types.h>
@@ -72,6 +72,11 @@
  * Due to this passthrough, some of the entry points exported by the
  * VNIC driver are never directly invoked. These entry points include
  * vnic_m_start, vnic_m_stop, vnic_m_promisc, vnic_m_multicst, etc.
+ *
+ * VNICs support multiple upper mac clients to enable support for
+ * multiple MAC addresses on the VNIC. When the VNIC is created the
+ * initial mac client is the primary upper mac. Any additional mac
+ * clients are secondary macs.
  */
 
 static int vnic_m_start(void *);
@@ -89,6 +94,7 @@ static void vnic_m_propinfo(void *, const char *, mac_prop_id_t,
 static mblk_t *vnic_m_tx(void *, mblk_t *);
 static boolean_t vnic_m_capab_get(void *, mac_capab_t, void *);
 static void vnic_notify_cb(void *, mac_notify_type_t);
+static void vnic_cleanup_secondary_macs(vnic_t *, int);
 
 static kmem_cache_t	*vnic_cache;
 static krwlock_t	vnic_lock;
@@ -624,6 +630,8 @@ vnic_dev_delete(datalink_id_t vnic_id, uint32_t flags, cred_t *credp)
 		return (rc);
 	}
 
+	vnic_cleanup_secondary_macs(vnic, vnic->vn_nhandles);
+
 	vnic->vn_enabled = B_FALSE;
 	(void) mod_hash_remove(vnic_hash, VNIC_HASH_KEY(vnic_id), &val);
 	ASSERT(vnic == (vnic_t *)val);
@@ -744,6 +752,20 @@ vnic_mac_client_handle(void *vnic_arg)
 	return (vnic->vn_mch);
 }
 
+/*
+ * Invoked when updating the primary MAC so that the secondary MACs are
+ * kept in sync.
+ */
+static void
+vnic_mac_secondary_update(void *vnic_arg)
+{
+	vnic_t *vn = vnic_arg;
+	int i;
+
+	for (i = 1; i <= vn->vn_nhandles; i++) {
+		mac_secondary_dup(vn->vn_mc_handles[0], vn->vn_mc_handles[i]);
+	}
+}
 
 /*
  * Return information about the specified capability.
@@ -778,6 +800,8 @@ vnic_m_capab_get(void *arg, mac_capab_t cap, void *cap_data)
 			vnic_capab->mcv_arg = vnic;
 			vnic_capab->mcv_mac_client_handle =
 			    vnic_mac_client_handle;
+			vnic_capab->mcv_mac_secondary_update =
+			    vnic_mac_secondary_update;
 		}
 		break;
 	}
@@ -844,6 +868,126 @@ vnic_m_unicst(void *arg, const uint8_t *macaddr)
 	return (mac_vnic_unicast_set(vnic->vn_mch, macaddr));
 }
 
+static void
+vnic_cleanup_secondary_macs(vnic_t *vn, int cnt)
+{
+	int i;
+
+	/* Remove existing secondaries (primary is at 0) */
+	for (i = 1; i <= cnt; i++) {
+		mac_rx_clear(vn->vn_mc_handles[i]);
+
+		/* unicast handle might not have been set yet */
+		if (vn->vn_mu_handles[i] != NULL)
+			mac_unicast_remove(vn->vn_mc_handles[i],
+			    vn->vn_mu_handles[i]);
+
+		mac_secondary_cleanup(vn->vn_mc_handles[i]);
+
+		mac_client_close(vn->vn_mc_handles[i], MAC_CLOSE_FLAGS_IS_VNIC);
+
+		vn->vn_mu_handles[i] = NULL;
+		vn->vn_mc_handles[i] = NULL;
+	}
+
+	vn->vn_nhandles = 0;
+}
+
+/*
+ * Setup secondary MAC addresses on the vnic. Due to limitations in the mac
+ * code, each mac address must be associated with a mac_client (and the
+ * flow that goes along with the client) so we need to create those clients
+ * here.
+ */
+static int
+vnic_set_secondary_macs(vnic_t *vn, mac_secondary_addr_t *msa)
+{
+	int i, err;
+	char primary_name[MAXNAMELEN];
+
+	/* First, remove pre-existing secondaries */
+	ASSERT(vn->vn_nhandles < MPT_MAXMACADDR);
+	vnic_cleanup_secondary_macs(vn, vn->vn_nhandles);
+
+	if (msa->ms_addrcnt == (uint32_t)-1)
+		msa->ms_addrcnt = 0;
+
+	vn->vn_nhandles = msa->ms_addrcnt;
+
+	(void) dls_mgmt_get_linkinfo(vn->vn_id, primary_name, NULL, NULL, NULL);
+
+	/*
+	 * Now add the new secondary MACs
+	 * Recall that the primary MAC address is the first element.
+	 * The secondary clients are named after the primary with their
+	 * index to distinguish them.
+	 */
+	for (i = 1; i <= vn->vn_nhandles; i++) {
+		uint8_t *addr;
+		mac_diag_t mac_diag;
+		char secondary_name[MAXNAMELEN];
+
+		(void) snprintf(secondary_name, sizeof (secondary_name),
+		    "%s%02d", primary_name, i);
+
+		err = mac_client_open(vn->vn_lower_mh, &vn->vn_mc_handles[i],
+		    secondary_name, MAC_OPEN_FLAGS_IS_VNIC);
+		if (err != 0) {
+			/* Remove any that we successfully added */
+			vnic_cleanup_secondary_macs(vn, --i);
+			return (err);
+		}
+
+		/*
+		 * Assign a MAC address to the VNIC
+		 *
+		 * Normally this would be done with vnic_unicast_add but since
+		 * we know these are fixed adddresses, and since we need to
+		 * save this in the proper array slot, we bypass that function
+		 * and go direct.
+		 */
+		addr = msa->ms_addrs[i - 1];
+		err = mac_unicast_add(vn->vn_mc_handles[i], addr, 0,
+		    &vn->vn_mu_handles[i], vn->vn_vid, &mac_diag);
+		if (err != 0) {
+			/* Remove any that we successfully added */
+			vnic_cleanup_secondary_macs(vn, i);
+			return (err);
+		}
+
+		/*
+		 * Setup the secondary the same way as the primary (i.e.
+		 * receiver function/argument (e.g. i_dls_link_rx, mac_pkt_drop,
+		 * etc.), the promisc list, and the resource controls).
+		 */
+		mac_secondary_dup(vn->vn_mc_handles[0], vn->vn_mc_handles[i]);
+	}
+
+	return (0);
+}
+
+static int
+vnic_get_secondary_macs(vnic_t *vn, uint_t pr_valsize, void *pr_val)
+{
+	int i;
+	mac_secondary_addr_t msa;
+
+	if (pr_valsize < sizeof (msa))
+		return (EINVAL);
+
+	/* Get existing addresses (primary is at 0) */
+	ASSERT(vn->vn_nhandles < MPT_MAXMACADDR);
+	for (i = 1; i <= vn->vn_nhandles; i++) {
+		ASSERT(vn->vn_mc_handles[i] != NULL);
+		mac_unicast_secondary_get(vn->vn_mc_handles[i],
+		    msa.ms_addrs[i - 1]);
+	}
+	msa.ms_addrcnt = vn->vn_nhandles;
+
+	bcopy(&msa, pr_val, sizeof (msa));
+	return (0);
+}
+
 /*
  * Callback functions for set/get of properties
  */
@@ -887,6 +1031,14 @@ vnic_m_setprop(void *m_driver, const char *pr_name, mac_prop_id_t pr_num,
 
 		bcopy(pr_val, &filtered, sizeof (filtered));
 		mac_set_promisc_filtered(vn->vn_mch, filtered);
+		break;
+	}
+	case MAC_PROP_SECONDARY_ADDRS: {
+		mac_secondary_addr_t msa;
+
+		bcopy(pr_val, &msa, sizeof (msa));
+		vnic_set_secondary_macs(vn, &msa);
+		break;
 	}
 	default:
 		err = ENOTSUP;
@@ -909,6 +1061,9 @@ vnic_m_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num,
 		ASSERT(pr_valsize >= sizeof (boolean_t));
 		bcopy(&out, pr_val, sizeof (boolean_t));
 		break;
+	case MAC_PROP_SECONDARY_ADDRS:
+		ret = vnic_get_secondary_macs(vn, pr_valsize, pr_val);
+		break;
 	default:
 		ret = EINVAL;
 		break;
@@ -968,7 +1123,8 @@ vnic_info(vnic_info_t *info, cred_t *credp)
 
 	bzero(&info->vn_resource_props, sizeof (mac_resource_props_t));
 	if (vnic->vn_mch != NULL)
-		mac_resource_ctl_get(vnic->vn_mch, &info->vn_resource_props);
+		mac_client_get_resources(vnic->vn_mch,
+		    &info->vn_resource_props);
 
 	rw_exit(&vnic_lock);
 	return (0);
diff --git a/usr/src/uts/common/sys/mac.h b/usr/src/uts/common/sys/mac.h
index 220446af65..38049b991e 100644
--- a/usr/src/uts/common/sys/mac.h
+++ b/usr/src/uts/common/sys/mac.h
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
+ * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
  */
 
 #ifndef	_SYS_MAC_H
@@ -129,6 +129,13 @@ typedef struct mac_propval_range_s {
  */
 #define	MAXMACADDRLEN		20
 
+#define	MPT_MAXMACADDR		32
+
+typedef struct mac_secondary_addr_s {
+	uint32_t	ms_addrcnt;
+	uint8_t		ms_addrs[MPT_MAXMACADDR][MAXMACADDRLEN];
+} mac_secondary_addr_t;
+
 typedef enum {
 	MAC_LOGTYPE_LINK = 1,
 	MAC_LOGTYPE_FLOW
@@ -207,6 +214,7 @@ typedef enum {
 	MAC_PROP_MAX_TXHWCLNT_AVAIL,
 	MAC_PROP_IB_LINKMODE,
 	MAC_PROP_VN_PROMISC_FILTERED,
+	MAC_PROP_SECONDARY_ADDRS,
 	MAC_PROP_PRIVATE = -1
 } mac_prop_id_t;
 
@@ -314,13 +322,18 @@ typedef struct mac_info_s {
  *
  * This capability allows the MAC layer to detect when a VNIC is being
  * access, and implement the required shortcuts.
+ *
+ * In addition, this capability is used to keep the VNIC's secondary
+ * mac_clients in sync when the primary MAC is updated.
  */
 
 typedef void *(*mac_client_handle_fn_t)(void *);
+typedef void (*mac_client_update_fn_t)(void *);
 
 typedef struct mac_capab_vnic_s {
 	void			*mcv_arg;
 	mac_client_handle_fn_t	mcv_mac_client_handle;
+	mac_client_update_fn_t	mcv_mac_secondary_update;
 } mac_capab_vnic_t;
 
 typedef void (*mac_rename_fn_t)(const char *, void *);
diff --git a/usr/src/uts/common/sys/mac_client.h b/usr/src/uts/common/sys/mac_client.h
index 40cd15a1b8..0fc4939503 100644
--- a/usr/src/uts/common/sys/mac_client.h
+++ b/usr/src/uts/common/sys/mac_client.h
@@ -22,6 +22,7 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2013 Joyent, Inc.  All rights reserved.
  */
 
 /*
@@ -135,6 +136,8 @@ extern void mac_multicast_remove(mac_client_handle_t, const uint8_t *);
 
 extern void mac_rx_set(mac_client_handle_t, mac_rx_t, void *);
 extern void mac_rx_clear(mac_client_handle_t);
+extern void mac_secondary_dup(mac_client_handle_t, mac_client_handle_t);
+extern void mac_secondary_cleanup(mac_client_handle_t);
 extern mac_tx_cookie_t mac_tx(mac_client_handle_t, mblk_t *,
     uintptr_t, uint16_t, mblk_t **);
 extern boolean_t mac_tx_is_flow_blocked(mac_client_handle_t, mac_tx_cookie_t);
@@ -158,6 +161,8 @@ extern int mac_unicast_primary_set(mac_handle_t, const uint8_t *);
 extern void mac_unicast_primary_get(mac_handle_t, uint8_t *);
 extern void mac_unicast_primary_info(mac_handle_t, char *, boolean_t *);
 
+extern void mac_unicast_secondary_get(mac_client_handle_t, uint8_t *);
+
 extern boolean_t mac_dst_get(mac_handle_t, uint8_t *);
 
 extern int mac_addr_random(mac_client_handle_t, uint_t, uint8_t *,
diff --git a/usr/src/uts/common/sys/mac_client_priv.h b/usr/src/uts/common/sys/mac_client_priv.h
index 0ddc1f074d..bed2e293b3 100644
--- a/usr/src/uts/common/sys/mac_client_priv.h
+++ b/usr/src/uts/common/sys/mac_client_priv.h
@@ -22,6 +22,7 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2013 Joyent, Inc.  All rights reserved.
  */
 
 /*
@@ -91,9 +92,6 @@ extern boolean_t mac_client_is_vlan_vnic(mac_client_handle_t);
 extern void mac_client_poll_enable(mac_client_handle_t);
 extern void mac_client_poll_disable(mac_client_handle_t);
 
-extern int mac_resource_ctl_set(mac_client_handle_t, mac_resource_props_t *);
-extern void mac_resource_ctl_get(mac_client_handle_t, mac_resource_props_t *);
-
 /*
  * Flow-related APIs for MAC clients.
  */
diff --git a/usr/src/uts/common/sys/vnic_impl.h b/usr/src/uts/common/sys/vnic_impl.h
index 2bb48a60c6..ffaa2939f5 100644
--- a/usr/src/uts/common/sys/vnic_impl.h
+++ b/usr/src/uts/common/sys/vnic_impl.h
@@ -21,6 +21,7 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
+ * Copyright 2013 Joyent, Inc.  All rights reserved.
  */
 
 #ifndef	_SYS_VNIC_IMPL_H
@@ -46,8 +47,10 @@ typedef struct vnic_s {
 
 	mac_handle_t		vn_mh;
 	mac_handle_t		vn_lower_mh;
-	mac_client_handle_t	vn_mch;
-	mac_unicast_handle_t	vn_muh;
+	uint_t			vn_nhandles; /* # of secondary mac handles */
+	/* The primary handle is always the first element in the array */
+	mac_client_handle_t	vn_mc_handles[MPT_MAXMACADDR];
+	mac_unicast_handle_t	vn_mu_handles[MPT_MAXMACADDR];
 	uint32_t		vn_margin;
 	int			vn_slot_id;
 	vnic_mac_addr_type_t	vn_addr_type;
@@ -63,6 +66,9 @@ typedef struct vnic_s {
 	uint32_t		vn_hcksum_txflags;
 } vnic_t;
 
+#define	vn_mch	vn_mc_handles[0]
+#define	vn_muh	vn_mu_handles[0]
+
 extern int vnic_dev_create(datalink_id_t, datalink_id_t, vnic_mac_addr_type_t *,
     int *, uchar_t *, int *, uint_t, uint16_t, vrid_t, int,
     mac_resource_props_t *, uint32_t, vnic_ioc_diag_t *, cred_t *);