OS-3053 VNICs should support a variable MTU20140529release-20140529

Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Rob Gulewich <robert.gulewich@joyent.com>

10 files changed, 214 insertions, 23 deletions

diff --git a/usr/src/lib/libdladm/common/libdladm.c b/usr/src/lib/libdladm/common/libdladm.c
index e0150d6529..2d2488182a 100644
--- a/usr/src/lib/libdladm/common/libdladm.c
+++ b/usr/src/lib/libdladm/common/libdladm.c
@@ -415,6 +415,9 @@ dladm_status2str(dladm_status_t status, char *buf)
 	case DLADM_STATUS_PORT_NOPROTO:
 		s = "local or remote port requires transport";
 		break;
+	case DLADM_STATUS_INVALID_MTU:
+		s = "MTU check failed, MTU outside of device's supported range";
+		break;
 	default:
 		s = "<unknown error>";
 		break;
diff --git a/usr/src/lib/libdladm/common/libdladm.h b/usr/src/lib/libdladm/common/libdladm.h
index 1cfb927a41..f3347a0ede 100644
--- a/usr/src/lib/libdladm/common/libdladm.h
+++ b/usr/src/lib/libdladm/common/libdladm.h
@@ -178,7 +178,8 @@ typedef enum {
 	DLADM_STATUS_INVALID_PKEY,
 	DLADM_STATUS_NO_IB_HW_RESOURCE,
 	DLADM_STATUS_INVALID_PKEY_TBL_SIZE,
-	DLADM_STATUS_PORT_NOPROTO
+	DLADM_STATUS_PORT_NOPROTO,
+	DLADM_STATUS_INVALID_MTU
 } dladm_status_t;
 
 typedef enum {
diff --git a/usr/src/lib/libdladm/common/libdlvnic.c b/usr/src/lib/libdladm/common/libdlvnic.c
index 1a866dcb06..4cd5d8e56d 100644
--- a/usr/src/lib/libdladm/common/libdlvnic.c
+++ b/usr/src/lib/libdladm/common/libdlvnic.c
@@ -91,6 +91,8 @@ dladm_vnic_diag2status(vnic_ioc_diag_t ioc_diag)
 		return (DLADM_STATUS_NO_HWRINGS);
 	case VNIC_IOC_DIAG_MACADDR_INVALID:
 		return (DLADM_STATUS_INVALIDMACADDR);
+	case VNIC_IOC_DIAG_MACMTU_INVALID:
+		return (DLADM_STATUS_INVALID_MTU);
 	default:
 		return (DLADM_STATUS_FAILED);
 	}
@@ -546,7 +548,7 @@ dladm_vnic_create(dladm_handle_t handle, const char *vnic, datalink_id_t linkid,
 
 done:
 	if (status == DLADM_STATUS_OK && proplist != NULL) {
-    		uint32_t flg;
+		uint32_t flg;
 
 		flg = (flags & DLADM_OPT_PERSIST) ?
 		    DLADM_OPT_PERSIST : DLADM_OPT_ACTIVE;
diff --git a/usr/src/uts/common/io/mac/mac.c b/usr/src/uts/common/io/mac/mac.c
index 61a5353365..e175f67313 100644
--- a/usr/src/uts/common/io/mac/mac.c
+++ b/usr/src/uts/common/io/mac/mac.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
+ * Copyright (c) 2014, Joyent, Inc.  All rights reserved.
  */
 
 /*
@@ -2685,6 +2685,100 @@ mac_margin_update(mac_handle_t mh, uint32_t margin)
 	return (margin_needed <= margin);
 }
 
+int
+mac_mtu_add(mac_handle_t mh, uint32_t *mtup, boolean_t current)
+{
+	mac_impl_t		*mip = (mac_impl_t *)mh;
+	mac_mtu_req_t		*prev, *cur;
+	mac_propval_range_t	mpr;
+	int			err;
+
+	i_mac_perim_enter(mip);
+	rw_enter(&mip->mi_rw_lock, RW_WRITER);
+
+	if (current == B_TRUE)
+		*mtup = mip->mi_sdu_max;
+	mpr.mpr_count = 1;
+	err = mac_prop_info(mh, MAC_PROP_MTU, "mtu", NULL, 0, &mpr, NULL);
+	if (err != 0) {
+		rw_exit(&mip->mi_rw_lock);
+		i_mac_perim_exit(mip);
+		return (err);
+	}
+
+	if (*mtup > mip->mi_sdu_max ||
+	    *mtup < mpr.mpr_range_uint32[0].mpur_min) {
+		rw_exit(&mip->mi_rw_lock);
+		i_mac_perim_exit(mip);
+		return (ENOTSUP);
+	}
+
+	prev = NULL;
+	for (cur = mip->mi_mtrp; cur != NULL; cur = cur->mtr_nextp) {
+		if (*mtup == cur->mtr_mtu) {
+			cur->mtr_ref++;
+			rw_exit(&mip->mi_rw_lock);
+			i_mac_perim_exit(mip);
+			return (0);
+		}
+
+		if (*mtup > cur->mtr_mtu)
+			break;
+
+		prev = cur;
+	}
+
+	cur = kmem_alloc(sizeof (mac_mtu_req_t), KM_SLEEP);
+	cur->mtr_mtu = *mtup;
+	cur->mtr_ref = 1;
+	if (prev != NULL) {
+		cur->mtr_nextp = prev->mtr_nextp;
+		prev->mtr_nextp = cur;
+	} else {
+		cur->mtr_nextp = mip->mi_mtrp;
+		mip->mi_mtrp = cur;
+	}
+
+	rw_exit(&mip->mi_rw_lock);
+	i_mac_perim_exit(mip);
+	return (0);
+}
+
+int
+mac_mtu_remove(mac_handle_t mh, uint32_t mtu)
+{
+	mac_impl_t *mip = (mac_impl_t *)mh;
+	mac_mtu_req_t *cur, *prev;
+
+	i_mac_perim_enter(mip);
+	rw_enter(&mip->mi_rw_lock, RW_WRITER);
+
+	prev = NULL;
+	for (cur = mip->mi_mtrp; cur != NULL; cur = cur->mtr_nextp) {
+		if (cur->mtr_mtu == mtu) {
+			ASSERT(cur->mtr_ref > 0);
+			cur->mtr_ref--;
+			if (cur->mtr_ref == 0) {
+				if (prev == NULL) {
+					mip->mi_mtrp = cur->mtr_nextp;
+				} else {
+					prev->mtr_nextp = cur->mtr_nextp;
+				}
+				kmem_free(cur, sizeof (mac_mtu_req_t));
+			}
+			rw_exit(&mip->mi_rw_lock);
+			i_mac_perim_exit(mip);
+			return (0);
+		}
+
+		prev = cur;
+	}
+
+	rw_exit(&mip->mi_rw_lock);
+	i_mac_perim_exit(mip);
+	return (ENOENT);
+}
+
 /*
  * MAC Type Plugin functions.
  */
diff --git a/usr/src/uts/common/io/mac/mac_client.c b/usr/src/uts/common/io/mac/mac_client.c
index 0f0d44cda5..88620518f1 100644
--- a/usr/src/uts/common/io/mac/mac_client.c
+++ b/usr/src/uts/common/io/mac/mac_client.c
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
+ * Copyright (c) 2014, Joyent, Inc.  All rights reserved.
  */
 
 /*
@@ -5258,6 +5258,14 @@ mac_set_mtu(mac_handle_t mh, uint_t new_mtu, uint_t *old_mtu_arg)
 		goto bail;
 	}
 
+	rw_enter(&mip->mi_rw_lock, RW_READER);
+	if (mip->mi_mtrp != NULL && new_mtu < mip->mi_mtrp->mtr_mtu) {
+		rv = EBUSY;
+		rw_exit(&mip->mi_rw_lock);
+		goto bail;
+	}
+	rw_exit(&mip->mi_rw_lock);
+
 	if (old_mtu != new_mtu) {
 		rv = mip->mi_callbacks->mc_setprop(mip->mi_driver,
 		    "mtu", MAC_PROP_MTU, sizeof (uint_t), &new_mtu);
diff --git a/usr/src/uts/common/io/vnic/vnic_dev.c b/usr/src/uts/common/io/vnic/vnic_dev.c
index e7ad2608dc..64ce41a001 100644
--- a/usr/src/uts/common/io/vnic/vnic_dev.c
+++ b/usr/src/uts/common/io/vnic/vnic_dev.c
@@ -500,10 +500,21 @@ vnic_dev_create(datalink_id_t vnic_id, datalink_id_t linkid,
 
 		mac_sdu_get(vnic->vn_lower_mh, &mac->m_min_sdu,
 		    &mac->m_max_sdu);
+		err = mac_mtu_add(vnic->vn_lower_mh, &mac->m_max_sdu, B_FALSE);
+		if (err != 0) {
+			VERIFY(mac_margin_remove(vnic->vn_lower_mh,
+			    vnic->vn_margin) == 0);
+			mac_free(mac);
+			if (diag != NULL)
+				*diag = VNIC_IOC_DIAG_MACMTU_INVALID;
+			goto bail;
+		}
+		vnic->vn_mtu = mac->m_max_sdu;
 	} else {
 		vnic->vn_margin = VLAN_TAGSZ;
 		mac->m_min_sdu = 1;
 		mac->m_max_sdu = ANCHOR_VNIC_MAX_MTU;
+		vnic->vn_mtu = ANCHOR_VNIC_MAX_MTU;
 	}
 
 	mac->m_margin = vnic->vn_margin;
@@ -511,8 +522,12 @@ vnic_dev_create(datalink_id_t vnic_id, datalink_id_t linkid,
 	err = mac_register(mac, &vnic->vn_mh);
 	mac_free(mac);
 	if (err != 0) {
-		VERIFY(is_anchor || mac_margin_remove(vnic->vn_lower_mh,
-		    vnic->vn_margin) == 0);
+		if (!is_anchor) {
+			VERIFY(mac_mtu_remove(vnic->vn_lower_mh,
+			    vnic->vn_mtu) == 0);
+			VERIFY(mac_margin_remove(vnic->vn_lower_mh,
+			    vnic->vn_margin) == 0);
+		}
 		goto bail;
 	}
 
@@ -527,6 +542,10 @@ vnic_dev_create(datalink_id_t vnic_id, datalink_id_t linkid,
 			}
 			err = mac_client_set_resources(vnic->vn_mch, mrp);
 			if (err != 0) {
+				VERIFY(mac_mtu_remove(vnic->vn_lower_mh,
+				    vnic->vn_mtu) == 0);
+				VERIFY(mac_margin_remove(vnic->vn_lower_mh,
+				    vnic->vn_margin) == 0);
 				(void) mac_unregister(vnic->vn_mh);
 				goto bail;
 			}
@@ -537,6 +556,12 @@ vnic_dev_create(datalink_id_t vnic_id, datalink_id_t linkid,
 	if (err != 0) {
 		VERIFY(is_anchor || mac_margin_remove(vnic->vn_lower_mh,
 		    vnic->vn_margin) == 0);
+		if (!is_anchor) {
+			VERIFY(mac_mtu_remove(vnic->vn_lower_mh,
+			    vnic->vn_mtu) == 0);
+			VERIFY(mac_margin_remove(vnic->vn_lower_mh,
+			    vnic->vn_margin) == 0);
+		}
 		(void) mac_unregister(vnic->vn_mh);
 		goto bail;
 	}
@@ -655,6 +680,7 @@ vnic_dev_delete(datalink_id_t vnic_id, uint32_t flags, cred_t *credp)
 			    vnic->vn_slot_id);
 		}
 		(void) mac_margin_remove(vnic->vn_lower_mh, vnic->vn_margin);
+		(void) mac_mtu_remove(vnic->vn_lower_mh, vnic->vn_mtu);
 		(void) mac_notify_remove(vnic->vn_mnh, B_TRUE);
 		(void) mac_unicast_remove(vnic->vn_mch, vnic->vn_muh);
 		mac_client_close(vnic->vn_mch, MAC_CLOSE_FLAGS_IS_VNIC);
@@ -1004,20 +1030,35 @@ vnic_m_setprop(void *m_driver, const char *pr_name, mac_prop_id_t pr_num,
 		uint32_t	mtu;
 
 		/* allow setting MTU only on an etherstub */
-		if (vn->vn_link_id != DATALINK_INVALID_LINKID) {
-			err = ENOTSUP;
-			break;
-		}
 
 		if (pr_valsize < sizeof (mtu)) {
 			err = EINVAL;
 			break;
 		}
 		bcopy(pr_val, &mtu, sizeof (mtu));
-		if (mtu < ANCHOR_VNIC_MIN_MTU || mtu > ANCHOR_VNIC_MAX_MTU) {
-			err = EINVAL;
-			break;
+
+		if (vn->vn_link_id == DATALINK_INVALID_LINKID) {
+			if (mtu < ANCHOR_VNIC_MIN_MTU ||
+			    mtu > ANCHOR_VNIC_MAX_MTU) {
+				err = EINVAL;
+				break;
+			}
+		} else {
+			err = mac_mtu_add(vn->vn_lower_mh, &mtu, B_FALSE);
+			/*
+			 * If it's not supported to set a value here, translate
+			 * that to EINVAL, so user land gets a better idea of
+			 * what went wrong. This realistically means that they
+			 * violated the output of prop info.
+			 */
+			if (err == ENOTSUP)
+				err = EINVAL;
+			if (err != 0)
+				break;
+			VERIFY(mac_mtu_remove(vn->vn_lower_mh,
+			    vn->vn_mtu) == 0);
 		}
+		vn->vn_mtu = mtu;
 		err = mac_maxsdu_update(vn->vn_mh, mtu);
 		break;
 	}
@@ -1078,14 +1119,43 @@ static void vnic_m_propinfo(void *m_driver, const char *pr_name,
 {
 	vnic_t		*vn = m_driver;
 
-	/* MTU setting allowed only on an etherstub */
-	if (vn->vn_link_id != DATALINK_INVALID_LINKID)
-		return;
-
 	switch (pr_num) {
 	case MAC_PROP_MTU:
-		mac_prop_info_set_range_uint32(prh,
-		    ANCHOR_VNIC_MIN_MTU, ANCHOR_VNIC_MAX_MTU);
+		if (vn->vn_link_id == DATALINK_INVALID_LINKID) {
+			mac_prop_info_set_range_uint32(prh,
+			    ANCHOR_VNIC_MIN_MTU, ANCHOR_VNIC_MAX_MTU);
+		} else {
+			uint32_t		max;
+			mac_perim_handle_t	mph;
+			mac_propval_range_t	range;
+
+			/*
+			 * The valid range for a VNIC's MTU is the minimum that
+			 * the device supports and the current value of the
+			 * device. A VNIC cannot increase the current MTU of the
+			 * device. Therefore we need to get the range from the
+			 * propinfo endpoint and current mtu from the
+			 * traditional property endpoint.
+			 */
+			mac_perim_enter_by_mh(vn->vn_lower_mh, &mph);
+			if (mac_get_prop(vn->vn_lower_mh, MAC_PROP_MTU, "mtu",
+			    &max, sizeof (uint32_t)) != 0) {
+				mac_perim_exit(mph);
+				return;
+			}
+
+			range.mpr_count = 1;
+			if (mac_prop_info(vn->vn_lower_mh, MAC_PROP_MTU, "mtu",
+			    NULL, 0, &range, NULL) != 0) {
+				mac_perim_exit(mph);
+				return;
+			}
+
+			mac_prop_info_set_default_uint32(prh, max);
+			mac_prop_info_set_range_uint32(prh,
+			    range.mpr_range_uint32[0].mpur_min, max);
+			mac_perim_exit(mph);
+		}
 		break;
 	}
 }
diff --git a/usr/src/uts/common/sys/mac.h b/usr/src/uts/common/sys/mac.h
index 38049b991e..5803ad58d4 100644
--- a/usr/src/uts/common/sys/mac.h
+++ b/usr/src/uts/common/sys/mac.h
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
+ * Copyright (c) 2014, Joyent, Inc.  All rights reserved.
  */
 
 #ifndef	_SYS_MAC_H
@@ -618,6 +618,9 @@ extern void			mac_margin_get(mac_handle_t, uint32_t *);
 extern int			mac_margin_remove(mac_handle_t, uint32_t);
 extern int			mac_margin_add(mac_handle_t, uint32_t *,
 				    boolean_t);
+extern int			mac_mtu_add(mac_handle_t, uint32_t *,
+				    boolean_t);
+extern int			mac_mtu_remove(mac_handle_t, uint32_t);
 extern int			mac_fastpath_disable(mac_handle_t);
 extern void			mac_fastpath_enable(mac_handle_t);
 extern void			mac_no_active(mac_handle_t);
diff --git a/usr/src/uts/common/sys/mac_impl.h b/usr/src/uts/common/sys/mac_impl.h
index 2eef66113d..d19c19e43d 100644
--- a/usr/src/uts/common/sys/mac_impl.h
+++ b/usr/src/uts/common/sys/mac_impl.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, Joyent, Inc.  All rights reserved.
+ * Copyright (c) 2014, Joyent, Inc.  All rights reserved.
  */
 
 #ifndef	_SYS_MAC_IMPL_H
@@ -69,6 +69,13 @@ struct mac_margin_req_s {
 	uint32_t		mmr_margin;
 };
 
+typedef struct mac_mtu_req_s mac_mtu_req_t;
+struct mac_mtu_req_s {
+	mac_mtu_req_t		*mtr_nextp;
+	uint_t			mtr_ref;
+	uint32_t		mtr_mtu;
+};
+
 /* Generic linked chain type */
 typedef	struct mac_chain_s {
 	struct mac_chain_s	*next;
@@ -534,6 +541,7 @@ struct mac_impl_s {
 	 * sorted: the first one has the greatest value.
 	 */
 	mac_margin_req_t	*mi_mmrp;
+	mac_mtu_req_t		*mi_mtrp;
 	char			**mi_priv_prop;
 	uint_t			mi_priv_prop_count;
 
diff --git a/usr/src/uts/common/sys/vnic.h b/usr/src/uts/common/sys/vnic.h
index 3a6f5279ee..e18673a6c1 100644
--- a/usr/src/uts/common/sys/vnic.h
+++ b/usr/src/uts/common/sys/vnic.h
@@ -55,7 +55,8 @@ typedef enum {
 	VNIC_IOC_DIAG_MACPREFIX_INVALID,
 	VNIC_IOC_DIAG_MACPREFIXLEN_INVALID,
 	VNIC_IOC_DIAG_MACMARGIN_INVALID,
-	VNIC_IOC_DIAG_NO_HWRINGS
+	VNIC_IOC_DIAG_NO_HWRINGS,
+	VNIC_IOC_DIAG_MACMTU_INVALID
 } vnic_ioc_diag_t;
 
 /*
diff --git a/usr/src/uts/common/sys/vnic_impl.h b/usr/src/uts/common/sys/vnic_impl.h
index ffaa2939f5..7e50091347 100644
--- a/usr/src/uts/common/sys/vnic_impl.h
+++ b/usr/src/uts/common/sys/vnic_impl.h
@@ -21,7 +21,7 @@
 /*
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
- * Copyright 2013 Joyent, Inc.  All rights reserved.
+ * Copyright 2014 Joyent, Inc.  All rights reserved.
  */
 
 #ifndef	_SYS_VNIC_IMPL_H
@@ -64,6 +64,7 @@ typedef struct vnic_s {
 	mac_notify_handle_t	vn_mnh;
 
 	uint32_t		vn_hcksum_txflags;
+	uint32_t		vn_mtu;
 } vnic_t;
 
 #define	vn_mch	vn_mc_handles[0]