OS-6847 vxlan header allocation should think about mblk chains

2 files changed, 45 insertions, 32 deletions

diff --git a/usr/src/uts/common/io/overlay/overlay.c b/usr/src/uts/common/io/overlay/overlay.c
index 36c4fd38b1..7aba941fd6 100644
--- a/usr/src/uts/common/io/overlay/overlay.c
+++ b/usr/src/uts/common/io/overlay/overlay.c
@@ -1054,25 +1054,16 @@ overlay_m_unicast(void *arg, const uint8_t *macaddr)
 }
 
 static inline void
-overlay_tx_checksum_shift(mblk_t *source, mblk_t *target)
+overlay_tx_checksum_shift(mblk_t *mp, uint16_t flags)
 {
-	uint32_t oflags, nflags = 0;
-
-	mac_hcksum_get(source, NULL, NULL, NULL, NULL, &oflags);
-	mac_hcksum_set(source, NULL, NULL, NULL, NULL, 0);
-
-	if ((oflags & HCK_IPV4_HDRCKSUM) != 0)
-		nflags |= HCK_INNER_IPV4_HDRCKSUM_NEEDED;
-	if ((oflags & HCK_FULLCKSUM) != 0) {
-		nflags |= HCK_INNER_FULLCKSUM_NEEDED;
-	} else if ((oflags & HCK_PARTIALCKSUM) != 0) {
-		nflags |= HCK_INNER_PSEUDO_NEEDED;
+	DB_CKSUMFLAGS(mp) &= ~HCK_FLAGS;
+	if ((flags & HCK_IPV4_HDRCKSUM) != 0)
+		DB_CKSUMFLAGS(mp) |= HCK_INNER_IPV4_HDRCKSUM_NEEDED;
+	if ((flags & HCK_FULLCKSUM) != 0) {
+		DB_CKSUMFLAGS(mp) |= HCK_INNER_FULLCKSUM_NEEDED;
+	} else if ((flags & HCK_PARTIALCKSUM) != 0) {
+		DB_CKSUMFLAGS(mp) |= HCK_INNER_PSEUDO_NEEDED;
 	}
-
-	/*
-	 * Manually or in the flags so we don't clobber existing information.
-	 */
-	DB_CKSUMFLAGS(target) |= nflags;
 }
 
 mblk_t *
@@ -1130,9 +1121,11 @@ overlay_m_tx(void *arg, mblk_t *mp_chain)
 		 * Make sure any checksum flags that ended up on mp from the
 		 * lower level are shifted over to emp as outer flags.
 		 */
-		overlay_tx_checksum_shift(mp, ep);
+		overlay_tx_checksum_shift(ep, DB_CKSUMFLAGS(mp));
+		if (ep != mp) {
+			ep->b_cont = mp;
+		}
 
-		ep->b_cont = mp;
 		ret = overlay_mux_tx(odd->odd_mux, &hdr, ep);
 		if (ret != 0)
 			goto out;
diff --git a/usr/src/uts/common/io/overlay/plugins/overlay_vxlan.c b/usr/src/uts/common/io/overlay/plugins/overlay_vxlan.c
index a381a0c793..60659ade8c 100644
--- a/usr/src/uts/common/io/overlay/plugins/overlay_vxlan.c
+++ b/usr/src/uts/common/io/overlay/plugins/overlay_vxlan.c
@@ -61,6 +61,20 @@ static uint16_t vxlan_defport = IPPORT_VXLAN;
  */
 boolean_t vxlan_fanout = B_TRUE;
 
+/*
+ * This represents the size in bytes that we want to allocate when allocating a
+ * vxlan header block. This is intended such that lower levels can try and use
+ * the message block that we allocate for the IP and UPD header. The hope is
+ * that even if this is tunneled, that this is enough space.
+ *
+ * The vxlan_noalloc_min value represents the minimum amount of space we need to
+ * consider not allocating a message block and just passing it down the stack in
+ * this form. This number assumes that we have a VLAN tag, so 18 byte Ethernet
+ * header, 20 byte IP header, 8 byte UDP header, and 8 byte VXLAN header.
+ */
+uint_t vxlan_alloc_size = 128;
+uint_t vxlan_noalloc_min = 54;
+
 static const char *vxlan_props[] = {
 	"vxlan/listen_ip",
 	"vxlan/listen_port",
@@ -172,22 +186,28 @@ vxlan_o_encap(void *arg, mblk_t *mp, ovep_encap_info_t *einfop,
 
 	ASSERT(einfop->ovdi_id < (1 << 24));
 
-	/*
-	 * This allocation could get hot. We may want to have a good way to
-	 * cache and handle this allocation the same way that IP does with
-	 * keeping around a message block per entry, or basically treating this
-	 * as an immutable message block in the system. Basically freemsg() will
-	 * be a nop, but we'll do the right thing with respect to the rest of
-	 * the chain.
-	 */
-	ob = allocb(VXLAN_HDR_LEN, 0);
-	if (ob == NULL)
-		return (ENOMEM);
-
+	if (DB_REF(mp) != 1 || mp->b_rptr - vxlan_noalloc_min < DB_BASE(mp)) {
+		/*
+		 * This allocation could get hot. We may want to have a good way to
+		 * cache and handle this allocation the same way that IP does with
+		 * keeping around a message block per entry, or basically treating this
+		 * as an immutable message block in the system. Basically freemsg() will
+		 * be a nop, but we'll do the right thing with respect to the rest of
+		 * the chain.
+		 */
+		ob = allocb(vxlan_alloc_size, 0);
+		if (ob == NULL)
+			return (ENOMEM);
+
+		ob->b_wptr = DB_LIM(ob);
+		ob->b_rptr = ob->b_wptr - VXLAN_HDR_LEN;
+	} else {
+		ob = mp;
+		mp->b_rptr -= VXLAN_HDR_LEN;
+	}
 	vxh = (vxlan_hdr_t *)ob->b_rptr;
 	vxh->vxlan_flags = ntohl(VXLAN_F_VDI);
 	vxh->vxlan_id = htonl((uint32_t)einfop->ovdi_id << VXLAN_ID_SHIFT);
-	ob->b_wptr += VXLAN_HDR_LEN;
 
 	/*
 	 * Make sure to set the fact that this is a VXLAN packet on this message