12913 i40e want configurable num_rx_groups

Reviewed by: Paul Winder <paul@winder.uk.net>
Reviewed by: Randy Fishel <randyf@sibernet.com>
Reviewed by: Robert Mustacchi <rm@fingolfin.org>
Approved by: Dan McDonald <danmcd@joyent.com>

4 files changed, 41 insertions, 13 deletions

diff --git a/usr/src/man/man7d/i40e.7d b/usr/src/man/man7d/i40e.7d
index c5ba043149..fd666dcb65 100644
--- a/usr/src/man/man7d/i40e.7d
+++ b/usr/src/man/man7d/i40e.7d
@@ -10,8 +10,9 @@
 .\"
 .\"
 .\" Copyright (c) 2018 Joyent, Inc.
+.\" Copyright 2020 Ryan Zezeski
 .\"
-.Dd April 15, 2020
+.Dd July 1, 2020
 .Dt I40E 7D
 .Os
 .Sh NAME
@@ -157,6 +158,25 @@ for the device.
 The default is always to enable them.
 It is not recommended to to disable them.
 .Ed
+.It Sy rx_num_groups
+.Bd -filled -compact
+Minimum:
+.Sy 1 |
+Maximum:
+.Sy 32
+.Ed
+.Bd -filled
+The
+.Sy rx_num_groups
+property determines the number of receive mac groups provided by the driver.
+Each group can handle all unicast traffic for a single MAC address, more groups
+means more unicast traffic that can be steered by hardware.
+However, more groups also means more demand for kernel memory.
+If you are not making heavy use of VNICs, or do not need the efficiency gains
+of hardware steering, then reducing this number can reduce kernel memory
+taken by
+.Nm i40e.
+.Ed
 .It Sy rx_ring_size
 .Bd -filled -compact
 Minimum:
diff --git a/usr/src/uts/common/io/i40e/i40e_gld.c b/usr/src/uts/common/io/i40e/i40e_gld.c
index 1bf40bbce3..e988793675 100644
--- a/usr/src/uts/common/io/i40e/i40e_gld.c
+++ b/usr/src/uts/common/io/i40e/i40e_gld.c
@@ -13,6 +13,7 @@
  * Copyright 2015 OmniTI Computer Consulting, Inc. All rights reserved.
  * Copyright (c) 2018, Joyent, Inc.
  * Copyright 2017 Tegile Systems, Inc.  All rights reserved.
+ * Copyright 2020 Ryan Zezeski
  */
 
 /*
@@ -565,7 +566,7 @@ i40e_fill_rx_group(void *arg, mac_ring_type_t rtype, const int index,
 	infop->mgi_addmac = i40e_group_add_mac;
 	infop->mgi_remmac = i40e_group_remove_mac;
 
-	ASSERT(i40e->i40e_num_rx_groups <= I40E_GROUP_MAX);
+	ASSERT3U(i40e->i40e_num_rx_groups, <=, I40E_MAX_NUM_RX_GROUPS);
 	infop->mgi_count = i40e->i40e_num_trqpairs_per_vsi;
 }
 
diff --git a/usr/src/uts/common/io/i40e/i40e_main.c b/usr/src/uts/common/io/i40e/i40e_main.c
index e9fae0e708..dd2edfd379 100644
--- a/usr/src/uts/common/io/i40e/i40e_main.c
+++ b/usr/src/uts/common/io/i40e/i40e_main.c
@@ -14,6 +14,7 @@
  * Copyright 2019 Joyent, Inc.
  * Copyright 2017 Tegile Systems, Inc.  All rights reserved.
  * Copyright 2020 RackTop Systems, Inc.
+ * Copyright 2020 Ryan Zezeski
  */
 
 /*
@@ -1220,7 +1221,7 @@ i40e_hw_to_instance(i40e_t *i40e, i40e_hw_t *hw)
 	}
 
 	if (i40e->i40e_num_rx_groups == 0) {
-		i40e->i40e_num_rx_groups = I40E_GROUP_MAX;
+		i40e->i40e_num_rx_groups = I40E_DEF_NUM_RX_GROUPS;
 	}
 }
 
@@ -1589,6 +1590,10 @@ i40e_init_properties(i40e_t *i40e)
 	    i40e->i40e_tx_ring_size - I40E_TX_MAX_COOKIE,
 	    I40E_DEF_TX_BLOCK_THRESH);
 
+	i40e->i40e_num_rx_groups = i40e_get_prop(i40e, "rx_num_groups",
+	    I40E_MIN_NUM_RX_GROUPS, I40E_MAX_NUM_RX_GROUPS,
+	    I40E_DEF_NUM_RX_GROUPS);
+
 	i40e->i40e_rx_ring_size = i40e_get_prop(i40e, "rx_ring_size",
 	    I40E_MIN_RX_RING_SIZE, I40E_MAX_RX_RING_SIZE,
 	    I40E_DEF_RX_RING_SIZE);
@@ -1776,7 +1781,6 @@ i40e_alloc_intrs(i40e_t *i40e, dev_info_t *devinfo)
 	}
 
 	i40e->i40e_intr_type = 0;
-	i40e->i40e_num_rx_groups = I40E_GROUP_MAX;
 
 	/*
 	 * We need to determine the number of queue pairs per traffic
diff --git a/usr/src/uts/common/io/i40e/i40e_sw.h b/usr/src/uts/common/io/i40e/i40e_sw.h
index 4bd0a58c2a..f487e411ac 100644
--- a/usr/src/uts/common/io/i40e/i40e_sw.h
+++ b/usr/src/uts/common/io/i40e/i40e_sw.h
@@ -13,6 +13,7 @@
  * Copyright 2015 OmniTI Computer Consulting, Inc. All rights reserved.
  * Copyright 2019 Joyent, Inc.
  * Copyright 2017 Tegile Systems, Inc.  All rights reserved.
+ * Copyright 2020 Ryan Zezeski
  */
 
 /*
@@ -89,6 +90,16 @@ extern "C" {
 #define	I40E_MAX_TX_RING_SIZE	4096
 #define	I40E_DEF_TX_RING_SIZE	1024
 
+/*
+ * Place an artificial limit on the max number of groups. The X710
+ * series supports up to 384 VSIs to be partitioned across PFs as the
+ * driver sees fit. But until we support more interrupts this seems
+ * like a good place to start.
+ */
+#define	I40E_MIN_NUM_RX_GROUPS	1
+#define	I40E_MAX_NUM_RX_GROUPS	32
+#define	I40E_DEF_NUM_RX_GROUPS	16
+
 #define	I40E_MIN_RX_RING_SIZE	64
 #define	I40E_MAX_RX_RING_SIZE	4096
 #define	I40E_DEF_RX_RING_SIZE	1024
@@ -269,14 +280,6 @@ typedef enum i40e_itr_index {
  */
 #define	I40E_DDI_PROP_LEN	64
 
-/*
- * Place an artificial limit on the max number of groups. The X710
- * series supports up to 384 VSIs to be partitioned across PFs as the
- * driver sees fit. But until we support more interrupts this seems
- * like a good place to start.
- */
-#define	I40E_GROUP_MAX		32
-
 #define	I40E_GROUP_NOMSIX	1
 #define	I40E_TRQPAIR_NOMSIX	1
 
@@ -834,7 +837,7 @@ typedef struct i40e {
 	/*
 	 * Device state, switch information, and resources.
 	 */
-	i40e_vsi_t		i40e_vsis[I40E_GROUP_MAX];
+	i40e_vsi_t		i40e_vsis[I40E_MAX_NUM_RX_GROUPS];
 	uint16_t		i40e_mac_seid;	 /* SEID of physical MAC */
 	uint16_t		i40e_veb_seid;	 /* switch atop MAC (SEID) */
 	uint16_t		i40e_vsi_avail;	 /* VSIs avail to this PF */