12693 Enable Forward Error Correction (FEC) configuration via dladm

Reviewed by: Garrett D'Amore <garrett@damore.org>
Reviewed by: Robert Mustacchi <rm@fingolfin.org>
Approved by: Dan McDonald <danmcd@joyent.com>

17 files changed, 758 insertions, 40 deletions

diff --git a/usr/src/lib/libdladm/common/linkprop.c b/usr/src/lib/libdladm/common/linkprop.c
index c33268c8f8..90edf1293f 100644
--- a/usr/src/lib/libdladm/common/linkprop.c
+++ b/usr/src/lib/libdladm/common/linkprop.c
@@ -22,6 +22,7 @@
  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2017 Joyent, Inc.
  * Copyright 2015 Garrett D'Amore <garrett@damore.org>
+ * Copyright 2020 RackTop Systems, Inc.
  */
 
 #include <stdlib.h>
@@ -154,13 +155,13 @@ static pd_getf_t	get_zone, get_autopush, get_rate_mod, get_rate,
 			get_bridge_pvid, get_protection, get_rxrings,
 			get_txrings, get_cntavail, get_secondary_macs,
 			get_allowedips, get_allowedcids, get_pool,
-			get_rings_range, get_linkmode_prop,
+			get_rings_range, get_linkmode_prop, get_bits,
 			get_promisc_filtered;
 
 static pd_setf_t	set_zone, set_rate, set_powermode, set_radio,
 			set_public_prop, set_resource, set_stp_prop,
 			set_bridge_forward, set_bridge_pvid, set_secondary_macs,
-			set_promisc_filtered;
+			set_promisc_filtered, set_public_bitprop;
 
 static pd_checkf_t	check_zone, check_autopush, check_rate, check_hoplimit,
 			check_encaplim, check_uint32, check_maxbw, check_cpus,
@@ -255,6 +256,10 @@ static link_attr_t link_attr[] = {
 
 	{ MAC_PROP_FLOWCTRL,	sizeof (link_flowctrl_t), "flowctrl"},
 
+	{ MAC_PROP_ADV_FEC_CAP,	sizeof (link_fec_t),	"adv_fec_cap"},
+
+	{ MAC_PROP_EN_FEC_CAP,	sizeof (link_fec_t),	"en_fec_cap"},
+
 	{ MAC_PROP_ZONE,	sizeof (dld_ioc_zid_t),	"zone"},
 
 	{ MAC_PROP_AUTOPUSH,	sizeof (struct dlautopush), "autopush"},
@@ -433,6 +438,12 @@ static  val_desc_t	link_flow_vals[] = {
 	{ "rx",		LINK_FLOWCTRL_RX	},
 	{ "bi",		LINK_FLOWCTRL_BI	}
 };
+static  val_desc_t	link_fec_vals[] = {
+	{ "none",	LINK_FEC_NONE		},
+	{ "auto",	LINK_FEC_AUTO		},
+	{ "rs",		LINK_FEC_RS		},
+	{ "base-r",	LINK_FEC_BASE_R		}
+};
 static  val_desc_t	link_priority_vals[] = {
 	{ "low",	MPL_LOW	},
 	{ "medium",	MPL_MEDIUM	},
@@ -551,6 +562,16 @@ static prop_desc_t	prop_table[] = {
 	    set_public_prop, NULL, get_flowctl, NULL,
 	    0, DATALINK_CLASS_PHYS, DL_ETHER },
 
+	{ "adv_fec_cap", { "", LINK_FEC_AUTO },
+	    link_fec_vals, VALCNT(link_fec_vals),
+	    NULL, NULL, get_bits, NULL,
+	    0, DATALINK_CLASS_PHYS, DL_ETHER },
+
+	{ "en_fec_cap", { "", LINK_FEC_AUTO },
+	    link_fec_vals, VALCNT(link_fec_vals),
+	    set_public_bitprop, NULL, get_bits, NULL,
+	    0, DATALINK_CLASS_PHYS, DL_ETHER },
+
 	{ "secondary-macs", { "--", 0 }, NULL, 0,
 	    set_secondary_macs, NULL,
 	    get_secondary_macs, check_secondary_macs, PD_CHECK_ALLOC,
@@ -3846,6 +3867,33 @@ done:
 	return (status);
 }
 
+static dladm_status_t
+set_public_bitprop(dladm_handle_t handle, prop_desc_t *pdp,
+    datalink_id_t linkid, val_desc_t *vdp, uint_t val_cnt, uint_t flags,
+    datalink_media_t media)
+{
+	uint_t		i, j;
+	val_desc_t	vd = { 0 };
+
+	if ((pdp->pd_flags & PD_CHECK_ALLOC) != 0)
+		return (DLADM_STATUS_BADARG);
+
+	for (i = 0; i < val_cnt; i++) {
+		for (j = 0; j < pdp->pd_noptval; j++) {
+			if (strcasecmp(vdp[i].vd_name,
+			    pdp->pd_optval[j].vd_name) == 0) {
+				vd.vd_val |= pdp->pd_optval[j].vd_val;
+				break;
+			}
+		}
+	}
+
+	if (vd.vd_val == 0)
+		return (DLADM_STATUS_BADARG);
+
+	return (set_public_prop(handle, pdp, linkid, &vd, 1, flags, media));
+}
+
 dladm_status_t
 i_dladm_macprop(dladm_handle_t handle, void *dip, boolean_t set)
 {
@@ -4158,6 +4206,34 @@ get_flowctl(dladm_handle_t handle, prop_desc_t *pdp,
 	return (DLADM_STATUS_OK);
 }
 
+static dladm_status_t
+get_bits(dladm_handle_t handle, prop_desc_t *pdp,
+    datalink_id_t linkid, char **prop_val, uint_t *val_cnt,
+    datalink_media_t media, uint_t flags, uint_t *perm_flags)
+{
+	uint32_t	v;
+	dladm_status_t	status;
+	uint_t		i, cnt;
+
+	status = i_dladm_get_public_prop(handle, linkid, pdp->pd_name, flags,
+	    perm_flags, &v, sizeof (v));
+	if (status != DLADM_STATUS_OK)
+		return (status);
+
+	cnt = 0;
+	for (i = 0; cnt < *val_cnt && i < pdp->pd_noptval; i++) {
+		if ((v & pdp->pd_optval[i].vd_val) != 0) {
+			(void) snprintf(prop_val[cnt++], DLADM_STRSIZE,
+			    pdp->pd_optval[i].vd_name);
+		}
+	}
+
+	if (i < pdp->pd_noptval)
+		return (DLADM_STATUS_BADVALCNT);
+
+	*val_cnt = cnt;
+	return (DLADM_STATUS_OK);
+}
 
 /* ARGSUSED */
 static dladm_status_t
diff --git a/usr/src/man/man1m/dladm.1m b/usr/src/man/man1m/dladm.1m
index f84c147caf..d6050c5114 100644
--- a/usr/src/man/man1m/dladm.1m
+++ b/usr/src/man/man1m/dladm.1m
@@ -42,12 +42,12 @@
 .\"
 .\" Copyright (c) 2008, Sun Microsystems, Inc. All Rights Reserved
 .\" Copyright 2016 Joyent, Inc.
+.\" Copyright 2020 RackTop Systems, Inc.
 .\"
-.TH DLADM 1M "Dec 16, 2016"
+.TH DLADM 1M "May 4, 2020"
 .SH NAME
 dladm \- administer data links
 .SH SYNOPSIS
-.LP
 .nf
 \fBdladm show-link\fR [\fB-P\fR] [\fB-s\fR [\fB-i\fR \fIinterval\fR]] [[\fB-p\fR] \fB-o\fR \fIfield\fR[,...]] [\fIlink\fR]
 \fBdladm rename-link\fR [\fB-R\fR \fIroot-dir\fR] \fIlink\fR \fInew-link\fR
@@ -179,7 +179,6 @@ dladm \- administer data links
 .fi
 
 .SH DESCRIPTION
-.LP
 The \fBdladm\fR command is used to administer data-links. A data-link is
 represented in the system as a \fBSTREAMS DLPI\fR (v2) interface which can be
 plumbed under protocol stacks such as \fBTCP/IP\fR. Each data-link relies on
@@ -332,7 +331,6 @@ characters.
 .RE
 
 .SS "Options"
-.LP
 Each \fBdladm\fR subcommand has its own set of options. However, many of the
 subcommands have the following as a common option:
 .sp
@@ -347,7 +345,6 @@ deletion, or renaming-should apply.
 .RE
 
 .SS "SUBCOMMANDS"
-.LP
 The following subcommands are supported:
 .sp
 .ne 2
@@ -4438,7 +4435,6 @@ display network usage for all links.
 .RE
 
 .SS "Parsable Output Format"
-.LP
 Many \fBdladm\fR subcommands have an option that displays output in a
 machine-parsable format. The output format is one or more lines of colon
 (\fB:\fR) delimited fields. The fields displayed are specific to the subcommand
@@ -4454,7 +4450,6 @@ by using shell \fBread\fR(1) functions with the environment variable
 \fBIFS=:\fR (see \fBEXAMPLES\fR, below). Note that escaping is not done when
 you request only a single field.
 .SS "General Link Properties"
-.LP
 The following general link properties are supported:
 .sp
 .ne 2
@@ -4725,7 +4720,6 @@ currently running on the system. By default, the zone binding is as per
 .RE
 
 .SS "Wifi Link Properties"
-.LP
 The following \fBWiFi\fR link properties are supported. Note that the ability
 to set a given property to a given value depends on the driver and hardware.
 .sp
@@ -4777,7 +4771,6 @@ is no fixed speed.
 .RE
 
 .SS "Ethernet Link Properties"
-.LP
 The following MII Properties, as documented in \fBieee802.3\fR(5), are
 supported in read-only mode:
 .RS +4
@@ -4938,6 +4931,75 @@ capabilities allowed by the device and the link partner.
 .sp
 .ne 2
 .na
+\fB\fBen_fec_cap\fR\fR
+.ad
+.sp .6
+.RS 4n
+Sets the Forward Error Correct (FEC) code(s) to be advertised by the
+device.
+Valid values are:
+.sp
+.ne 2
+.na
+\fB\fBnone\fR\fR
+.ad
+.sp .6
+.RS 4n
+Allow the device not to use FEC.
+.RE
+
+.sp
+.ne 2
+.na
+\fB\fBauto\fR\fR
+.ad
+.sp .6
+.RS 4n
+The device will automatically decide which FEC code to use.
+.RE
+
+.sp
+.ne 2
+.na
+\fB\fBrs\fR\fR
+.ad
+.sp .6
+.RS 4n
+Allow Reed-Solomon FEC code.
+.RE
+
+.sp
+.ne 2
+.na
+\fB\fBbase-r\fR\fR
+.ad
+.sp .6
+.RS 4n
+Allow Base-R (also known as FireCode) code.
+.RE
+
+Valid input is either \fBauto\fR as a single value, or a comma separated
+combination of \fBnone\fR, \fBrs\fR and \fBbase-r\fR.
+The default value is \fBauto\fR.
+.sp
+.LP
+Note the actual FEC settings and combinations are constrained by the
+capabilities allowed by the device and the link partner.
+.RE
+
+.sp
+.ne 2
+.na
+\fB\fBadv_fec_cap\fR\fR
+.ad
+.sp .6
+.RS 4n
+(read only) The current negotiated Forward Error Correction code.
+.RE
+
+.sp
+.ne 2
+.na
 \fB\fBsecondary-macs\fR\fR
 .ad
 .sp .6
@@ -4992,7 +5054,6 @@ The default value is \fBvlanonly\fR.
 .RE
 
 .SS "IP Tunnel Link Properties"
-.LP
 The following IP tunnel link properties are supported.
 .sp
 .ne 2
@@ -5019,7 +5080,6 @@ default value is 4. A value of 0 disables the encapsulation limit.
 .RE
 
 .SH EXAMPLES
-.LP
 \fBExample 1 \fRConfiguring an Aggregation
 .sp
 .LP
@@ -5494,7 +5554,6 @@ interface. See \fBifconfig\fR(1M) for a description of how IPv6 addresses are
 configured on 6to4 tunnel links.
 
 .SH ATTRIBUTES
-.LP
 See \fBattributes\fR(5) for descriptions of the following attributes:
 .sp
 .LP
@@ -5527,12 +5586,10 @@ Interface Stability	Committed
 .TE
 
 .SH SEE ALSO
-.LP
 \fBacctadm\fR(1M), \fBautopush\fR(1M), \fBifconfig\fR(1M), \fBipsecconf\fR(1M),
 \fBndd\fR(1M), \fBpsrset\fR(1M), \fBwpad\fR(1M), \fBzonecfg\fR(1M),
 \fBattributes\fR(5), \fBieee802.3\fR(5), \fBdlpi\fR(7P)
 .SH NOTES
-.LP
 The preferred method of referring to an aggregation in the aggregation
 subcommands is by its link name. Referring to an aggregation by its integer
 \fIkey\fR is supported for backward compatibility, but is not necessary. When
diff --git a/usr/src/man/man9e/mac.9e b/usr/src/man/man9e/mac.9e
index ffeea417ca..3a3f2ae90a 100644
--- a/usr/src/man/man9e/mac.9e
+++ b/usr/src/man/man9e/mac.9e
@@ -10,8 +10,9 @@
 .\"
 .\"
 .\" Copyright 2019 Joyent, Inc.
+.\" Copyright 2020 RackTop Systems, Inc.
 .\"
-.Dd July 22, 2019
+.Dd May 11, 2020
 .Dt MAC 9E
 .Os
 .Sh NAME
@@ -890,6 +891,57 @@ it has configured the device, not what the device has actually
 negotiated.
 When setting the property, it should update the hardware and allow the link to
 potentially perform auto-negotiation again.
+.It Sy MAC_PROP_EN_FEC_CAP
+.Bd -filled -compact
+Type:
+.Sy link_fec_t |
+Permissions:
+.Sy Read/Write
+.Ed
+.Pp
+The
+.Sy MAC_PROP_EN_FEC_CAP
+property indicates which Forward Error Correction (FEC) code is advertised
+by the device.
+.Pp
+The
+.Sy link_fec_t
+is an enumeration that may be a combination of the following bit values:
+.Bl -tag -width Ds
+.It Sy LINK_FEC_NONE
+No FEC over the link.
+.It Sy LINK_FEC_AUTO
+The FEC coding to use is auto-negotiated,
+.Sy LINK_FEC_AUTO
+cannot be set along with any of the other values.
+This is the default setting the device driver should use.
+.It Sy LINK_FEC_RS
+The link may use Reed-Solomon FEC coding.
+.It Sy LINK_FEC_BASE_R
+The link may use Base-R coding, also common referred to as FireCode.
+.El
+.Pp
+When setting the property, it should update the hardware with the requested, or
+combination of requested codings.
+If a particular combination of codings is not supported by the hardware,
+the device driver should return
+.Er EINVAL .
+When retrieving this property, the device driver should return the current
+value of the property.
+.It Sy MAC_PROP_ADV_FEC_CAP
+.Bd -filled -compact
+Type:
+.Sy link_fec_t |
+Permissions:
+.Sy Read-Only
+.Ed
+.Pp
+The
+.Sy MAC_PROP_ADV_FEC_CAP
+has the same values as
+.Sy MAC_PROP_EN_FEC_CAP .
+The property indicates which Forward Error Correction (FEC) code has been
+negotiated over the link.
 .El
 .Pp
 The remaining properties are all about various auto-negotiation link
diff --git a/usr/src/uts/common/io/cxgbe/common/common.h b/usr/src/uts/common/io/cxgbe/common/common.h
index c7de2c4ebf..b8d77ebda3 100644
--- a/usr/src/uts/common/io/cxgbe/common/common.h
+++ b/usr/src/uts/common/io/cxgbe/common/common.h
@@ -20,6 +20,10 @@
  * release for licensing terms and conditions.
  */
 
+/*
+ * Copyright 2020 RackTop Systems, Inc.
+ */
+
 #ifndef __CHELSIO_COMMON_H
 #define __CHELSIO_COMMON_H
 
@@ -103,9 +107,16 @@ enum {
 typedef unsigned char cc_pause_t;
 
 enum {
-	FEC_AUTO	= 1 << 0,	/* IEEE 802.3 "automatic" */
-	FEC_RS		= 1 << 1,	/* Reed-Solomon */
-	FEC_BASER_RS	= 1 << 2,	/* BaseR/Reed-Solomon */
+	FEC_RS		= 1 << 0,	/* Reed-Solomon */
+	FEC_BASER_RS	= 1 << 1,	/* Base-R, aka Firecode */
+	FEC_NONE	= 1 << 2,	/* no FEC */
+
+	/*
+	 * Pseudo FECs that translate to real FECs.  The firmware knows nothing
+	 * about these and they start at M_FW_PORT_CAP32_FEC + 1.  AUTO should
+	 * be set all by itself.
+	 */
+	FEC_AUTO	= 1 << 5,
 };
 typedef unsigned char cc_fec_t;
 
diff --git a/usr/src/uts/common/io/cxgbe/common/t4_hw.c b/usr/src/uts/common/io/cxgbe/common/t4_hw.c
index ae88f36f15..4bb48f1b3a 100644
--- a/usr/src/uts/common/io/cxgbe/common/t4_hw.c
+++ b/usr/src/uts/common/io/cxgbe/common/t4_hw.c
@@ -20,6 +20,10 @@
  * release for licensing terms and conditions.
  */
 
+/*
+ * Copyright 2020 RackTop Systems, Inc.
+ */
+
 #include "common.h"
 #include "t4_regs.h"
 #include "t4_regs_values.h"
@@ -4645,20 +4649,57 @@ static inline cc_fec_t fwcap_to_cc_fec(fw_port_cap32_t fw_fec)
 	if (fw_fec & FW_PORT_CAP32_FEC_BASER_RS)
 		cc_fec |= FEC_BASER_RS;
 
-	return cc_fec;
+	if (cc_fec == 0)
+		cc_fec = FEC_NONE;
+
+	return (cc_fec);
 }
 
 /* Translate Common Code Forward Error Correction specification to Firmware */
-static inline fw_port_cap32_t cc_to_fwcap_fec(cc_fec_t cc_fec)
+static inline boolean_t
+cc_to_fwcap_fec(fw_port_cap32_t *fw_fecp, cc_fec_t cc_fec,
+    struct link_config *lc)
 {
 	fw_port_cap32_t fw_fec = 0;
 
-	if (cc_fec & FEC_RS)
+	if ((cc_fec & FEC_AUTO) != 0) {
+		if ((lc->pcaps & FW_PORT_CAP32_SPEED_100G) == 0)
+			fw_fec |= FW_PORT_CAP32_FEC_BASER_RS;
+
+		if ((lc->pcaps & FW_PORT_CAP32_FORCE_FEC) != 0)
+			fw_fec |= FW_PORT_CAP32_FEC_NO_FEC;
+
+		fw_fec |= FW_PORT_CAP32_FEC_RS;
+
+		*fw_fecp = fw_fec;
+		return (B_TRUE);
+	}
+
+	if ((cc_fec & FEC_RS) != 0)
 		fw_fec |= FW_PORT_CAP32_FEC_RS;
-	if (cc_fec & FEC_BASER_RS)
+
+	if ((cc_fec & FEC_BASER_RS) != 0 &&
+	    (lc->pcaps & FW_PORT_CAP32_SPEED_100G) == 0)
 		fw_fec |= FW_PORT_CAP32_FEC_BASER_RS;
 
-	return fw_fec;
+	if ((cc_fec & FEC_NONE) != 0) {
+		if ((lc->pcaps & FW_PORT_CAP32_FORCE_FEC) != 0) {
+			fw_fec |= FW_PORT_CAP32_FORCE_FEC;
+			fw_fec |= FW_PORT_CAP32_FEC_NO_FEC;
+		}
+
+		*fw_fecp = fw_fec;
+		return (B_TRUE);
+	}
+
+	if (fw_fec == 0)
+		return (B_FALSE);
+
+	if ((lc->pcaps & FW_PORT_CAP32_FORCE_FEC) != 0)
+		fw_fec |= FW_PORT_CAP32_FORCE_FEC;
+
+	*fw_fecp = fw_fec;
+	return (B_TRUE);
 }
 
 /**
@@ -4692,11 +4733,18 @@ fw_port_cap32_t t4_link_acaps(struct adapter *adapter, unsigned int port,
 	 * the Transceiver Module EPROM FEC parameters.  Otherwise we
 	 * use whatever is in the current Requested FEC settings.
 	 */
-	if (lc->requested_fec & FEC_AUTO)
-		cc_fec = fwcap_to_cc_fec(lc->def_acaps);
-	else
-		cc_fec = lc->requested_fec;
-	fw_fec = cc_to_fwcap_fec(cc_fec);
+	if (fec_supported(lc->pcaps)) {
+		if (lc->requested_fec & FEC_AUTO)
+			cc_fec = fwcap_to_cc_fec(lc->def_acaps);
+		else
+			cc_fec = lc->requested_fec;
+
+		if (!cc_to_fwcap_fec(&fw_fec, cc_fec, lc))
+			return (0);
+	} else {
+		fw_fec = 0;
+		cc_fec = FEC_NONE;
+	}
 
 	/* Figure out what our Requested Port Capabilities are going to be.
 	 * Note parallel structure in t4_handle_get_port_info() and
@@ -9641,12 +9689,17 @@ static void init_link_config(struct link_config *lc, fw_port_cap32_t pcaps,
 	lc->speed = 0;
 	lc->requested_fc = lc->fc = PAUSE_RX | PAUSE_TX;
 
-	/*
-	 * For Forward Error Control, we default to whatever the Firmware
-	 * tells us the Link is currently advertising.
-	 */
-	lc->requested_fec = FEC_AUTO;
-	lc->fec = fwcap_to_cc_fec(lc->def_acaps);
+	if (fec_supported(pcaps)) {
+		/*
+		 * For Forward Error Control, we default to whatever the Firmware
+		 * tells us the Link is currently advertising.
+		 */
+		lc->requested_fec = FEC_AUTO;
+		lc->fec = fwcap_to_cc_fec(lc->def_acaps);
+	} else {
+		lc->requested_fec = FEC_NONE;
+		lc->fec = FEC_NONE;
+	}
 
 	/* If the Port is capable of Auto-Negtotiation, initialize it as
 	 * "enabled" and copy over all of the Physical Port Capabilities
diff --git a/usr/src/uts/common/io/cxgbe/firmware/t4fw_interface.h b/usr/src/uts/common/io/cxgbe/firmware/t4fw_interface.h
index d705c73891..b998e85bae 100644
--- a/usr/src/uts/common/io/cxgbe/firmware/t4fw_interface.h
+++ b/usr/src/uts/common/io/cxgbe/firmware/t4fw_interface.h
@@ -11,6 +11,10 @@
  * release for licensing terms and conditions.
  */
 
+/*
+ * Copyright 2020 RackTop Systems, Inc.
+ */
+
 #ifndef _T4FW_INTERFACE_H_
 #define _T4FW_INTERFACE_H_
 
@@ -7204,11 +7208,12 @@ enum fw_port_mdi {
 #define	FW_PORT_CAP32_MDISTRAIGHT	0x00400000UL
 #define	FW_PORT_CAP32_FEC_RS		0x00800000UL
 #define	FW_PORT_CAP32_FEC_BASER_RS	0x01000000UL
-#define	FW_PORT_CAP32_FEC_RESERVED1	0x02000000UL
+#define	FW_PORT_CAP32_FEC_NO_FEC	0x02000000UL
 #define	FW_PORT_CAP32_FEC_RESERVED2	0x04000000UL
 #define	FW_PORT_CAP32_FEC_RESERVED3	0x08000000UL
 #define	FW_PORT_CAP32_FORCE_PAUSE	0x10000000UL
-#define	FW_PORT_CAP32_RESERVED2		0xe0000000UL
+#define	FW_PORT_CAP32_FORCE_FEC		0x20000000UL
+#define	FW_PORT_CAP32_RESERVED2		0xc0000000UL
 
 #define S_FW_PORT_CAP32_SPEED	0
 #define M_FW_PORT_CAP32_SPEED	0xfff
@@ -7254,7 +7259,7 @@ enum fw_port_mdi32 {
     (((x) >> S_FW_PORT_CAP32_MDI) & M_FW_PORT_CAP32_MDI)
 
 #define S_FW_PORT_CAP32_FEC	23
-#define M_FW_PORT_CAP32_FEC	0x1f
+#define M_FW_PORT_CAP32_FEC	0x5f
 #define V_FW_PORT_CAP32_FEC(x)	((x) << S_FW_PORT_CAP32_FEC)
 #define G_FW_PORT_CAP32_FEC(x) \
     (((x) >> S_FW_PORT_CAP32_FEC) & M_FW_PORT_CAP32_FEC)
@@ -7269,6 +7274,15 @@ enum fw_port_mdi32 {
 #define CAP32_FC(__cap32) \
 	(V_FW_PORT_CAP32_FC(M_FW_PORT_CAP32_FC) & __cap32)
 
+#ifdef _KERNEL
+static inline boolean_t
+fec_supported(uint32_t caps)
+{
+	return ((caps & (FW_PORT_CAP32_SPEED_25G | FW_PORT_CAP32_SPEED_50G |
+	    FW_PORT_CAP32_SPEED_100G)) != 0);
+}
+#endif
+
 enum fw_port_action {
 	FW_PORT_ACTION_L1_CFG		= 0x0001,
 	FW_PORT_ACTION_L2_CFG		= 0x0002,
diff --git a/usr/src/uts/common/io/cxgbe/t4nex/t4_mac.c b/usr/src/uts/common/io/cxgbe/t4nex/t4_mac.c
index 59c0ddde8d..9b4ffd8325 100644
--- a/usr/src/uts/common/io/cxgbe/t4nex/t4_mac.c
+++ b/usr/src/uts/common/io/cxgbe/t4nex/t4_mac.c
@@ -20,6 +20,10 @@
  * release for licensing terms and conditions.
  */
 
+/*
+ * Copyright 2020 RackTop Systems, Inc.
+ */
+
 #include <sys/ddi.h>
 #include <sys/sunddi.h>
 #include <sys/dlpi.h>
@@ -930,6 +934,62 @@ t4_mc_getcapab(void *arg, mac_capab_t cap, void *data)
 	return (status);
 }
 
+static link_fec_t
+fec_to_link_fec(cc_fec_t cc_fec)
+{
+	link_fec_t link_fec = 0;
+
+	if ((cc_fec & (FEC_RS | FEC_BASER_RS)) == (FEC_RS | FEC_BASER_RS))
+		return (LINK_FEC_AUTO);
+
+	if ((cc_fec & FEC_NONE) != 0)
+		link_fec |= LINK_FEC_NONE;
+
+	if ((cc_fec & FEC_AUTO) != 0)
+		link_fec |= LINK_FEC_AUTO;
+
+	if ((cc_fec & FEC_RS) != 0)
+		link_fec |= LINK_FEC_RS;
+
+	if ((cc_fec & FEC_BASER_RS) != 0)
+		link_fec |= LINK_FEC_BASE_R;
+
+	return (link_fec);
+}
+
+static int
+link_fec_to_fec(int v)
+{
+	int fec = 0;
+
+	if ((v & LINK_FEC_AUTO) != 0) {
+		fec = FEC_AUTO;
+		v &= ~LINK_FEC_AUTO;
+	} else {
+		if ((v & LINK_FEC_NONE) != 0) {
+			fec = FEC_NONE;
+			v &= ~LINK_FEC_NONE;
+		}
+
+		if ((v & LINK_FEC_RS) != 0) {
+			fec |= FEC_RS;
+			v &= ~LINK_FEC_RS;
+		}
+
+		if ((v & LINK_FEC_BASE_R) != 0) {
+			fec |= FEC_BASER_RS;
+			v &= ~LINK_FEC_BASE_R;
+		}
+	}
+
+	if (v != 0)
+		return (-1);
+
+	ASSERT3S(fec, !=, 0);
+
+	return (fec);
+}
+
 /* ARGSUSED */
 static int
 t4_mc_setprop(void *arg, const char *name, mac_prop_id_t id, uint_t size,
@@ -941,7 +1001,9 @@ t4_mc_setprop(void *arg, const char *name, mac_prop_id_t id, uint_t size,
 	uint8_t v8 = *(uint8_t *)val;
 	uint32_t v32 = *(uint32_t *)val;
 	int old, new = 0, relink = 0, rx_mode = 0, rc = 0;
+	boolean_t down_link = B_TRUE;
 	link_flowctrl_t fc;
+	link_fec_t fec;
 
 	/*
 	 * Save a copy of link_config. This can be used to restore link_config
@@ -1009,6 +1071,30 @@ t4_mc_setprop(void *arg, const char *name, mac_prop_id_t id, uint_t size,
 		}
 		break;
 
+	case MAC_PROP_EN_FEC_CAP:
+		if (!fec_supported(lc->pcaps)) {
+			rc = ENOTSUP;
+			break;
+		}
+
+		fec = *(link_fec_t *)val;
+		new = link_fec_to_fec(fec);
+		if (new < 0) {
+			rc = EINVAL;
+		} else if (new != lc->requested_fec) {
+			lc->requested_fec = new;
+			relink = 1;
+			/*
+			 * For fec, do not preemptively force the link
+			 * down. If changing fec causes the link state
+			 * to transition, then appropriate asynchronous
+			 * events are generated which correctly reflect
+			 * the link state.
+			 */
+			down_link = B_FALSE;
+		}
+		break;
+
 	case MAC_PROP_EN_10GFDX_CAP:
 		if (lc->pcaps & FW_PORT_CAP32_ANEG && is_10G_port(pi)) {
 			old = lc->acaps & FW_PORT_CAP32_SPEED_10G;
@@ -1062,7 +1148,8 @@ t4_mc_setprop(void *arg, const char *name, mac_prop_id_t id, uint_t size,
 
 	if (isset(&sc->open_device_map, pi->port_id) != 0) {
 		if (relink != 0) {
-			t4_os_link_changed(pi->adapter, pi->port_id, 0);
+			if (down_link)
+				t4_os_link_changed(pi->adapter, pi->port_id, 0);
 			rc = begin_synchronized_op(pi, 1, 1);
 			if (rc != 0)
 				return (rc);
@@ -1143,6 +1230,20 @@ t4_mc_getprop(void *arg, const char *name, mac_prop_id_t id, uint_t size,
 			*(link_flowctrl_t *)val = LINK_FLOWCTRL_NONE;
 		break;
 
+	case MAC_PROP_ADV_FEC_CAP:
+		if (!fec_supported(lc->pcaps))
+			return (ENOTSUP);
+
+		*(link_fec_t *)val = fec_to_link_fec(lc->fec);
+		break;
+
+	case MAC_PROP_EN_FEC_CAP:
+		if (!fec_supported(lc->pcaps))
+			return (ENOTSUP);
+
+		*(link_fec_t *)val = fec_to_link_fec(lc->requested_fec);
+		break;
+
 	case MAC_PROP_ADV_100GFDX_CAP:
 	case MAC_PROP_EN_100GFDX_CAP:
 		*u = !!(lc->acaps & FW_PORT_CAP32_SPEED_100G);
@@ -1212,6 +1313,15 @@ t4_mc_propinfo(void *arg, const char *name, mac_prop_id_t id,
 		mac_prop_info_set_default_link_flowctrl(ph, LINK_FLOWCTRL_BI);
 		break;
 
+	case MAC_PROP_EN_FEC_CAP:
+		mac_prop_info_set_default_fec(ph, LINK_FEC_AUTO);
+		break;
+
+	case MAC_PROP_ADV_FEC_CAP:
+		mac_prop_info_set_perm(ph, MAC_PROP_PERM_READ);
+		mac_prop_info_set_default_fec(ph, LINK_FEC_AUTO);
+		break;
+
 	case MAC_PROP_EN_10GFDX_CAP:
 		if (lc->pcaps & FW_PORT_CAP32_ANEG &&
 		    lc->pcaps & FW_PORT_CAP32_SPEED_10G)
diff --git a/usr/src/uts/common/io/mac/mac.c b/usr/src/uts/common/io/mac/mac.c
index 0a52043a15..707fca24d0 100644
--- a/usr/src/uts/common/io/mac/mac.c
+++ b/usr/src/uts/common/io/mac/mac.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2020 Joyent, Inc.
  * Copyright 2015 Garrett D'Amore <garrett@damore.org>
+ * Copyright 2020 RackTop Systems, Inc.
  */
 
 /*
@@ -3336,6 +3337,10 @@ mac_prop_check_size(mac_prop_id_t id, uint_t valsize, boolean_t is_range)
 	case MAC_PROP_FLOWCTRL:
 		minsize = sizeof (link_flowctrl_t);
 		break;
+	case MAC_PROP_ADV_FEC_CAP:
+	case MAC_PROP_EN_FEC_CAP:
+		minsize = sizeof (link_fec_t);
+		break;
 	case MAC_PROP_ADV_5000FDX_CAP:
 	case MAC_PROP_EN_5000FDX_CAP:
 	case MAC_PROP_ADV_2500FDX_CAP:
@@ -3524,6 +3529,28 @@ mac_set_prop(mac_handle_t mh, mac_prop_id_t id, char *name, void *val,
 		break;
 	}
 
+	case MAC_PROP_ADV_FEC_CAP:
+	case MAC_PROP_EN_FEC_CAP: {
+		link_fec_t fec;
+
+		ASSERT(valsize >= sizeof (link_fec_t));
+
+		/*
+		 * fec cannot be zero, and auto must be set exclusively.
+		 */
+		bcopy(val, &fec, sizeof (link_fec_t));
+		if (fec == 0)
+			return (EINVAL);
+		if ((fec & LINK_FEC_AUTO) != 0 && (fec & ~LINK_FEC_AUTO) != 0)
+			return (EINVAL);
+
+		if (mip->mi_callbacks->mc_callbacks & MC_SETPROP) {
+			err = mip->mi_callbacks->mc_setprop(mip->mi_driver,
+			    name, id, valsize, val);
+		}
+		break;
+	}
+
 	default:
 		/* For other driver properties, call driver's callback */
 		if (mip->mi_callbacks->mc_callbacks & MC_SETPROP) {
diff --git a/usr/src/uts/common/io/mac/mac_provider.c b/usr/src/uts/common/io/mac/mac_provider.c
index ce986fd4bf..bfaf232d25 100644
--- a/usr/src/uts/common/io/mac/mac_provider.c
+++ b/usr/src/uts/common/io/mac/mac_provider.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2019 Joyent, Inc.
  * Copyright 2017 OmniTI Computer Consulting, Inc. All rights reserved.
+ * Copyright 2020 RackTop Systems, Inc.
  */
 
 #include <sys/types.h>
@@ -1527,6 +1528,22 @@ mac_prop_info_set_default_link_flowctrl(mac_prop_info_handle_t ph,
 }
 
 void
+mac_prop_info_set_default_fec(mac_prop_info_handle_t ph, link_fec_t val)
+{
+	mac_prop_info_state_t *pr = (mac_prop_info_state_t *)ph;
+
+	/* nothing to do if the caller doesn't want the default value */
+	if (pr->pr_default == NULL)
+		return;
+
+	ASSERT(pr->pr_default_size >= sizeof (link_fec_t));
+
+	bcopy(&val, pr->pr_default, sizeof (val));
+
+	pr->pr_flags |= MAC_PROP_INFO_DEFAULT;
+}
+
+void
 mac_prop_info_set_range_uint32(mac_prop_info_handle_t ph, uint32_t min,
     uint32_t max)
 {
diff --git a/usr/src/uts/common/io/mlxcx/mlxcx.c b/usr/src/uts/common/io/mlxcx/mlxcx.c
index 9fae7c5f77..2aefac33db 100644
--- a/usr/src/uts/common/io/mlxcx/mlxcx.c
+++ b/usr/src/uts/common/io/mlxcx/mlxcx.c
@@ -1756,6 +1756,11 @@ mlxcx_setup_ports(mlxcx_t *mlxp)
 			mutex_exit(&p->mlp_mtx);
 			goto err;
 		}
+		if (!mlxcx_cmd_query_port_fec(mlxp, p)) {
+			mutex_exit(&p->mlp_mtx);
+			goto err;
+		}
+		p->mlp_fec_requested = LINK_FEC_AUTO;
 
 		mutex_exit(&p->mlp_mtx);
 	}
diff --git a/usr/src/uts/common/io/mlxcx/mlxcx.h b/usr/src/uts/common/io/mlxcx/mlxcx.h
index 52240df3a3..06277d033c 100644
--- a/usr/src/uts/common/io/mlxcx/mlxcx.h
+++ b/usr/src/uts/common/io/mlxcx/mlxcx.h
@@ -346,6 +346,8 @@ typedef struct mlxcx_port {
 	mlxcx_eth_proto_t	mlp_max_proto;
 	mlxcx_eth_proto_t	mlp_admin_proto;
 	mlxcx_eth_proto_t	mlp_oper_proto;
+	mlxcx_pplm_fec_active_t	mlp_fec_active;
+	link_fec_t		mlp_fec_requested;
 
 	mlxcx_eth_inline_mode_t	mlp_wqe_min_inline;
 
@@ -1320,7 +1322,12 @@ extern boolean_t mlxcx_cmd_access_register(mlxcx_t *, mlxcx_cmd_reg_opmod_t,
     mlxcx_register_id_t, mlxcx_register_data_t *);
 extern boolean_t mlxcx_cmd_query_port_mtu(mlxcx_t *, mlxcx_port_t *);
 extern boolean_t mlxcx_cmd_query_port_status(mlxcx_t *, mlxcx_port_t *);
+extern boolean_t mlxcx_cmd_modify_port_status(mlxcx_t *, mlxcx_port_t *,
+    mlxcx_port_status_t);
 extern boolean_t mlxcx_cmd_query_port_speed(mlxcx_t *, mlxcx_port_t *);
+extern boolean_t mlxcx_cmd_query_port_fec(mlxcx_t *, mlxcx_port_t *);
+extern boolean_t mlxcx_cmd_modify_port_fec(mlxcx_t *, mlxcx_port_t *,
+    mlxcx_pplm_fec_caps_t);
 
 extern boolean_t mlxcx_cmd_set_port_mtu(mlxcx_t *, mlxcx_port_t *);
 
diff --git a/usr/src/uts/common/io/mlxcx/mlxcx_cmd.c b/usr/src/uts/common/io/mlxcx/mlxcx_cmd.c
index 30fb7ca8ef..f059b856a6 100644
--- a/usr/src/uts/common/io/mlxcx/mlxcx_cmd.c
+++ b/usr/src/uts/common/io/mlxcx/mlxcx_cmd.c
@@ -12,6 +12,7 @@
 /*
  * Copyright 2020, The University of Queensland
  * Copyright (c) 2018, Joyent, Inc.
+ * Copyright 2020 RackTop Systems, Inc.
  */
 
 /*
@@ -1594,6 +1595,8 @@ mlxcx_reg_name(mlxcx_register_id_t rid)
 		return ("MCIA");
 	case MLXCX_REG_PPCNT:
 		return ("PPCNT");
+	case MLXCX_REG_PPLM:
+		return ("PPLM");
 	default:
 		return ("???");
 	}
@@ -1640,6 +1643,9 @@ mlxcx_cmd_access_register(mlxcx_t *mlxp, mlxcx_cmd_reg_opmod_t opmod,
 	case MLXCX_REG_PPCNT:
 		dsize = sizeof (mlxcx_reg_ppcnt_t);
 		break;
+	case MLXCX_REG_PPLM:
+		dsize = sizeof (mlxcx_reg_pplm_t);
+		break;
 	default:
 		dsize = 0;
 		VERIFY(0);
@@ -1776,6 +1782,25 @@ mlxcx_cmd_query_port_status(mlxcx_t *mlxp, mlxcx_port_t *mlp)
 }
 
 boolean_t
+mlxcx_cmd_modify_port_status(mlxcx_t *mlxp, mlxcx_port_t *mlp,
+    mlxcx_port_status_t status)
+{
+	mlxcx_register_data_t data;
+	boolean_t ret;
+
+	ASSERT(mutex_owned(&mlp->mlp_mtx));
+	bzero(&data, sizeof (data));
+	data.mlrd_paos.mlrd_paos_local_port = mlp->mlp_num + 1;
+	data.mlrd_paos.mlrd_paos_admin_status = status;
+	set_bit32(&data.mlrd_paos.mlrd_paos_flags, MLXCX_PAOS_ADMIN_ST_EN);
+
+	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE,
+	    MLXCX_REG_PAOS, &data);
+
+	return (ret);
+}
+
+boolean_t
 mlxcx_cmd_query_port_speed(mlxcx_t *mlxp, mlxcx_port_t *mlp)
 {
 	mlxcx_register_data_t data;
@@ -1809,6 +1834,82 @@ mlxcx_cmd_query_port_speed(mlxcx_t *mlxp, mlxcx_port_t *mlp)
 }
 
 boolean_t
+mlxcx_cmd_query_port_fec(mlxcx_t *mlxp, mlxcx_port_t *mlp)
+{
+	mlxcx_register_data_t data;
+	boolean_t ret;
+
+	ASSERT(mutex_owned(&mlp->mlp_mtx));
+	bzero(&data, sizeof (data));
+	data.mlrd_pplm.mlrd_pplm_local_port = mlp->mlp_num + 1;
+
+	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ,
+	    MLXCX_REG_PPLM, &data);
+
+	if (ret) {
+		mlp->mlp_fec_active =
+		    from_be24(data.mlrd_pplm.mlrd_pplm_fec_mode_active);
+	}
+
+	return (ret);
+}
+
+boolean_t
+mlxcx_cmd_modify_port_fec(mlxcx_t *mlxp, mlxcx_port_t *mlp,
+    mlxcx_pplm_fec_caps_t fec)
+{
+	mlxcx_register_data_t data_in, data_out;
+	mlxcx_pplm_fec_caps_t caps;
+	mlxcx_reg_pplm_t *pplm_in, *pplm_out;
+	boolean_t ret;
+
+	ASSERT(mutex_owned(&mlp->mlp_mtx));
+	bzero(&data_in, sizeof (data_in));
+	pplm_in = &data_in.mlrd_pplm;
+	pplm_in->mlrd_pplm_local_port = mlp->mlp_num + 1;
+
+	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ,
+	    MLXCX_REG_PPLM, &data_in);
+
+	if (!ret)
+		return (B_FALSE);
+
+	bzero(&data_out, sizeof (data_out));
+	pplm_out = &data_out.mlrd_pplm;
+	pplm_out->mlrd_pplm_local_port = mlp->mlp_num + 1;
+
+	caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap,
+	    MLXCX_PPLM_CAP_56G);
+	set_bits32(&pplm_out->mlrd_pplm_fec_override_admin,
+	    MLXCX_PPLM_CAP_56G, fec & caps);
+
+	caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap,
+	    MLXCX_PPLM_CAP_100G);
+	set_bits32(&pplm_out->mlrd_pplm_fec_override_admin,
+	    MLXCX_PPLM_CAP_100G, fec & caps);
+
+	caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap,
+	    MLXCX_PPLM_CAP_50G);
+	set_bits32(&pplm_out->mlrd_pplm_fec_override_admin,
+	    MLXCX_PPLM_CAP_50G, fec & caps);
+
+	caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap,
+	    MLXCX_PPLM_CAP_25G);
+	set_bits32(&pplm_out->mlrd_pplm_fec_override_admin,
+	    MLXCX_PPLM_CAP_25G, fec & caps);
+
+	caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap,
+	    MLXCX_PPLM_CAP_10_40G);
+	set_bits32(&pplm_out->mlrd_pplm_fec_override_admin,
+	    MLXCX_PPLM_CAP_10_40G, fec & caps);
+
+	ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE,
+	    MLXCX_REG_PPLM, &data_out);
+
+	return (ret);
+}
+
+boolean_t
 mlxcx_cmd_modify_nic_vport_ctx(mlxcx_t *mlxp, mlxcx_port_t *mlp,
     mlxcx_modify_nic_vport_ctx_fields_t fields)
 {
diff --git a/usr/src/uts/common/io/mlxcx/mlxcx_gld.c b/usr/src/uts/common/io/mlxcx/mlxcx_gld.c
index 5d15ec1fbb..2521641a00 100644
--- a/usr/src/uts/common/io/mlxcx/mlxcx_gld.c
+++ b/usr/src/uts/common/io/mlxcx/mlxcx_gld.c
@@ -80,6 +80,53 @@ mlxcx_speed_to_bits(mlxcx_eth_proto_t v)
 	}
 }
 
+static link_fec_t
+mlxcx_fec_to_link_fec(mlxcx_pplm_fec_active_t mlxcx_fec)
+{
+	if ((mlxcx_fec & MLXCX_PPLM_FEC_ACTIVE_NONE) != 0)
+		return (LINK_FEC_NONE);
+
+	if ((mlxcx_fec & MLXCX_PPLM_FEC_ACTIVE_FIRECODE) != 0)
+		return (LINK_FEC_BASE_R);
+
+	if ((mlxcx_fec & (MLXCX_PPLM_FEC_ACTIVE_RS528 |
+	    MLXCX_PPLM_FEC_ACTIVE_RS271 | MLXCX_PPLM_FEC_ACTIVE_RS544 |
+	    MLXCX_PPLM_FEC_ACTIVE_RS272)) != 0)
+		return (LINK_FEC_RS);
+
+	return (LINK_FEC_NONE);
+}
+
+static boolean_t
+mlxcx_link_fec_cap(link_fec_t fec, mlxcx_pplm_fec_caps_t *pfecp)
+{
+	mlxcx_pplm_fec_caps_t pplm_fec = 0;
+
+	if ((fec & LINK_FEC_AUTO) != 0) {
+		pplm_fec = MLXCX_PPLM_FEC_CAP_AUTO;
+		fec &= ~LINK_FEC_AUTO;
+	} else if ((fec & LINK_FEC_NONE) != 0) {
+		pplm_fec = MLXCX_PPLM_FEC_CAP_NONE;
+		fec &= ~LINK_FEC_NONE;
+	} else if ((fec & LINK_FEC_RS) != 0) {
+		pplm_fec |= MLXCX_PPLM_FEC_CAP_RS;
+		fec &= ~LINK_FEC_RS;
+	} else if ((fec & LINK_FEC_BASE_R) != 0) {
+		pplm_fec |= MLXCX_PPLM_FEC_CAP_FIRECODE;
+		fec &= ~LINK_FEC_BASE_R;
+	}
+
+	/*
+	 * Only one fec option is allowed.
+	 */
+	if (fec != 0)
+		return (B_FALSE);
+
+	*pfecp = pplm_fec;
+
+	return (B_TRUE);
+}
+
 static int
 mlxcx_mac_stat_rfc_2863(mlxcx_t *mlxp, mlxcx_port_t *port, uint_t stat,
     uint64_t *val)
@@ -1091,6 +1138,14 @@ mlxcx_mac_propinfo(void *arg, const char *pr_name, mac_prop_id_t pr_num,
 		mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ);
 		mac_prop_info_set_default_uint8(prh, 1);
 		break;
+	case MAC_PROP_ADV_FEC_CAP:
+		mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ);
+		mac_prop_info_set_default_fec(prh, LINK_FEC_AUTO);
+		break;
+	case MAC_PROP_EN_FEC_CAP:
+		mac_prop_info_set_perm(prh, MAC_PROP_PERM_RW);
+		mac_prop_info_set_default_fec(prh, LINK_FEC_AUTO);
+		break;
 	case MAC_PROP_ADV_100GFDX_CAP:
 	case MAC_PROP_EN_100GFDX_CAP:
 		mac_prop_info_set_perm(prh, MAC_PROP_PERM_READ);
@@ -1150,6 +1205,9 @@ mlxcx_mac_setprop(void *arg, const char *pr_name, mac_prop_id_t pr_num,
 	uint32_t new_mtu, new_hw_mtu, old_mtu;
 	mlxcx_buf_shard_t *sh;
 	boolean_t allocd = B_FALSE;
+	boolean_t relink = B_FALSE;
+	link_fec_t fec;
+	mlxcx_pplm_fec_caps_t cap_fec;
 
 	mutex_enter(&port->mlp_mtx);
 
@@ -1198,11 +1256,57 @@ mlxcx_mac_setprop(void *arg, const char *pr_name, mac_prop_id_t pr_num,
 			break;
 		}
 		break;
+
+	case MAC_PROP_EN_FEC_CAP:
+		bcopy(pr_val, &fec, sizeof (fec));
+		if (!mlxcx_link_fec_cap(fec, &cap_fec)) {
+			ret = EINVAL;
+			break;
+		}
+
+		/*
+		 * Don't change the FEC if it is already at the requested
+		 * setting AND the port is up.
+		 * When the port is down, always set the FEC and attempt
+		 * to retrain the link.
+		 */
+		if (fec == port->mlp_fec_requested &&
+		    fec == mlxcx_fec_to_link_fec(port->mlp_fec_active) &&
+		    port->mlp_oper_status != MLXCX_PORT_STATUS_DOWN)
+			break;
+
+		/*
+		 * The most like cause of this failing is an invalid
+		 * or unsupported fec option.
+		 */
+		if (!mlxcx_cmd_modify_port_fec(mlxp, port, cap_fec)) {
+			ret = EINVAL;
+			break;
+		}
+
+		port->mlp_fec_requested = fec;
+
+		/*
+		 * For FEC to become effective, the link needs to go back
+		 * to training and negotiation state. This happens when
+		 * the link transitions from down to up, force a relink.
+		 */
+		relink = B_TRUE;
+		break;
+
 	default:
 		ret = ENOTSUP;
 		break;
 	}
 
+	if (relink) {
+		if (!mlxcx_cmd_modify_port_status(mlxp, port,
+		    MLXCX_PORT_STATUS_DOWN) ||
+		    !mlxcx_cmd_modify_port_status(mlxp, port,
+		    MLXCX_PORT_STATUS_UP)) {
+			ret = EIO;
+		}
+	}
 	mutex_exit(&port->mlp_mtx);
 
 	return (ret);
@@ -1260,6 +1364,21 @@ mlxcx_mac_getprop(void *arg, const char *pr_name, mac_prop_id_t pr_num,
 		}
 		*(uint8_t *)pr_val = port->mlp_autoneg;
 		break;
+	case MAC_PROP_ADV_FEC_CAP:
+		if (pr_valsize < sizeof (link_fec_t)) {
+			ret = EOVERFLOW;
+			break;
+		}
+		*(link_fec_t *)pr_val =
+		    mlxcx_fec_to_link_fec(port->mlp_fec_active);
+		break;
+	case MAC_PROP_EN_FEC_CAP:
+		if (pr_valsize < sizeof (link_fec_t)) {
+			ret = EOVERFLOW;
+			break;
+		}
+		*(link_fec_t *)pr_val = port->mlp_fec_requested;
+		break;
 	case MAC_PROP_MTU:
 		if (pr_valsize < sizeof (uint32_t)) {
 			ret = EOVERFLOW;
diff --git a/usr/src/uts/common/io/mlxcx/mlxcx_intr.c b/usr/src/uts/common/io/mlxcx/mlxcx_intr.c
index 4dc4291b08..aed691897b 100644
--- a/usr/src/uts/common/io/mlxcx/mlxcx_intr.c
+++ b/usr/src/uts/common/io/mlxcx/mlxcx_intr.c
@@ -355,6 +355,7 @@ mlxcx_update_link_state(mlxcx_t *mlxp, mlxcx_port_t *port)
 	mutex_enter(&port->mlp_mtx);
 	(void) mlxcx_cmd_query_port_status(mlxp, port);
 	(void) mlxcx_cmd_query_port_speed(mlxp, port);
+	(void) mlxcx_cmd_query_port_fec(mlxp, port);
 
 	switch (port->mlp_oper_status) {
 	case MLXCX_PORT_STATUS_UP:
diff --git a/usr/src/uts/common/io/mlxcx/mlxcx_reg.h b/usr/src/uts/common/io/mlxcx/mlxcx_reg.h
index 6d09abea5c..abd717842d 100644
--- a/usr/src/uts/common/io/mlxcx/mlxcx_reg.h
+++ b/usr/src/uts/common/io/mlxcx/mlxcx_reg.h
@@ -2464,6 +2464,59 @@ typedef struct {
 } mlxcx_reg_ppcnt_t;
 
 typedef enum {
+	MLXCX_PPLM_FEC_CAP_AUTO			= 0,
+	MLXCX_PPLM_FEC_CAP_NONE			= (1 << 0),
+	MLXCX_PPLM_FEC_CAP_FIRECODE		= (1 << 1),
+	MLXCX_PPLM_FEC_CAP_RS			= (1 << 2),
+} mlxcx_pplm_fec_caps_t;
+
+typedef enum {
+	MLXCX_PPLM_FEC_ACTIVE_NONE		= (1 << 0),
+	MLXCX_PPLM_FEC_ACTIVE_FIRECODE		= (1 << 1),
+	MLXCX_PPLM_FEC_ACTIVE_RS528		= (1 << 2),
+	MLXCX_PPLM_FEC_ACTIVE_RS271		= (1 << 3),
+	MLXCX_PPLM_FEC_ACTIVE_RS544		= (1 << 7),
+	MLXCX_PPLM_FEC_ACTIVE_RS272		= (1 << 9),
+} mlxcx_pplm_fec_active_t;
+
+/* CSTYLED */
+#define	MLXCX_PPLM_CAP_56G		(bitdef_t){ 16, 0x000f0000 }
+/* CSTYLED */
+#define	MLXCX_PPLM_CAP_100G		(bitdef_t){ 12, 0x0000f000 }
+/* CSTYLED */
+#define	MLXCX_PPLM_CAP_50G		(bitdef_t){ 8, 0x00000f00 }
+/* CSTYLED */
+#define	MLXCX_PPLM_CAP_25G		(bitdef_t){ 4, 0x000000f0 }
+/* CSTYLED */
+#define	MLXCX_PPLM_CAP_10_40G		(bitdef_t){ 0, 0x0000000f }
+
+typedef struct {
+	uint8_t		mlrd_pplm_rsvd;
+	uint8_t		mlrd_pplm_local_port;
+	uint8_t		mlrd_pplm_rsvd1[11];
+	uint24be_t	mlrd_pplm_fec_mode_active;
+	bits32_t	mlrd_pplm_fec_override_cap;
+	bits32_t	mlrd_pplm_fec_override_admin;
+	uint16be_t	mlrd_pplm_fec_override_cap_400g_8x;
+	uint16be_t	mlrd_pplm_fec_override_cap_200g_4x;
+	uint16be_t	mlrd_pplm_fec_override_cap_100g_2x;
+	uint16be_t	mlrd_pplm_fec_override_cap_50g_1x;
+	uint16be_t	mlrd_pplm_fec_override_admin_400g_8x;
+	uint16be_t	mlrd_pplm_fec_override_admin_200g_4x;
+	uint16be_t	mlrd_pplm_fec_override_admin_100g_2x;
+	uint16be_t	mlrd_pplm_fec_override_admin_50g_1x;
+	uint8_t		mlrd_pplm_rsvd2[8];
+	uint16be_t	mlrd_pplm_fec_override_cap_hdr;
+	uint16be_t	mlrd_pplm_fec_override_cap_edr;
+	uint16be_t	mlrd_pplm_fec_override_cap_fdr;
+	uint16be_t	mlrd_pplm_fec_override_cap_fdr10;
+	uint16be_t	mlrd_pplm_fec_override_admin_hdr;
+	uint16be_t	mlrd_pplm_fec_override_admin_edr;
+	uint16be_t	mlrd_pplm_fec_override_admin_fdr;
+	uint16be_t	mlrd_pplm_fec_override_admin_fdr10;
+} mlxcx_reg_pplm_t;
+
+typedef enum {
 	MLXCX_REG_PMTU		= 0x5003,
 	MLXCX_REG_PTYS		= 0x5004,
 	MLXCX_REG_PAOS		= 0x5006,
@@ -2472,6 +2525,7 @@ typedef enum {
 	MLXCX_REG_MLCR		= 0x902B,
 	MLXCX_REG_MCIA		= 0x9014,
 	MLXCX_REG_PPCNT		= 0x5008,
+	MLXCX_REG_PPLM		= 0x5023,
 } mlxcx_register_id_t;
 
 typedef union {
@@ -2482,6 +2536,7 @@ typedef union {
 	mlxcx_reg_pmaos_t		mlrd_pmaos;
 	mlxcx_reg_mcia_t		mlrd_mcia;
 	mlxcx_reg_ppcnt_t		mlrd_ppcnt;
+	mlxcx_reg_pplm_t		mlrd_pplm;
 } mlxcx_register_data_t;
 
 typedef enum {
diff --git a/usr/src/uts/common/sys/mac.h b/usr/src/uts/common/sys/mac.h
index 2ce448fc3d..00d9901719 100644
--- a/usr/src/uts/common/sys/mac.h
+++ b/usr/src/uts/common/sys/mac.h
@@ -23,6 +23,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2018 Joyent, Inc.
  * Copyright (c) 2015 Garrett D'Amore <garrett@damore.org>
+ * Copyright 2020 RackTop Systems, Inc.
  */
 
 #ifndef	_SYS_MAC_H
@@ -88,6 +89,13 @@ typedef enum {
 } link_flowctrl_t;
 
 typedef enum {
+	LINK_FEC_NONE		= 1 << 0,
+	LINK_FEC_AUTO		= 1 << 1,
+	LINK_FEC_RS		= 1 << 2,
+	LINK_FEC_BASE_R		= 1 << 3
+} link_fec_t;
+
+typedef enum {
 	LINK_TAGMODE_VLANONLY = 0,
 	LINK_TAGMODE_NORMAL
 } link_tagmode_t;
@@ -230,6 +238,8 @@ typedef enum {
 	MAC_PROP_EN_25GFDX_CAP,
 	MAC_PROP_ADV_50GFDX_CAP,
 	MAC_PROP_EN_50GFDX_CAP,
+	MAC_PROP_EN_FEC_CAP,
+	MAC_PROP_ADV_FEC_CAP,
 	MAC_PROP_PRIVATE = -1
 } mac_prop_id_t;
 
diff --git a/usr/src/uts/common/sys/mac_provider.h b/usr/src/uts/common/sys/mac_provider.h
index 9cc5c1ad5c..2cb326814a 100644
--- a/usr/src/uts/common/sys/mac_provider.h
+++ b/usr/src/uts/common/sys/mac_provider.h
@@ -22,6 +22,7 @@
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2018, Joyent, Inc.
+ * Copyright 2020 RackTop Systems, Inc.
  */
 
 #ifndef	_SYS_MAC_PROVIDER_H
@@ -630,6 +631,8 @@ extern void			mac_prop_info_set_default_uint32(
 				    mac_prop_info_handle_t, uint32_t);
 extern void			mac_prop_info_set_default_link_flowctrl(
 				    mac_prop_info_handle_t, link_flowctrl_t);
+extern void			mac_prop_info_set_default_fec(
+				    mac_prop_info_handle_t, link_fec_t);
 extern void			mac_prop_info_set_range_uint32(
 				    mac_prop_info_handle_t,
 				    uint32_t, uint32_t);