path: root/usr/src/uts/common/io/i40e/i40e_sw.h

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2015 OmniTI Computer Consulting, Inc. All rights reserved.
 * Copyright 2019 Joyent, Inc.
 * Copyright 2017 Tegile Systems, Inc.  All rights reserved.
 * Copyright 2020 Ryan Zezeski
 * Copyright 2020 RackTop Systems, Inc.
 */

/*
 * Please see i40e_main.c for an introduction to the device driver, its layout,
 * and more.
 */

#ifndef	_I40E_SW_H
#define	_I40E_SW_H

#ifdef __cplusplus
extern "C" {
#endif

#include <sys/types.h>
#include <sys/conf.h>
#include <sys/debug.h>
#include <sys/stropts.h>
#include <sys/stream.h>
#include <sys/strsun.h>
#include <sys/strlog.h>
#include <sys/kmem.h>
#include <sys/stat.h>
#include <sys/kstat.h>
#include <sys/modctl.h>
#include <sys/errno.h>
#include <sys/dlpi.h>
#include <sys/mac_provider.h>
#include <sys/mac_ether.h>
#include <sys/vlan.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/pci.h>
#include <sys/pcie.h>
#include <sys/sdt.h>
#include <sys/ethernet.h>
#include <sys/pattr.h>
#include <sys/strsubr.h>
#include <sys/netlb.h>
#include <sys/random.h>
#include <inet/common.h>
#include <inet/tcp.h>
#include <inet/ip.h>
#include <inet/mi.h>
#include <inet/nd.h>
#include <netinet/udp.h>
#include <netinet/sctp.h>
#include <sys/bitmap.h>
#include <sys/cpuvar.h>
#include <sys/ddifm.h>
#include <sys/fm/protocol.h>
#include <sys/fm/util.h>
#include <sys/disp.h>
#include <sys/fm/io/ddi.h>
#include <sys/list.h>
#include <sys/debug.h>
#include <sys/sdt.h>
#include <sys/ddi_ufm.h>
#include "i40e_type.h"
#include "i40e_osdep.h"
#include "i40e_prototype.h"
#include "i40e_xregs.h"

#define	I40E_MODULE_NAME "i40e"

#define	I40E_ADAPTER_REGSET	1

/*
 * Configuration constants. Note that the hardware defines a minimum bound of 32
 * descriptors and requires that the programming of the descriptor lengths be
 * aligned in units of 32 descriptors.
 */
#define	I40E_MIN_TX_RING_SIZE	64
#define	I40E_MAX_TX_RING_SIZE	4096
#define	I40E_DEF_TX_RING_SIZE	1024

/*
 * Place an artificial limit on the max number of groups. The X710
 * series supports up to 384 VSIs to be partitioned across PFs as the
 * driver sees fit. But until we support more interrupts this seems
 * like a good place to start.
 */
#define	I40E_MIN_NUM_RX_GROUPS	1
#define	I40E_MAX_NUM_RX_GROUPS	32
#define	I40E_DEF_NUM_RX_GROUPS	16

#define	I40E_MIN_RX_RING_SIZE	64
#define	I40E_MAX_RX_RING_SIZE	4096
#define	I40E_DEF_RX_RING_SIZE	1024

#define	I40E_DESC_ALIGN		32

/*
 * Sizes used for asynchronous processing of the adminq. We allocate a fixed
 * size buffer for each instance of the device during attach time, rather than
 * allocating and freeing one during interrupt processing.
 *
 * We also define the descriptor size of the admin queue here.
 */
#define	I40E_ADMINQ_BUFSZ	4096
#define	I40E_MAX_ADMINQ_SIZE	1024
#define	I40E_DEF_ADMINQ_SIZE	256

/*
 * Note, while the min and maximum values are based upon the sizing of the ring
 * itself, the default is taken from ixgbe without much thought. It's basically
 * been cargo culted. See i40e_transceiver.c for a bit more information.
 */
#define	I40E_MIN_RX_LIMIT_PER_INTR	16
#define	I40E_MAX_RX_LIMIT_PER_INTR	4096
#define	I40E_DEF_RX_LIMIT_PER_INTR	256

/*
 * Valid MTU ranges. Note that the XL710's maximum payload is actually 9728.
 * However, we need to adjust for the ETHERFCSL (4 bytes) and the Ethernet VLAN
 * header size (18 bytes) to get the actual maximum frame we can use. If
 * different adapters end up with different sizes, we should make this value a
 * bit more dynamic.
 */
#define	I40E_MAX_MTU	9706
#define	I40E_MIN_MTU	ETHERMIN
#define	I40E_DEF_MTU	ETHERMTU

/*
 * Interrupt throttling related values. Interrupt throttling values are defined
 * in two microsecond increments. Note that a value of zero basically says do no
 * ITR activity. A helpful way to think about these is that setting the ITR to a
 * value will allow a certain number of interrupts per second.
 *
 * Our default values for RX allow 20k interrupts per second while our default
 * values for TX allow for 5k interrupts per second. For other class interrupts,
 * we limit ourselves to a rate of 2k/s.
 */
#define	I40E_MIN_ITR		0x0000
#define	I40E_MAX_ITR		0x0FF0
#define	I40E_DEF_RX_ITR		0x0019
#define	I40E_DEF_TX_ITR		0x0064
#define	I40E_DEF_OTHER_ITR	0x00FA

/*
 * Indexes into the three ITR registers that we have.
 */
typedef enum i40e_itr_index {
	I40E_ITR_INDEX_RX	= 0x0,
	I40E_ITR_INDEX_TX	= 0x1,
	I40E_ITR_INDEX_OTHER	= 0x2,
	I40E_ITR_INDEX_NONE	= 0x3
} i40e_itr_index_t;

/*
 * The hardware claims to support LSO up to 256 KB, but due to the limitations
 * imposed by the IP header for non-jumbo frames, we cap it at 64 KB.
 */
#define	I40E_LSO_MAXLEN	(64 * 1024)

#define	I40E_CYCLIC_PERIOD NANOSEC	/* 1 second */
#define	I40E_DRAIN_RX_WAIT	(500 * MILLISEC)	/* In us */

/*
 * All the other queue types for are defined by the common code. However, this
 * is the constant to indicate that it's terminated.
 */
#define	I40E_QUEUE_TYPE_EOL	0x7FF

/*
 * See the comments in i40e_transceiver.c as to the purpose of this value and
 * how it's used to ensure that the IP header is eventually aligned when it's
 * received by the OS.
 */
#define	I40E_BUF_IPHDR_ALIGNMENT	2

/*
 * The XL710 controller has a total of eight buffers available for the
 * transmission of any single frame. This is defined in 8.4.1 - Transmit
 * Packet in System Memory.
 */
#define	I40E_TX_MAX_COOKIE	8

/*
 * An LSO frame can be as large as 64KB, so we allow a DMA bind to span more
 * cookies than a non-LSO frame.  The key here to is to select a value such
 * that once the HW has chunked up the LSO frame into MSS-sized segments that no
 * single segment spans more than 8 cookies (see comments for
 * I40E_TX_MAX_COOKIE)
 */
#define	I40E_TX_LSO_MAX_COOKIE	32

/*
 * Sizing to determine the amount of available descriptors at which we'll
 * consider ourselves blocked. Also, when we have these available, we'll then
 * consider ourselves available to transmit to MAC again. Strictly speaking, the
 * MAX is based on the ring size. The default sizing is based on ixgbe.
 */
#define	I40E_MIN_TX_BLOCK_THRESH	I40E_TX_MAX_COOKIE
#define	I40E_DEF_TX_BLOCK_THRESH	I40E_MIN_TX_BLOCK_THRESH

/*
 * Sizing for DMA thresholds. These are used to indicate whether or not we
 * should perform a bcopy or a DMA binding of a given message block. The range
 * allows for setting things such that we'll always do a bcopy (a high value) or
 * always perform a DMA binding (a low value).
 */
#define	I40E_MIN_RX_DMA_THRESH		0
#define	I40E_DEF_RX_DMA_THRESH		256
#define	I40E_MAX_RX_DMA_THRESH		INT32_MAX

#define	I40E_MIN_TX_DMA_THRESH		0
#define	I40E_DEF_TX_DMA_THRESH		256
#define	I40E_MAX_TX_DMA_THRESH		INT32_MAX

/*
 * The max size of each individual tx buffer is 16KB - 1.
 * See table 8-17
 */
#define	I40E_MAX_TX_BUFSZ		0x0000000000003FFFull

/*
 * Resource sizing counts. There are various aspects of hardware where we may
 * have some variable number of elements that we need to handle. Such as the
 * hardware capabilities and switch capacities. We cannot know a priori how many
 * elements to do, so instead we take a starting guess and then will grow it up
 * to an upper bound on a number of elements, to limit memory consumption in
 * case of a hardware bug.
 */
#define	I40E_HW_CAP_DEFAULT	40
#define	I40E_SWITCH_CAP_DEFAULT	25

/*
 * Host Memory Context related constants.
 */
#define	I40E_HMC_RX_CTX_UNIT		128
#define	I40E_HMC_RX_DBUFF_MIN		1024
#define	I40E_HMC_RX_DBUFF_MAX		(16 * 1024 - 128)
#define	I40E_HMC_RX_DTYPE_NOSPLIT	0
#define	I40E_HMC_RX_DSIZE_32BYTE	1
#define	I40E_HMC_RX_CRCSTRIP_ENABLE	1
#define	I40E_HMC_RX_FC_DISABLE		0
#define	I40E_HMC_RX_L2TAGORDER		1
#define	I40E_HMC_RX_HDRSPLIT_DISABLE	0
#define	I40E_HMC_RX_INVLAN_DONTSTRIP	0
#define	I40E_HMC_RX_TPH_DISABLE		0
#define	I40E_HMC_RX_LOWRXQ_NOINTR	0
#define	I40E_HMC_RX_PREFENA		1

#define	I40E_HMC_TX_CTX_UNIT		128
#define	I40E_HMC_TX_NEW_CONTEXT		1
#define	I40E_HMC_TX_FC_DISABLE		0
#define	I40E_HMC_TX_TS_DISABLE		0
#define	I40E_HMC_TX_FD_DISABLE		0
#define	I40E_HMC_TX_ALT_VLAN_DISABLE	0
#define	I40E_HMC_TX_WB_ENABLE		1
#define	I40E_HMC_TX_TPH_DISABLE		0

/*
 * This defines the error mask that we care about from rx descriptors. Currently
 * we're only concerned with the general errors and oversize errors.
 */
#define	I40E_RX_ERR_BITS	((1 << I40E_RX_DESC_ERROR_RXE_SHIFT) | \
	(1 << I40E_RX_DESC_ERROR_OVERSIZE_SHIFT))

/*
 * Property sizing macros for firmware versions, etc. They need to be large
 * enough to hold 32-bit quantities transformed to strings as %d.%d or %x.
 */
#define	I40E_DDI_PROP_LEN	64

#define	I40E_GROUP_NOMSIX	1
#define	I40E_TRQPAIR_NOMSIX	1

/*
 * It seems reasonable to cast this to void because the only reason that we
 * should be getting a DDI_FAILURE is due to the fact that we specify addresses
 * out of range. Because we specify no offset or address, it shouldn't happen.
 */
#ifdef	DEBUG
#define	I40E_DMA_SYNC(handle, flag)	ASSERT0(ddi_dma_sync( \
					    (handle)->dmab_dma_handle, 0, 0, \
					    (flag)))
#else	/* !DEBUG */
#define	I40E_DMA_SYNC(handle, flag)	((void) ddi_dma_sync( \
					    (handle)->dmab_dma_handle, 0, 0, \
					    (flag)))
#endif	/* DEBUG */

/*
 * Constants related to ring startup and teardown. These refer to the amount of
 * time that we're willing to wait for a ring to spin up and spin down.
 */
#define	I40E_RING_WAIT_NTRIES	10
#define	I40E_RING_WAIT_PAUSE	10	/* ms */
#define	I40E_RING_ENABLE_GAP	50	/* ms */

/*
 * Printed Board Assembly (PBA) length. These are derived from Table 6-2.
 */
#define	I40E_PBANUM_LENGTH	12
#define	I40E_PBANUM_STRLEN	13

/*
 * Define the maximum number of queues for a traffic class. These values come
 * from the 'Number and offset of queue pairs per TCs' section of the 'Add VSI
 * Command Buffer' table. For the 710 controller family this is table 7-62
 * (r2.5) and for the 722 this is table 38-216 (r2.0).
 */
#define	I40E_710_MAX_TC_QUEUES	64
#define	I40E_722_MAX_TC_QUEUES	128

/*
 * Define the size of the HLUT table size. The HLUT table can either be 128 or
 * 512 bytes. We always set the table size to be 512 bytes in i40e_chip_start().
 * Note, this should not be confused with the common code's macro
 * I40E_HASH_LUT_SIZE_512 which is the bit pattern needed to tell the card to
 * use a 512 byte HLUT.
 */
#define	I40E_HLUT_TABLE_SIZE	512

/*
 * Bit flags for attach_progress
 */
typedef enum i40e_attach_state {
	I40E_ATTACH_PCI_CONFIG	= 0x0001,	/* PCI config setup */
	I40E_ATTACH_REGS_MAP	= 0x0002,	/* Registers mapped */
	I40E_ATTACH_PROPS	= 0x0004,	/* Properties initialized */
	I40E_ATTACH_ALLOC_INTR	= 0x0008,	/* Interrupts allocated */
	I40E_ATTACH_ALLOC_RINGSLOCKS	= 0x0010, /* Rings & locks allocated */
	I40E_ATTACH_ADD_INTR	= 0x0020,	/* Intr handlers added */
	I40E_ATTACH_COMMON_CODE	= 0x0040,	/* Intel code initialized */
	I40E_ATTACH_INIT	= 0x0080,	/* Device initialized */
	I40E_ATTACH_STATS	= 0x0200,	/* Kstats created */
	I40E_ATTACH_MAC		= 0x0800,	/* MAC registered */
	I40E_ATTACH_ENABLE_INTR	= 0x1000,	/* DDI interrupts enabled */
	I40E_ATTACH_FM_INIT	= 0x2000,	/* FMA initialized */
	I40E_ATTACH_LINK_TIMER	= 0x4000,	/* link check timer */
	I40E_ATTACH_UFM_INIT	= 0x8000,	/* DDI UFM initialized */
} i40e_attach_state_t;


/*
 * State flags that what's going on in in the device. Some of these state flags
 * indicate some aspirational work that needs to happen in the driver.
 *
 * I40E_UNKNOWN:	The device has yet to be started.
 * I40E_INITIALIZED:	The device has been fully attached.
 * I40E_STARTED:	The device has come out of the GLDV3 start routine.
 * I40E_SUSPENDED:	The device is suspended and I/O among other things
 *			should not occur. This happens because of an actual
 *			DDI_SUSPEND or interrupt adjustments.
 * I40E_STALL:		The tx stall detection logic has found a stall.
 * I40E_OVERTEMP:	The device has encountered a temperature alarm.
 * I40E_INTR_ADJUST:	Our interrupts are being manipulated and therefore we
 *			shouldn't be manipulating their state.
 * I40E_ERROR:		We've detected an FM error and degraded the device.
 */
typedef enum i40e_state {
	I40E_UNKNOWN		= 0x00,
	I40E_INITIALIZED	= 0x01,
	I40E_STARTED		= 0x02,
	I40E_SUSPENDED		= 0x04,
	I40E_STALL		= 0x08,
	I40E_OVERTEMP		= 0x20,
	I40E_INTR_ADJUST	= 0x40,
	I40E_ERROR		= 0x80
} i40e_state_t;


/*
 * Definitions for common Intel things that we use and some slightly more usable
 * names.
 */
typedef struct i40e_hw i40e_hw_t;
typedef struct i40e_aqc_switch_resource_alloc_element_resp i40e_switch_rsrc_t;

/*
 * Handles and addresses of DMA buffers.
 */
typedef struct i40e_dma_buffer {
	caddr_t		dmab_address;		/* Virtual address */
	uint64_t	dmab_dma_address;	/* DMA (Hardware) address */
	ddi_acc_handle_t dmab_acc_handle;	/* Data access handle */
	ddi_dma_handle_t dmab_dma_handle;	/* DMA handle */
	size_t		dmab_size;		/* Buffer size */
	size_t		dmab_len;		/* Data length in the buffer */
} i40e_dma_buffer_t;

/*
 * RX Control Block
 */
typedef struct i40e_rx_control_block {
	mblk_t			*rcb_mp;
	uint32_t		rcb_ref;
	i40e_dma_buffer_t	rcb_dma;
	frtn_t			rcb_free_rtn;
	struct i40e_rx_data	*rcb_rxd;
} i40e_rx_control_block_t;

typedef enum {
	I40E_TX_NONE,
	I40E_TX_COPY,
	I40E_TX_DMA,
	I40E_TX_DESC,
} i40e_tx_type_t;

typedef struct i40e_tx_desc i40e_tx_desc_t;
typedef struct i40e_tx_context_desc i40e_tx_context_desc_t;
typedef union i40e_32byte_rx_desc i40e_rx_desc_t;

struct i40e_dma_bind_info {
	caddr_t dbi_paddr;
	size_t dbi_len;
};

typedef struct i40e_tx_control_block {
	struct i40e_tx_control_block	*tcb_next;
	mblk_t				*tcb_mp;
	i40e_tx_type_t			tcb_type;
	ddi_dma_handle_t		tcb_dma_handle;
	ddi_dma_handle_t		tcb_lso_dma_handle;
	i40e_dma_buffer_t		tcb_dma;
	struct i40e_dma_bind_info	*tcb_bind_info;
	uint_t				tcb_bind_ncookies;
	boolean_t			tcb_used_lso;
} i40e_tx_control_block_t;

/*
 * Receive ring data (used below).
 */
typedef struct i40e_rx_data {
	struct i40e	*rxd_i40e;

	/*
	 * RX descriptor ring definitions
	 */
	i40e_dma_buffer_t rxd_desc_area;	/* DMA buffer of rx desc ring */
	i40e_rx_desc_t *rxd_desc_ring;		/* Rx desc ring */
	uint32_t rxd_desc_next;			/* Index of next rx desc */

	/*
	 * RX control block list definitions
	 */
	kmutex_t		rxd_free_lock;	/* Lock to protect free data */
	i40e_rx_control_block_t	*rxd_rcb_area;	/* Array of control blocks */
	i40e_rx_control_block_t	**rxd_work_list; /* Work list of rcbs */
	i40e_rx_control_block_t	**rxd_free_list; /* Free list of rcbs */
	uint32_t		rxd_rcb_free;	/* Number of free rcbs */

	/*
	 * RX software ring settings
	 */
	uint32_t	rxd_ring_size;		/* Rx descriptor ring size */
	uint32_t	rxd_free_list_size;	/* Rx free list size */

	/*
	 * RX outstanding data. This is used to keep track of outstanding loaned
	 * descriptors after we've shut down receiving information. Note these
	 * are protected by the i40e_t`i40e_rx_pending_lock.
	 */
	uint32_t	rxd_rcb_pending;
	boolean_t	rxd_shutdown;
} i40e_rx_data_t;

/*
 * Structures for unicast and multicast addresses. Note that we keep the VSI id
 * around for unicast addresses, since they may belong to different VSIs.
 * However, since all multicast addresses belong to the default VSI, we don't
 * duplicate that information.
 */
typedef struct i40e_uaddr {
	uint8_t iua_mac[ETHERADDRL];
	int	iua_vsi;
} i40e_uaddr_t;

typedef struct i40e_maddr {
	uint8_t ima_mac[ETHERADDRL];
} i40e_maddr_t;

/*
 * Collection of RX statistics on a given queue.
 */
typedef struct i40e_rxq_stat {
	/*
	 * The i40e hardware does not maintain statistics on a per-ring basis,
	 * only on a per-PF and per-VSI level. As such, to satisfy the GLDv3, we
	 * need to maintain our own stats for packets and bytes.
	 */
	kstat_named_t	irxs_bytes;	/* Bytes in on queue */
	kstat_named_t	irxs_packets;	/* Packets in on queue */

	/*
	 * The following set of stats cover non-checksum data path issues.
	 */
	kstat_named_t	irxs_rx_desc_error;	/* Error bit set on desc */
	kstat_named_t	irxs_rx_copy_nomem;	/* allocb failure for copy */
	kstat_named_t	irxs_rx_intr_limit;	/* Hit i40e_rx_limit_per_intr */
	kstat_named_t	irxs_rx_bind_norcb;	/* No replacement rcb free */
	kstat_named_t	irxs_rx_bind_nomp;	/* No mblk_t in bind rcb */

	/*
	 * The following set of statistics covers rx checksum related activity.
	 * These are all primarily set in i40e_rx_hcksum. If rx checksum
	 * activity is disabled, then these should all be zero.
	 */
	kstat_named_t	irxs_hck_v4hdrok;	/* Valid IPv4 Header */
	kstat_named_t	irxs_hck_l4hdrok;	/* Valid L4 Header */
	kstat_named_t	irxs_hck_unknown;	/* !pinfo.known */
	kstat_named_t	irxs_hck_nol3l4p;	/* Missing L3L4P bit in desc */
	kstat_named_t	irxs_hck_iperr;		/* IPE error bit set */
	kstat_named_t	irxs_hck_eiperr;	/* EIPE error bit set */
	kstat_named_t	irxs_hck_l4err;		/* L4E error bit set */
	kstat_named_t	irxs_hck_v6skip;	/* IPv6 case hw fails on */
	kstat_named_t	irxs_hck_set;		/* Total times we set cksum */
	kstat_named_t	irxs_hck_miss;		/* Times with zero cksum bits */
} i40e_rxq_stat_t;

/*
 * Collection of TX Statistics on a given queue
 */
typedef struct i40e_txq_stat {
	kstat_named_t	itxs_bytes;		/* Bytes out on queue */
	kstat_named_t	itxs_packets;		/* Packets out on queue */
	kstat_named_t	itxs_descriptors;	/* Descriptors issued */
	kstat_named_t	itxs_recycled;		/* Descriptors reclaimed */
	kstat_named_t	itxs_force_copy;	/* non-TSO force copy */
	kstat_named_t	itxs_tso_force_copy;	/* TSO force copy */
	/*
	 * Various failure conditions.
	 */
	kstat_named_t	itxs_hck_meoifail;	/* ether offload failures */
	kstat_named_t	itxs_hck_nol2info;	/* Missing l2 info */
	kstat_named_t	itxs_hck_nol3info;	/* Missing l3 info */
	kstat_named_t	itxs_hck_nol4info;	/* Missing l4 info */
	kstat_named_t	itxs_hck_badl3;		/* Not IPv4/IPv6 */
	kstat_named_t	itxs_hck_badl4;		/* Bad L4 Paylaod */
	kstat_named_t	itxs_lso_nohck;		/* Missing offloads for LSO */
	kstat_named_t	itxs_bind_fails;	/* DMA bind failures */
	kstat_named_t	itxs_tx_short;		/* Tx chain too short */

	kstat_named_t	itxs_err_notcb;		/* No tcb's available */
	kstat_named_t	itxs_err_nodescs;	/* No tcb's available */
	kstat_named_t	itxs_err_context;	/* Total context failures */

	kstat_named_t	itxs_num_unblocked;	/* Number of MAC unblocks */
} i40e_txq_stat_t;

/*
 * An instance of an XL710 transmit/receive queue pair. This currently
 * represents a combination of both a transmit and receive ring, though they
 * should really be split apart into separate logical structures. Unfortunately,
 * during initial work we mistakenly joined them together.
 */
typedef struct i40e_trqpair {
	struct i40e *itrq_i40e;

	/* interrupt control structures */
	kmutex_t itrq_intr_lock;
	kcondvar_t itrq_intr_cv;
	boolean_t itrq_intr_busy;	/* Busy processing interrupt */
	boolean_t itrq_intr_quiesce;	/* Interrupt quiesced */

	hrtime_t irtq_time_stopped;	/* Time when ring was stopped */

	/* Receive-side structures. */
	kmutex_t itrq_rx_lock;
	mac_ring_handle_t itrq_macrxring; /* Receive ring handle. */
	i40e_rx_data_t *itrq_rxdata;	/* Receive ring rx data. */
	uint64_t itrq_rxgen;		/* Generation number for mac/GLDv3. */
	uint32_t itrq_index;		/* Queue index in the PF */
	uint32_t itrq_rx_intrvec;	/* Receive interrupt vector. */
	boolean_t itrq_intr_poll;	/* True when polling */

	/* Receive-side stats. */
	i40e_rxq_stat_t	itrq_rxstat;
	kstat_t	*itrq_rxkstat;

	/* Transmit-side structures. */
	kmutex_t itrq_tx_lock;
	kcondvar_t itrq_tx_cv;
	uint_t itrq_tx_active;		/* No. of active i40e_ring_tx()'s */
	boolean_t itrq_tx_quiesce;	/* Tx is quiesced */
	mac_ring_handle_t itrq_mactxring; /* Transmit ring handle. */
	uint32_t itrq_tx_intrvec;	/* Transmit interrupt vector. */
	boolean_t itrq_tx_blocked;	/* Does MAC think we're blocked? */

	/*
	 * TX data sizing
	 */
	uint32_t		itrq_tx_ring_size;
	uint32_t		itrq_tx_free_list_size;

	/*
	 * TX descriptor ring data
	 */
	i40e_dma_buffer_t	itrq_desc_area;	/* DMA buffer of tx desc ring */
	i40e_tx_desc_t		*itrq_desc_ring; /* TX Desc ring */
	volatile uint32_t	*itrq_desc_wbhead; /* TX write-back index */
	uint32_t		itrq_desc_head;	/* Last index hw freed */
	uint32_t		itrq_desc_tail;	/* Index of next free desc */
	uint32_t		itrq_desc_free;	/* Number of free descriptors */

	/*
	 * TX control block (tcb) data
	 */
	kmutex_t		itrq_tcb_lock;
	i40e_tx_control_block_t	*itrq_tcb_area;	/* Array of control blocks */
	i40e_tx_control_block_t	**itrq_tcb_work_list;	/* In use tcb */
	i40e_tx_control_block_t	**itrq_tcb_free_list;	/* Available tcb */
	uint32_t		itrq_tcb_free;	/* Count of free tcb */

	/* Transmit-side stats. */
	i40e_txq_stat_t		itrq_txstat;
	kstat_t			*itrq_txkstat;

} i40e_trqpair_t;

/*
 * VSI statistics.
 *
 * This mirrors the i40e_eth_stats structure but transforms it into a kstat.
 * Note that the stock statistic structure also includes entries for tx
 * discards. However, this is not actually implemented for the VSI (see Table
 * 7-221), hence why we don't include the member which would always have a value
 * of zero. This choice was made to minimize confusion to someone looking at
 * these, as a value of zero does not necessarily equate to the fact that it's
 * not implemented.
 */
typedef struct i40e_vsi_stats {
	uint64_t ivs_rx_bytes;			/* gorc */
	uint64_t ivs_rx_unicast;		/* uprc */
	uint64_t ivs_rx_multicast;		/* mprc */
	uint64_t ivs_rx_broadcast;		/* bprc */
	uint64_t ivs_rx_discards;		/* rdpc */
	uint64_t ivs_rx_unknown_protocol;	/* rupp */
	uint64_t ivs_tx_bytes;			/* gotc */
	uint64_t ivs_tx_unicast;		/* uptc */
	uint64_t ivs_tx_multicast;		/* mptc */
	uint64_t ivs_tx_broadcast;		/* bptc */
	uint64_t ivs_tx_errors;			/* tepc */
} i40e_vsi_stats_t;

typedef struct i40e_vsi_kstats {
	kstat_named_t	ivk_rx_bytes;
	kstat_named_t	ivk_rx_unicast;
	kstat_named_t	ivk_rx_multicast;
	kstat_named_t	ivk_rx_broadcast;
	kstat_named_t	ivk_rx_discards;
	kstat_named_t	ivk_rx_unknown_protocol;
	kstat_named_t	ivk_tx_bytes;
	kstat_named_t	ivk_tx_unicast;
	kstat_named_t	ivk_tx_multicast;
	kstat_named_t	ivk_tx_broadcast;
	kstat_named_t	ivk_tx_errors;
} i40e_vsi_kstats_t;

/*
 * For pf statistics, we opt not to use the standard statistics as defined by
 * the Intel common code. This also currently combines statistics that are
 * global across the entire device.
 */
typedef struct i40e_pf_stats {
	uint64_t ips_rx_bytes;			/* gorc */
	uint64_t ips_rx_unicast;		/* uprc */
	uint64_t ips_rx_multicast;		/* mprc */
	uint64_t ips_rx_broadcast;		/* bprc */
	uint64_t ips_tx_bytes;			/* gotc */
	uint64_t ips_tx_unicast;		/* uptc */
	uint64_t ips_tx_multicast;		/* mptc */
	uint64_t ips_tx_broadcast;		/* bptc */

	uint64_t ips_rx_size_64;		/* prc64 */
	uint64_t ips_rx_size_127;		/* prc127 */
	uint64_t ips_rx_size_255;		/* prc255 */
	uint64_t ips_rx_size_511;		/* prc511 */
	uint64_t ips_rx_size_1023;		/* prc1023 */
	uint64_t ips_rx_size_1522;		/* prc1522 */
	uint64_t ips_rx_size_9522;		/* prc9522 */

	uint64_t ips_tx_size_64;		/* ptc64 */
	uint64_t ips_tx_size_127;		/* ptc127 */
	uint64_t ips_tx_size_255;		/* ptc255 */
	uint64_t ips_tx_size_511;		/* ptc511 */
	uint64_t ips_tx_size_1023;		/* ptc1023 */
	uint64_t ips_tx_size_1522;		/* ptc1522 */
	uint64_t ips_tx_size_9522;		/* ptc9522 */

	uint64_t ips_link_xon_rx;		/* lxonrxc */
	uint64_t ips_link_xoff_rx;		/* lxoffrxc */
	uint64_t ips_link_xon_tx;		/* lxontxc */
	uint64_t ips_link_xoff_tx;		/* lxofftxc */
	uint64_t ips_priority_xon_rx[8];	/* pxonrxc[8] */
	uint64_t ips_priority_xoff_rx[8];	/* pxoffrxc[8] */
	uint64_t ips_priority_xon_tx[8];	/* pxontxc[8] */
	uint64_t ips_priority_xoff_tx[8];	/* pxofftxc[8] */
	uint64_t ips_priority_xon_2_xoff[8];	/* rxon2offcnt[8] */

	uint64_t ips_crc_errors;		/* crcerrs */
	uint64_t ips_illegal_bytes;		/* illerrc */
	uint64_t ips_mac_local_faults;		/* mlfc */
	uint64_t ips_mac_remote_faults;		/* mrfc */
	uint64_t ips_rx_length_errors;		/* rlec */
	uint64_t ips_rx_undersize;		/* ruc */
	uint64_t ips_rx_fragments;		/* rfc */
	uint64_t ips_rx_oversize;		/* roc */
	uint64_t ips_rx_jabber;			/* rjc */
	uint64_t ips_rx_discards;		/* rdpc */
	uint64_t ips_rx_vm_discards;		/* ldpc */
	uint64_t ips_rx_short_discards;		/* mspdc */
	uint64_t ips_tx_dropped_link_down;	/* tdold */
	uint64_t ips_rx_unknown_protocol;	/* rupp */
	uint64_t ips_rx_err1;			/* rxerr1 */
	uint64_t ips_rx_err2;			/* rxerr2 */
} i40e_pf_stats_t;

typedef struct i40e_pf_kstats {
	kstat_named_t ipk_rx_bytes;		/* gorc */
	kstat_named_t ipk_rx_unicast;		/* uprc */
	kstat_named_t ipk_rx_multicast;		/* mprc */
	kstat_named_t ipk_rx_broadcast;		/* bprc */
	kstat_named_t ipk_tx_bytes;		/* gotc */
	kstat_named_t ipk_tx_unicast;		/* uptc */
	kstat_named_t ipk_tx_multicast;		/* mptc */
	kstat_named_t ipk_tx_broadcast;		/* bptc */

	kstat_named_t ipk_rx_size_64;		/* prc64 */
	kstat_named_t ipk_rx_size_127;		/* prc127 */
	kstat_named_t ipk_rx_size_255;		/* prc255 */
	kstat_named_t ipk_rx_size_511;		/* prc511 */
	kstat_named_t ipk_rx_size_1023;		/* prc1023 */
	kstat_named_t ipk_rx_size_1522;		/* prc1522 */
	kstat_named_t ipk_rx_size_9522;		/* prc9522 */

	kstat_named_t ipk_tx_size_64;		/* ptc64 */
	kstat_named_t ipk_tx_size_127;		/* ptc127 */
	kstat_named_t ipk_tx_size_255;		/* ptc255 */
	kstat_named_t ipk_tx_size_511;		/* ptc511 */
	kstat_named_t ipk_tx_size_1023;		/* ptc1023 */
	kstat_named_t ipk_tx_size_1522;		/* ptc1522 */
	kstat_named_t ipk_tx_size_9522;		/* ptc9522 */

	kstat_named_t ipk_link_xon_rx;		/* lxonrxc */
	kstat_named_t ipk_link_xoff_rx;		/* lxoffrxc */
	kstat_named_t ipk_link_xon_tx;		/* lxontxc */
	kstat_named_t ipk_link_xoff_tx;		/* lxofftxc */
	kstat_named_t ipk_priority_xon_rx[8];	/* pxonrxc[8] */
	kstat_named_t ipk_priority_xoff_rx[8];	/* pxoffrxc[8] */
	kstat_named_t ipk_priority_xon_tx[8];	/* pxontxc[8] */
	kstat_named_t ipk_priority_xoff_tx[8];	/* pxofftxc[8] */
	kstat_named_t ipk_priority_xon_2_xoff[8];	/* rxon2offcnt[8] */

	kstat_named_t ipk_crc_errors;		/* crcerrs */
	kstat_named_t ipk_illegal_bytes;	/* illerrc */
	kstat_named_t ipk_mac_local_faults;	/* mlfc */
	kstat_named_t ipk_mac_remote_faults;	/* mrfc */
	kstat_named_t ipk_rx_length_errors;	/* rlec */
	kstat_named_t ipk_rx_undersize;		/* ruc */
	kstat_named_t ipk_rx_fragments;		/* rfc */
	kstat_named_t ipk_rx_oversize;		/* roc */
	kstat_named_t ipk_rx_jabber;		/* rjc */
	kstat_named_t ipk_rx_discards;		/* rdpc */
	kstat_named_t ipk_rx_vm_discards;	/* ldpc */
	kstat_named_t ipk_rx_short_discards;	/* mspdc */
	kstat_named_t ipk_tx_dropped_link_down;	/* tdold */
	kstat_named_t ipk_rx_unknown_protocol;	/* rupp */
	kstat_named_t ipk_rx_err1;		/* rxerr1 */
	kstat_named_t ipk_rx_err2;		/* rxerr2 */
} i40e_pf_kstats_t;

/*
 * Resources that are pooled and specific to a given i40e_t.
 */
typedef struct i40e_func_rsrc {
	uint_t	ifr_nrx_queue;
	uint_t	ifr_nrx_queue_used;
	uint_t	ifr_ntx_queue;
	uint_t	ifr_trx_queue_used;
	uint_t	ifr_nvsis;
	uint_t	ifr_nvsis_used;
	uint_t	ifr_nmacfilt;
	uint_t	ifr_nmacfilt_used;
	uint_t	ifr_nmcastfilt;
	uint_t	ifr_nmcastfilt_used;
} i40e_func_rsrc_t;

typedef struct i40e_vsi {
	uint16_t		iv_seid;
	uint16_t		iv_number;
	kstat_t			*iv_kstats;
	i40e_vsi_stats_t	iv_stats;
	uint16_t		iv_stats_id;
} i40e_vsi_t;

/*
 * While irg_index and irg_grp_hdl aren't used anywhere, they are
 * still useful for debugging.
 */
typedef struct i40e_rx_group {
	uint32_t		irg_index;    /* index in i40e_rx_groups[] */
	uint16_t		irg_vsi_seid; /* SEID of VSI for this group */
	mac_group_handle_t	irg_grp_hdl;  /* handle to mac_group_t */
	struct i40e		*irg_i40e;    /* ref to i40e_t */
} i40e_rx_group_t;

/*
 * Main i40e per-instance state.
 */
typedef struct i40e {
	list_node_t	i40e_glink;		/* Global list link */
	list_node_t	i40e_dlink;		/* Device list link */
	kmutex_t	i40e_general_lock;	/* General device lock */

	/*
	 * General Data and management
	 */
	dev_info_t	*i40e_dip;
	int		i40e_instance;
	int		i40e_fm_capabilities;
	uint_t		i40e_state;
	i40e_attach_state_t i40e_attach_progress;
	mac_handle_t	i40e_mac_hdl;
	ddi_periodic_t	i40e_periodic_id;

	/*
	 * Pointers to common code data structures and memory for the common
	 * code.
	 */
	struct i40e_hw				i40e_hw_space;
	struct i40e_osdep			i40e_osdep_space;
	struct i40e_aq_get_phy_abilities_resp	i40e_phy;
	void					*i40e_aqbuf;

#define	I40E_DEF_VSI_IDX	0
#define	I40E_DEF_VSI(i40e)	((i40e)->i40e_vsis[I40E_DEF_VSI_IDX])
#define	I40E_DEF_VSI_SEID(i40e)	(I40E_DEF_VSI(i40e).iv_seid)

	/*
	 * Device state, switch information, and resources.
	 */
	i40e_vsi_t		i40e_vsis[I40E_MAX_NUM_RX_GROUPS];
	uint16_t		i40e_mac_seid;	 /* SEID of physical MAC */
	uint16_t		i40e_veb_seid;	 /* switch atop MAC (SEID) */
	uint16_t		i40e_vsi_avail;	 /* VSIs avail to this PF */
	uint16_t		i40e_vsi_used;	 /* VSIs used by this PF */
	struct i40e_device	*i40e_device;
	i40e_func_rsrc_t	i40e_resources;
	uint16_t		i40e_switch_rsrc_alloc;
	uint16_t		i40e_switch_rsrc_actual;
	i40e_switch_rsrc_t	*i40e_switch_rsrcs;
	i40e_uaddr_t		*i40e_uaddrs;
	i40e_maddr_t		*i40e_maddrs;
	int			i40e_mcast_promisc_count;
	boolean_t		i40e_promisc_on;
	link_state_t		i40e_link_state;
	uint32_t		i40e_link_speed;	/* In Mbps */
	link_duplex_t		i40e_link_duplex;
	link_fec_t		i40e_fec_requested;
	uint_t			i40e_sdu;
	uint_t			i40e_frame_max;

	/*
	 * Transmit and receive information, tunables, and MAC info.
	 */
	i40e_trqpair_t	*i40e_trqpairs;
	boolean_t	i40e_mr_enable;
	uint_t		i40e_num_trqpairs; /* total TRQPs (per PF) */
	uint_t		i40e_num_trqpairs_per_vsi; /* TRQPs per VSI */
	uint_t		i40e_other_itr;

	i40e_rx_group_t	*i40e_rx_groups;
	uint_t		i40e_num_rx_groups;
	int		i40e_num_rx_descs;
	uint32_t	i40e_rx_ring_size;
	uint32_t	i40e_rx_buf_size;
	boolean_t	i40e_rx_hcksum_enable;
	uint32_t	i40e_rx_dma_min;
	uint32_t	i40e_rx_limit_per_intr;
	uint_t		i40e_rx_itr;

	int		i40e_num_tx_descs;
	uint32_t	i40e_tx_ring_size;
	uint32_t	i40e_tx_buf_size;
	uint32_t	i40e_tx_block_thresh;
	boolean_t	i40e_tx_hcksum_enable;
	boolean_t	i40e_tx_lso_enable;
	uint32_t	i40e_tx_dma_min;
	uint_t		i40e_tx_itr;

	/*
	 * Interrupt state
	 */
	uint_t		i40e_intr_pri;
	uint_t		i40e_intr_force;
	uint_t		i40e_intr_type;
	int		i40e_intr_cap;
	uint32_t	i40e_intr_count;
	uint32_t	i40e_intr_count_max;
	uint32_t	i40e_intr_count_min;
	size_t		i40e_intr_size;
	ddi_intr_handle_t *i40e_intr_handles;
	ddi_cb_handle_t	i40e_callback_handle;

	/*
	 * DMA attributes. See i40e_transceiver.c for why we have copies of them
	 * in the i40e_t.
	 */
	ddi_dma_attr_t		i40e_static_dma_attr;
	ddi_dma_attr_t		i40e_txbind_dma_attr;
	ddi_dma_attr_t		i40e_txbind_lso_dma_attr;
	ddi_device_acc_attr_t	i40e_desc_acc_attr;
	ddi_device_acc_attr_t	i40e_buf_acc_attr;

	/*
	 * The following two fields are used to protect and keep track of
	 * outstanding, loaned buffers to MAC. If we have these, we can't
	 * detach as we have active DMA memory outstanding.
	 */
	kmutex_t	i40e_rx_pending_lock;
	kcondvar_t	i40e_rx_pending_cv;
	uint32_t	i40e_rx_pending;

	/*
	 * PF statistics and VSI statistics.
	 */
	kmutex_t		i40e_stat_lock;
	kstat_t			*i40e_pf_kstat;
	i40e_pf_stats_t		i40e_pf_stat;

	/*
	 * Misc. stats and counters that should maybe one day be kstats.
	 */
	uint64_t	i40e_s_link_status_errs;
	uint32_t	i40e_s_link_status_lasterr;

	/*
	 * LED information. Note this state is only modified in
	 * i40e_gld_set_led() which is protected by MAC's serializer lock.
	 */
	uint32_t	i40e_led_status;
	boolean_t	i40e_led_saved;

	/* DDI UFM handle */
	ddi_ufm_handle_t	*i40e_ufmh;
} i40e_t;

/*
 * The i40e_device represents a PCI device which encapsulates multiple physical
 * functions which are represented as an i40e_t. This is used to track the use
 * of pooled resources throughout all of the various devices.
 */
typedef struct i40e_device {
	list_node_t	id_link;
	dev_info_t	*id_parent;
	uint_t		id_pci_bus;
	uint_t		id_pci_device;
	uint_t		id_nfuncs;	/* Total number of functions */
	uint_t		id_nreg;	/* Total number present */
	list_t		id_i40e_list;	/* List of i40e_t's registered */
	i40e_switch_rsrc_t	*id_rsrcs; /* Switch resources for this PF */
	uint_t		id_rsrcs_alloc;	/* Total allocated resources */
	uint_t		id_rsrcs_act;	/* Actual number of resources */
} i40e_device_t;

/* Values for the interrupt forcing on the NIC. */
#define	I40E_INTR_NONE			0
#define	I40E_INTR_MSIX			1
#define	I40E_INTR_MSI			2
#define	I40E_INTR_LEGACY		3

/* Hint that we don't want to do any polling... */
#define	I40E_POLL_NULL			-1

/*
 * Logging functions.
 */
/*PRINTFLIKE2*/
extern void i40e_error(i40e_t *, const char *, ...) __KPRINTFLIKE(2);
/*PRINTFLIKE2*/
extern void i40e_notice(i40e_t *, const char *, ...) __KPRINTFLIKE(2);
/*PRINTFLIKE2*/
extern void i40e_log(i40e_t *, const char *, ...) __KPRINTFLIKE(2);

/*
 * General link handling functions.
 */
extern void i40e_link_check(i40e_t *);
extern void i40e_update_mtu(i40e_t *);

/*
 * FMA functions.
 */
extern int i40e_check_acc_handle(ddi_acc_handle_t);
extern int i40e_check_dma_handle(ddi_dma_handle_t);
extern void i40e_fm_ereport(i40e_t *, char *);

/*
 * Interrupt handlers and interrupt handler setup.
 */
extern void i40e_intr_chip_init(i40e_t *);
extern void i40e_intr_chip_fini(i40e_t *);
extern uint_t i40e_intr_msix(void *, void *);
extern uint_t i40e_intr_msi(void *, void *);
extern uint_t i40e_intr_legacy(void *, void *);
extern void i40e_intr_io_enable_all(i40e_t *);
extern void i40e_intr_io_disable_all(i40e_t *);
extern void i40e_intr_io_clear_cause(i40e_t *);
extern void i40e_intr_rx_queue_disable(i40e_trqpair_t *);
extern void i40e_intr_rx_queue_enable(i40e_trqpair_t *);
extern void i40e_intr_set_itr(i40e_t *, i40e_itr_index_t, uint_t);
extern void i40e_intr_quiesce(i40e_trqpair_t *);

/*
 * Receive-side functions
 */
extern mblk_t *i40e_ring_rx(i40e_trqpair_t *, int);
extern mblk_t *i40e_ring_rx_poll(void *, int);
extern void i40e_rx_recycle(caddr_t);
extern boolean_t i40e_ring_tx_quiesce(i40e_trqpair_t *);

/*
 * Transmit-side functions
 */
mblk_t *i40e_ring_tx(void *, mblk_t *);
extern void i40e_tx_recycle_ring(i40e_trqpair_t *);
extern void i40e_tx_cleanup_ring(i40e_trqpair_t *);

/*
 * Statistics functions.
 */
extern boolean_t i40e_stats_init(i40e_t *);
extern void i40e_stats_fini(i40e_t *);
extern boolean_t i40e_stat_vsi_init(i40e_t *, uint_t);
extern void i40e_stat_vsi_fini(i40e_t *, uint_t);
extern boolean_t i40e_stats_trqpair_init(i40e_trqpair_t *);
extern void i40e_stats_trqpair_fini(i40e_trqpair_t *);
extern int i40e_m_stat(void *, uint_t, uint64_t *);
extern int i40e_rx_ring_stat(mac_ring_driver_t, uint_t, uint64_t *);
extern int i40e_tx_ring_stat(mac_ring_driver_t, uint_t, uint64_t *);

/*
 * MAC/GLDv3 functions, and functions called by MAC/GLDv3 support code.
 */
extern boolean_t i40e_register_mac(i40e_t *);
extern boolean_t i40e_start(i40e_t *);
extern void i40e_stop(i40e_t *);
extern int i40e_setup_ring(i40e_trqpair_t *);
extern boolean_t i40e_shutdown_ring(i40e_trqpair_t *);

/*
 * DMA & buffer functions and attributes
 */
extern void i40e_init_dma_attrs(i40e_t *, boolean_t);
extern boolean_t i40e_alloc_ring_mem(i40e_trqpair_t *);
extern void i40e_free_ring_mem(i40e_trqpair_t *, boolean_t);

#ifdef __cplusplus
}
#endif

#endif /* _I40E_SW_H */