path: root/usr/src/uts/intel/io/imc/imc.h

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2019 Joyent, Inc.
 */

#ifndef _INTEL_IMC_H
#define	_INTEL_IMC_H

#include <sys/types.h>
#include <sys/bitmap.h>
#include <sys/list.h>
#include <sys/sunddi.h>

/*
 * This header file contains the definitions used for the various generations of
 * the Intel IMC driver.
 */

#ifdef __cplusplus
extern "C" {
#endif

/*
 * The maximum number of sockets that the IMC driver supports. This is currently
 * determined by the Purley platforms (Skylake) which support up to 8 CPUs.
 */
#define	IMC_MAX_SOCKETS		8

/*
 * The maximum number of memory controllers that exist per socket. Currently all
 * supported platforms (Sandy Bridge -> Skylake) support at most two.
 */
#define	IMC_MAX_IMCPERSOCK	2

/*
 * The maximum number of channels that exist per IMC. Currently Skylake supports
 * 3 per IMC. On certain configurations of Haswell/Broadwell, there is only a
 * single IMC which supports all 4 channels.
 */
#define	IMC_MAX_CHANPERMC	4

/*
 * The maximum number of DIMMs that exist per channel. On Skylake this is two
 * DIMMs. However, Sandy Bridge through Broadwell support three.
 */
#define	IMC_MAX_DIMMPERCHAN	3

/*
 * The maximum number of rank disable bits per DIMM. This is currently
 * consistent across all generations that have these bits.
 */
#define	IMC_MAX_RANK_DISABLE	4

/*
 * The number of different PCI buses that we need to record for a given
 * platform. Pre-Skylake there are only two that are required, one for the IIO
 * and one for the non-IIO. On Skylake, more PCI buses are used.
 */
#define	IMC_MAX_PCIBUSES	3

/*
 * Macros to take apart the node id for a given processor. These assume that
 * we're reading the nodeid from the UBox and not from the SAD control.
 */
#define	IMC_NODEID_UBOX_MASK(x)		((x) & 0x7)

/*
 * On Ivy Bridge through Broadwell, the node id that is found in the SAD targets
 * has the HA indicator as NodeID[2]. This means that the actual target node of
 * the socket is NodeID[3] | NodeID[1:0].
 */
#define	IMC_NODEID_IVY_BRD_UPPER(x)	BITX(x, 3, 3)
#define	IMC_NODEID_IVY_BRD_LOWER(x)	BITX(x, 1, 0)
#define	IMC_NODEID_IVY_BRD_HA(x)	BITX(x, 2, 2)

/*
 * Macros to take apart the MCMTR register bits that we care about.
 */
#define	IMC_MCMTR_CLOSED_PAGE(x)	BITX(x, 0, 0)
#define	IMC_MCMTR_LOCKSTEP(x)		BITX(x, 1, 1)
#define	IMC_MCMTR_ECC_ENABLED(x)	BITX(x, 2, 2)

#define	IMC_MCMTR_DDR4_HAS_BRD(x)	BITX(x, 14, 14)

/*
 * Macros to take apart the dimmmtr_* registers in different generations. While
 * there are similarities, these often end up different between generations and
 * chips. These macros use a range of CPUs that they're valid for in the name.
 * Macros with no suffix are valid for all currently supported CPUs.
 */

#define	IMC_REG_MC_MTR0		0x80
#define	IMC_REG_MC_MTR1		0x84
#define	IMC_REG_MC_MTR2		0x88

#define	IMC_MTR_CA_WIDTH(x)	BITX(x, 1, 0)
#define	IMC_MTR_CA_BASE		10
#define	IMC_MTR_CA_MIN		10
#define	IMC_MTR_CA_MAX		12

#define	IMC_MTR_RA_WIDTH(x)	BITX(x, 4, 2)
#define	IMC_MTR_RA_BASE		12
#define	IMC_MTR_RA_MIN		13
#define	IMC_MTR_RA_MAX		18

#define	IMC_MTR_DENSITY_IVY_BRD(x)	BITX(x, 6, 5)
#define	IMC_MTR_DENSITY_SKX(x)		BITX(x, 7, 5)

#define	IMC_MTR_WIDTH_IVB_HAS(x)	BITX(x, 8, 7)
#define	IMC_MTR_WIDTH_BRD_SKX(x)	BITX(x, 9, 8)

#define	IMC_MTR_DDR_RANKS(x)		BITX(x, 13, 12)
#define	IMC_MTR_DDR_RANKS_MAX		4
#define	IMC_MTR_DDR_RANKS_MAX_HAS_SKX	8

#define	IMC_MTR_PRESENT_SNB_BRD(x)	BITX(x, 14, 14)
#define	IMC_MTR_PRESENT_SKYLAKE(x)	BITX(x, 15, 15)

#define	IMC_MTR_RANK_DISABLE(x)		BITX(x, 19, 16)

#define	IMC_MTR_DDR4_ENABLE_HAS_BRD(x)	BITX(x, 20, 20)
#define	IMC_MTR_HDRL_HAS_SKX(x)		BITX(x, 21, 21)
#define	IMC_MTR_HDRL_PARITY_HAS_SKX(x)	BITX(x, 22, 22)
#define	IMC_MTR_3DSRANKS_HAS_SKX(x)	BITX(x, 24, 23)

/*
 * Data for the RASENABLES register.
 */
#define	IMC_MC_MIRROR_SNB_BRD(x)	BITX(x, 0, 0)

/*
 * The maximum number of SAD rules that exist on all supported platforms.
 */
#define	IMC_MAX_SAD_RULES	24

/*
 * The maximum number of targets that can be interleaved in a sad rule.
 */
#define	IMC_MAX_SAD_INTERLEAVE	8

/*
 * The maximum number of route entries that exist in SAD. This is only used on
 * SKX.
 */
#define	IMC_MAX_SAD_MCROUTES	6

/*
 * Definitions used to decode the MC Route table. Note that at this time this is
 * very Skylake specific (as it's the only platform it's supported on).
 */
#define	IMC_REG_SKX_SAD_MC_ROUTE_TABLE	0xb4
#define	IMC_MC_ROUTE_RING_BITS		3
#define	IMC_MC_ROUTE_RING_MASK		0x7
#define	IMC_MC_ROUTE_CHAN_BITS		2
#define	IMC_MC_ROUTE_CHAN_MASK		0x3
#define	IMC_MC_ROUTE_CHAN_OFFSET	18

/*
 * Definitions to help decode TOLM (top of low memory) and TOHM (top of high
 * memory). The way this is done varies based on generation. These regions are
 * currently always 64-MByte aligned
 *
 * On Sandy Bridge and Ivy Bridge the low four bits of TOLM are bits 31:28. TOHM
 * is a single register. Bits 20:0 map to bits 45:25. Both registers represent
 * the upper limit (as in one higher than the max DRAM value).
 *
 * On Haswell through Skylake, TOLM is represented as a 32-bit quantity. No
 * shifting is required. However, only bits 31:26 are present. TOHM is spread
 * out among two registers. The lower 32-bits is masked in a similar fashion. In
 * both cases, these registers represent an inclusive range where we don't care
 * about other bits. To deal with this we'll increment the lowest bit we care
 * about to make it an exclusive range.
 *
 * Based on the above, we have opted to make both ranges in the IMC driver
 * normalized to an _exclusive_ value.
 *
 * Ivy Bridge has the values in both the CBo SAD and a VT-d section; however, we
 * use the CBo SAD which is why it looks like Sandy Bridge and not Haswell.
 */

#define	IMC_TOLM_SNB_IVY_MASK		0xf
#define	IMC_TOLM_SNB_IVY_SHIFT		28
#define	IMC_TOHM_SNB_IVY_MASK		0x1fffff
#define	IMC_TOHM_SNB_IVY_SHIFT		25

#define	IMC_TOLM_HAS_SKX_MASK		0xfc000000
#define	IMC_TOLM_HAS_SKY_EXCL		(1 << 26)
#define	IMC_TOHM_LOW_HAS_SKX_MASK	0xfc000000
#define	IMC_TOHM_HAS_SKY_EXCL		(1 << 26)

/*
 * Definitions to decode SAD values. These are sometimes subtlety different
 * across generations.
 */
#define	IMC_SAD_DRAM_RULE_ENABLE(x)		BITX(x, 0, 0)

#define	IMC_SAD_DRAM_INTERLEAVE_SNB_BRD(x)	BITX(x, 1, 1)
#define	IMC_SAD_DRAM_INTERLEAVE_SNB_BRD_8t6XOR	0
#define	IMC_SAD_DRAM_INTERLEAVE_SNB_BRD_8t6	1

#define	IMC_SAD_DRAM_INTERLEAVE_SKX(x)		BITX(x, 2, 1)
#define	IMC_SAD_DRAM_INTERLEAVE_SKX_8t6		0
#define	IMC_SAD_DRAM_INTERLEAVE_SKX_10t8	1
#define	IMC_SAD_DRAM_INTERLEAVE_SKX_14t12	2
#define	IMC_SAD_DRAM_INTERLEAVE_SKX_32t30	3

#define	IMC_SAD_DRAM_ATTR_SNB_BRD(x)		BITX(x, 3, 2)
#define	IMC_SAD_DRAM_ATTR_SKX(x)		BITX(x, 4, 3)
#define	IMC_SAD_DRAM_ATTR_DRAM			0
#define	IMC_SAD_DRAM_ATTR_MMCFG			1
#define	IMC_SAD_DRAM_ATTR_NXM			2

#define	IMC_SAD_DRAM_MOD23_SKX(x)		BITX(x, 6, 5)
#define	IMC_SAD_DRAM_MOD23_MOD3			0
#define	IMC_SAD_DRAM_MOD23_MOD2_C01		1
#define	IMC_SAD_DRAM_MOD23_MOD2_C12		2
#define	IMC_SAD_DRAM_MOD23_MOD2_C02		3

#define	IMC_SAD_DRAM_LIMIT_SNB_BRD(x)		BITX(x, 25, 6)
#define	IMC_SAD_DRAM_LIMIT_SKX(x)		BITX(x, 26, 7)
#define	IMC_SAD_DRAM_LIMIT_SHIFT		26
#define	IMC_SAD_DRAM_LIMIT_EXCLUSIVE		(1 << IMC_SAD_DRAM_LIMIT_SHIFT)

#define	IMC_SAD_DRAM_A7_IVB_BRD(x)		BITX(x, 26, 26)
#define	IMC_SAD_DRAM_MOD3_SKX(x)		BITX(x, 27, 27)
#define	IMC_SAD_DRAM_MOD3_MODE_SKX(x)		BITX(x, 31, 30)
#define	IMC_SAD_DRAM_MOD3_MODE_45t6		0
#define	IMC_SAD_DRAM_MOD3_MODE_45t8		1
#define	IMC_SAD_DRAM_MOD3_MODE_45t12		2

#define	IMC_SAD_ILEAVE_SNB_MASK			0x7
#define	IMC_SAD_ILEAVE_SNB_LEN			3
#define	IMC_SAD_ILEAVE_IVB_SKX_MASK		0xf
#define	IMC_SAD_ILEAVE_IVB_SKX_LEN		4

/*
 * The interleave targets on Skylake use the upper bit to indicate whether it is
 * referring to a local memory controller or if it actually refers to another
 * node that is far away. The maximum value includes the upper bit which is used
 * to indicate whether it is remote or far.
 */
#define	IMC_SAD_ILEAVE_SKX_LOCAL(x)		BITX(x, 3, 3)
#define	IMC_SAD_ILEAVE_SKX_TARGET(x)		BITX(x, 2, 0)
#define	IMC_SAD_ILEAVE_SKX_MAX			0xf

/*
 * Maximum number of TAD tables that we need to consider. On Sandy Bridge
 * through Broadwell this is based on the number of home agents that are present
 * in the system. On Sandy Bridge there is one, on others, there are up to two.
 * On Skylake, there is one TAD per IMC.
 */
#define	IMC_MAX_TAD	2

/*
 * Maximum number of TAD rules on any of the supported processors.
 */
#define	IMC_MAX_TAD_RULES	12

/*
 * Maximum number of interleave targets. Note, this only applies to Sandy Bridge
 * through Broadwell. Skylake gets this information in another form.
 */
#define	IMC_MAX_TAD_TARGETS	4

/*
 * Offset between the base TAD rule and the corresponding wayness rule on
 * Skylake.
 */
#define	IMC_SKX_WAYNESS_OFFSET	0x30

/*
 * Various macros to decode the TAD rules.
 */
#define	IMC_TAD_LIMIT(x)		BITX(x, 31, 12)
#define	IMC_TAD_LIMIT_SHIFT		26
#define	IMC_TAD_LIMIT_EXCLUSIVE		(1 << IMC_TAD_LIMIT_SHIFT)

#define	IMC_TAD_SOCK_WAY(x)		BITX(x, 11, 10)
#define	IMC_TAD_SOCK_WAY_1		0
#define	IMC_TAD_SOCK_WAY_2		1
#define	IMC_TAD_SOCK_WAY_4		2
#define	IMC_TAD_SOCK_WAY_8		3
#define	IMC_TAD_CHAN_WAY(x)		BITX(x, 9, 8)
#define	IMC_TAD_TARG3(x)		BITX(x, 7, 6)
#define	IMC_TAD_TARG2(x)		BITX(x, 5, 4)
#define	IMC_TAD_TARG1(x)		BITX(x, 3, 2)
#define	IMC_TAD_TARG0(x)		BITX(x, 1, 0)

#define	IMC_TAD_SNB_BRD_NTARGETS	4

/*
 * These are registers specific to the Skylake and newer TAD BASE registers.
 */
#define	IMC_TAD_BASE_BASE(x)		BITX(x, 31, 12)
#define	IMC_TAD_BASE_SHIFT		26

#define	IMC_TAD_BASE_CHAN_GRAN(x)	BITX(x, 7, 6)
#define	IMC_TAD_BASE_CHAN_GRAN_64B	0
#define	IMC_TAD_BASE_CHAN_GRAN_256B	1
#define	IMC_TAD_BASE_CHAN_GRAN_4KB	2

#define	IMC_TAD_BASE_SOCK_GRAN(x)	BITX(x, 5, 4)
#define	IMC_TAD_BASE_SOCK_GRAN_64B	0
#define	IMC_TAD_BASE_SOCK_GRAN_256B	1
#define	IMC_TAD_BASE_SOCK_GRAN_4KB	2
#define	IMC_TAD_BASE_SOCK_GRAN_1GB	3

#define	IMC_TADCHAN_OFFSET_SNB_BRD(x)	BITX(x, 25, 6)
#define	IMC_TADCHAN_OFFSET_SKX(x)	BITX(x, 23, 4)
#define	IMC_TADCHAN_OFFSET_SHIFT	26

/*
 * Macros to get at various TAD features.
 */
#define	IMC_TAD_SYSDEF_LOCKSTEP(x)	BITX(x, 7, 7)
#define	IMC_TAD_SYSDEF2_SHIFTUP(x)	BITX(x, 22, 22)
#define	IMC_TAD_SYSDEF2_CHANHASH(x)	BITX(x, 21, 21)

/*
 * Maximum number of different wayness entries that exist across the various IMC
 * generations. Each wayness then has a maximum number of target entries.
 */
#define	IMC_MAX_RANK_WAYS		5
#define	IMC_MAX_RANK_INTERLEAVES	8

/*
 * Macros to take apart the rank interleave wayness and offset registers.
 */
#define	IMC_RIR_WAYNESS_ENABLED(x)	BITX(x, 31, 31)
#define	IMC_RIR_WAYNESS_WAY(x)		BITX(x, 29, 28)
#define	IMC_RIR_LIMIT_HAS_SKX(x)	BITX(x, 11, 1)
#define	IMC_RIR_LIMIT_SNB_IVB(x)	BITX(x, 10, 1)
#define	IMC_RIR_LIMIT_SHIFT		29
#define	IMC_RIR_LIMIT_EXCLUSIVE		(1 << IMC_RIR_LIMIT_SHIFT)

/*
 * Currently, everything other than Broadwell has the same value for the target
 * offset.
 */
#define	IMC_RIR_OFFSET_TARGET_BRD(x)		BITX(x, 23, 20)
#define	IMC_RIR_OFFSET_TARGET(x)		BITX(x, 19, 16)
#define	IMC_RIR_OFFSET_OFFSET_HAS_SKX(x)	BITX(x, 15, 2)
#define	IMC_RIR_OFFSET_OFFSET_SNB_IVB(x)	BITX(x, 14, 2)
#define	IMC_RIR_OFFSET_SHIFT			29

/*
 * Definitions to cover manipulations of open and closed pages.
 */
#define	IMC_PAGE_BITS_CLOSED	6
#define	IMC_PAGE_BITS_OPEN	13

/*
 * Macros to decode and understand the CPUBUSNO registers in the UBOX_DECS.
 */
#define	IMC_UBOX_CPUBUSNO_0(x)			BITX(x, 7, 0)
#define	IMC_UBOX_CPUBUSNO_1(x)			BITX(x, 15, 8)
#define	IMC_UBOX_CPUBUSNO_2(x)			BITX(x, 23, 16)

/*
 * Hardware generations supported by the IMC driver.
 */
typedef enum {
	IMC_GEN_UNKNOWN = 0,
	IMC_GEN_SANDY,
	IMC_GEN_IVY,
	IMC_GEN_HASWELL,
	IMC_GEN_BROADWELL,
	/*
	 * IMC_GEN_SKYLAKE also covers Cascade Lake. The two are similar to the
	 * point of even having the same PCI IDs for all of the devices. The
	 * only difference in the cpuid signature between them is the stepping,
	 * hence we do not have a separate Cascade Lake target here, as it's
	 * really the same as Skylake.
	 */
	IMC_GEN_SKYLAKE
} imc_gen_t;

/*
 * Generation specific limits.
 */
typedef struct imc_gen_data {
	uint_t	igd_max_sockets;
	uint_t	igd_max_imcs;
	uint_t	igd_max_channels;
	uint_t	igd_max_dimms;
	uint_t	igd_max_ranks;
	uint_t	igd_mtr_offsets[IMC_MAX_DIMMPERCHAN];
	uint_t	igd_mcmtr_offset;
	uint_t	igd_topo_offset;
	uint_t	igd_num_mcroutes;
	uint_t	igd_tolm_offset;
	uint_t	igd_tohm_low_offset;
	uint_t	igd_tohm_hi_offset;
	uint_t	igd_sad_dram_offset;
	uint_t	igd_sad_ndram_rules;
	uint_t	igd_sad_nodeid_offset;
	uint_t	igd_tad_nrules;
	uint_t	igd_tad_rule_offset;
	uint_t	igd_tad_chan_offset;
	uint_t	igd_tad_sysdef;
	uint_t	igd_tad_sysdef2;
	uint_t	igd_mc_mirror;
	uint_t	igd_rir_nways;
	uint_t	igd_rir_way_offset;
	uint_t	igd_rir_nileaves;
	uint_t	igd_rir_ileave_offset;
	uint_t	igd_ubox_cpubusno_offset;
} imc_gen_data_t;

/*
 * Different types of PCI devices that show up on the core that we may need to
 * attach to.
 */
typedef enum {
	IMC_TYPE_UNKNOWN = 0,
	IMC_TYPE_MC0_M2M,	/* SKX Only */
	IMC_TYPE_MC1_M2M,	/* SKX Only */
	IMC_TYPE_MC0_MAIN0,
	IMC_TYPE_MC0_MAIN1,
	IMC_TYPE_MC1_MAIN0,
	IMC_TYPE_MC1_MAIN1,
	IMC_TYPE_MC0_CHANNEL0,
	IMC_TYPE_MC0_CHANNEL1,
	IMC_TYPE_MC0_CHANNEL2,
	IMC_TYPE_MC0_CHANNEL3,
	IMC_TYPE_MC1_CHANNEL0,
	IMC_TYPE_MC1_CHANNEL1,
	IMC_TYPE_MC1_CHANNEL2,
	IMC_TYPE_MC1_CHANNEL3,
	IMC_TYPE_SAD_DRAM,
	IMC_TYPE_SAD_MMIO,
	/*
	 * We want to note which device has the TOLM and TOHM registers.
	 * Unfortunately this is a rather complicated affair. On Sandy Bridge
	 * they are a part of the IMC_TYPE_SAD_MMIO. On Ivy Bridge, it's on its
	 * own dedicated device on the CBo.
	 *
	 * On Haswell onward, these move to the VT-D misc. registers. On Haswell
	 * and Broadwell, only one of these exist in the system. However, on
	 * Skylake these exist per socket.
	 */
	IMC_TYPE_SAD_MISC,
	IMC_TYPE_VTD_MISC,
	/*
	 * On SKX this exists on a per-core basis. It contains the memory
	 * controller routing table.
	 */
	IMC_TYPE_SAD_MCROUTE,
	IMC_TYPE_UBOX,
	IMC_TYPE_UBOX_CPUBUSNO,
	IMC_TYPE_HA0,
	IMC_TYPE_HA1,
} imc_type_t;

/*
 * Each entry in the stub table represents a device that we might attach to in a
 * given generation. This is only defined in the kernel to make it easier to
 * build the imc decoder in userland for testing.
 */
#ifdef	_KERNEL
typedef struct imc_stub_table {
	imc_gen_t	imcs_gen;
	imc_type_t	imcs_type;
	uint16_t	imcs_devid;
	uint16_t	imcs_pcidev;
	uint16_t	imcs_pcifunc;
	const char	*imcs_desc;
} imc_stub_table_t;

typedef struct imc_stub {
	avl_node_t		istub_link;
	dev_info_t		*istub_dip;
	uint16_t		istub_vid;
	uint16_t		istub_did;
	uint16_t		istub_bus;
	uint16_t		istub_dev;
	uint16_t		istub_func;
	ddi_acc_handle_t	istub_cfgspace;
	const imc_stub_table_t	*istub_table;
} imc_stub_t;
#else
typedef struct imc_stub {
	void	*istub_unused;
} imc_stub_t;
#endif	/* _KERNEL */

typedef enum {
	IMC_F_UNSUP_PLATFORM	= (1 << 0),
	IMC_F_SCAN_DISPATCHED	= (1 << 1),
	IMC_F_SCAN_COMPLETE	= (1 << 2),
	IMC_F_ATTACH_DISPATCHED	= (1 << 3),
	IMC_F_ATTACH_COMPLETE	= (1 << 4),
	IMC_F_MCREG_FAILED	= (1 << 5),
	IMC_F_VALIDATE_FAILED	= (1 << 6)
} imc_flags_t;

#define	IMC_F_ALL_FLAGS	(IMC_F_UNSUP_PLATFORM | IMC_F_SCAN_DISPATCHED | \
    IMC_F_SCAN_COMPLETE | IMC_F_ATTACH_DISPATCHED | IMC_F_ATTACH_COMPLETE | \
    IMC_F_MCREG_FAILED | IMC_F_VALIDATE_FAILED)

typedef enum imc_dimm_type {
	IMC_DIMM_UNKNOWN,
	IMC_DIMM_DDR3,
	IMC_DIMM_DDR4,
	IMC_DIMM_NVDIMM
} imc_dimm_type_t;

typedef enum imc_dimm_valid {
	IMC_DIMM_V_VALID	= 0,
	IMC_DIMM_V_BAD_PCI_READ	= (1 << 0),
	IMC_DIMM_V_BAD_ROWS	= (1 << 1),
	IMC_DIMM_V_BAD_COLUMNS	= (1 << 2),
	IMC_DIMM_V_BAD_DENSITY	= (1 <<	3),
	IMC_DIMM_V_BAD_WIDTH	= (1 << 4),
	IMC_DIMM_V_BAD_RANKS	= (1 << 5)
} imc_dimm_valid_t;

typedef struct imc_dimm {
	imc_dimm_valid_t	idimm_valid;
	boolean_t	idimm_present;
	uint8_t		idimm_3dsranks;
	boolean_t	idimm_hdrl_parity;
	boolean_t	idimm_hdrl;
	boolean_t	idimm_ranks_disabled[IMC_MAX_RANK_DISABLE];
	uint8_t		idimm_nbanks;
	uint8_t		idimm_nranks;
	uint8_t		idimm_width;
	uint8_t		idimm_density; /* In GiB */
	uint8_t		idimm_nrows;
	uint8_t		idimm_ncolumns;
	/* Synthesized */
	uint64_t	idimm_size;
	/* Raw data */
	uint32_t	idimm_mtr;
} imc_dimm_t;

typedef struct imc_rank_ileave_entry {
	uint8_t		irle_target;
	uint64_t	irle_offset;
} imc_rank_ileave_entry_t;

typedef struct imc_rank_ileave {
	boolean_t		irle_enabled;
	uint32_t		irle_raw;
	uint8_t			irle_nways;
	uint8_t			irle_nwaysbits;
	uint64_t		irle_limit;
	uint_t			irle_nentries;
	imc_rank_ileave_entry_t	irle_entries[IMC_MAX_RANK_INTERLEAVES];
} imc_rank_ileave_t;

typedef enum imc_channel_valid {
	IMC_CHANNEL_V_VALID		= 0,
	IMC_CHANNEL_V_BAD_PCI_READ	= 1 << 0,
} imc_channel_valid_t;

typedef struct imc_channel {
	imc_channel_valid_t	ich_valid;
	imc_stub_t		*ich_desc;
	uint_t			ich_ndimms;
	imc_dimm_t		ich_dimms[IMC_MAX_DIMMPERCHAN];
	uint_t			ich_ntad_offsets;
	uint32_t		ich_tad_offsets_raw[IMC_MAX_TAD_RULES];
	uint64_t		ich_tad_offsets[IMC_MAX_TAD_RULES];
	uint_t			ich_nrankileaves;
	imc_rank_ileave_t	ich_rankileaves[IMC_MAX_RANK_WAYS];
} imc_channel_t;

typedef struct imc_controller {
	imc_stub_t	*icn_main0;
	imc_stub_t	*icn_main1;
	imc_stub_t	*icn_m2m;
	boolean_t	icn_invalid;
	imc_dimm_type_t	icn_dimm_type;
	boolean_t	icn_ecc;
	boolean_t	icn_lockstep;
	boolean_t	icn_closed;
	uint32_t	icn_topo;
	uint_t		icn_nchannels;
	imc_channel_t	icn_channels[IMC_MAX_CHANPERMC];
} imc_mc_t;

typedef enum imc_sad_rule_type {
	IMC_SAD_TYPE_DRAM,
	IMC_SAD_TYPE_MMCFG,
	IMC_SAD_TYPE_NXM
} imc_sad_rule_type_t;

typedef enum imc_sad_rule_imode {
	IMC_SAD_IMODE_8t6,
	IMC_SAD_IMODE_8t6XOR,
	IMC_SAD_IMODE_10t8,
	IMC_SAD_IMODE_14t12,
	IMC_SAD_IMODE_32t30
} imc_sad_rule_imode_t;

typedef enum imc_sad_rule_mod_mode {
	IMC_SAD_MOD_MODE_NONE,
	IMC_SAD_MOD_MODE_45t6,
	IMC_SAD_MOD_MODE_45t8,
	IMC_SAD_MOD_MODE_45t12
} imc_sad_rule_mod_mode_t;

typedef enum imc_sad_rule_mod_type {
	IMC_SAD_MOD_TYPE_NONE,
	IMC_SAD_MOD_TYPE_MOD3,
	IMC_SAD_MOD_TYPE_MOD2_01,
	IMC_SAD_MOD_TYPE_MOD2_12,
	IMC_SAD_MOD_TYPE_MOD2_02
} imc_sad_rule_mod_type_t;

typedef struct imc_sad_mcroute_entry {
	uint8_t	ismce_imc;		/* ID of the target IMC */
	uint8_t	ismce_pchannel;		/* ID of the target physical channel */
} imc_sad_mcroute_entry_t;

typedef struct imc_sad_mcroute_table {
	uint32_t		ismc_raw_mcroute;
	uint_t			ismc_nroutes;
	imc_sad_mcroute_entry_t	ismc_mcroutes[IMC_MAX_SAD_MCROUTES];
} imc_sad_mcroute_table_t;

/*
 * This rule represents a single SAD entry.
 */
typedef struct imc_sad_rule {
	uint32_t		isr_raw_dram;
	uint32_t		isr_raw_interleave;
	boolean_t		isr_enable;
	boolean_t		isr_a7mode;
	boolean_t		isr_need_mod3;
	uint64_t		isr_limit;
	imc_sad_rule_type_t	isr_type;
	imc_sad_rule_imode_t	isr_imode;
	imc_sad_rule_mod_mode_t	isr_mod_mode;
	imc_sad_rule_mod_type_t	isr_mod_type;
	uint_t			isr_ntargets;
	uint8_t			isr_targets[IMC_MAX_SAD_INTERLEAVE];
} imc_sad_rule_t;

typedef enum imc_sad_flags {
	IMC_SAD_MCROUTE_VALID	= 1 << 0,
} imc_sad_flags_t;

typedef enum imc_sad_valid {
	IMC_SAD_V_VALID		= 0,
	IMC_SAD_V_BAD_PCI_READ	= 1 << 0,
	IMC_SAD_V_BAD_MCROUTE	= 1 << 1,
	IMC_SAD_V_BAD_DRAM_ATTR	= 1 << 2,
	IMC_SAD_V_BAD_MOD3	= 1 << 3,
} imc_sad_valid_t;

typedef struct imc_sad {
	imc_sad_flags_t	isad_flags;
	imc_sad_valid_t	isad_valid;
	imc_stub_t	*isad_dram;
	imc_stub_t	*isad_mmio;
	imc_stub_t	*isad_tolh;
	uint64_t	isad_tolm;
	uint64_t	isad_tohm;
	uint_t		isad_nrules;
	imc_sad_rule_t	isad_rules[IMC_MAX_SAD_RULES];
	imc_sad_mcroute_table_t isad_mcroute;
} imc_sad_t;

typedef enum imc_tad_gran {
	IMC_TAD_GRAN_64B = 0,
	IMC_TAD_GRAN_256B,
	IMC_TAD_GRAN_4KB,
	IMC_TAD_GRAN_1GB
} imc_tad_gran_t;

typedef struct imc_tad_rule {
	uint64_t	itr_base;
	uint64_t	itr_limit;
	uint32_t	itr_raw;
	uint32_t	itr_raw_gran;
	uint8_t		itr_sock_way;
	uint8_t		itr_chan_way;
	imc_tad_gran_t	itr_sock_gran;
	imc_tad_gran_t	itr_chan_gran;
	uint_t		itr_ntargets;
	uint8_t		itr_targets[IMC_MAX_TAD_TARGETS];
} imc_tad_rule_t;

typedef enum imc_tad_valid {
	IMC_TAD_V_VALID		= 1 << 0,
	IMC_TAD_V_BAD_PCI_READ	= 1 << 1,
	IMC_TAD_V_BAD_CHAN_GRAN	= 1 << 2
} imc_tad_valid_t;

typedef enum imc_tad_flags {
	IMC_TAD_FLAG_CHANSHIFT	= 1 << 0,
	IMC_TAD_FLAG_CHANHASH	= 1 << 1,
	IMC_TAD_FLAG_MIRROR	= 1 << 2,
	IMC_TAD_FLAG_LOCKSTEP	= 1 << 3
} imc_tad_flags_t;

typedef struct imc_tad {
	imc_tad_valid_t	itad_valid;
	imc_stub_t	*itad_stub;
	imc_tad_flags_t	itad_flags;
	uint_t		itad_nrules;
	imc_tad_rule_t	itad_rules[IMC_MAX_TAD_RULES];
} imc_tad_t;

typedef enum imc_socket_valid {
	IMC_SOCKET_V_VALID	= 0,
	IMC_SOCKET_V_BAD_NODEID	= 1 << 0
} imc_socket_valid_t;

typedef struct imc_socket {
	imc_socket_valid_t	isock_valid;
	uint_t			isock_bus[IMC_MAX_PCIBUSES];
	uint_t			isock_nbus;
	uint_t			isock_gen;
	nvlist_t		*isock_nvl;
	char			*isock_buf;
	size_t			isock_buflen;
	imc_sad_t		isock_sad;
	uint_t			isock_ntad;
	imc_tad_t		isock_tad[IMC_MAX_TAD];
	imc_stub_t		*isock_ubox;
	imc_stub_t		*isock_cpubusno;
	uint32_t		isock_nodeid;
	uint_t			isock_nimc;
	imc_mc_t		isock_imcs[IMC_MAX_IMCPERSOCK];
} imc_socket_t;

typedef struct imc {
	/*
	 * The initial members here are only used in the kernel. This is done to
	 * make it easier for us to be able to define a version of this to use
	 * in testing.
	 */
#ifdef	_KERNEL
	dev_info_t	*imc_dip;
	kmutex_t	imc_lock;
	imc_flags_t	imc_flags;
	const imc_gen_data_t	*imc_gen_data;
	ddi_taskq_t	*imc_taskq;
	uint_t		imc_nscanned;
	avl_tree_t	imc_stubs;
	nvlist_t	*imc_decoder_dump;
	char		*imc_decoder_buf;
	size_t		imc_decoder_len;
#endif	/* _KERNEL */
	imc_gen_t	imc_gen;

	/*
	 * Data about the memory in the system
	 */
	uint_t		imc_nsockets;
	imc_socket_t	imc_sockets[IMC_MAX_SOCKETS];

#ifdef _KERNEL
	/*
	 * The imc_sockets[] array is organized based on increasing PCI Bus ID.
	 * This array maps the socket id that user land thinks of back to the
	 * actual underlying socket in case hardware does not put them in order.
	 */
	imc_socket_t	*imc_spointers[IMC_MAX_SOCKETS];

	/*
	 * Store the IIO global VT-D misc. device. While there are sometimes
	 * multiple on the system, we only keep a single one around.
	 */
	imc_stub_t	*imc_gvtd_misc;
#endif
} imc_t;


/*
 * Decoder failure reasons
 */
typedef enum imc_decode_failure {
	IMC_DECODE_F_NONE = 0,
	/*
	 * Indicates that the memory address fell into a reserved legacy range.
	 * The legacy range index is stored in the failure data.
	 */
	IMC_DECODE_F_LEGACY_RANGE,
	/*
	 * Indicates that we had bad socket data. The socket in question is
	 * noted in the failure data.
	 */
	IMC_DECODE_F_BAD_SOCKET,
	/*
	 * Indicates that we had bad SAD data. The socket the SAD is associated
	 * with is noted in the failure data.
	 */
	IMC_DECODE_F_BAD_SAD,
	/*
	 * Indicates that the address was not contained in conventional, low,
	 * or high memory.
	 */
	IMC_DECODE_F_OUTSIDE_DRAM,
	/*
	 * Indicates that no valid SAD rule was found for the address.
	 */
	IMC_DECODE_F_NO_SAD_RULE,
	/*
	 * Indicates that the SAD interleave target was beyond the valid index.
	 */
	IMC_DECODE_F_BAD_SAD_INTERLEAVE,
	/*
	 * Indicates that the route suggested a remote processor we can't find.
	 */
	IMC_DECODE_F_BAD_REMOTE_MC_ROUTE,
	/*
	 * Indicates that we ended up in a loop trying to find the right socket
	 * to use.
	 */
	IMC_DECODE_F_SAD_SEARCH_LOOP,
	/*
	 * Indicates that we encountered a SAD rule that asked for inconsistent
	 * mod rules.
	 */
	IMC_DECODE_F_SAD_BAD_MOD,
	/*
	 * Indicates that the socket or tad rule we found doesn't actually point
	 * to something that we know about.
	 */
	IMC_DECODE_F_SAD_BAD_SOCKET,
	IMC_DECODE_F_SAD_BAD_TAD,
	/*
	 * Indicates that we could not find a matching tad rule.
	 */
	IMC_DECODE_F_NO_TAD_RULE,
	/*
	 * Indicates that we encountered the TAD channel 3-way interleave that
	 * we don't support.
	 */
	IMC_DECODE_F_TAD_3_ILEAVE,
	/*
	 * Indicates that we had a bad target index.
	 */
	IMC_DECODE_F_TAD_BAD_TARGET_INDEX,
	/*
	 * Indicates that we have a bad channel ID.
	 */
	IMC_DECODE_F_BAD_CHANNEL_ID,
	/*
	 * Indicates that the TAD rule offset in the channel interleave was
	 * incorrect.
	 */
	IMC_DECODE_F_BAD_CHANNEL_TAD_OFFSET,
	/*
	 * We couldn't find a valid rank interleave rule.
	 */
	IMC_DECODE_F_NO_RIR_RULE,
	/*
	 * Indicates that the index of the rank interleaving target was bad.
	 */
	IMC_DECODE_F_BAD_RIR_ILEAVE_TARGET,
	/*
	 * Indicates that the calculated DIMM represents an invalid DIMM that is
	 * beyond the number of supported DIMMS per channel on the platform.
	 */
	IMC_DECODE_F_BAD_DIMM_INDEX,
	/*
	 * Indicates that the specified DIMM is not preset; however, it is a
	 * valid DIMM number.
	 */
	IMC_DECODE_F_DIMM_NOT_PRESENT,
	/*
	 * Indicates that the specified rank on the DIMM is more than the number
	 * of ranks that the DIMM has.
	 */
	IMC_DECODE_F_BAD_DIMM_RANK,
	/*
	 * Indicates that the channel offset is larger than the system address,
	 * meaning that we would end up with an underflow if we continued. The
	 * equivalent is true for the rank address.
	 */
	IMC_DECODE_F_CHANOFF_UNDERFLOW,
	IMC_DECODE_F_RANKOFF_UNDERFLOW,
} imc_decode_failure_t;

/*
 * Decoder state tracking
 */
typedef struct imc_decode_state {
	imc_decode_failure_t	ids_fail;
	uint64_t		ids_fail_data;
	uint64_t		ids_pa;
	uint64_t		ids_chanaddr;
	uint64_t		ids_rankaddr;
	uint32_t		ids_nodeid;
	uint32_t		ids_tadid;
	uint32_t		ids_channelid;
	uint32_t		ids_physrankid;
	uint32_t		ids_dimmid;
	uint32_t		ids_rankid;
	const imc_socket_t	*ids_socket;
	const imc_sad_t		*ids_sad;
	const imc_sad_rule_t	*ids_sad_rule;
	const imc_tad_t		*ids_tad;
	const imc_tad_rule_t	*ids_tad_rule;
	const imc_mc_t		*ids_mc;
	const imc_channel_t	*ids_chan;
	const imc_rank_ileave_t	*ids_rir;
	const imc_dimm_t	*ids_dimm;
} imc_decode_state_t;

#ifdef	_KERNEL

/*
 * Functions needed for the stub drivers.
 */
extern int imc_attach_stub(dev_info_t *, ddi_attach_cmd_t);
extern int imc_detach_stub(dev_info_t *, ddi_detach_cmd_t);

/*
 * Decoder related functions
 */
extern void imc_decoder_init(imc_t *);

extern nvlist_t *imc_dump_decoder(imc_t *);
#else	/* !_KERNEL */
extern boolean_t imc_restore_decoder(nvlist_t *, imc_t *);
#endif	/* _KERNEL */

extern boolean_t imc_decode_pa(const imc_t *, uint64_t, imc_decode_state_t *);


#ifdef __cplusplus
}
#endif

#endif /* _INTEL_IMC_H */