6594519 Need support for ACPI System Resource Affinity Table (SRAT)

6621201 Need support to read ACPI System Locality Information Table (SLIT)
6688471 x86/x64 lgroup platform support code needs cleaning

6 files changed, 1776 insertions, 913 deletions

diff --git a/usr/src/uts/i86pc/os/acpi_fw.h b/usr/src/uts/i86pc/os/acpi_fw.h
index b1a3c5c925..89e067d4b9 100644
--- a/usr/src/uts/i86pc/os/acpi_fw.h
+++ b/usr/src/uts/i86pc/os/acpi_fw.h
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -138,6 +138,11 @@ struct srat {
 #define	SRAT_HOT_PLUG	  (2)
 #define	SRAT_NON_VOLATILE (4)
 
+/*
+ * Pointer to System Resource Affinity Table (SRAT)
+ */
+extern struct srat	*srat_ptr;
+
 struct slit {
 	struct table_header hdr;
 	uint64_t number;
@@ -145,6 +150,11 @@ struct slit {
 };
 
 /*
+ * Pointer to System Locality Information Table (SLIT)
+ */
+extern struct slit	*slit_ptr;
+
+/*
  * Arbitrary limit on number of localities we handle; if
  * this limit is raised to more than UINT16_MAX, make sure
  * process_slit() knows how to handle it.
diff --git a/usr/src/uts/i86pc/os/cpuid.c b/usr/src/uts/i86pc/os/cpuid.c
index 39b99de95b..61e51263c3 100644
--- a/usr/src/uts/i86pc/os/cpuid.c
+++ b/usr/src/uts/i86pc/os/cpuid.c
@@ -2620,6 +2620,19 @@ cpuid_get_dtlb_nent(cpu_t *cpu, size_t pagesize)
 	return (dtlb_nent);
 }
 
+uint_t
+cpuid_get_apicid(cpu_t *cpu)
+{
+	struct cpuid_info *cpi;
+
+	if (cpu == NULL)
+		cpu = CPU;
+	cpi = cpu->cpu_m.mcpu_cpi;
+
+	ASSERT(cpuid_checkpass(cpu, 1));
+	return (CPI_APIC_ID(cpi));
+}
+
 /*
  * Return 0 if the erratum is not present or not applicable, positive
  * if it is, and negative if the status of the erratum is unknown.
diff --git a/usr/src/uts/i86pc/os/fakebop.c b/usr/src/uts/i86pc/os/fakebop.c
index 2a11950faa..3ba83270a0 100644
--- a/usr/src/uts/i86pc/os/fakebop.c
+++ b/usr/src/uts/i86pc/os/fakebop.c
@@ -122,6 +122,14 @@ static void build_firmware_properties(void);
 static int early_allocation = 1;
 
 /*
+ * Pointers to where System Resource Affinity Table (SRAT) and
+ * System Locality Information Table (SLIT) are mapped into virtual memory
+ */
+struct srat	*srat_ptr = NULL;
+struct slit	*slit_ptr = NULL;
+
+
+/*
  * Allocate aligned physical memory at boot time. This allocator allocates
  * from the highest possible addresses. This avoids exhausting memory that
  * would be useful for DMA buffers.
@@ -2008,11 +2016,11 @@ build_firmware_properties(void)
 	if ((tp = find_fw_table("APIC")) != NULL)
 		process_madt((struct madt *)tp);
 
-	if ((tp = find_fw_table("SRAT")) != NULL)
-		process_srat((struct srat *)tp);
+	if ((srat_ptr = (struct srat *)find_fw_table("SRAT")) != NULL)
+		process_srat(srat_ptr);
 
-	if (tp = find_fw_table("SLIT"))
-		process_slit((struct slit *)tp);
+	if (slit_ptr = (struct slit *)find_fw_table("SLIT"))
+		process_slit(slit_ptr);
 #else /* __xpv */
 	enumerate_xen_cpus();
 #endif /* __xpv */
diff --git a/usr/src/uts/i86pc/os/lgrpplat.c b/usr/src/uts/i86pc/os/lgrpplat.c
index 927c6d5ad5..24316c3aea 100644
--- a/usr/src/uts/i86pc/os/lgrpplat.c
+++ b/usr/src/uts/i86pc/os/lgrpplat.c
@@ -27,6 +27,98 @@
 #pragma ident	"%Z%%M%	%I%	%E% SMI"
 
 
+/*
+ * LOCALITY GROUP (LGROUP) PLATFORM SUPPORT FOR X86/AMD64 PLATFORMS
+ * ================================================================
+ * Multiprocessor AMD and Intel systems may have Non Uniform Memory Access
+ * (NUMA).  A NUMA machine consists of one or more "nodes" that each consist of
+ * one or more CPUs and some local memory.  The CPUs in each node can access
+ * the memory in the other nodes but at a higher latency than accessing their
+ * local memory.  Typically, a system with only one node has Uniform Memory
+ * Access (UMA), but it may be possible to have a one node system that has
+ * some global memory outside of the node which is higher latency.
+ *
+ * Module Description
+ * ------------------
+ * This module provides a platform interface for determining which CPUs and
+ * which memory (and how much) are in a NUMA node and how far each node is from
+ * each other.  The interface is used by the Virtual Memory (VM) system and the
+ * common lgroup framework.  The VM system uses the plat_*() routines to fill
+ * in its memory node (memnode) array with the physical address range spanned
+ * by each NUMA node to know which memory belongs to which node, so it can
+ * build and manage a physical page free list for each NUMA node and allocate
+ * local memory from each node as needed.  The common lgroup framework uses the
+ * exported lgrp_plat_*() routines to figure out which CPUs and memory belong
+ * to each node (leaf lgroup) and how far each node is from each other, so it
+ * can build the latency (lgroup) topology for the machine in order to optimize
+ * for locality.  Also, an lgroup platform handle instead of lgroups are used
+ * in the interface with this module, so this module shouldn't need to know
+ * anything about lgroups.  Instead, it just needs to know which CPUs, memory,
+ * etc. are in each NUMA node, how far each node is from each other, and to use
+ * a unique lgroup platform handle to refer to each node through the interface.
+ *
+ * Determining NUMA Configuration
+ * ------------------------------
+ * By default, this module will try to determine the NUMA configuration of the
+ * machine by reading the ACPI System Resource Affinity Table (SRAT) and System
+ * Locality Information Table (SLIT).  The SRAT contains info to tell which
+ * CPUs and memory are local to a given proximity domain (NUMA node).  The SLIT
+ * is a matrix that gives the distance between each system locality (which is
+ * a NUMA node and should correspond to proximity domains in the SRAT).  For
+ * more details on the SRAT and SLIT, please refer to an ACPI 3.0 or newer
+ * specification.
+ *
+ * If the SRAT doesn't exist on a system with AMD Opteron processors, we
+ * examine registers in PCI configuration space to determine how many nodes are
+ * in the system and which CPUs and memory are in each node.
+ * do while booting the kernel.
+ *
+ * NOTE: Using these PCI configuration space registers to determine this
+ *       locality info is not guaranteed to work or be compatible across all
+ *	 Opteron processor families.
+ *
+ * If the SLIT does not exist or look right, the kernel will probe to determine
+ * the distance between nodes as long as the NUMA CPU and memory configuration
+ * has been determined (see lgrp_plat_probe() for details).
+ *
+ * Data Structures
+ * ---------------
+ * The main data structures used by this code are the following:
+ *
+ * - lgrp_plat_cpu_node[]		APIC ID to node ID mapping table
+ *					indexed by hashed APIC ID (only used
+ *					for SRAT)
+ *
+ * - lgrp_plat_lat_stats.latencies[][]	Table of latencies between same and
+ *					different nodes indexed by node ID
+ *
+ * - lgrp_plat_node_cnt			Number of NUMA nodes in system
+ *
+ * - lgrp_plat_node_domain[]		Node ID to proximity domain ID mapping
+ *					table indexed by node ID (only used
+ *					for SRAT)
+ *
+ * - lgrp_plat_node_memory[]		Table with physical address range for
+ *					each node indexed by node ID
+ *
+ * The code is implemented to make the following always be true:
+ *
+ *	lgroup platform handle == node ID == memnode ID
+ *
+ * Moreover, it allows for the proximity domain ID to be equal to all of the
+ * above as long as the proximity domains IDs are numbered from 0 to <number of
+ * nodes - 1>.  This is done by hashing each proximity domain ID into the range
+ * from 0 to <number of nodes - 1>.  Then proximity ID N will hash into node ID
+ * N and proximity domain ID N will be entered into lgrp_plat_node_domain[N]
+ * and be assigned node ID N.  If the proximity domain IDs aren't numbered
+ * from 0 to <number of nodes - 1>, then hashing the proximity domain IDs into
+ * lgrp_plat_node_domain[] will still work for assigning proximity domain IDs
+ * to node IDs.  However, the proximity domain IDs may not map to the
+ * equivalent node ID since we want to keep the node IDs numbered from 0 to
+ * <number of nodes - 1> to minimize cost of searching and potentially space.
+ */
+
+
 #include <sys/archsystm.h>	/* for {in,out}{b,w,l}() */
 #include <sys/cmn_err.h>
 #include <sys/controlregs.h>
@@ -42,216 +134,161 @@
 #include <sys/param.h>
 #include <sys/pghw.h>
 #include <sys/promif.h>		/* for prom_printf() */
+#include <sys/sysmacros.h>
 #include <sys/systm.h>
 #include <sys/thread.h>
 #include <sys/types.h>
 #include <sys/var.h>
 #include <sys/x86_archext.h>	/* for x86_feature and X86_AMD */
-#include <sys/sysmacros.h>
 #include <vm/hat_i86.h>
 #include <vm/seg_kmem.h>
 #include <vm/vm_dep.h>
 
+#include "acpi_fw.h"		/* for SRAT and SLIT */
 
-/*
- * lgroup platform support for x86 platforms.
- */
 
 #define	MAX_NODES		8
 #define	NLGRP			(MAX_NODES * (MAX_NODES - 1) + 1)
 
-#define	LGRP_PLAT_CPU_TO_NODE(cpu) (pg_plat_hw_instance_id(cpu, PGHW_CHIP))
-
+/*
+ * Constants for configuring probing
+ */
 #define	LGRP_PLAT_PROBE_NROUNDS		64	/* default laps for probing */
 #define	LGRP_PLAT_PROBE_NSAMPLES	1	/* default samples to take */
 #define	LGRP_PLAT_PROBE_NREADS		256	/* number of vendor ID reads */
 
 /*
- * Multiprocessor Opteron machines have Non Uniform Memory Access (NUMA).
- *
- * Until this code supports reading System Resource Affinity Table (SRAT),
- * we need to examine registers in PCI configuration space to determine how
- * many nodes are in the system and which CPUs and memory are in each node.
- * This could be determined by probing all memory from each CPU, but that is
- * too expensive to do while booting the kernel.
- *
- * NOTE: Using these PCI configuration space registers to determine this
- *       locality info is not guaranteed to work on future generations of
- *	 Opteron processor.
- */
-
-/*
- * Opteron DRAM Address Map in PCI configuration space gives base and limit
- * of physical memory in each node.  The following constants and macros define
- * their contents, structure, and access.
+ * Flags for probing
  */
+#define	LGRP_PLAT_PROBE_ENABLE		0x1	/* enable probing */
+#define	LGRP_PLAT_PROBE_PGCPY		0x2	/* probe using page copy */
+#define	LGRP_PLAT_PROBE_VENDOR		0x4	/* probe vendor ID register */
 
 /*
- * How many bits to shift Opteron DRAM Address Map base and limit registers
- * to get actual value
+ * Hash CPU APIC ID into CPU to node mapping table using max_ncpus
+ * to minimize span of entries used
  */
-#define	OPT_DRAMADDR_HI_LSHIFT_ADDR	40	/* shift left for address */
-#define	OPT_DRAMADDR_LO_LSHIFT_ADDR	8	/* shift left for address */
-
-#define	OPT_DRAMADDR_HI_MASK_ADDR	0x000000FF /* address bits 47-40 */
-#define	OPT_DRAMADDR_LO_MASK_ADDR	0xFFFF0000 /* address bits 39-24 */
-
-#define	OPT_DRAMADDR_LO_MASK_OFF	0xFFFFFF /* offset for address */
+#define	CPU_NODE_HASH(apicid)		((apicid) % max_ncpus)
 
 /*
- * Macros to derive addresses from Opteron DRAM Address Map registers
+ * Hash proximity domain ID into node to domain mapping table using to minimize
+ * span of entries used
  */
-#define	OPT_DRAMADDR_HI(reg) \
-	(((u_longlong_t)reg & OPT_DRAMADDR_HI_MASK_ADDR) << \
-	    OPT_DRAMADDR_HI_LSHIFT_ADDR)
-
-#define	OPT_DRAMADDR_LO(reg) \
-	(((u_longlong_t)reg & OPT_DRAMADDR_LO_MASK_ADDR) << \
-	    OPT_DRAMADDR_LO_LSHIFT_ADDR)
+#define	NODE_DOMAIN_HASH(domain)	((domain) % lgrp_plat_node_cnt)
 
-#define	OPT_DRAMADDR(high, low) \
-	(OPT_DRAMADDR_HI(high) | OPT_DRAMADDR_LO(low))
 
 /*
- * Bit masks defining what's in Opteron DRAM Address Map base register
+ * CPU APIC ID to node ID mapping structure (only used with SRAT)
  */
-#define	OPT_DRAMBASE_LO_MASK_RE		0x1	/* read enable */
-#define	OPT_DRAMBASE_LO_MASK_WE		0x2	/* write enable */
-#define	OPT_DRAMBASE_LO_MASK_INTRLVEN	0x700	/* interleave */
+typedef	struct cpu_node_map {
+	int		exists;
+	uint_t		node;
+	uint32_t	apicid;
+	uint32_t	prox_domain;
+} cpu_node_map_t;
 
 /*
- * Bit masks defining what's in Opteron DRAM Address Map limit register
+ * Latency statistics
  */
-#define	OPT_DRAMLIMIT_LO_MASK_DSTNODE	0x7		/* destination node */
-#define	OPT_DRAMLIMIT_LO_MASK_INTRLVSEL	0x700		/* interleave select */
-
+typedef struct lgrp_plat_latency_stats {
+	hrtime_t	latencies[MAX_NODES][MAX_NODES];
+	hrtime_t	latency_max;
+	hrtime_t	latency_min;
+} lgrp_plat_latency_stats_t;
 
 /*
- * Opteron Node ID register in PCI configuration space contains
- * number of nodes in system, etc. for Opteron K8.  The following
- * constants and macros define its contents, structure, and access.
+ * Memory configuration for probing
  */
+typedef struct lgrp_plat_probe_mem_config {
+	size_t	probe_memsize;		/* how much memory to probe per node */
+	caddr_t	probe_va[MAX_NODES];	/* where memory mapped for probing */
+	pfn_t	probe_pfn[MAX_NODES];	/* physical pages to map for probing */
+} lgrp_plat_probe_mem_config_t;
 
 /*
- * Bit masks defining what's in Opteron Node ID register
+ * Statistics kept for probing
  */
-#define	OPT_NODE_MASK_ID	0x7	/* node ID */
-#define	OPT_NODE_MASK_CNT	0x70	/* node count */
-#define	OPT_NODE_MASK_IONODE	0x700	/* Hypertransport I/O hub node ID */
-#define	OPT_NODE_MASK_LCKNODE	0x7000	/* lock controller node ID */
-#define	OPT_NODE_MASK_CPUCNT	0xF0000	/* CPUs in system (0 means 1 CPU)  */
+typedef struct lgrp_plat_probe_stats {
+	hrtime_t	flush_cost;
+	hrtime_t	probe_cost;
+	hrtime_t	probe_cost_total;
+	hrtime_t	probe_error_code;
+	hrtime_t	probe_errors[MAX_NODES][MAX_NODES];
+	int		probe_suspect[MAX_NODES][MAX_NODES];
+	hrtime_t	probe_max[MAX_NODES][MAX_NODES];
+	hrtime_t	probe_min[MAX_NODES][MAX_NODES];
+} lgrp_plat_probe_stats_t;
 
 /*
- * How many bits in Opteron Node ID register to shift right to get actual value
+ * Node to proximity domain ID mapping structure (only used with SRAT)
  */
-#define	OPT_NODE_RSHIFT_CNT	0x4	/* shift right for node count value */
+typedef	struct node_domain_map {
+	int		exists;
+	uint32_t	prox_domain;
+} node_domain_map_t;
 
 /*
- * Macros to get values from Opteron Node ID register
+ * Node ID and starting and ending page for physical memory in node
  */
-#define	OPT_NODE_CNT(reg) \
-	((reg & OPT_NODE_MASK_CNT) >> OPT_NODE_RSHIFT_CNT)
+typedef	struct node_phys_addr_map {
+	pfn_t		start;
+	pfn_t		end;
+	int		exists;
+	uint32_t	prox_domain;
+} node_phys_addr_map_t;
 
-/*
- * Macro to setup PCI Extended Configuration Space (ECS) address to give to
- * "in/out" instructions
- *
- * NOTE: Should only be used in lgrp_plat_init() before MMIO setup because any
- *	 other uses should just do MMIO to access PCI ECS.
- *	 Must enable special bit in Northbridge Configuration Register on
- *	 Greyhound for extended CF8 space access to be able to access PCI ECS
- *	 using "in/out" instructions and restore special bit after done
- *	 accessing PCI ECS.
- */
-#define	OPT_PCI_ECS_ADDR(bus, device, function, reg) \
-	(PCI_CONE | (((bus) & 0xff) << 16) | (((device & 0x1f)) << 11)  | \
-	    (((function) & 0x7) << 8) | ((reg) & 0xfc) | \
-	    ((((reg) >> 8) & 0xf) << 24))
 
 /*
- * PCI configuration space registers accessed by specifying
- * a bus, device, function, and offset.  The following constants
- * define the values needed to access Opteron K8 configuration
- * info to determine its node topology
+ * CPU APIC ID to node ID mapping table (only used for SRAT)
  */
-
-#define	OPT_PCS_BUS_CONFIG	0	/* Hypertransport config space bus */
+static cpu_node_map_t			lgrp_plat_cpu_node[NCPU];
 
 /*
- * Opteron PCI configuration space register function values
+ * Latency statistics
  */
-#define	OPT_PCS_FUNC_HT		0	/* Hypertransport configuration */
-#define	OPT_PCS_FUNC_ADDRMAP	1	/* Address map configuration */
-#define	OPT_PCS_FUNC_DRAM	2	/* DRAM configuration */
-#define	OPT_PCS_FUNC_MISC	3	/* Miscellaneous configuration */
+lgrp_plat_latency_stats_t		lgrp_plat_lat_stats;
 
 /*
- * PCI Configuration Space register offsets
+ * Whether memory is interleaved across nodes causing MPO to be disabled
  */
-#define	OPT_PCS_OFF_VENDOR	0x0	/* device/vendor ID register */
-#define	OPT_PCS_OFF_DRAMBASE_HI	0x140	/* DRAM Base register (node 0) */
-#define	OPT_PCS_OFF_DRAMBASE_LO	0x40	/* DRAM Base register (node 0) */
-#define	OPT_PCS_OFF_NODEID	0x60	/* Node ID register */
+static int				lgrp_plat_mem_intrlv = 0;
 
 /*
- * Opteron PCI Configuration Space device IDs for nodes
+ * Node ID to proximity domain ID mapping table (only used for SRAT)
  */
-#define	OPT_PCS_DEV_NODE0		24	/* device number for node 0 */
-
+static node_domain_map_t		lgrp_plat_node_domain[MAX_NODES];
 
 /*
- * Bookkeeping for latencies seen during probing (used for verification)
+ * Physical address range for memory in each node
  */
-typedef	struct lgrp_plat_latency_acct {
-	hrtime_t	la_value;	/* latency value */
-	int		la_count;	/* occurrences */
-} lgrp_plat_latency_acct_t;
-
+static node_phys_addr_map_t		lgrp_plat_node_memory[MAX_NODES];
 
 /*
- * Choices for probing to determine lgroup topology
+ * Statistics gotten from probing
  */
-typedef	enum lgrp_plat_probe_op {
-	LGRP_PLAT_PROBE_PGCPY,		/* Use page copy */
-	LGRP_PLAT_PROBE_VENDOR		/* Read vendor ID on Northbridge */
-} lgrp_plat_probe_op_t;
-
+static lgrp_plat_probe_stats_t		lgrp_plat_probe_stats;
 
 /*
- * Opteron DRAM address map gives base and limit for physical memory in a node
+ * Memory configuration for probing
  */
-typedef	struct opt_dram_addr_map {
-	uint32_t	base_hi;
-	uint32_t	base_lo;
-	uint32_t	limit_hi;
-	uint32_t	limit_lo;
-} opt_dram_addr_map_t;
-
+static lgrp_plat_probe_mem_config_t	lgrp_plat_probe_mem_config;
 
 /*
- * Starting and ending page for physical memory in node
+ * Error code from processing ACPI SRAT
  */
-typedef	struct phys_addr_map {
-	pfn_t	start;
-	pfn_t	end;
-	int	exists;
-} phys_addr_map_t;
-
+static int				lgrp_plat_srat_error = 0;
 
 /*
- * Opteron DRAM address map for each node
+ * Error code from processing ACPI SLIT
  */
-struct opt_dram_addr_map	opt_dram_map[MAX_NODES];
+static int				lgrp_plat_slit_error = 0;
 
 /*
- * Node ID register contents for each node
+ * Allocate lgroup array statically
  */
-uint_t				opt_node_info[MAX_NODES];
+static lgrp_t				lgrp_space[NLGRP];
+static int				nlgrps_alloc;
 
-/*
- * Whether memory is interleaved across nodes causing MPO to be disabled
- */
-int			lgrp_plat_mem_intrlv = 0;
 
 /*
  * Number of nodes in system
@@ -259,140 +296,212 @@ int			lgrp_plat_mem_intrlv = 0;
 uint_t			lgrp_plat_node_cnt = 1;
 
 /*
- * Physical address range for memory in each node
+ * Configuration Parameters for Probing
+ * - lgrp_plat_probe_flags	Flags to specify enabling probing, probe
+ *				operation, etc.
+ * - lgrp_plat_probe_nrounds	How many rounds of probing to do
+ * - lgrp_plat_probe_nsamples	Number of samples to take when probing each
+ *				node
+ * - lgrp_plat_probe_nreads	Number of times to read vendor ID from
+ *				Northbridge for each probe
  */
-phys_addr_map_t		lgrp_plat_node_memory[MAX_NODES];
+uint_t			lgrp_plat_probe_flags = 0;
+int			lgrp_plat_probe_nrounds = LGRP_PLAT_PROBE_NROUNDS;
+int			lgrp_plat_probe_nsamples = LGRP_PLAT_PROBE_NSAMPLES;
+int			lgrp_plat_probe_nreads = LGRP_PLAT_PROBE_NREADS;
 
 /*
- * Probe costs (individual and total) and flush cost
+ * Enable use of ACPI System Resource Affinity Table (SRAT) and System
+ * Locality Information Table (SLIT)
  */
-hrtime_t		lgrp_plat_flush_cost = 0;
-hrtime_t		lgrp_plat_probe_cost = 0;
-hrtime_t		lgrp_plat_probe_cost_total = 0;
+int			lgrp_plat_srat_enable = 1;
+int			lgrp_plat_slit_enable = 1;
 
 /*
- * Error code for latency adjustment and verification
+ * Static array to hold lgroup statistics
  */
-int			lgrp_plat_probe_error_code = 0;
+struct lgrp_stats	lgrp_stats[NLGRP];
 
-/*
- * How much latencies were off from minimum values gotten
- */
-hrtime_t		lgrp_plat_probe_errors[MAX_NODES][MAX_NODES];
 
 /*
- * Unique probe latencies and number of occurrences of each
+ * Forward declarations of platform interface routines
  */
-lgrp_plat_latency_acct_t	lgrp_plat_probe_lat_acct[MAX_NODES];
+void		plat_build_mem_nodes(struct memlist *list);
 
-/*
- * Size of memory buffer in each node for probing
- */
-size_t			lgrp_plat_probe_memsize = 0;
+int		plat_lgrphand_to_mem_node(lgrp_handle_t hand);
 
-/*
- * Virtual address of page in each node for probing
- */
-caddr_t			lgrp_plat_probe_memory[MAX_NODES];
+lgrp_handle_t	plat_mem_node_to_lgrphand(int mnode);
 
-/*
- * Number of unique latencies in probe times
- */
-int			lgrp_plat_probe_nlatencies = 0;
+int		plat_mnode_xcheck(pfn_t pfncnt);
 
-/*
- * How many rounds of probing to do
- */
-int			lgrp_plat_probe_nrounds = LGRP_PLAT_PROBE_NROUNDS;
+int		plat_pfn_to_mem_node(pfn_t pfn);
 
 /*
- * Number of samples to take when probing each node
+ * Forward declarations of lgroup platform interface routines
  */
-int			lgrp_plat_probe_nsamples = LGRP_PLAT_PROBE_NSAMPLES;
+lgrp_t		*lgrp_plat_alloc(lgrp_id_t lgrpid);
 
-/*
- * Number of times to read vendor ID from Northbridge for each probe.
- */
-int			lgrp_plat_probe_nreads = LGRP_PLAT_PROBE_NREADS;
+void		lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg);
 
-/*
- * How to probe to determine lgroup topology
- */
-lgrp_plat_probe_op_t	lgrp_plat_probe_op = LGRP_PLAT_PROBE_VENDOR;
+lgrp_handle_t	lgrp_plat_cpu_to_hand(processorid_t id);
 
-/*
- * PFN of page in each node for probing
- */
-pfn_t			lgrp_plat_probe_pfn[MAX_NODES];
+void		lgrp_plat_init(void);
 
-/*
- * Whether probe time was suspect (ie. not within tolerance of value that it
- * should match)
- */
-int			lgrp_plat_probe_suspect[MAX_NODES][MAX_NODES];
+int		lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to);
 
-/*
- * How long it takes to access memory from each node
- */
-hrtime_t		lgrp_plat_probe_times[MAX_NODES][MAX_NODES];
+void		lgrp_plat_main_init(void);
 
-/*
- * Min and max node memory probe times seen
- */
-hrtime_t		lgrp_plat_probe_time_max = 0;
-hrtime_t		lgrp_plat_probe_time_min = -1;
-hrtime_t		lgrp_plat_probe_max[MAX_NODES][MAX_NODES];
-hrtime_t		lgrp_plat_probe_min[MAX_NODES][MAX_NODES];
+int		lgrp_plat_max_lgrps(void);
 
+pgcnt_t		lgrp_plat_mem_size(lgrp_handle_t plathand,
+    lgrp_mem_query_t query);
 
-/*
- * Allocate lgrp and lgrp stat arrays statically.
- */
-static lgrp_t	lgrp_space[NLGRP];
-static int	nlgrps_alloc;
+lgrp_handle_t	lgrp_plat_pfn_to_hand(pfn_t pfn);
+
+void		lgrp_plat_probe(void);
+
+lgrp_handle_t	lgrp_plat_root_hand(void);
 
-struct lgrp_stats lgrp_stats[NLGRP];
 
 /*
- * Supported AMD processor families
+ * Forward declarations of local routines
  */
-#define	AMD_FAMILY_HAMMER	15
-#define	AMD_FAMILY_GREYHOUND	16
+static int	is_opteron(void);
+
+static int	lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node);
+
+static int	lgrp_plat_domain_to_node(node_domain_map_t *node_domain,
+    uint32_t domain);
+
+static void	lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory,
+    lgrp_plat_latency_stats_t *lat_stats,
+    lgrp_plat_probe_stats_t *probe_stats);
+
+static int	lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory,
+    lgrp_plat_latency_stats_t *lat_stats);
+
+static pgcnt_t	lgrp_plat_mem_size_default(lgrp_handle_t, lgrp_mem_query_t);
+
+static int	lgrp_plat_node_domain_update(node_domain_map_t *node_domain,
+    uint32_t domain);
+
+static int	lgrp_plat_node_memory_update(node_domain_map_t *node_domain,
+    node_phys_addr_map_t *node_memory, uintptr_t start, uintptr_t end,
+    uint32_t domain);
+
+static hrtime_t	lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node,
+    lgrp_plat_probe_mem_config_t *probe_mem_config,
+    lgrp_plat_latency_stats_t *lat_stats,
+    lgrp_plat_probe_stats_t *probe_stats);
+
+static int	lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt,
+    node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats);
+
+static int	lgrp_plat_process_srat(struct srat *tp, uint_t *node_cnt,
+    node_domain_map_t *node_domain, cpu_node_map_t *cpu_node,
+    node_phys_addr_map_t *node_memory);
+
+static int	lgrp_plat_srat_domains(struct srat *tp);
+
+static void	lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory,
+    lgrp_plat_latency_stats_t *lat_stats);
+
+static void	opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv,
+    node_phys_addr_map_t *node_memory);
+
+static hrtime_t	opt_probe_vendor(int dest_node, int nreads);
+
 
 /*
- * Whether to have is_opteron() return 1 even when processor isn't
- * supported
+ * PLATFORM INTERFACE ROUTINES
  */
-uint_t	is_opteron_override = 0;
 
 /*
- * AMD processor family for current CPU
+ * Configure memory nodes for machines with more than one node (ie NUMA)
  */
-uint_t	opt_family = 0;
+void
+plat_build_mem_nodes(struct memlist *list)
+{
+	pfn_t		cur_start;	/* start addr of subrange */
+	pfn_t		cur_end;	/* end addr of subrange */
+	pfn_t		start;		/* start addr of whole range */
+	pfn_t		end;		/* end addr of whole range */
+
+	/*
+	 * Boot install lists are arranged <addr, len>, ...
+	 */
+	while (list) {
+		int	node;
 
-uint_t	opt_probe_func = OPT_PCS_FUNC_DRAM;
+		start = list->address >> PAGESHIFT;
+		end = (list->address + list->size - 1) >> PAGESHIFT;
 
+		if (start > physmax) {
+			list = list->next;
+			continue;
+		}
+		if (end > physmax)
+			end = physmax;
 
-/*
- * Determine whether we're running on a supported AMD Opteron since reading
- * node count and DRAM address map registers may have different format or
- * may not be supported in future processor families
- */
-int
-is_opteron(void)
-{
+		/*
+		 * When there is only one memnode, just add memory to memnode
+		 */
+		if (max_mem_nodes == 1) {
+			mem_node_add_slice(start, end);
+			list = list->next;
+			continue;
+		}
 
-	if (x86_vendor != X86_VENDOR_AMD)
-		return (0);
+		/*
+		 * mem_node_add_slice() expects to get a memory range that
+		 * is within one memnode, so need to split any memory range
+		 * that spans multiple memnodes into subranges that are each
+		 * contained within one memnode when feeding them to
+		 * mem_node_add_slice()
+		 */
+		cur_start = start;
+		do {
+			node = plat_pfn_to_mem_node(cur_start);
 
-	opt_family = cpuid_getfamily(CPU);
-	if (opt_family == AMD_FAMILY_HAMMER ||
-	    opt_family == AMD_FAMILY_GREYHOUND || is_opteron_override)
-		return (1);
-	else
-		return (0);
+			/*
+			 * Panic if DRAM address map registers or SRAT say
+			 * memory in node doesn't exist or address from
+			 * boot installed memory list entry isn't in this node.
+			 * This shouldn't happen and rest of code can't deal
+			 * with this if it does.
+			 */
+			if (node < 0 || node >= lgrp_plat_node_cnt ||
+			    !lgrp_plat_node_memory[node].exists ||
+			    cur_start < lgrp_plat_node_memory[node].start ||
+			    cur_start > lgrp_plat_node_memory[node].end) {
+				cmn_err(CE_PANIC, "Don't know which memnode "
+				    "to add installed memory address 0x%lx\n",
+				    cur_start);
+			}
+
+			/*
+			 * End of current subrange should not span memnodes
+			 */
+			cur_end = end;
+			if (lgrp_plat_node_memory[node].exists &&
+			    cur_end > lgrp_plat_node_memory[node].end)
+				cur_end = lgrp_plat_node_memory[node].end;
+
+			mem_node_add_slice(cur_start, cur_end);
+
+			/*
+			 * Next subrange starts after end of current one
+			 */
+			cur_start = cur_end + 1;
+		} while (cur_end < end);
+
+		list = list->next;
+	}
+	mem_node_physalign = 0;
+	mem_node_pfn_shift = 0;
 }
 
+
 int
 plat_lgrphand_to_mem_node(lgrp_handle_t hand)
 {
@@ -402,15 +511,6 @@ plat_lgrphand_to_mem_node(lgrp_handle_t hand)
 	return ((int)hand);
 }
 
-lgrp_handle_t
-plat_mem_node_to_lgrphand(int mnode)
-{
-	if (max_mem_nodes == 1)
-		return (LGRP_DEFAULT_HANDLE);
-
-	return ((lgrp_handle_t)mnode);
-}
-
 
 /*
  * plat_mnode_xcheck: checks the node memory ranges to see if there is a pfncnt
@@ -475,6 +575,17 @@ plat_mnode_xcheck(pfn_t pfncnt)
 	return (0);
 }
 
+
+lgrp_handle_t
+plat_mem_node_to_lgrphand(int mnode)
+{
+	if (max_mem_nodes == 1)
+		return (LGRP_DEFAULT_HANDLE);
+
+	return ((lgrp_handle_t)mnode);
+}
+
+
 int
 plat_pfn_to_mem_node(pfn_t pfn)
 {
@@ -495,93 +606,64 @@ plat_pfn_to_mem_node(pfn_t pfn)
 			return (node);
 	}
 
+	/*
+	 * Didn't find memnode where this PFN lives which should never happen
+	 */
 	ASSERT(node < lgrp_plat_node_cnt);
 	return (-1);
 }
 
+
 /*
- * Configure memory nodes for machines with more than one node (ie NUMA)
+ * LGROUP PLATFORM INTERFACE ROUTINES
  */
-void
-plat_build_mem_nodes(struct memlist *list)
-{
-	pfn_t		cur_start;	/* start addr of subrange */
-	pfn_t		cur_end;	/* end addr of subrange */
-	pfn_t		start;		/* start addr of whole range */
-	pfn_t		end;		/* end addr of whole range */
 
-	/*
-	 * Boot install lists are arranged <addr, len>, ...
-	 */
-	while (list) {
-		int	node;
+/*
+ * Allocate additional space for an lgroup.
+ */
+/* ARGSUSED */
+lgrp_t *
+lgrp_plat_alloc(lgrp_id_t lgrpid)
+{
+	lgrp_t *lgrp;
 
-		start = list->address >> PAGESHIFT;
-		end = (list->address + list->size - 1) >> PAGESHIFT;
+	lgrp = &lgrp_space[nlgrps_alloc++];
+	if (lgrpid >= NLGRP || nlgrps_alloc > NLGRP)
+		return (NULL);
+	return (lgrp);
+}
 
-		if (start > physmax) {
-			list = list->next;
-			continue;
-		}
-		if (end > physmax)
-			end = physmax;
 
-		/*
-		 * When there is only one memnode, just add memory to memnode
-		 */
-		if (max_mem_nodes == 1) {
-			mem_node_add_slice(start, end);
-			list = list->next;
-			continue;
-		}
+/*
+ * Platform handling for (re)configuration changes
+ */
+/* ARGSUSED */
+void
+lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg)
+{
+}
 
-		/*
-		 * mem_node_add_slice() expects to get a memory range that
-		 * is within one memnode, so need to split any memory range
-		 * that spans multiple memnodes into subranges that are each
-		 * contained within one memnode when feeding them to
-		 * mem_node_add_slice()
-		 */
-		cur_start = start;
-		do {
-			node = plat_pfn_to_mem_node(cur_start);
 
-			/*
-			 * Panic if DRAM address map registers or SRAT say
-			 * memory in node doesn't exist or address from
-			 * boot installed memory list entry isn't in this node.
-			 * This shouldn't happen and rest of code can't deal
-			 * with this if it does.
-			 */
-			if (node < 0 || node >= lgrp_plat_node_cnt ||
-			    !lgrp_plat_node_memory[node].exists ||
-			    cur_start < lgrp_plat_node_memory[node].start ||
-			    cur_start > lgrp_plat_node_memory[node].end) {
-				cmn_err(CE_PANIC, "Don't know which memnode "
-				    "to add installed memory address 0x%lx\n",
-				    cur_start);
-			}
+/*
+ * Return the platform handle for the lgroup containing the given CPU
+ */
+/* ARGSUSED */
+lgrp_handle_t
+lgrp_plat_cpu_to_hand(processorid_t id)
+{
+	lgrp_handle_t	hand;
 
-			/*
-			 * End of current subrange should not span memnodes
-			 */
-			cur_end = end;
-			if (lgrp_plat_node_memory[node].exists &&
-			    cur_end > lgrp_plat_node_memory[node].end)
-				cur_end = lgrp_plat_node_memory[node].end;
+	if (lgrp_plat_node_cnt == 1)
+		return (LGRP_DEFAULT_HANDLE);
 
-			mem_node_add_slice(cur_start, cur_end);
+	hand = (lgrp_handle_t)lgrp_plat_cpu_to_node(cpu[id],
+	    lgrp_plat_cpu_node);
 
-			/*
-			 * Next subrange starts after end of current one
-			 */
-			cur_start = cur_end + 1;
-		} while (cur_end < end);
+	ASSERT(hand != (lgrp_handle_t)-1);
+	if (hand == (lgrp_handle_t)-1)
+		return (LGRP_NULL_HANDLE);
 
-		list = list->next;
-	}
-	mem_node_physalign = 0;
-	mem_node_pfn_shift = 0;
+	return (hand);
 }
 
 
@@ -597,188 +679,614 @@ lgrp_plat_init(void)
 	 */
 	lgrp_plat_node_cnt = max_mem_nodes = 1;
 #else	/* __xpv */
-	uint_t		bus;
-	uint_t		dev;
-	uint_t		node;
-	uint_t		off_hi;
-	uint_t		off_lo;
-	uint64_t	nb_cfg_reg;
-
-	extern lgrp_load_t	lgrp_expand_proc_thresh;
-	extern lgrp_load_t	lgrp_expand_proc_diff;
+	uint_t	probe_op;
 
 	/*
-	 * Initialize as a UMA machine if this isn't an Opteron
+	 * Initialize as a UMA machine
 	 */
-	if (!is_opteron() || lgrp_topo_ht_limit() == 1) {
+	if (lgrp_topo_ht_limit() == 1) {
 		lgrp_plat_node_cnt = max_mem_nodes = 1;
 		return;
 	}
 
 	/*
-	 * Read configuration registers from PCI configuration space to
-	 * determine node information, which memory is in each node, etc.
+	 * Determine which CPUs and memory are local to each other and number
+	 * of NUMA nodes by reading ACPI System Resource Affinity Table (SRAT)
+	 */
+	lgrp_plat_srat_error = lgrp_plat_process_srat(srat_ptr,
+	    &lgrp_plat_node_cnt, lgrp_plat_node_domain, lgrp_plat_cpu_node,
+	    lgrp_plat_node_memory);
+
+	/*
+	 * Try to use PCI config space registers on Opteron if SRAT doesn't
+	 * exist or there is some error processing the SRAT
+	 */
+	if (lgrp_plat_srat_error != 0 && is_opteron())
+		opt_get_numa_config(&lgrp_plat_node_cnt, &lgrp_plat_mem_intrlv,
+		    lgrp_plat_node_memory);
+
+	/*
+	 * Don't bother to setup system for multiple lgroups and only use one
+	 * memory node when memory is interleaved between any nodes or there is
+	 * only one NUMA node
 	 *
-	 * Write to PCI configuration space address register to specify
-	 * which configuration register to read and read/write PCI
-	 * configuration space data register to get/set contents
+	 * NOTE: May need to change this for Dynamic Reconfiguration (DR)
+	 *	 when and if it happens for x86/x64
 	 */
-	bus = OPT_PCS_BUS_CONFIG;
-	dev = OPT_PCS_DEV_NODE0;
-	off_hi = OPT_PCS_OFF_DRAMBASE_HI;
-	off_lo = OPT_PCS_OFF_DRAMBASE_LO;
+	if (lgrp_plat_mem_intrlv || lgrp_plat_node_cnt == 1) {
+		lgrp_plat_node_cnt = max_mem_nodes = 1;
+		(void) lgrp_topo_ht_limit_set(1);
+		return;
+	}
 
 	/*
-	 * Read node ID register for node 0 to get node count
+	 * Leaf lgroups on x86/x64 architectures contain one physical
+	 * processor chip. Tune lgrp_expand_proc_thresh and
+	 * lgrp_expand_proc_diff so that lgrp_choose() will spread
+	 * things out aggressively.
 	 */
-	opt_node_info[0] = pci_getl_func(bus, dev, OPT_PCS_FUNC_HT,
-	    OPT_PCS_OFF_NODEID);
-	lgrp_plat_node_cnt = OPT_NODE_CNT(opt_node_info[0]) + 1;
+	lgrp_expand_proc_thresh = LGRP_LOADAVG_THREAD_MAX / 2;
+	lgrp_expand_proc_diff = 0;
 
 	/*
-	 * For Greyhound, PCI Extended Configuration Space must be enabled to
-	 * read high DRAM address map base and limit registers
+	 * There should be one memnode (physical page free list(s)) for
+	 * each node
 	 */
-	if (opt_family == AMD_FAMILY_GREYHOUND) {
-		nb_cfg_reg = rdmsr(MSR_AMD_NB_CFG);
-		if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0)
-			wrmsr(MSR_AMD_NB_CFG,
-			    nb_cfg_reg | AMD_GH_NB_CFG_EN_ECS);
+	max_mem_nodes = lgrp_plat_node_cnt;
+
+	/*
+	 * Determine how far each NUMA node is from each other by
+	 * reading ACPI System Locality Information Table (SLIT) if it
+	 * exists
+	 */
+	lgrp_plat_slit_error = lgrp_plat_process_slit(slit_ptr,
+	    lgrp_plat_node_cnt, lgrp_plat_node_memory,
+	    &lgrp_plat_lat_stats);
+	if (lgrp_plat_slit_error == 0)
+		return;
+
+	/*
+	 * Probe to determine latency between NUMA nodes when SLIT
+	 * doesn't exist or make sense
+	 */
+	lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_ENABLE;
+
+	/*
+	 * Specify whether to probe using vendor ID register or page copy
+	 * if hasn't been specified already or is overspecified
+	 */
+	probe_op = lgrp_plat_probe_flags &
+	    (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR);
+
+	if (probe_op == 0 ||
+	    probe_op == (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR)) {
+		lgrp_plat_probe_flags &=
+		    ~(LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR);
+		if (is_opteron())
+			lgrp_plat_probe_flags |=
+			    LGRP_PLAT_PROBE_VENDOR;
+		else
+			lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_PGCPY;
 	}
 
-	for (node = 0; node < lgrp_plat_node_cnt; node++) {
-		uint32_t	base_hi;
-		uint32_t	base_lo;
-		uint32_t	limit_hi;
-		uint32_t	limit_lo;
+	/*
+	 * Probing errors can mess up the lgroup topology and
+	 * force us fall back to a 2 level lgroup topology.
+	 * Here we bound how tall the lgroup topology can grow
+	 * in hopes of avoiding any anamolies in probing from
+	 * messing up the lgroup topology by limiting the
+	 * accuracy of the latency topology.
+	 *
+	 * Assume that nodes will at least be configured in a
+	 * ring, so limit height of lgroup topology to be less
+	 * than number of nodes on a system with 4 or more
+	 * nodes
+	 */
+	if (lgrp_plat_node_cnt >= 4 && lgrp_topo_ht_limit() ==
+	    lgrp_topo_ht_limit_default())
+		(void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1);
+#endif	/* __xpv */
+}
+
+
+/*
+ * Return latency between "from" and "to" lgroups
+ *
+ * This latency number can only be used for relative comparison
+ * between lgroups on the running system, cannot be used across platforms,
+ * and may not reflect the actual latency.  It is platform and implementation
+ * specific, so platform gets to decide its value.  It would be nice if the
+ * number was at least proportional to make comparisons more meaningful though.
+ */
+/* ARGSUSED */
+int
+lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to)
+{
+	lgrp_handle_t	src, dest;
+	int		node;
+
+	if (max_mem_nodes == 1)
+		return (0);
+
+	/*
+	 * Return max latency for root lgroup
+	 */
+	if (from == LGRP_DEFAULT_HANDLE || to == LGRP_DEFAULT_HANDLE)
+		return (lgrp_plat_lat_stats.latency_max);
+
+	src = from;
+	dest = to;
+
+	/*
+	 * Return 0 for nodes (lgroup platform handles) out of range
+	 */
+	if (src < 0 || src >= MAX_NODES || dest < 0 || dest >= MAX_NODES)
+		return (0);
+
+	/*
+	 * Probe from current CPU if its lgroup latencies haven't been set yet
+	 * and we are trying to get latency from current CPU to some node
+	 */
+	node = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node);
+	ASSERT(node >= 0 && node < lgrp_plat_node_cnt);
+	if (lgrp_plat_lat_stats.latencies[src][src] == 0 && node == src)
+		lgrp_plat_probe();
+
+	return (lgrp_plat_lat_stats.latencies[src][dest]);
+}
+
+
+/*
+ * Platform-specific initialization
+ */
+void
+lgrp_plat_main_init(void)
+{
+	int	curnode;
+	int	ht_limit;
+	int	i;
+
+	/*
+	 * Print a notice that MPO is disabled when memory is interleaved
+	 * across nodes....Would do this when it is discovered, but can't
+	 * because it happens way too early during boot....
+	 */
+	if (lgrp_plat_mem_intrlv)
+		cmn_err(CE_NOTE,
+		    "MPO disabled because memory is interleaved\n");
 
+	/*
+	 * Don't bother to do any probing if it is disabled, there is only one
+	 * node, or the height of the lgroup topology less than or equal to 2
+	 */
+	ht_limit = lgrp_topo_ht_limit();
+	if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) ||
+	    max_mem_nodes == 1 || ht_limit <= 2) {
 		/*
-		 * Read node ID register (except for node 0 which we just read)
+		 * Setup lgroup latencies for 2 level lgroup topology
+		 * (ie. local and remote only) if they haven't been set yet
 		 */
-		if (node > 0) {
-			opt_node_info[node] = pci_getl_func(bus, dev,
-			    OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID);
-		}
+		if (ht_limit == 2 && lgrp_plat_lat_stats.latency_min == -1 &&
+		    lgrp_plat_lat_stats.latency_max == 0)
+			lgrp_plat_2level_setup(lgrp_plat_node_memory,
+			    &lgrp_plat_lat_stats);
+		return;
+	}
 
+	if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) {
 		/*
-		 * Read DRAM base and limit registers which specify
-		 * physical memory range of each node
+		 * Should have been able to probe from CPU 0 when it was added
+		 * to lgroup hierarchy, but may not have been able to then
+		 * because it happens so early in boot that gethrtime() hasn't
+		 * been initialized.  (:-(
 		 */
-		if (opt_family != AMD_FAMILY_GREYHOUND)
-			base_hi = 0;
-		else {
-			outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev,
-			    OPT_PCS_FUNC_ADDRMAP, off_hi));
-			base_hi = opt_dram_map[node].base_hi =
-			    inl(PCI_CONFDATA);
-		}
-		base_lo = opt_dram_map[node].base_lo = pci_getl_func(bus, dev,
-		    OPT_PCS_FUNC_ADDRMAP, off_lo);
+		curnode = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node);
+		ASSERT(curnode >= 0 && curnode < lgrp_plat_node_cnt);
+		if (lgrp_plat_lat_stats.latencies[curnode][curnode] == 0)
+			lgrp_plat_probe();
 
-		if (opt_dram_map[node].base_lo & OPT_DRAMBASE_LO_MASK_INTRLVEN)
-			lgrp_plat_mem_intrlv++;
+		return;
+	}
 
-		off_hi += 4;	/* high limit register offset */
-		if (opt_family != AMD_FAMILY_GREYHOUND)
-			limit_hi = 0;
-		else {
-			outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev,
-			    OPT_PCS_FUNC_ADDRMAP, off_hi));
-			limit_hi = opt_dram_map[node].limit_hi =
-			    inl(PCI_CONFDATA);
-		}
+	/*
+	 * When probing memory, use one page for every sample to determine
+	 * lgroup topology and taking multiple samples
+	 */
+	if (lgrp_plat_probe_mem_config.probe_memsize == 0)
+		lgrp_plat_probe_mem_config.probe_memsize = PAGESIZE *
+		    lgrp_plat_probe_nsamples;
 
-		off_lo += 4;	/* low limit register offset */
-		limit_lo = opt_dram_map[node].limit_lo = pci_getl_func(bus,
-		    dev, OPT_PCS_FUNC_ADDRMAP, off_lo);
+	/*
+	 * Map memory in each node needed for probing to determine latency
+	 * topology
+	 */
+	for (i = 0; i < lgrp_plat_node_cnt; i++) {
+		int	mnode;
 
 		/*
-		 * Increment device number to next node and register offsets
-		 * for DRAM base register of next node
+		 * Skip this node and leave its probe page NULL
+		 * if it doesn't have any memory
 		 */
-		off_hi += 4;
-		off_lo += 4;
-		dev++;
+		mnode = plat_lgrphand_to_mem_node((lgrp_handle_t)i);
+		if (!mem_node_config[mnode].exists) {
+			lgrp_plat_probe_mem_config.probe_va[i] = NULL;
+			continue;
+		}
 
 		/*
-		 * Both read and write enable bits must be enabled in DRAM
-		 * address map base register for physical memory to exist in
-		 * node
+		 * Allocate one kernel virtual page
 		 */
-		if ((base_lo & OPT_DRAMBASE_LO_MASK_RE) == 0 ||
-		    (base_lo & OPT_DRAMBASE_LO_MASK_WE) == 0) {
-			/*
-			 * Mark node memory as non-existent and set start and
-			 * end addresses to be same in lgrp_plat_node_memory[]
-			 */
-			lgrp_plat_node_memory[node].exists = 0;
-			lgrp_plat_node_memory[node].start =
-			    lgrp_plat_node_memory[node].end = (pfn_t)-1;
-			continue;
+		lgrp_plat_probe_mem_config.probe_va[i] = vmem_alloc(heap_arena,
+		    lgrp_plat_probe_mem_config.probe_memsize, VM_NOSLEEP);
+		if (lgrp_plat_probe_mem_config.probe_va[i] == NULL) {
+			cmn_err(CE_WARN,
+			    "lgrp_plat_main_init: couldn't allocate memory");
+			return;
 		}
 
 		/*
-		 * Get PFN for first page in each node,
-		 * so we can probe memory to determine latency topology
+		 * Get PFN for first page in each node
 		 */
-		lgrp_plat_probe_pfn[node] =
-		    btop(OPT_DRAMADDR(base_hi, base_lo));
+		lgrp_plat_probe_mem_config.probe_pfn[i] =
+		    mem_node_config[mnode].physbase;
 
 		/*
-		 * Mark node memory as existing and remember physical address
-		 * range of each node for use later
+		 * Map virtual page to first page in node
 		 */
-		lgrp_plat_node_memory[node].exists = 1;
+		hat_devload(kas.a_hat, lgrp_plat_probe_mem_config.probe_va[i],
+		    lgrp_plat_probe_mem_config.probe_memsize,
+		    lgrp_plat_probe_mem_config.probe_pfn[i],
+		    PROT_READ | PROT_WRITE | HAT_PLAT_NOCACHE,
+		    HAT_LOAD_NOCONSIST);
+	}
 
-		lgrp_plat_node_memory[node].start =
-		    btop(OPT_DRAMADDR(base_hi, base_lo));
+	/*
+	 * Probe from current CPU
+	 */
+	lgrp_plat_probe();
+}
 
-		lgrp_plat_node_memory[node].end =
-		    btop(OPT_DRAMADDR(limit_hi, limit_lo) |
-		    OPT_DRAMADDR_LO_MASK_OFF);
+
+/*
+ * Return the maximum number of lgrps supported by the platform.
+ * Before lgrp topology is known it returns an estimate based on the number of
+ * nodes. Once topology is known it returns the actual maximim number of lgrps
+ * created. Since x86/x64 doesn't support Dynamic Reconfiguration (DR) and
+ * dynamic addition of new nodes, this number may not grow during system
+ * lifetime (yet).
+ */
+int
+lgrp_plat_max_lgrps(void)
+{
+	return (lgrp_topo_initialized ?
+	    lgrp_alloc_max + 1 :
+	    lgrp_plat_node_cnt * (lgrp_plat_node_cnt - 1) + 1);
+}
+
+
+/*
+ * Return the number of free pages in an lgroup.
+ *
+ * For query of LGRP_MEM_SIZE_FREE, return the number of base pagesize
+ * pages on freelists.  For query of LGRP_MEM_SIZE_AVAIL, return the
+ * number of allocatable base pagesize pages corresponding to the
+ * lgroup (e.g. do not include page_t's, BOP_ALLOC()'ed memory, ..)
+ * For query of LGRP_MEM_SIZE_INSTALL, return the amount of physical
+ * memory installed, regardless of whether or not it's usable.
+ */
+pgcnt_t
+lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query)
+{
+	int	mnode;
+	pgcnt_t npgs = (pgcnt_t)0;
+	extern struct memlist *phys_avail;
+	extern struct memlist *phys_install;
+
+
+	if (plathand == LGRP_DEFAULT_HANDLE)
+		return (lgrp_plat_mem_size_default(plathand, query));
+
+	if (plathand != LGRP_NULL_HANDLE) {
+		mnode = plat_lgrphand_to_mem_node(plathand);
+		if (mnode >= 0 && mem_node_config[mnode].exists) {
+			switch (query) {
+			case LGRP_MEM_SIZE_FREE:
+				npgs = MNODE_PGCNT(mnode);
+				break;
+			case LGRP_MEM_SIZE_AVAIL:
+				npgs = mem_node_memlist_pages(mnode,
+				    phys_avail);
+				break;
+			case LGRP_MEM_SIZE_INSTALL:
+				npgs = mem_node_memlist_pages(mnode,
+				    phys_install);
+				break;
+			default:
+				break;
+			}
+		}
 	}
+	return (npgs);
+}
+
+
+/*
+ * Return the platform handle of the lgroup that contains the physical memory
+ * corresponding to the given page frame number
+ */
+/* ARGSUSED */
+lgrp_handle_t
+lgrp_plat_pfn_to_hand(pfn_t pfn)
+{
+	int	mnode;
+
+	if (max_mem_nodes == 1)
+		return (LGRP_DEFAULT_HANDLE);
+
+	if (pfn > physmax)
+		return (LGRP_NULL_HANDLE);
+
+	mnode = plat_pfn_to_mem_node(pfn);
+	if (mnode < 0)
+		return (LGRP_NULL_HANDLE);
+
+	return (MEM_NODE_2_LGRPHAND(mnode));
+}
+
+
+/*
+ * Probe memory in each node from current CPU to determine latency topology
+ *
+ * The probing code will probe the vendor ID register on the Northbridge of
+ * Opteron processors and probe memory for other processors by default.
+ *
+ * Since probing is inherently error prone, the code takes laps across all the
+ * nodes probing from each node to each of the other nodes some number of
+ * times.  Furthermore, each node is probed some number of times before moving
+ * onto the next one during each lap.  The minimum latency gotten between nodes
+ * is kept as the latency between the nodes.
+ *
+ * After all that,  the probe times are adjusted by normalizing values that are
+ * close to each other and local latencies are made the same.  Lastly, the
+ * latencies are verified to make sure that certain conditions are met (eg.
+ * local < remote, latency(a, b) == latency(b, a), etc.).
+ *
+ * If any of the conditions aren't met, the code will export a NUMA
+ * configuration with the local CPUs and memory given by the SRAT or PCI config
+ * space registers and one remote memory latency since it can't tell exactly
+ * how far each node is from each other.
+ */
+void
+lgrp_plat_probe(void)
+{
+	int				from;
+	int				i;
+	lgrp_plat_latency_stats_t	*lat_stats;
+	hrtime_t			probe_time;
+	int				to;
+
+	if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) ||
+	    max_mem_nodes == 1 || lgrp_topo_ht_limit() <= 2)
+		return;
 
 	/*
-	 * Restore PCI Extended Configuration Space enable bit
+	 * Determine ID of node containing current CPU
 	 */
-	if (opt_family == AMD_FAMILY_GREYHOUND) {
-		if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0)
-			wrmsr(MSR_AMD_NB_CFG, nb_cfg_reg);
+	from = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node);
+	ASSERT(from >= 0 && from < lgrp_plat_node_cnt);
+	if (srat_ptr && lgrp_plat_srat_enable && !lgrp_plat_srat_error)
+		ASSERT(lgrp_plat_node_domain[from].exists);
+
+	/*
+	 * Don't need to probe if got times already
+	 */
+	lat_stats = &lgrp_plat_lat_stats;
+	if (lat_stats->latencies[from][from] != 0)
+		return;
+
+	/*
+	 * Read vendor ID in Northbridge or read and write page(s)
+	 * in each node from current CPU and remember how long it takes,
+	 * so we can build latency topology of machine later.
+	 * This should approximate the memory latency between each node.
+	 */
+	for (i = 0; i < lgrp_plat_probe_nrounds; i++) {
+		for (to = 0; to < lgrp_plat_node_cnt; to++) {
+			/*
+			 * Get probe time and bail out if can't get it yet
+			 */
+			probe_time = lgrp_plat_probe_time(to,
+			    lgrp_plat_cpu_node, &lgrp_plat_probe_mem_config,
+			    &lgrp_plat_lat_stats, &lgrp_plat_probe_stats);
+			if (probe_time == 0)
+				return;
+
+			/*
+			 * Keep lowest probe time as latency between nodes
+			 */
+			if (lat_stats->latencies[from][to] == 0 ||
+			    probe_time < lat_stats->latencies[from][to])
+				lat_stats->latencies[from][to] = probe_time;
+
+			/*
+			 * Update overall minimum and maximum probe times
+			 * across all nodes
+			 */
+			if (probe_time < lat_stats->latency_min ||
+			    lat_stats->latency_min == -1)
+				lat_stats->latency_min = probe_time;
+			if (probe_time > lat_stats->latency_max)
+				lat_stats->latency_max = probe_time;
+		}
 	}
 
 	/*
-	 * Only use one memory node if memory is interleaved between any nodes
+	 * - Fix up latencies such that local latencies are same,
+	 *   latency(i, j) == latency(j, i), etc. (if possible)
+	 *
+	 * - Verify that latencies look ok
+	 *
+	 * - Fallback to just optimizing for local and remote if
+	 *   latencies didn't look right
 	 */
-	if (lgrp_plat_mem_intrlv) {
-		lgrp_plat_node_cnt = max_mem_nodes = 1;
-		(void) lgrp_topo_ht_limit_set(1);
-	} else {
-		max_mem_nodes = lgrp_plat_node_cnt;
+	lgrp_plat_latency_adjust(lgrp_plat_node_memory, &lgrp_plat_lat_stats,
+	    &lgrp_plat_probe_stats);
+	lgrp_plat_probe_stats.probe_error_code =
+	    lgrp_plat_latency_verify(lgrp_plat_node_memory,
+	    &lgrp_plat_lat_stats);
+	if (lgrp_plat_probe_stats.probe_error_code)
+		lgrp_plat_2level_setup(lgrp_plat_node_memory,
+		    &lgrp_plat_lat_stats);
+}
 
-		/*
-		 * Probing errors can mess up the lgroup topology and force us
-		 * fall back to a 2 level lgroup topology.  Here we bound how
-		 * tall the lgroup topology can grow in hopes of avoiding any
-		 * anamolies in probing from messing up the lgroup topology
-		 * by limiting the accuracy of the latency topology.
-		 *
-		 * Assume that nodes will at least be configured in a ring,
-		 * so limit height of lgroup topology to be less than number
-		 * of nodes on a system with 4 or more nodes
-		 */
-		if (lgrp_plat_node_cnt >= 4 &&
-		    lgrp_topo_ht_limit() == lgrp_topo_ht_limit_default())
-			(void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1);
+
+/*
+ * Return platform handle for root lgroup
+ */
+lgrp_handle_t
+lgrp_plat_root_hand(void)
+{
+	return (LGRP_DEFAULT_HANDLE);
+}
+
+
+/*
+ * INTERNAL ROUTINES
+ */
+
+
+/*
+ * Update CPU to node mapping for given CPU and proximity domain (and returns
+ * negative numbers for errors and positive ones for success)
+ */
+static int
+lgrp_plat_cpu_node_update(node_domain_map_t *node_domain,
+    cpu_node_map_t *cpu_node, uint32_t apicid, uint32_t domain)
+{
+	uint_t	i;
+	uint_t	start;
+	int	node;
+
+	/*
+	 * Get node number for proximity domain
+	 */
+	node = lgrp_plat_domain_to_node(node_domain, domain);
+	if (node == -1) {
+		node = lgrp_plat_node_domain_update(node_domain, domain);
+		if (node == -1)
+			return (-1);
 	}
 
 	/*
-	 * Lgroups on Opteron architectures have but a single physical
-	 * processor. Tune lgrp_expand_proc_thresh and lgrp_expand_proc_diff
-	 * so that lgrp_choose() will spread things out aggressively.
+	 * Hash given CPU APIC ID into CPU to node mapping table/array and
+	 * enter it and its corresponding node and proximity domain IDs into
+	 * first non-existent or matching entry
 	 */
-	lgrp_expand_proc_thresh = LGRP_LOADAVG_THREAD_MAX / 2;
-	lgrp_expand_proc_diff = 0;
-#endif	/* __xpv */
+	i = start = CPU_NODE_HASH(apicid);
+	do {
+		if (cpu_node[i].exists) {
+			/*
+			 * Update already existing entry for CPU
+			 */
+			if (cpu_node[i].apicid == apicid) {
+				/*
+				 * Just return when everything same
+				 */
+				if (cpu_node[i].prox_domain == domain &&
+				    cpu_node[i].node == node)
+					return (1);
+
+				/*
+				 * Assert that proximity domain and node IDs
+				 * should be same and return error on non-debug
+				 * kernel
+				 */
+				ASSERT(cpu_node[i].prox_domain == domain &&
+				    cpu_node[i].node == node);
+				return (-1);
+			}
+		} else {
+			/*
+			 * Create new entry for CPU
+			 */
+			cpu_node[i].exists = 1;
+			cpu_node[i].apicid = apicid;
+			cpu_node[i].prox_domain = domain;
+			cpu_node[i].node = node;
+			return (0);
+		}
+		i = CPU_NODE_HASH(i + 1);
+	} while (i != start);
+
+	/*
+	 * Ran out of supported number of entries which shouldn't happen....
+	 */
+	ASSERT(i != start);
+	return (-1);
+}
+
+
+/*
+ * Get node ID for given CPU ID
+ */
+static int
+lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node)
+{
+	uint32_t	apicid;
+	uint_t		i;
+	uint_t		start;
+
+	if (cp == NULL)
+		return (-1);
+
+	/*
+	 * SRAT doesn't exist, isn't enabled, or there was an error processing
+	 * it, so return chip ID for Opteron and -1 otherwise.
+	 */
+	if (srat_ptr == NULL || !lgrp_plat_srat_enable ||
+	    lgrp_plat_srat_error) {
+		if (is_opteron())
+			return (pg_plat_hw_instance_id(cp, PGHW_CHIP));
+		return (-1);
+	}
+
+	/*
+	 * SRAT does exist, so get APIC ID for given CPU and map that to its
+	 * node ID
+	 */
+	apicid = cpuid_get_apicid(cp);
+	i = start = CPU_NODE_HASH(apicid);
+	do {
+		if (cpu_node[i].apicid == apicid && cpu_node[i].exists)
+			return (cpu_node[i].node);
+		i = CPU_NODE_HASH(i + 1);
+	} while (i != start);
+	return (-1);
+}
+
+
+/*
+ * Return node number for given proximity domain/system locality
+ */
+static int
+lgrp_plat_domain_to_node(node_domain_map_t *node_domain, uint32_t domain)
+{
+	uint_t	node;
+	uint_t	start;
+
+	/*
+	 * Hash proximity domain ID into node to domain mapping table (array),
+	 * search for entry with matching proximity domain ID, and return index
+	 * of matching entry as node ID.
+	 */
+	node = start = NODE_DOMAIN_HASH(domain);
+	do {
+		if (node_domain[node].prox_domain == domain &&
+		    node_domain[node].exists)
+			return (node);
+		node = NODE_DOMAIN_HASH(node + 1);
+	} while (node != start);
+	return (-1);
 }
 
 
@@ -797,7 +1305,8 @@ int	lgrp_plat_probe_lt_shift = LGRP_LAT_TOLERANCE_SHIFT;
  * latencies be same
  */
 static void
-lgrp_plat_latency_adjust(void)
+lgrp_plat_latency_adjust(node_phys_addr_map_t *node_memory,
+    lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats)
 {
 	int				i;
 	int				j;
@@ -812,19 +1321,28 @@ lgrp_plat_latency_adjust(void)
 	int				lat_corrected[MAX_NODES][MAX_NODES];
 
 	/*
-	 * Nothing to do when this is an UMA machine
+	 * Nothing to do when this is an UMA machine or don't have args needed
 	 */
 	if (max_mem_nodes == 1)
 		return;
 
+	ASSERT(node_memory != NULL && lat_stats != NULL &&
+	    probe_stats != NULL);
+
 	/*
 	 * Make sure that latencies are symmetric between any two nodes
 	 * (ie. latency(node0, node1) == latency(node1, node0))
 	 */
-	for (i = 0; i < lgrp_plat_node_cnt; i++)
+	for (i = 0; i < lgrp_plat_node_cnt; i++) {
+		if (!node_memory[i].exists)
+			continue;
+
 		for (j = 0; j < lgrp_plat_node_cnt; j++) {
-			t1 = lgrp_plat_probe_times[i][j];
-			t2 = lgrp_plat_probe_times[j][i];
+			if (!node_memory[j].exists)
+				continue;
+
+			t1 = lat_stats->latencies[i][j];
+			t2 = lat_stats->latencies[j][i];
 
 			if (t1 == 0 || t2 == 0 || t1 == t2)
 				continue;
@@ -838,25 +1356,26 @@ lgrp_plat_latency_adjust(void)
 			 */
 			if (t1 > t2) {
 				t = t2;
-				lgrp_plat_probe_errors[i][j] += t1 - t2;
+				probe_stats->probe_errors[i][j] += t1 - t2;
 				if (t1 - t2 > t2 >> lgrp_plat_probe_lt_shift) {
-					lgrp_plat_probe_suspect[i][j]++;
-					lgrp_plat_probe_suspect[j][i]++;
+					probe_stats->probe_suspect[i][j]++;
+					probe_stats->probe_suspect[j][i]++;
 				}
 			} else if (t2 > t1) {
 				t = t1;
-				lgrp_plat_probe_errors[j][i] += t2 - t1;
+				probe_stats->probe_errors[j][i] += t2 - t1;
 				if (t2 - t1 > t1 >> lgrp_plat_probe_lt_shift) {
-					lgrp_plat_probe_suspect[i][j]++;
-					lgrp_plat_probe_suspect[j][i]++;
+					probe_stats->probe_suspect[i][j]++;
+					probe_stats->probe_suspect[j][i]++;
 				}
 			}
 
-			lgrp_plat_probe_times[i][j] =
-			    lgrp_plat_probe_times[j][i] = t;
+			lat_stats->latencies[i][j] =
+			    lat_stats->latencies[j][i] = t;
 			lgrp_config(cflag, t1, t);
 			lgrp_config(cflag, t2, t);
 		}
+	}
 
 	/*
 	 * Keep track of which latencies get corrected
@@ -870,13 +1389,17 @@ lgrp_plat_latency_adjust(void)
 	 * are about the same distance apart and make the latencies be the same
 	 * if they are close enough together
 	 */
-	for (i = 0; i < lgrp_plat_node_cnt; i++)
+	for (i = 0; i < lgrp_plat_node_cnt; i++) {
+		if (!node_memory[i].exists)
+			continue;
 		for (j = 0; j < lgrp_plat_node_cnt; j++) {
+			if (!node_memory[j].exists)
+				continue;
 			/*
 			 * Pick one pair of nodes (i, j)
 			 * and get latency between them
 			 */
-			t1 = lgrp_plat_probe_times[i][j];
+			t1 = lat_stats->latencies[i][j];
 
 			/*
 			 * Skip this pair of nodes if there isn't a latency
@@ -885,8 +1408,12 @@ lgrp_plat_latency_adjust(void)
 			if (t1 == 0)
 				continue;
 
-			for (k = 0; k < lgrp_plat_node_cnt; k++)
+			for (k = 0; k < lgrp_plat_node_cnt; k++) {
+				if (!node_memory[k].exists)
+					continue;
 				for (l = 0; l < lgrp_plat_node_cnt; l++) {
+					if (!node_memory[l].exists)
+						continue;
 					/*
 					 * Pick another pair of nodes (k, l)
 					 * not same as (i, j) and get latency
@@ -895,7 +1422,7 @@ lgrp_plat_latency_adjust(void)
 					if (k == i && l == j)
 						continue;
 
-					t2 = lgrp_plat_probe_times[k][l];
+					t2 = lat_stats->latencies[k][l];
 
 					/*
 					 * Skip this pair of nodes if there
@@ -943,13 +1470,15 @@ lgrp_plat_latency_adjust(void)
 						t1 = t2 = t;
 					}
 
-					lgrp_plat_probe_times[i][j] =
-					    lgrp_plat_probe_times[k][l] = t;
+					lat_stats->latencies[i][j] =
+					    lat_stats->latencies[k][l] = t;
 
 					lat_corrected[i][j] =
 					    lat_corrected[k][l] = 1;
 				}
+			}
 		}
+	}
 
 	/*
 	 * Local latencies should be same
@@ -959,7 +1488,9 @@ lgrp_plat_latency_adjust(void)
 	min = -1;
 	max = 0;
 	for (i = 0; i < lgrp_plat_node_cnt; i++) {
-		t = lgrp_plat_probe_times[i][i];
+		if (!node_memory[i].exists)
+			continue;
+		t = lat_stats->latencies[i][i];
 		if (t == 0)
 			continue;
 		if (min == -1 || t < min)
@@ -971,7 +1502,10 @@ lgrp_plat_latency_adjust(void)
 		for (i = 0; i < lgrp_plat_node_cnt; i++) {
 			int	local;
 
-			local = lgrp_plat_probe_times[i][i];
+			if (!node_memory[i].exists)
+				continue;
+
+			local = lat_stats->latencies[i][i];
 			if (local == 0)
 				continue;
 
@@ -981,28 +1515,33 @@ lgrp_plat_latency_adjust(void)
 			 * probe times are corrected by
 			 */
 			if (local - min > min >> lgrp_plat_probe_lt_shift)
-				lgrp_plat_probe_suspect[i][i]++;
+				probe_stats->probe_suspect[i][i]++;
 
-			lgrp_plat_probe_errors[i][i] += local - min;
+			probe_stats->probe_errors[i][i] += local - min;
 
 			/*
 			 * Make local latencies be minimum
 			 */
 			lgrp_config(LGRP_CONFIG_LAT_CHANGE, i, min);
-			lgrp_plat_probe_times[i][i] = min;
+			lat_stats->latencies[i][i] = min;
 		}
 	}
 
 	/*
 	 * Determine max probe time again since just adjusted latencies
 	 */
-	lgrp_plat_probe_time_max = 0;
-	for (i = 0; i < lgrp_plat_node_cnt; i++)
+	lat_stats->latency_max = 0;
+	for (i = 0; i < lgrp_plat_node_cnt; i++) {
+		if (!node_memory[i].exists)
+			continue;
 		for (j = 0; j < lgrp_plat_node_cnt; j++) {
-			t = lgrp_plat_probe_times[i][j];
-			if (t > lgrp_plat_probe_time_max)
-				lgrp_plat_probe_time_max = t;
+			if (!node_memory[j].exists)
+				continue;
+			t = lat_stats->latencies[i][j];
+			if (t > lat_stats->latency_max)
+				lat_stats->latency_max = t;
 		}
+	}
 }
 
 
@@ -1020,48 +1559,55 @@ lgrp_plat_latency_adjust(void)
  *	-1	Not symmetric
  *	-2	Local latencies not same
  *	-3	Local >= remote
- *	-4	Wrong number of latencies
- *	-5	Not enough occurrences of given latency
  */
 static int
-lgrp_plat_latency_verify(void)
+lgrp_plat_latency_verify(node_phys_addr_map_t *node_memory,
+    lgrp_plat_latency_stats_t *lat_stats)
 {
 	int				i;
 	int				j;
-	lgrp_plat_latency_acct_t	*l;
-	int				probed;
 	u_longlong_t			t1;
 	u_longlong_t			t2;
 
+	ASSERT(node_memory != NULL && lat_stats != NULL);
+
 	/*
 	 * Nothing to do when this is an UMA machine, lgroup topology is
 	 * limited to 2 levels, or there aren't any probe times yet
 	 */
 	if (max_mem_nodes == 1 || lgrp_topo_levels < 2 ||
-	    (lgrp_plat_probe_time_max == 0 && lgrp_plat_probe_time_min == -1))
+	    lat_stats->latencies[0][0] == 0)
 		return (0);
 
 	/*
 	 * Make sure that latencies are symmetric between any two nodes
 	 * (ie. latency(node0, node1) == latency(node1, node0))
 	 */
-	for (i = 0; i < lgrp_plat_node_cnt; i++)
+	for (i = 0; i < lgrp_plat_node_cnt; i++) {
+		if (!node_memory[i].exists)
+			continue;
 		for (j = 0; j < lgrp_plat_node_cnt; j++) {
-			t1 = lgrp_plat_probe_times[i][j];
-			t2 = lgrp_plat_probe_times[j][i];
+			if (!node_memory[j].exists)
+				continue;
+			t1 = lat_stats->latencies[i][j];
+			t2 = lat_stats->latencies[j][i];
 
 			if (t1 == 0 || t2 == 0 || t1 == t2)
 				continue;
 
 			return (-1);
 		}
+	}
 
 	/*
 	 * Local latencies should be same
 	 */
-	t1 = lgrp_plat_probe_times[0][0];
+	t1 = lat_stats->latencies[0][0];
 	for (i = 1; i < lgrp_plat_node_cnt; i++) {
-		t2 = lgrp_plat_probe_times[i][i];
+		if (!node_memory[i].exists)
+			continue;
+
+		t2 = lat_stats->latencies[i][i];
 		if (t2 == 0)
 			continue;
 
@@ -1078,131 +1624,151 @@ lgrp_plat_latency_verify(void)
 	 * Local latencies should be less than remote
 	 */
 	if (t1) {
-		for (i = 0; i < lgrp_plat_node_cnt; i++)
+		for (i = 0; i < lgrp_plat_node_cnt; i++) {
+			if (!node_memory[i].exists)
+				continue;
 			for (j = 0; j < lgrp_plat_node_cnt; j++) {
-				t2 = lgrp_plat_probe_times[i][j];
+				if (!node_memory[j].exists)
+					continue;
+				t2 = lat_stats->latencies[i][j];
 				if (i == j || t2 == 0)
 					continue;
 
 				if (t1 >= t2)
 					return (-3);
 			}
+		}
 	}
 
-	/*
-	 * Rest of checks are not very useful for machines with less than
-	 * 4 nodes (which means less than 3 latencies on Opteron)
-	 */
-	if (lgrp_plat_node_cnt < 4)
-		return (0);
+	return (0);
+}
 
-	/*
-	 * Need to see whether done probing in order to verify number of
-	 * latencies are correct
-	 */
-	probed = 0;
-	for (i = 0; i < lgrp_plat_node_cnt; i++)
-		if (lgrp_plat_probe_times[i][i])
-			probed++;
 
-	if (probed != lgrp_plat_node_cnt)
-		return (0);
+/*
+ * Return the number of free, allocatable, or installed
+ * pages in an lgroup
+ * This is a copy of the MAX_MEM_NODES == 1 version of the routine
+ * used when MPO is disabled (i.e. single lgroup) or this is the root lgroup
+ */
+/* ARGSUSED */
+static pgcnt_t
+lgrp_plat_mem_size_default(lgrp_handle_t lgrphand, lgrp_mem_query_t query)
+{
+	struct memlist *mlist;
+	pgcnt_t npgs = 0;
+	extern struct memlist *phys_avail;
+	extern struct memlist *phys_install;
 
-	/*
-	 * Determine number of unique latencies seen in probe times,
-	 * their values, and number of occurrences of each
-	 */
-	lgrp_plat_probe_nlatencies = 0;
-	bzero(lgrp_plat_probe_lat_acct,
-	    MAX_NODES * sizeof (lgrp_plat_latency_acct_t));
-	for (i = 0; i < lgrp_plat_node_cnt; i++) {
-		for (j = 0; j < lgrp_plat_node_cnt; j++) {
-			int	k;
+	switch (query) {
+	case LGRP_MEM_SIZE_FREE:
+		return ((pgcnt_t)freemem);
+	case LGRP_MEM_SIZE_AVAIL:
+		memlist_read_lock();
+		for (mlist = phys_avail; mlist; mlist = mlist->next)
+			npgs += btop(mlist->size);
+		memlist_read_unlock();
+		return (npgs);
+	case LGRP_MEM_SIZE_INSTALL:
+		memlist_read_lock();
+		for (mlist = phys_install; mlist; mlist = mlist->next)
+			npgs += btop(mlist->size);
+		memlist_read_unlock();
+		return (npgs);
+	default:
+		return ((pgcnt_t)0);
+	}
+}
 
-			/*
-			 * Look at each probe time
-			 */
-			t1 = lgrp_plat_probe_times[i][j];
-			if (t1 == 0)
-				continue;
 
-			/*
-			 * Account for unique latencies
-			 */
-			for (k = 0; k < lgrp_plat_node_cnt; k++) {
-				l = &lgrp_plat_probe_lat_acct[k];
-				if (t1 == l->la_value) {
-					/*
-					 * Increment number of occurrences
-					 * if seen before
-					 */
-					l->la_count++;
-					break;
-				} else if (l->la_value == 0) {
-					/*
-					 * Record latency if haven't seen before
-					 */
-					l->la_value = t1;
-					l->la_count++;
-					lgrp_plat_probe_nlatencies++;
-					break;
-				}
-			}
-		}
-	}
+/*
+ * Update node to proximity domain mappings for given domain and return node ID
+ */
+static int
+lgrp_plat_node_domain_update(node_domain_map_t *node_domain, uint32_t domain)
+{
+	uint_t	node;
+	uint_t	start;
 
 	/*
-	 * Number of latencies should be relative to number of
-	 * nodes in system:
-	 * - Same as nodes when nodes <= 2
-	 * - Less than nodes when nodes > 2
-	 * - Greater than 2 when nodes >= 4
+	 * Hash proximity domain ID into node to domain mapping table (array)
+	 * and add entry for it into first non-existent or matching entry found
 	 */
-	if ((lgrp_plat_node_cnt <= 2 &&
-	    lgrp_plat_probe_nlatencies != lgrp_plat_node_cnt) ||
-	    (lgrp_plat_node_cnt > 2 &&
-	    lgrp_plat_probe_nlatencies >= lgrp_plat_node_cnt) ||
-	    (lgrp_plat_node_cnt >= 4 && lgrp_topo_levels >= 3 &&
-	    lgrp_plat_probe_nlatencies <= 2))
-		return (-4);
+	node = start = NODE_DOMAIN_HASH(domain);
+	do {
+		/*
+		 * Entry doesn't exist yet, so create one for this proximity
+		 * domain and return node ID which is index into mapping table.
+		 */
+		if (!node_domain[node].exists) {
+			node_domain[node].exists = 1;
+			node_domain[node].prox_domain = domain;
+			return (node);
+		}
+
+		/*
+		 * Entry exists for this proximity domain already, so just
+		 * return node ID (index into table).
+		 */
+		if (node_domain[node].prox_domain == domain)
+			return (node);
+		node = NODE_DOMAIN_HASH(node + 1);
+	} while (node != start);
 
 	/*
-	 * There should be more than one occurrence of every latency
-	 * as long as probing is complete
+	 * Ran out of supported number of entries which shouldn't happen....
 	 */
-	for (i = 0; i < lgrp_plat_probe_nlatencies; i++) {
-		l = &lgrp_plat_probe_lat_acct[i];
-		if (l->la_count <= 1)
-			return (-5);
-	}
-	return (0);
+	ASSERT(node != start);
+	return (-1);
 }
 
 
 /*
- * Set lgroup latencies for 2 level lgroup topology
+ * Update node memory information for given proximity domain with specified
+ * starting and ending physical address range (and return positive numbers for
+ * success and negative ones for errors)
  */
-static void
-lgrp_plat_2level_setup(void)
+static int
+lgrp_plat_node_memory_update(node_domain_map_t *node_domain,
+    node_phys_addr_map_t *node_memory, uintptr_t start, uintptr_t end,
+    uint32_t domain)
 {
-	int	i;
+	int	node;
 
-	if (lgrp_plat_node_cnt >= 4)
-		cmn_err(CE_NOTE,
-		    "MPO only optimizing for local and remote\n");
-	for (i = 0; i < lgrp_plat_node_cnt; i++) {
-		int	j;
+	/*
+	 * Get node number for proximity domain
+	 */
+	node = lgrp_plat_domain_to_node(node_domain, domain);
+	if (node == -1) {
+		node = lgrp_plat_node_domain_update(node_domain, domain);
+		if (node == -1)
+			return (-1);
+	}
 
-		for (j = 0; j < lgrp_plat_node_cnt; j++) {
-			if (i == j)
-				lgrp_plat_probe_times[i][j] = 2;
-			else
-				lgrp_plat_probe_times[i][j] = 3;
-		}
+	/*
+	 * Create entry in table for node if it doesn't exist
+	 */
+	if (!node_memory[node].exists) {
+		node_memory[node].exists = 1;
+		node_memory[node].start = btop(start);
+		node_memory[node].end = btop(end);
+		node_memory[node].prox_domain = domain;
+		return (0);
 	}
-	lgrp_plat_probe_time_min = 2;
-	lgrp_plat_probe_time_max = 3;
-	lgrp_config(LGRP_CONFIG_FLATTEN, 2, 0);
+
+	/*
+	 * Entry already exists for this proximity domain
+	 *
+	 * There may be more than one SRAT memory entry for a domain, so we may
+	 * need to update existing start or end address for the node.
+	 */
+	if (node_memory[node].prox_domain == domain) {
+		if (btop(start) < node_memory[node].start)
+			node_memory[node].start = btop(start);
+		if (btop(end) > node_memory[node].end)
+			node_memory[node].end = btop(end);
+		return (1);
+	}
+	return (-2);
 }
 
 
@@ -1210,38 +1776,37 @@ lgrp_plat_2level_setup(void)
  * Return time needed to probe from current CPU to memory in given node
  */
 static hrtime_t
-lgrp_plat_probe_time(int to)
+lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node,
+    lgrp_plat_probe_mem_config_t *probe_mem_config,
+    lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats)
 {
-	caddr_t		buf;
-	uint_t		dev;
-	/* LINTED: set but not used in function */
-	volatile uint_t	dev_vendor;
-	hrtime_t	elapsed;
-	hrtime_t	end;
-	int		from;
-	int		i;
-	int		ipl;
-	hrtime_t	max;
-	hrtime_t	min;
-	hrtime_t	start;
-	int		cnt;
-	extern int	use_sse_pagecopy;
+	caddr_t			buf;
+	hrtime_t		elapsed;
+	hrtime_t		end;
+	int			from;
+	int			i;
+	int			ipl;
+	hrtime_t		max;
+	hrtime_t		min;
+	hrtime_t		start;
+	extern int		use_sse_pagecopy;
 
 	/*
 	 * Determine ID of node containing current CPU
 	 */
-	from = LGRP_PLAT_CPU_TO_NODE(CPU);
+	from = lgrp_plat_cpu_to_node(CPU, cpu_node);
+	ASSERT(from >= 0 && from < lgrp_plat_node_cnt);
 
 	/*
 	 * Do common work for probing main memory
 	 */
-	if (lgrp_plat_probe_op == LGRP_PLAT_PROBE_PGCPY) {
+	if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_PGCPY) {
 		/*
 		 * Skip probing any nodes without memory and
 		 * set probe time to 0
 		 */
-		if (lgrp_plat_probe_memory[to] == NULL) {
-			lgrp_plat_probe_times[from][to] = 0;
+		if (probe_mem_config->probe_va[to] == NULL) {
+			lat_stats->latencies[from][to] = 0;
 			return (0);
 		}
 
@@ -1249,10 +1814,11 @@ lgrp_plat_probe_time(int to)
 		 * Invalidate caches once instead of once every sample
 		 * which should cut cost of probing by a lot
 		 */
-		lgrp_plat_flush_cost = gethrtime();
+		probe_stats->flush_cost = gethrtime();
 		invalidate_cache();
-		lgrp_plat_flush_cost = gethrtime() - lgrp_plat_flush_cost;
-		lgrp_plat_probe_cost_total += lgrp_plat_flush_cost;
+		probe_stats->flush_cost = gethrtime() -
+		    probe_stats->flush_cost;
+		probe_stats->probe_cost_total += probe_stats->flush_cost;
 	}
 
 	/*
@@ -1262,23 +1828,26 @@ lgrp_plat_probe_time(int to)
 	max = 0;
 	min = -1;
 	for (i = 0; i < lgrp_plat_probe_nsamples; i++) {
-		lgrp_plat_probe_cost = gethrtime();
+		probe_stats->probe_cost = gethrtime();
 
 		/*
 		 * Can't measure probe time if gethrtime() isn't working yet
 		 */
-		if (lgrp_plat_probe_cost == 0 && gethrtime() == 0)
+		if (probe_stats->probe_cost == 0 && gethrtime() == 0)
 			return (0);
 
-		switch (lgrp_plat_probe_op) {
-
-		case LGRP_PLAT_PROBE_PGCPY:
-		default:
+		if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) {
+			/*
+			 * Measure how long it takes to read vendor ID from
+			 * Northbridge
+			 */
+			elapsed = opt_probe_vendor(to, lgrp_plat_probe_nreads);
+		} else {
 			/*
 			 * Measure how long it takes to copy page
 			 * on top of itself
 			 */
-			buf = lgrp_plat_probe_memory[to] + (i * PAGESIZE);
+			buf = probe_mem_config->probe_va[to] + (i * PAGESIZE);
 
 			kpreempt_disable();
 			ipl = splhigh();
@@ -1291,30 +1860,11 @@ lgrp_plat_probe_time(int to)
 			elapsed = end - start;
 			splx(ipl);
 			kpreempt_enable();
-			break;
-
-		case LGRP_PLAT_PROBE_VENDOR:
-			/*
-			 * Measure how long it takes to read vendor ID from
-			 * Northbridge
-			 */
-			dev = OPT_PCS_DEV_NODE0 + to;
-			kpreempt_disable();
-			ipl = spl8();
-			outl(PCI_CONFADD, PCI_CADDR1(0, dev, opt_probe_func,
-			    OPT_PCS_OFF_VENDOR));
-			start = gethrtime();
-			for (cnt = 0; cnt < lgrp_plat_probe_nreads; cnt++)
-				dev_vendor = inl(PCI_CONFDATA);
-			end = gethrtime();
-			elapsed = (end - start) / lgrp_plat_probe_nreads;
-			splx(ipl);
-			kpreempt_enable();
-			break;
 		}
 
-		lgrp_plat_probe_cost = gethrtime() - lgrp_plat_probe_cost;
-		lgrp_plat_probe_cost_total += lgrp_plat_probe_cost;
+		probe_stats->probe_cost = gethrtime() -
+		    probe_stats->probe_cost;
+		probe_stats->probe_cost_total += probe_stats->probe_cost;
 
 		if (min == -1 || elapsed < min)
 			min = elapsed;
@@ -1326,396 +1876,677 @@ lgrp_plat_probe_time(int to)
 	 * Update minimum and maximum probe times between
 	 * these two nodes
 	 */
-	if (min < lgrp_plat_probe_min[from][to] ||
-	    lgrp_plat_probe_min[from][to] == 0)
-		lgrp_plat_probe_min[from][to] = min;
+	if (min < probe_stats->probe_min[from][to] ||
+	    probe_stats->probe_min[from][to] == 0)
+		probe_stats->probe_min[from][to] = min;
 
-	if (max > lgrp_plat_probe_max[from][to])
-		lgrp_plat_probe_max[from][to] = max;
+	if (max > probe_stats->probe_max[from][to])
+		probe_stats->probe_max[from][to] = max;
 
 	return (min);
 }
 
 
 /*
- * Probe memory in each node from current CPU to determine latency topology
+ * Read ACPI System Locality Information Table (SLIT) to determine how far each
+ * NUMA node is from each other
  */
-void
-lgrp_plat_probe(void)
+static int
+lgrp_plat_process_slit(struct slit *tp, uint_t node_cnt,
+    node_phys_addr_map_t *node_memory, lgrp_plat_latency_stats_t *lat_stats)
 {
-	int		from;
 	int		i;
-	hrtime_t	probe_time;
-	int		to;
+	int		j;
+	int		localities;
+	hrtime_t	max;
+	hrtime_t	min;
+	int		retval;
+	uint8_t		*slit_entries;
 
-	if (max_mem_nodes == 1 || lgrp_topo_ht_limit() <= 2)
-		return;
+	if (tp == NULL || !lgrp_plat_slit_enable)
+		return (1);
+
+	if (lat_stats == NULL)
+		return (2);
+
+	localities = tp->number;
+	if (localities != node_cnt)
+		return (3);
+
+	min = lat_stats->latency_min;
+	max = lat_stats->latency_max;
 
 	/*
-	 * Determine ID of node containing current CPU
+	 * Fill in latency matrix based on SLIT entries
 	 */
-	from = LGRP_PLAT_CPU_TO_NODE(CPU);
+	slit_entries = tp->entry;
+	for (i = 0; i < localities; i++) {
+		for (j = 0; j < localities; j++) {
+			uint8_t	latency;
+
+			latency = slit_entries[(i * localities) + j];
+			lat_stats->latencies[i][j] = latency;
+			if (latency < min)
+				min = latency;
+			if (latency > max)
+				max = latency;
+		}
+	}
 
 	/*
-	 * Don't need to probe if got times already
+	 * Verify that latencies/distances given in SLIT look reasonable
 	 */
-	if (lgrp_plat_probe_times[from][from] != 0)
-		return;
+	retval = lgrp_plat_latency_verify(node_memory, lat_stats);
+
+	if (retval) {
+		/*
+		 * Reinitialize (zero) latency table since SLIT doesn't look
+		 * right
+		 */
+		for (i = 0; i < localities; i++) {
+			for (j = 0; j < localities; j++)
+				lat_stats->latencies[i][j] = 0;
+		}
+	} else {
+		/*
+		 * Update min and max latencies seen since SLIT looks valid
+		 */
+		lat_stats->latency_min = min;
+		lat_stats->latency_max = max;
+	}
+
+	return (retval);
+}
+
+
+/*
+ * Read ACPI System Resource Affinity Table (SRAT) to determine which CPUs
+ * and memory are local to each other in the same NUMA node
+ */
+static int
+lgrp_plat_process_srat(struct srat *tp, uint_t *node_cnt,
+    node_domain_map_t *node_domain, cpu_node_map_t *cpu_node,
+    node_phys_addr_map_t *node_memory)
+{
+	struct srat_item	*end;
+	int			i;
+	struct srat_item	*item;
+
+	if (tp == NULL || !lgrp_plat_srat_enable)
+		return (1);
 
 	/*
-	 * Read vendor ID in Northbridge or read and write page(s)
-	 * in each node from current CPU and remember how long it takes,
-	 * so we can build latency topology of machine later.
-	 * This should approximate the memory latency between each node.
+	 * Determine number of nodes by counting number of proximity domains in
+	 * SRAT
 	 */
-	for (i = 0; i < lgrp_plat_probe_nrounds; i++)
-		for (to = 0; to < lgrp_plat_node_cnt; to++) {
-			/*
-			 * Get probe time and bail out if can't get it yet
-			 */
-			probe_time = lgrp_plat_probe_time(to);
-			if (probe_time == 0)
-				return;
+	if (node_cnt) {
+		int	nodes;
+
+		nodes = lgrp_plat_srat_domains(tp);
+		if (nodes < 0) {
+			*node_cnt = 1;
+			return (2);
+		}
+		*node_cnt = nodes;
+	}
+
+	/*
+	 * Walk through SRAT, examining each CPU and memory entry to determine
+	 * which CPUs and memory belong to which node.
+	 */
+	item = tp->list;
+	end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp);
+	while (item < end) {
+		uint32_t	apic_id;
+		uint32_t	domain;
+		uint64_t	end;
+		uint64_t	length;
+		uint64_t	start;
+
+		switch (item->type) {
+		case SRAT_PROCESSOR:	/* CPU entry */
+			if (!(item->i.p.flags & SRAT_ENABLED) ||
+			    cpu_node == NULL)
+				break;
 
 			/*
-			 * Keep lowest probe time as latency between nodes
+			 * Calculate domain (node) ID and fill in APIC ID to
+			 * domain/node mapping table
 			 */
-			if (lgrp_plat_probe_times[from][to] == 0 ||
-			    probe_time < lgrp_plat_probe_times[from][to])
-				lgrp_plat_probe_times[from][to] = probe_time;
+			domain = item->i.p.domain1;
+			for (i = 0; i < 3; i++) {
+				domain += item->i.p.domain2[i] <<
+				    ((i + 1) * 8);
+			}
+			apic_id = item->i.p.apic_id;
+
+			if (lgrp_plat_cpu_node_update(node_domain, cpu_node,
+			    apic_id, domain) < 0)
+				return (3);
+			break;
+
+		case SRAT_MEMORY:	/* memory entry */
+			if (!(item->i.m.flags & SRAT_ENABLED) ||
+			    node_memory == NULL)
+				break;
 
 			/*
-			 * Update overall minimum and maximum probe times
-			 * across all nodes
+			 * Get domain (node) ID and fill in domain/node
+			 * to memory mapping table
 			 */
-			if (probe_time < lgrp_plat_probe_time_min ||
-			    lgrp_plat_probe_time_min == -1)
-				lgrp_plat_probe_time_min = probe_time;
-			if (probe_time > lgrp_plat_probe_time_max)
-				lgrp_plat_probe_time_max = probe_time;
+			domain = item->i.m.domain;
+			start = item->i.m.base_addr;
+			length = item->i.m.len;
+			end = start + length - 1;
+
+			if (lgrp_plat_node_memory_update(node_domain,
+			    node_memory, start, end, domain) < 0)
+				return (4);
+			break;
+
+		default:
+			break;
 		}
 
-	/*
-	 * - Fix up latencies such that local latencies are same,
-	 *   latency(i, j) == latency(j, i), etc. (if possible)
-	 *
-	 * - Verify that latencies look ok
-	 *
-	 * - Fallback to just optimizing for local and remote if
-	 *   latencies didn't look right
-	 */
-	lgrp_plat_latency_adjust();
-	lgrp_plat_probe_error_code = lgrp_plat_latency_verify();
-	if (lgrp_plat_probe_error_code)
-		lgrp_plat_2level_setup();
+		item = (struct srat_item *)((uintptr_t)item + item->len);
+	}
+	return (0);
 }
 
 
 /*
- * Platform-specific initialization
+ * Return number of proximity domains given in ACPI SRAT
  */
-void
-lgrp_plat_main_init(void)
+static int
+lgrp_plat_srat_domains(struct srat *tp)
 {
-	int	curnode;
-	int	ht_limit;
-	int	i;
+	int			domain_cnt;
+	struct srat_item	*end;
+	int			i;
+	struct srat_item	*item;
+	node_domain_map_t	node_domain[MAX_NODES];
 
-	/*
-	 * Print a notice that MPO is disabled when memory is interleaved
-	 * across nodes....Would do this when it is discovered, but can't
-	 * because it happens way too early during boot....
-	 */
-	if (lgrp_plat_mem_intrlv)
-		cmn_err(CE_NOTE,
-		    "MPO disabled because memory is interleaved\n");
+
+	if (tp == NULL || !lgrp_plat_srat_enable)
+		return (1);
 
 	/*
-	 * Don't bother to do any probing if there is only one node or the
-	 * height of the lgroup topology less than or equal to 2
+	 * Walk through SRAT, examining each CPU and memory entry to determine
+	 * proximity domain ID for each.
 	 */
-	ht_limit = lgrp_topo_ht_limit();
-	if (max_mem_nodes == 1 || ht_limit <= 2) {
+	domain_cnt = 0;
+	item = tp->list;
+	end = (struct srat_item *)(tp->hdr.len + (uintptr_t)tp);
+	bzero(node_domain, MAX_NODES * sizeof (node_domain_map_t));
+	while (item < end) {
+		uint32_t	domain;
+		boolean_t	overflow;
+		uint_t		start;
+
+		switch (item->type) {
+		case SRAT_PROCESSOR:	/* CPU entry */
+			if (!(item->i.p.flags & SRAT_ENABLED))
+				break;
+			domain = item->i.p.domain1;
+			for (i = 0; i < 3; i++) {
+				domain += item->i.p.domain2[i] <<
+				    ((i + 1) * 8);
+			}
+			break;
+
+		case SRAT_MEMORY:	/* memory entry */
+			if (!(item->i.m.flags & SRAT_ENABLED))
+				break;
+			domain = item->i.m.domain;
+			break;
+
+		default:
+			break;
+		}
+
 		/*
-		 * Setup lgroup latencies for 2 level lgroup topology
-		 * (ie. local and remote only) if they haven't been set yet
+		 * Count and keep track of which proximity domain IDs seen
 		 */
-		if (ht_limit == 2 && lgrp_plat_probe_time_min == -1 &&
-		    lgrp_plat_probe_time_max == 0)
-			lgrp_plat_2level_setup();
-		return;
-	}
+		start = i = domain % MAX_NODES;
+		overflow = B_TRUE;
+		do {
+			/*
+			 * Create entry for proximity domain and increment
+			 * count when no entry exists where proximity domain
+			 * hashed
+			 */
+			if (!node_domain[i].exists) {
+				node_domain[i].exists = 1;
+				node_domain[i].prox_domain = domain;
+				domain_cnt++;
+				overflow = B_FALSE;
+				break;
+			}
+
+			/*
+			 * Nothing to do when proximity domain seen already
+			 * and its entry exists
+			 */
+			if (node_domain[i].prox_domain == domain) {
+				overflow = B_FALSE;
+				break;
+			}
+
+			/*
+			 * Entry exists where proximity domain hashed, but for
+			 * different proximity domain so keep search for empty
+			 * slot to put it or matching entry whichever comes
+			 * first.
+			 */
+			i = (i + 1) % MAX_NODES;
+		} while (i != start);
 
-	if (lgrp_plat_probe_op == LGRP_PLAT_PROBE_VENDOR) {
 		/*
-		 * Should have been able to probe from CPU 0 when it was added
-		 * to lgroup hierarchy, but may not have been able to then
-		 * because it happens so early in boot that gethrtime() hasn't
-		 * been initialized.  (:-(
+		 * Didn't find empty or matching entry which means have more
+		 * proximity domains than supported nodes (:-(
 		 */
-		curnode = LGRP_PLAT_CPU_TO_NODE(CPU);
-		if (lgrp_plat_probe_times[curnode][curnode] == 0)
-			lgrp_plat_probe();
+		ASSERT(overflow != B_TRUE);
+		if (overflow == B_TRUE)
+			return (-1);
 
-		return;
+		item = (struct srat_item *)((uintptr_t)item + item->len);
 	}
+	return (domain_cnt);
+}
 
-	/*
-	 * When probing memory, use one page for every sample to determine
-	 * lgroup topology and taking multiple samples
-	 */
-	if (lgrp_plat_probe_memsize == 0)
-		lgrp_plat_probe_memsize = PAGESIZE *
-		    lgrp_plat_probe_nsamples;
 
-	/*
-	 * Map memory in each node needed for probing to determine latency
-	 * topology
-	 */
+/*
+ * Set lgroup latencies for 2 level lgroup topology
+ */
+static void
+lgrp_plat_2level_setup(node_phys_addr_map_t *node_memory,
+    lgrp_plat_latency_stats_t *lat_stats)
+{
+	int	i;
+
+	ASSERT(node_memory != NULL && lat_stats != NULL);
+
+	if (lgrp_plat_node_cnt >= 4)
+		cmn_err(CE_NOTE,
+		    "MPO only optimizing for local and remote\n");
 	for (i = 0; i < lgrp_plat_node_cnt; i++) {
-		int	mnode;
+		int	j;
 
-		/*
-		 * Skip this node and leave its probe page NULL
-		 * if it doesn't have any memory
-		 */
-		mnode = plat_lgrphand_to_mem_node((lgrp_handle_t)i);
-		if (!mem_node_config[mnode].exists) {
-			lgrp_plat_probe_memory[i] = NULL;
+		if (!node_memory[i].exists)
 			continue;
+		for (j = 0; j < lgrp_plat_node_cnt; j++) {
+			if (!node_memory[j].exists)
+				continue;
+			if (i == j)
+				lat_stats->latencies[i][j] = 2;
+			else
+				lat_stats->latencies[i][j] = 3;
 		}
+	}
+	lat_stats->latency_min = 2;
+	lat_stats->latency_max = 3;
+	lgrp_config(LGRP_CONFIG_FLATTEN, 2, 0);
+}
 
-		/*
-		 * Allocate one kernel virtual page
-		 */
-		lgrp_plat_probe_memory[i] = vmem_alloc(heap_arena,
-		    lgrp_plat_probe_memsize, VM_NOSLEEP);
-		if (lgrp_plat_probe_memory[i] == NULL) {
-			cmn_err(CE_WARN,
-			    "lgrp_plat_main_init: couldn't allocate memory");
-			return;
-		}
 
-		/*
-		 * Map virtual page to first page in node
-		 */
-		hat_devload(kas.a_hat, lgrp_plat_probe_memory[i],
-		    lgrp_plat_probe_memsize,
-		    lgrp_plat_probe_pfn[i],
-		    PROT_READ | PROT_WRITE | HAT_PLAT_NOCACHE,
-		    HAT_LOAD_NOCONSIST);
-	}
+/*
+ * The following Opteron specific constants, macros, types, and routines define
+ * PCI configuration space registers and how to read them to determine the NUMA
+ * configuration of *supported* Opteron processors.  They provide the same
+ * information that may be gotten from the ACPI System Resource Affinity Table
+ * (SRAT) if it exists on the machine of interest.
+ *
+ * The AMD BIOS and Kernel Developer's Guide (BKDG) for the processor family
+ * of interest describes all of these registers and their contents.  The main
+ * registers used by this code to determine the NUMA configuration of the
+ * machine are the node ID register for the number of NUMA nodes and the DRAM
+ * address map registers for the physical address range of each node.
+ *
+ * NOTE: The format and how to determine the NUMA configuration using PCI
+ *	 config space registers may change or may not be supported in future
+ *	 Opteron processor families.
+ */
 
-	/*
-	 * Probe from current CPU
-	 */
-	lgrp_plat_probe();
-}
+/*
+ * How many bits to shift Opteron DRAM Address Map base and limit registers
+ * to get actual value
+ */
+#define	OPT_DRAMADDR_HI_LSHIFT_ADDR	40	/* shift left for address */
+#define	OPT_DRAMADDR_LO_LSHIFT_ADDR	8	/* shift left for address */
+
+#define	OPT_DRAMADDR_HI_MASK_ADDR	0x000000FF /* address bits 47-40 */
+#define	OPT_DRAMADDR_LO_MASK_ADDR	0xFFFF0000 /* address bits 39-24 */
+
+#define	OPT_DRAMADDR_LO_MASK_OFF	0xFFFFFF /* offset for address */
 
 /*
- * Allocate additional space for an lgroup.
+ * Macros to derive addresses from Opteron DRAM Address Map registers
  */
-/* ARGSUSED */
-lgrp_t *
-lgrp_plat_alloc(lgrp_id_t lgrpid)
-{
-	lgrp_t *lgrp;
+#define	OPT_DRAMADDR_HI(reg) \
+	(((u_longlong_t)reg & OPT_DRAMADDR_HI_MASK_ADDR) << \
+	    OPT_DRAMADDR_HI_LSHIFT_ADDR)
 
-	lgrp = &lgrp_space[nlgrps_alloc++];
-	if (lgrpid >= NLGRP || nlgrps_alloc > NLGRP)
-		return (NULL);
-	return (lgrp);
-}
+#define	OPT_DRAMADDR_LO(reg) \
+	(((u_longlong_t)reg & OPT_DRAMADDR_LO_MASK_ADDR) << \
+	    OPT_DRAMADDR_LO_LSHIFT_ADDR)
+
+#define	OPT_DRAMADDR(high, low) \
+	(OPT_DRAMADDR_HI(high) | OPT_DRAMADDR_LO(low))
 
 /*
- * Platform handling for (re)configuration changes
+ * Bit masks defining what's in Opteron DRAM Address Map base register
  */
-/* ARGSUSED */
-void
-lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg)
-{
-}
+#define	OPT_DRAMBASE_LO_MASK_RE		0x1	/* read enable */
+#define	OPT_DRAMBASE_LO_MASK_WE		0x2	/* write enable */
+#define	OPT_DRAMBASE_LO_MASK_INTRLVEN	0x700	/* interleave */
 
 /*
- * Return the platform handle for the lgroup containing the given CPU
+ * Bit masks defining what's in Opteron DRAM Address Map limit register
  */
-/* ARGSUSED */
-lgrp_handle_t
-lgrp_plat_cpu_to_hand(processorid_t id)
-{
-	if (lgrp_plat_node_cnt == 1)
-		return (LGRP_DEFAULT_HANDLE);
+#define	OPT_DRAMLIMIT_LO_MASK_DSTNODE	0x7		/* destination node */
+#define	OPT_DRAMLIMIT_LO_MASK_INTRLVSEL	0x700		/* interleave select */
 
-	return ((lgrp_handle_t)LGRP_PLAT_CPU_TO_NODE(cpu[id]));
-}
 
 /*
- * Return the platform handle of the lgroup that contains the physical memory
- * corresponding to the given page frame number
+ * Opteron Node ID register in PCI configuration space contains
+ * number of nodes in system, etc. for Opteron K8.  The following
+ * constants and macros define its contents, structure, and access.
  */
-/* ARGSUSED */
-lgrp_handle_t
-lgrp_plat_pfn_to_hand(pfn_t pfn)
-{
-	int	mnode;
 
-	if (max_mem_nodes == 1)
-		return (LGRP_DEFAULT_HANDLE);
+/*
+ * Bit masks defining what's in Opteron Node ID register
+ */
+#define	OPT_NODE_MASK_ID	0x7	/* node ID */
+#define	OPT_NODE_MASK_CNT	0x70	/* node count */
+#define	OPT_NODE_MASK_IONODE	0x700	/* Hypertransport I/O hub node ID */
+#define	OPT_NODE_MASK_LCKNODE	0x7000	/* lock controller node ID */
+#define	OPT_NODE_MASK_CPUCNT	0xF0000	/* CPUs in system (0 means 1 CPU)  */
 
-	if (pfn > physmax)
-		return (LGRP_NULL_HANDLE);
+/*
+ * How many bits in Opteron Node ID register to shift right to get actual value
+ */
+#define	OPT_NODE_RSHIFT_CNT	0x4	/* shift right for node count value */
 
-	mnode = plat_pfn_to_mem_node(pfn);
-	if (mnode < 0)
-		return (LGRP_NULL_HANDLE);
+/*
+ * Macros to get values from Opteron Node ID register
+ */
+#define	OPT_NODE_CNT(reg) \
+	((reg & OPT_NODE_MASK_CNT) >> OPT_NODE_RSHIFT_CNT)
 
-	return (MEM_NODE_2_LGRPHAND(mnode));
-}
+/*
+ * Macro to setup PCI Extended Configuration Space (ECS) address to give to
+ * "in/out" instructions
+ *
+ * NOTE: Should only be used in lgrp_plat_init() before MMIO setup because any
+ *	 other uses should just do MMIO to access PCI ECS.
+ *	 Must enable special bit in Northbridge Configuration Register on
+ *	 Greyhound for extended CF8 space access to be able to access PCI ECS
+ *	 using "in/out" instructions and restore special bit after done
+ *	 accessing PCI ECS.
+ */
+#define	OPT_PCI_ECS_ADDR(bus, device, function, reg) \
+	(PCI_CONE | (((bus) & 0xff) << 16) | (((device & 0x1f)) << 11)  | \
+	    (((function) & 0x7) << 8) | ((reg) & 0xfc) | \
+	    ((((reg) >> 8) & 0xf) << 24))
 
 /*
- * Return the maximum number of lgrps supported by the platform.
- * Before lgrp topology is known it returns an estimate based on the number of
- * nodes. Once topology is known it returns the actual maximim number of lgrps
- * created. Since x86 doesn't support dynamic addition of new nodes, this number
- * may not grow during system lifetime.
+ * PCI configuration space registers accessed by specifying
+ * a bus, device, function, and offset.  The following constants
+ * define the values needed to access Opteron K8 configuration
+ * info to determine its node topology
  */
-int
-lgrp_plat_max_lgrps()
-{
-	return (lgrp_topo_initialized ?
-	    lgrp_alloc_max + 1 :
-	    lgrp_plat_node_cnt * (lgrp_plat_node_cnt - 1) + 1);
-}
+
+#define	OPT_PCS_BUS_CONFIG	0	/* Hypertransport config space bus */
 
 /*
- * Return the number of free, allocatable, or installed
- * pages in an lgroup
- * This is a copy of the MAX_MEM_NODES == 1 version of the routine
- * used when MPO is disabled (i.e. single lgroup) or this is the root lgroup
+ * Opteron PCI configuration space register function values
  */
-/* ARGSUSED */
-static pgcnt_t
-lgrp_plat_mem_size_default(lgrp_handle_t lgrphand, lgrp_mem_query_t query)
-{
-	struct memlist *mlist;
-	pgcnt_t npgs = 0;
-	extern struct memlist *phys_avail;
-	extern struct memlist *phys_install;
+#define	OPT_PCS_FUNC_HT		0	/* Hypertransport configuration */
+#define	OPT_PCS_FUNC_ADDRMAP	1	/* Address map configuration */
+#define	OPT_PCS_FUNC_DRAM	2	/* DRAM configuration */
+#define	OPT_PCS_FUNC_MISC	3	/* Miscellaneous configuration */
 
-	switch (query) {
-	case LGRP_MEM_SIZE_FREE:
-		return ((pgcnt_t)freemem);
-	case LGRP_MEM_SIZE_AVAIL:
-		memlist_read_lock();
-		for (mlist = phys_avail; mlist; mlist = mlist->next)
-			npgs += btop(mlist->size);
-		memlist_read_unlock();
-		return (npgs);
-	case LGRP_MEM_SIZE_INSTALL:
-		memlist_read_lock();
-		for (mlist = phys_install; mlist; mlist = mlist->next)
-			npgs += btop(mlist->size);
-		memlist_read_unlock();
-		return (npgs);
-	default:
-		return ((pgcnt_t)0);
-	}
-}
+/*
+ * PCI Configuration Space register offsets
+ */
+#define	OPT_PCS_OFF_VENDOR	0x0	/* device/vendor ID register */
+#define	OPT_PCS_OFF_DRAMBASE_HI	0x140	/* DRAM Base register (node 0) */
+#define	OPT_PCS_OFF_DRAMBASE_LO	0x40	/* DRAM Base register (node 0) */
+#define	OPT_PCS_OFF_NODEID	0x60	/* Node ID register */
 
 /*
- * Return the number of free pages in an lgroup.
- *
- * For query of LGRP_MEM_SIZE_FREE, return the number of base pagesize
- * pages on freelists.  For query of LGRP_MEM_SIZE_AVAIL, return the
- * number of allocatable base pagesize pages corresponding to the
- * lgroup (e.g. do not include page_t's, BOP_ALLOC()'ed memory, ..)
- * For query of LGRP_MEM_SIZE_INSTALL, return the amount of physical
- * memory installed, regardless of whether or not it's usable.
+ * Opteron PCI Configuration Space device IDs for nodes
  */
-pgcnt_t
-lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query)
-{
-	int	mnode;
-	pgcnt_t npgs = (pgcnt_t)0;
-	extern struct memlist *phys_avail;
-	extern struct memlist *phys_install;
+#define	OPT_PCS_DEV_NODE0		24	/* device number for node 0 */
 
 
-	if (plathand == LGRP_DEFAULT_HANDLE)
-		return (lgrp_plat_mem_size_default(plathand, query));
+/*
+ * Opteron DRAM address map gives base and limit for physical memory in a node
+ */
+typedef	struct opt_dram_addr_map {
+	uint32_t	base_hi;
+	uint32_t	base_lo;
+	uint32_t	limit_hi;
+	uint32_t	limit_lo;
+} opt_dram_addr_map_t;
 
-	if (plathand != LGRP_NULL_HANDLE) {
-		mnode = plat_lgrphand_to_mem_node(plathand);
-		if (mnode >= 0 && mem_node_config[mnode].exists) {
-			switch (query) {
-			case LGRP_MEM_SIZE_FREE:
-				npgs = MNODE_PGCNT(mnode);
-				break;
-			case LGRP_MEM_SIZE_AVAIL:
-				npgs = mem_node_memlist_pages(mnode,
-				    phys_avail);
-				break;
-			case LGRP_MEM_SIZE_INSTALL:
-				npgs = mem_node_memlist_pages(mnode,
-				    phys_install);
-				break;
-			default:
-				break;
-			}
-		}
-	}
-	return (npgs);
-}
 
 /*
- * Return latency between "from" and "to" lgroups
- *
- * This latency number can only be used for relative comparison
- * between lgroups on the running system, cannot be used across platforms,
- * and may not reflect the actual latency.  It is platform and implementation
- * specific, so platform gets to decide its value.  It would be nice if the
- * number was at least proportional to make comparisons more meaningful though.
+ * Supported AMD processor families
  */
-/* ARGSUSED */
-int
-lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to)
+#define	AMD_FAMILY_HAMMER	15
+#define	AMD_FAMILY_GREYHOUND	16
+
+/*
+ * Whether to have is_opteron() return 1 even when processor isn't supported
+ */
+uint_t	is_opteron_override = 0;
+
+/*
+ * AMD processor family for current CPU
+ */
+uint_t	opt_family = 0;
+
+
+/*
+ * Determine whether we're running on a supported AMD Opteron since reading
+ * node count and DRAM address map registers may have different format or
+ * may not be supported across processor families
+ */
+static int
+is_opteron(void)
 {
-	lgrp_handle_t	src, dest;
 
-	if (max_mem_nodes == 1)
+	if (x86_vendor != X86_VENDOR_AMD)
+		return (0);
+
+	opt_family = cpuid_getfamily(CPU);
+	if (opt_family == AMD_FAMILY_HAMMER ||
+	    opt_family == AMD_FAMILY_GREYHOUND || is_opteron_override)
+		return (1);
+	else
 		return (0);
+}
+
+
+/*
+ * Determine NUMA configuration for Opteron from registers that live in PCI
+ * configuration space
+ */
+static void
+opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv,
+    node_phys_addr_map_t *node_memory)
+{
+	uint_t				bus;
+	uint_t				dev;
+	struct opt_dram_addr_map	dram_map[MAX_NODES];
+	uint_t				node;
+	uint_t				node_info[MAX_NODES];
+	uint_t				off_hi;
+	uint_t				off_lo;
+	uint64_t			nb_cfg_reg;
 
 	/*
-	 * Return max latency for root lgroup
+	 * Read configuration registers from PCI configuration space to
+	 * determine node information, which memory is in each node, etc.
+	 *
+	 * Write to PCI configuration space address register to specify
+	 * which configuration register to read and read/write PCI
+	 * configuration space data register to get/set contents
 	 */
-	if (from == LGRP_DEFAULT_HANDLE || to == LGRP_DEFAULT_HANDLE)
-		return (lgrp_plat_probe_time_max);
+	bus = OPT_PCS_BUS_CONFIG;
+	dev = OPT_PCS_DEV_NODE0;
+	off_hi = OPT_PCS_OFF_DRAMBASE_HI;
+	off_lo = OPT_PCS_OFF_DRAMBASE_LO;
 
-	src = from;
-	dest = to;
+	/*
+	 * Read node ID register for node 0 to get node count
+	 */
+	node_info[0] = pci_getl_func(bus, dev, OPT_PCS_FUNC_HT,
+	    OPT_PCS_OFF_NODEID);
+	*node_cnt = OPT_NODE_CNT(node_info[0]) + 1;
 
 	/*
-	 * Return 0 for nodes (lgroup platform handles) out of range
+	 * If number of nodes is more than maximum supported, then set node
+	 * count to 1 and treat system as UMA instead of NUMA.
 	 */
-	if (src < 0 || src >= MAX_NODES || dest < 0 || dest >= MAX_NODES)
-		return (0);
+	if (*node_cnt > MAX_NODES) {
+		*node_cnt = 1;
+		return;
+	}
 
 	/*
-	 * Probe from current CPU if its lgroup latencies haven't been set yet
-	 * and we are trying to get latency from current CPU to some node
+	 * For Greyhound, PCI Extended Configuration Space must be enabled to
+	 * read high DRAM address map base and limit registers
 	 */
-	if (lgrp_plat_probe_times[src][src] == 0 &&
-	    LGRP_PLAT_CPU_TO_NODE(CPU) == src)
-		lgrp_plat_probe();
+	if (opt_family == AMD_FAMILY_GREYHOUND) {
+		nb_cfg_reg = rdmsr(MSR_AMD_NB_CFG);
+		if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0)
+			wrmsr(MSR_AMD_NB_CFG,
+			    nb_cfg_reg | AMD_GH_NB_CFG_EN_ECS);
+	}
+
+	for (node = 0; node < *node_cnt; node++) {
+		uint32_t	base_hi;
+		uint32_t	base_lo;
+		uint32_t	limit_hi;
+		uint32_t	limit_lo;
+
+		/*
+		 * Read node ID register (except for node 0 which we just read)
+		 */
+		if (node > 0) {
+			node_info[node] = pci_getl_func(bus, dev,
+			    OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID);
+		}
+
+		/*
+		 * Read DRAM base and limit registers which specify
+		 * physical memory range of each node
+		 */
+		if (opt_family != AMD_FAMILY_GREYHOUND)
+			base_hi = 0;
+		else {
+			outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev,
+			    OPT_PCS_FUNC_ADDRMAP, off_hi));
+			base_hi = dram_map[node].base_hi =
+			    inl(PCI_CONFDATA);
+		}
+		base_lo = dram_map[node].base_lo = pci_getl_func(bus, dev,
+		    OPT_PCS_FUNC_ADDRMAP, off_lo);
+
+		if ((dram_map[node].base_lo & OPT_DRAMBASE_LO_MASK_INTRLVEN) &&
+		    mem_intrlv)
+			*mem_intrlv = *mem_intrlv + 1;
+
+		off_hi += 4;	/* high limit register offset */
+		if (opt_family != AMD_FAMILY_GREYHOUND)
+			limit_hi = 0;
+		else {
+			outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev,
+			    OPT_PCS_FUNC_ADDRMAP, off_hi));
+			limit_hi = dram_map[node].limit_hi =
+			    inl(PCI_CONFDATA);
+		}
+
+		off_lo += 4;	/* low limit register offset */
+		limit_lo = dram_map[node].limit_lo = pci_getl_func(bus,
+		    dev, OPT_PCS_FUNC_ADDRMAP, off_lo);
 
-	return (lgrp_plat_probe_times[src][dest]);
+		/*
+		 * Increment device number to next node and register offsets
+		 * for DRAM base register of next node
+		 */
+		off_hi += 4;
+		off_lo += 4;
+		dev++;
+
+		/*
+		 * Both read and write enable bits must be enabled in DRAM
+		 * address map base register for physical memory to exist in
+		 * node
+		 */
+		if ((base_lo & OPT_DRAMBASE_LO_MASK_RE) == 0 ||
+		    (base_lo & OPT_DRAMBASE_LO_MASK_WE) == 0) {
+			/*
+			 * Mark node memory as non-existent and set start and
+			 * end addresses to be same in node_memory[]
+			 */
+			node_memory[node].exists = 0;
+			node_memory[node].start = node_memory[node].end =
+			    (pfn_t)-1;
+			continue;
+		}
+
+		/*
+		 * Mark node memory as existing and remember physical address
+		 * range of each node for use later
+		 */
+		node_memory[node].exists = 1;
+
+		node_memory[node].start = btop(OPT_DRAMADDR(base_hi, base_lo));
+
+		node_memory[node].end = btop(OPT_DRAMADDR(limit_hi, limit_lo) |
+		    OPT_DRAMADDR_LO_MASK_OFF);
+	}
+
+	/*
+	 * Restore PCI Extended Configuration Space enable bit
+	 */
+	if (opt_family == AMD_FAMILY_GREYHOUND) {
+		if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0)
+			wrmsr(MSR_AMD_NB_CFG, nb_cfg_reg);
+	}
 }
 
+
 /*
- * Return platform handle for root lgroup
+ * Return average amount of time to read vendor ID register on Northbridge
+ * N times on specified destination node from current CPU
  */
-lgrp_handle_t
-lgrp_plat_root_hand(void)
+static hrtime_t
+opt_probe_vendor(int dest_node, int nreads)
 {
-	return (LGRP_DEFAULT_HANDLE);
+	int		cnt;
+	uint_t		dev;
+	/* LINTED: set but not used in function */
+	volatile uint_t	dev_vendor;
+	hrtime_t	elapsed;
+	hrtime_t	end;
+	int		ipl;
+	hrtime_t	start;
+
+	dev = OPT_PCS_DEV_NODE0 + dest_node;
+	kpreempt_disable();
+	ipl = spl8();
+	outl(PCI_CONFADD, PCI_CADDR1(0, dev, OPT_PCS_FUNC_DRAM,
+	    OPT_PCS_OFF_VENDOR));
+	start = gethrtime();
+	for (cnt = 0; cnt < nreads; cnt++)
+		dev_vendor = inl(PCI_CONFDATA);
+	end = gethrtime();
+	elapsed = (end - start) / nreads;
+	splx(ipl);
+	kpreempt_enable();
+	return (elapsed);
 }
diff --git a/usr/src/uts/i86pc/os/mlsetup.c b/usr/src/uts/i86pc/os/mlsetup.c
index c099fc9764..6ba5f6e1f6 100644
--- a/usr/src/uts/i86pc/os/mlsetup.c
+++ b/usr/src/uts/i86pc/os/mlsetup.c
@@ -321,11 +321,6 @@ mlsetup(struct regs *rp)
 	pci_cfgspace_init();
 #endif
 
-	/*
-	 * Initialize the lgrp framework
-	 */
-	lgrp_init();
-
 	rp->r_fp = 0;	/* terminate kernel stack traces! */
 
 	prom_init("kernel", (void *)NULL);
@@ -337,6 +332,11 @@ mlsetup(struct regs *rp)
 
 	max_ncpus = boot_max_ncpus = boot_ncpus;
 
+	/*
+	 * Initialize the lgrp framework
+	 */
+	lgrp_init();
+
 	if (boothowto & RB_HALT) {
 		prom_printf("unix: kernel halted by -h flag\n");
 		prom_enter_mon();
diff --git a/usr/src/uts/intel/sys/x86_archext.h b/usr/src/uts/intel/sys/x86_archext.h
index af217929a1..7a65c28743 100644
--- a/usr/src/uts/intel/sys/x86_archext.h
+++ b/usr/src/uts/intel/sys/x86_archext.h
@@ -539,6 +539,7 @@ extern uint_t cpuid_get_ncpu_sharing_last_cache(struct cpu *);
 extern id_t cpuid_get_last_lvl_cacheid(struct cpu *);
 extern int cpuid_get_chipid(struct cpu *);
 extern id_t cpuid_get_coreid(struct cpu *);
+extern uint_t cpuid_get_apicid(struct cpu *);
 extern int cpuid_get_pkgcoreid(struct cpu *);
 extern int cpuid_get_clogid(struct cpu *);
 extern int cpuid_is_cmt(struct cpu *);