6395227 Need to support s3 on MP machines

6621792 nv_sata doesn't resume on MP ultra 40 system
6631154 cpr_wakecode.s has unguarded amd specific MSRs
6631159 cpr_wakecode.s will call APIC initialization, even if no APIC, or APIC disabled

5 files changed, 118 insertions, 93 deletions

diff --git a/usr/src/uts/common/cpr/cpr_main.c b/usr/src/uts/common/cpr/cpr_main.c
index 96b1d29534..22e1b702f7 100644
--- a/usr/src/uts/common/cpr/cpr_main.c
+++ b/usr/src/uts/common/cpr/cpr_main.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -380,21 +380,15 @@ cpr_lock_mgr(void (*service)(void))
 int
 cpr_suspend_cpus(void)
 {
-	cpu_t *bootcpu;
 	int	ret = 0;
 	extern void *i_cpr_save_context(void *arg);
 
 	mutex_enter(&cpu_lock);
 
 	/*
-	 * if bootcpu is offline bring it back online
-	 */
-	bootcpu = i_cpr_bootcpu();
-
-	/*
 	 * the machine could not have booted without a bootcpu
 	 */
-	ASSERT(bootcpu != NULL);
+	ASSERT(i_cpr_bootcpu() != NULL);
 
 	/*
 	 * bring all the offline cpus online
@@ -789,11 +783,6 @@ cpr_resume_cpus(void)
 	start_cpus();
 	mutex_exit(&cpu_lock);
 
-	/*
-	 * clear the affinity set in cpr_suspend_cpus()
-	 */
-	affinity_clear();
-
 	i_cpr_post_resume_cpus();
 
 	mutex_enter(&cpu_lock);
@@ -804,14 +793,14 @@ cpr_resume_cpus(void)
 	cpu_pause_func = NULL;
 
 	/*
-	 * offline all the cpus that were brought online during suspend
+	 * clear the affinity set in cpr_suspend_cpus()
 	 */
-	cpr_restore_offline();
+	affinity_clear();
 
 	/*
-	 * clear the affinity set in cpr_suspend_cpus()
+	 * offline all the cpus that were brought online during suspend
 	 */
-	affinity_clear();
+	cpr_restore_offline();
 
 	mutex_exit(&cpu_lock);
 }
@@ -839,14 +828,14 @@ cpr_unpause_cpus(void)
 	start_cpus();
 
 	/*
-	 * offline all the cpus that were brought online during suspend
+	 * clear the affinity set in cpr_suspend_cpus()
 	 */
-	cpr_restore_offline();
+	affinity_clear();
 
 	/*
-	 * clear the affinity set in cpr_suspend_cpus()
+	 * offline all the cpus that were brought online during suspend
 	 */
-	affinity_clear();
+	cpr_restore_offline();
 
 	mutex_exit(&cpu_lock);
 }
diff --git a/usr/src/uts/common/cpr/cpr_mod.c b/usr/src/uts/common/cpr/cpr_mod.c
index 0799bca9f1..008cf5d73c 100644
--- a/usr/src/uts/common/cpr/cpr_mod.c
+++ b/usr/src/uts/common/cpr/cpr_mod.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -190,14 +190,7 @@ cpr(int fcn, void *mdep)
 	case AD_LOOPBACK_SUSPEND_TO_RAM_FAIL:
 	case AD_FORCE_SUSPEND_TO_RAM:
 	case AD_DEVICE_SUSPEND_TO_RAM:
-		/*
-		 * if MP then do not support suspend to RAM, however override
-		 * the MP restriction if cpr_mp_enable has been set
-		 */
-		if (ncpus > 1 && cpr_mp_enable == 0)
-			return (ENOTSUP);
-		else
-			cpr_sleeptype = CPR_TORAM;
+		cpr_sleeptype = CPR_TORAM;
 		break;
 #endif
 	}
diff --git a/usr/src/uts/common/io/sata/adapters/nv_sata/nv_sata.c b/usr/src/uts/common/io/sata/adapters/nv_sata/nv_sata.c
index f6d5870c5c..65c7af2fb5 100644
--- a/usr/src/uts/common/io/sata/adapters/nv_sata/nv_sata.c
+++ b/usr/src/uts/common/io/sata/adapters/nv_sata/nv_sata.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -406,6 +406,7 @@ nv_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 	ddi_acc_handle_t pci_conf_handle;
 	nv_ctl_t *nvc;
 	uint8_t subclass;
+	uint32_t reg32;
 
 	switch (cmd) {
 
@@ -600,12 +601,41 @@ nv_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
 		NVLOG((NVDBG_INIT, nvc, NULL,
 		    "nv_attach(): DDI_RESUME inst %d", inst));
 
+		if (pci_config_setup(dip, &pci_conf_handle) != DDI_SUCCESS) {
+			return (DDI_FAILURE);
+		}
+
+		/*
+		 * If a device is attached after a suspend/resume, sometimes
+		 * the command register is zero, as it might not be set by
+		 * BIOS or a parent.  Set it again here.
+		 */
+		command = pci_config_get16(pci_conf_handle, PCI_CONF_COMM);
+
+		if (command == 0) {
+			pci_config_put16(pci_conf_handle, PCI_CONF_COMM,
+			    PCI_COMM_IO|PCI_COMM_MAE|PCI_COMM_ME);
+		}
+
+		/*
+		 * Need to set bit 2 to 1 at config offset 0x50
+		 * to enable access to the bar5 registers.
+		 */
+		reg32 = pci_config_get32(pci_conf_handle, NV_SATA_CFG_20);
+
+		if ((reg32 & NV_BAR5_SPACE_EN) != NV_BAR5_SPACE_EN) {
+			pci_config_put32(pci_conf_handle, NV_SATA_CFG_20,
+			    reg32 | NV_BAR5_SPACE_EN);
+		}
+
 		nvc->nvc_state &= ~NV_CTRL_SUSPEND;
 
 		for (i = 0; i < NV_MAX_PORTS(nvc); i++) {
 			nv_resume(&(nvc->nvc_port[i]));
 		}
 
+		pci_config_teardown(&pci_conf_handle);
+
 		return (DDI_SUCCESS);
 
 	default:
diff --git a/usr/src/uts/i86pc/ml/cpr_wakecode.s b/usr/src/uts/i86pc/ml/cpr_wakecode.s
index 93dffd9c51..32baeaeb39 100644
--- a/usr/src/uts/i86pc/ml/cpr_wakecode.s
+++ b/usr/src/uts/i86pc/ml/cpr_wakecode.s
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 	
@@ -282,8 +282,8 @@ wc_rm_end(void)
 / using the following value blows up machines! - DO NOT USE
 /	D16 movl	0xffc, %esp
 
-#define LED     1
-#define SERIAL  1
+#define LED     0
+#define SERIAL  0
 
 #if     LED
 	D16 movl        $0x80, %edx
@@ -717,7 +717,13 @@ kernel_wc_code:
 	pushq   $0              /* null frame pointer terminates stack trace */
 	movq    %rsp, %rbp      /* stack aligned on 16-byte boundary */
 
-	call    *ap_mlsetup
+	/*
+	 * skip iff function pointer is NULL
+	 */
+	cmpq	$0, ap_mlsetup
+	je	2f
+	call	*ap_mlsetup
+2:
 
 	call    *cpr_start_cpu_func
 
@@ -976,10 +982,18 @@ kernel_wc_code:
 	movb	$0xdc, %al
 	outb	$0x80
 
-	movl	$MSR_AMD_EFER, %ecx	/ re-enable NX bit
+	/*
+	 * Before proceeding, enable usage of the page table NX bit if
+	 * that's how the page tables are set up.
+	 */
+	movl    x86_feature, %ecx
+	andl   	 $X86_NX, %ecx
+	jz      1f
+	movl    $MSR_AMD_EFER, %ecx
 	rdmsr
-	orl	$AMD_EFER_NXE, %eax
+	orl     $AMD_EFER_NXE, %eax
 	wrmsr
+1:
 
 	movl	WC_CR4(%ebx), %eax	/ restore full cr4 (with Global Enable)
 	movl	%eax, %cr4
@@ -1003,9 +1017,12 @@ kernel_wc_code:
 	movw	WC_GS(%ebx), %gs
 
 	/*
-	 * APIC initialization 
+	 * APIC initialization, skip iff function pointer is NULL
 	 */
-	call    *ap_mlsetup
+	cmpl	$0, ap_mlsetup
+	je	2f
+	call	*ap_mlsetup
+2:
 
 	call    *cpr_start_cpu_func
 
diff --git a/usr/src/uts/i86pc/os/cpr_impl.c b/usr/src/uts/i86pc/os/cpr_impl.c
index 939df37716..a32b18511f 100644
--- a/usr/src/uts/i86pc/os/cpr_impl.c
+++ b/usr/src/uts/i86pc/os/cpr_impl.c
@@ -19,7 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -93,6 +93,7 @@ static int i_cpr_platform_alloc(psm_state_request_t *req);
 static void i_cpr_platform_free(psm_state_request_t *req);
 static int i_cpr_save_apic(psm_state_request_t *req);
 static int i_cpr_restore_apic(psm_state_request_t *req);
+static int wait_for_set(cpuset_t *set, int who);
 
 #if defined(__amd64)
 static void restore_stack(wc_cpu_t *cpup);
@@ -260,8 +261,7 @@ i_cpr_pre_resume_cpus()
 	 * just do the initialization to wake the other cpus
 	 */
 	unsigned who;
-	int cpuid = i_cpr_bootcpuid();
-	int started_cpu;
+	int boot_cpuid = i_cpr_bootcpuid();
 	uint32_t		code_length = 0;
 	caddr_t			wakevirt = rm_platter_va;
 	/*LINTED*/
@@ -292,14 +292,17 @@ i_cpr_pre_resume_cpus()
 
 	affinity_set(CPU_CURRENT);
 
-	cpu_ready_set = 0;
+	/*
+	 * mark the boot cpu as being ready, since we are running on that cpu
+	 */
+	CPUSET_ONLY(cpu_ready_set, boot_cpuid);
 
 	for (who = 0; who < ncpus; who++) {
 
 		wc_cpu_t	*cpup = wc_other_cpus + who;
 		wc_desctbr_t	gdt;
 
-		if (who == cpuid)
+		if (who == boot_cpuid)
 			continue;
 
 		if (!CPU_IN_SET(mp_cpus, who))
@@ -320,8 +323,6 @@ i_cpr_pre_resume_cpus()
 
 		init_real_mode_platter(who, code_length, cpup->wc_cr4, gdt);
 
-		started_cpu = 1;
-
 		if ((err = mach_cpuid_start(who, rm_platter_va)) != 0) {
 			cmn_err(CE_WARN, "cpu%d: failed to start during "
 			    "suspend/resume error %d", who, err);
@@ -331,57 +332,18 @@ i_cpr_pre_resume_cpus()
 		PMD(PMD_SX, ("%s() #1 waiting for procset 0x%lx\n", str,
 		    (ulong_t)procset))
 
-/*
- * This conditional compile only affects the MP case.
- */
-#ifdef	MP_PM
-		for (delays = 0; !CPU_IN_SET(procset, who); delays++) {
-			if (delays == 500) {
-				/*
-				 * After five seconds, things are probably
-				 * looking a bit bleak - explain the hang.
-				 */
-				cmn_err(CE_NOTE, "cpu%d: started, "
-				    "but not running in the kernel yet", who);
-				PMD(PMD_SX, ("%s() %d cpu started "
-				    "but not running in the kernel yet\n",
-				    str, who))
-			} else if (delays > 2000) {
-				/*
-				 * We waited at least 20 seconds, bail ..
-				 */
-				cmn_err(CE_WARN, "cpu%d: timed out", who);
-				PMD(PMD_SX, ("%s() %d cpu timed out\n",
-				    str, who))
-				started_cpu = 0;
-			}
-
-			/*
-			 * wait at least 10ms, then check again..
-			 */
-			delay(USEC_TO_TICK_ROUNDUP(10000));
-		}
-#else
-		while (!CPU_IN_SET(procset, who)) {
-			;
-		}
-
-#endif	/*	MP_PM	*/
+		if (!wait_for_set(&procset, who))
+			continue;
 
 		PMD(PMD_SX, ("%s() %d cpu started\n", str, who))
 
-		if (!started_cpu)
-			continue;
-
-		PMD(PMD_SX, ("%s() tsc_ready = %d\n", str,
-		    get_tsc_ready()))
+		PMD(PMD_SX, ("%s() tsc_ready = %d\n", str, get_tsc_ready()))
 
 		if (tsc_gethrtime_enable) {
 			PMD(PMD_SX, ("%s() calling tsc_sync_master\n", str))
 			tsc_sync_master(who);
 		}
 
-
 		PMD(PMD_SX, ("%s() waiting for cpu_ready_set %ld\n", str,
 		    cpu_ready_set))
 		/*
@@ -389,11 +351,8 @@ i_cpr_pre_resume_cpus()
 		 * cpus to start serially instead of in parallel, so that
 		 * they do not contend with each other in wc_rm_start()
 		 */
-		while (!CPU_IN_SET(cpu_ready_set, who)) {
-			PMD(PMD_SX, ("%s() waiting for "
-			    "cpu_ready_set %ld\n", str, cpu_ready_set))
-			;
-		}
+		if (!wait_for_set(&cpu_ready_set, who))
+			continue;
 
 		/*
 		 * do not need to re-initialize dtrace using dtrace_cpu_init
@@ -405,6 +364,7 @@ i_cpr_pre_resume_cpus()
 	affinity_clear();
 
 	PMD(PMD_SX, ("%s() all cpus now ready\n", str))
+
 }
 
 static void
@@ -1094,3 +1054,39 @@ i_cpr_restore_configuration(dev_info_t *dip)
 {
 	acpica_ddi_restore_resources(dip);
 }
+
+static int
+wait_for_set(cpuset_t *set, int who)
+{
+	int delays;
+	char *str = "wait_for_set";
+
+	for (delays = 0; !CPU_IN_SET(*set, who); delays++) {
+		if (delays == 500) {
+			/*
+			 * After five seconds, things are probably
+			 * looking a bit bleak - explain the hang.
+			 */
+			cmn_err(CE_NOTE, "cpu%d: started, "
+			    "but not running in the kernel yet", who);
+			PMD(PMD_SX, ("%s() %d cpu started "
+			    "but not running in the kernel yet\n",
+			    str, who))
+		} else if (delays > 2000) {
+			/*
+			 * We waited at least 20 seconds, bail ..
+			 */
+			cmn_err(CE_WARN, "cpu%d: timed out", who);
+			PMD(PMD_SX, ("%s() %d cpu timed out\n",
+			    str, who))
+			return (0);
+		}
+
+		/*
+		 * wait at least 10ms, then check again..
+		 */
+		drv_usecwait(10000);
+	}
+
+	return (1);
+}