6555476 pcacheset_cmp is inefficient

11 files changed, 123 insertions, 33 deletions

diff --git a/usr/src/uts/common/sys/systm.h b/usr/src/uts/common/sys/systm.h
index 0223b57cc9..017555524b 100644
--- a/usr/src/uts/common/sys/systm.h
+++ b/usr/src/uts/common/sys/systm.h
@@ -246,6 +246,7 @@ extern void longjmp(label_t *)
 #pragma unknown_control_flow(setjmp)
 #endif
 
+void prefetch64(caddr_t);
 caddr_t caller(void);
 caddr_t callee(void);
 int getpcstack(pc_t *, int);
diff --git a/usr/src/uts/common/syscall/poll.c b/usr/src/uts/common/syscall/poll.c
index 2fdac673b3..86a44d7c09 100644
--- a/usr/src/uts/common/syscall/poll.c
+++ b/usr/src/uts/common/syscall/poll.c
@@ -20,7 +20,7 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
@@ -32,8 +32,6 @@
  * under license from the Regents of the University of California.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/param.h>
 #include <sys/isa_defs.h>
 #include <sys/types.h>
@@ -546,7 +544,7 @@ poll_common(pollfd_t *fds, nfds_t nfds, timespec_t *tsp, k_sigset_t *ksetp)
 			rval = -1;
 		else
 			rval = cv_waituntil_sig(&pcp->pc_cv, &pcp->pc_lock,
-				rqtp, timecheck);
+			    rqtp, timecheck);
 		mutex_exit(&pcp->pc_lock);
 		/*
 		 * If we have received a signal or timed out
@@ -808,8 +806,8 @@ retry:
 					if (pevents) {
 						struct plist *t;
 						t = kmem_zalloc(
-							sizeof (struct plist),
-							    KM_SLEEP);
+						    sizeof (struct plist),
+						    KM_SLEEP);
 						t->pp = pkevp->portkev_port;
 						t->pevents = pevents;
 						if (plhead == NULL) {
@@ -996,6 +994,11 @@ pcacheset_cmp(pollfd_t *current, pollfd_t *cached, pollfd_t *newlist, int n)
 	int    ix;
 
 	for (ix = 0; ix < n; ix++) {
+		/* Prefetch 64 bytes worth of 8-byte elements */
+		if ((ix & 0x7) == 0) {
+			prefetch64((caddr_t)&current[ix + 8]);
+			prefetch64((caddr_t)&cached[ix + 8]);
+		}
 		if (current[ix].fd == cached[ix].fd) {
 			/*
 			 * Filter out invalid poll events while we are in
@@ -1600,11 +1603,11 @@ pcacheset_resolve(pollstate_t *ps, nfds_t nfds, int *fdcntp, int which)
 			if (current[count].events & ~VALID_POLL_EVENTS) {
 				if (newlist != NULL) {
 					newlist[count].events =
-						current[count].events &=
-							VALID_POLL_EVENTS;
+					    current[count].events &=
+					    VALID_POLL_EVENTS;
 				} else {
 					current[count].events &=
-						VALID_POLL_EVENTS;
+					    VALID_POLL_EVENTS;
 				}
 			}
 			/*
@@ -1631,7 +1634,7 @@ pcacheset_resolve(pollstate_t *ps, nfds_t nfds, int *fdcntp, int which)
 							pcache_update_xref(pcp,
 							    tmpfd, (ssize_t)i,
 							    which);
-						    break;
+							break;
 						}
 					}
 					ASSERT(i <= old_nfds);
@@ -1736,7 +1739,7 @@ pcacheset_resolve(pollstate_t *ps, nfds_t nfds, int *fdcntp, int which)
 			/* filter out invalid events */
 			if (current[i].events & ~VALID_POLL_EVENTS) {
 				newlist[i].events = current[i].events =
-				current[i].events & VALID_POLL_EVENTS;
+				    current[i].events & VALID_POLL_EVENTS;
 			}
 			if ((fd = current[i].fd) < 0) {
 				current[i].revents = 0;
@@ -2135,7 +2138,7 @@ pcacheset_cache_list(pollstate_t *ps, pollfd_t *fds, int *fdcntp, int which)
 		 */
 		if (pollfdp[i].events & ~VALID_POLL_EVENTS) {
 			newfdlist[i].events = pollfdp[i].events =
-			pollfdp[i].events & VALID_POLL_EVENTS;
+			    pollfdp[i].events & VALID_POLL_EVENTS;
 		}
 		if (fd < 0) {
 			pollfdp[i].revents = 0;
diff --git a/usr/src/uts/i86pc/ml/amd64.il b/usr/src/uts/i86pc/ml/amd64.il
index 03bdcf0b0d..9639af663c 100644
--- a/usr/src/uts/i86pc/ml/amd64.il
+++ b/usr/src/uts/i86pc/ml/amd64.il
@@ -177,3 +177,12 @@
 	xorl	%eax, %eax
 	bsrw	%di, %ax
 	.end
+
+/
+/ prefetch 64 bytes
+/
+	.inline	prefetch64,8
+	prefetcht0 (%rdi)
+	prefetcht0 32(%rdi)
+	.end
+
diff --git a/usr/src/uts/i86pc/ml/ia32.il b/usr/src/uts/i86pc/ml/ia32.il
index 74af6303d5..2236e8e1a6 100644
--- a/usr/src/uts/i86pc/ml/ia32.il
+++ b/usr/src/uts/i86pc/ml/ia32.il
@@ -24,8 +24,6 @@
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /
 / Inline functions specific to the i86pc kernel running on bare metal.
 /
@@ -178,3 +176,13 @@
 	movw	(%esp), %cx
 	bsrw	%cx, %ax
 	.end
+
+/
+/ prefetch 64 bytes
+/
+	.inline	prefetch64,4
+	movl  (%esp), %eax
+	prefetcht0 (%eax)
+	prefetcht0 32(%eax)
+	.end
+
diff --git a/usr/src/uts/i86xpv/ml/amd64.il b/usr/src/uts/i86xpv/ml/amd64.il
index 28f774674e..bca65ed47f 100644
--- a/usr/src/uts/i86xpv/ml/amd64.il
+++ b/usr/src/uts/i86xpv/ml/amd64.il
@@ -20,12 +20,10 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /
 / Inline functions for the x64 kernel running on the hypervisor
 /
@@ -88,3 +86,12 @@
 	shlq	$32, %rdx
 	orq	%rdx, %rax
 	.end
+
+/
+/ prefetch 64 bytes
+/
+	.inline	prefetch64,8
+	prefetcht0 (%rdi)
+	prefetcht0 32(%rdi)
+	.end
+
diff --git a/usr/src/uts/i86xpv/ml/ia32.il b/usr/src/uts/i86xpv/ml/ia32.il
index 64e7515b3e..b6e3cecb4e 100644
--- a/usr/src/uts/i86xpv/ml/ia32.il
+++ b/usr/src/uts/i86xpv/ml/ia32.il
@@ -20,12 +20,10 @@
  */
 
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /
 / Inline functions for the x86 kernel running on the hypervisor
 /
@@ -101,3 +99,12 @@
 	.inline	__rdtsc_insn, 0
 	rdtsc
 	.end
+
+/
+/ prefetch 64 bytes
+/
+	.inline	prefetch64, 4
+	movl  (%esp), %eax
+	prefetcht0 (%eax)
+	prefetcht0 32(%eax)
+	.end
diff --git a/usr/src/uts/intel/amd64/ml/amd64.il b/usr/src/uts/intel/amd64/ml/amd64.il
index 4053f81ebd..297fa134f2 100644
--- a/usr/src/uts/intel/amd64/ml/amd64.il
+++ b/usr/src/uts/intel/amd64/ml/amd64.il
@@ -205,3 +205,11 @@
         mfence
         swapgs
 	.end
+
+/
+/ prefetch 64 bytes
+/
+	.inline	prefetch64,8
+	prefetcht0 (%rdi)
+	prefetcht0 32(%rdi)
+	.end
diff --git a/usr/src/uts/intel/asm/cpu.h b/usr/src/uts/intel/asm/cpu.h
index 0f285f82b1..0b2bb92ba1 100644
--- a/usr/src/uts/intel/asm/cpu.h
+++ b/usr/src/uts/intel/asm/cpu.h
@@ -19,15 +19,13 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef _ASM_CPU_H
 #define	_ASM_CPU_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #ifdef	__cplusplus
 extern "C" {
 #endif
@@ -114,6 +112,27 @@ extern __inline__ void __swapgs(void)
 
 #endif	/* !__lint && __GNUC__ */
 
+#if !defined(__lint) && defined(__GNUC__)
+
+#if defined(__i386) || defined(__amd64)
+
+/*
+ * prefetch 64 bytes
+ */
+
+extern __inline__ void prefetch64(caddr_t addr)
+{
+	__asm__ __volatile__(
+	    "prefetcht0 (%0)"
+	    "prefetcht0 32(%0)"
+	    : /* no output */
+	    : "r" (addr));
+}
+
+#endif	/* __i386 || __amd64 */
+
+#endif	/* !__lint && __GNUC__ */
+
 #ifdef	__cplusplus
 }
 #endif
diff --git a/usr/src/uts/intel/ia32/ml/ia32.il b/usr/src/uts/intel/ia32/ml/ia32.il
index aa3fbcfeb2..a89c8a7d10 100644
--- a/usr/src/uts/intel/ia32/ml/ia32.il
+++ b/usr/src/uts/intel/ia32/ml/ia32.il
@@ -184,3 +184,12 @@
 	rep			/ our compiler doesn't support "pause" yet,
 	nop			/ so we're using "F3 90" opcode directly
 	.end
+
+/
+/ prefetch 64 bytes
+/
+	.inline	prefetch64,4
+	movl	(%esp), %eax
+	prefetcht0 (%eax)
+	prefetcht0 32(%eax)
+	.end
diff --git a/usr/src/uts/sparc/asm/flush.h b/usr/src/uts/sparc/asm/flush.h
index b9e8d7e93e..417b61b18a 100644
--- a/usr/src/uts/sparc/asm/flush.h
+++ b/usr/src/uts/sparc/asm/flush.h
@@ -20,15 +20,13 @@
  */
 
 /*
- * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #ifndef _ASM_FLUSH_H
 #define	_ASM_FLUSH_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 #include <sys/types.h>
 
 #ifdef	__cplusplus
@@ -44,8 +42,25 @@ doflush(void *addr)
 	__asm__ __volatile__(
 	    "andn	%0, 3, %0\n\t"
 	    "flush	%0\n\t"
-	: "=r" (addr)
-	: "0" (addr));
+	    : "=r" (addr)
+	    : "0" (addr));
+#else
+#error	"port me"
+#endif
+}
+
+#endif	/* !__lint && __GNUC__ */
+
+#if !defined(__lint) && defined(__GNUC__)
+
+extern __inline__ void
+prefetch64(caddr_t addr)
+{
+#if defined(__sparcv9)
+	__asm__ __volatile__(
+	    "prefetch	[%0], #n_writes\n\t"
+	    : /* no output */
+	    : "0" (addr));
 #else
 #error	"port me"
 #endif
diff --git a/usr/src/uts/sparc/ml/sparc.il b/usr/src/uts/sparc/ml/sparc.il
index 0f24a41d0a..4b8a186588 100644
--- a/usr/src/uts/sparc/ml/sparc.il
+++ b/usr/src/uts/sparc/ml/sparc.il
@@ -1,13 +1,12 @@
 !
-! Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
+! Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 ! Use is subject to license terms.
 !
 ! CDDL HEADER START
 !
 ! The contents of this file are subject to the terms of the
-! Common Development and Distribution License, Version 1.0 only
-! (the "License").  You may not use this file except in compliance
-! with the License.
+! Common Development and Distribution License (the "License").
+! You may not use this file except in compliance with the License.
 !
 ! You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 ! or http://www.opensolaris.org/os/licensing.
@@ -22,7 +21,6 @@
 !
 ! CDDL HEADER END
 !
-#ident	"%Z%%M%	%I%	%E% SMI"
 !
 ! In-line functions for sparc kernels.
 !
@@ -52,3 +50,9 @@
 	andn	%o0, 3, %o0
 	flush	%o0
 	.end
+
+! prefetch 64 bytes into L2-cache
+
+	.inline	prefetch64,8
+	prefetch [%o0], #n_writes
+	.end