6944589 need pause API

PSARC/2010/293 smt_pause

18 files changed, 233 insertions, 15 deletions

diff --git a/usr/src/head/synch.h b/usr/src/head/synch.h
index bf73cb33cf..89efe9c687 100644
--- a/usr/src/head/synch.h
+++ b/usr/src/head/synch.h
@@ -20,8 +20,7 @@
  */
 
 /*
- * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #ifndef	_SYNCH_H
@@ -262,6 +261,13 @@ int _mutex_held();
 
 #endif	/* __STDC__ */
 
+/* Pause API */
+#ifdef	__STDC__
+void smt_pause(void);
+#else	/* __STDC__ */
+void smt_pause();
+#endif	/* __STDC__ */
+
 #endif /* _ASM */
 
 #ifdef	__cplusplus
diff --git a/usr/src/lib/libc/amd64/threads/machdep.c b/usr/src/lib/libc/amd64/threads/machdep.c
index 2e8b9edc2c..b1638e0b31 100644
--- a/usr/src/lib/libc/amd64/threads/machdep.c
+++ b/usr/src/lib/libc/amd64/threads/machdep.c
@@ -20,8 +20,7 @@
  */
 
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #include "lint.h"
@@ -197,3 +196,9 @@ __csigsetjmp(sigjmp_buf env, int savemask, gregset_t rs)
 
 	return (0);
 }
+
+void
+smt_pause(void)
+{
+	SMT_PAUSE();
+}
diff --git a/usr/src/lib/libc/capabilities/Makefile.com b/usr/src/lib/libc/capabilities/Makefile.com
index 76f6d38661..f8e9e46331 100644
--- a/usr/src/lib/libc/capabilities/Makefile.com
+++ b/usr/src/lib/libc/capabilities/Makefile.com
@@ -32,3 +32,5 @@ OBJCAP =	pics/objcap.o
 SYMCAP =	pics/symcap.o
 
 CLOBBERFILES +=	$(OBJCAP) $(SYMCAP)
+
+LIBCBASE =	$(SRC)/lib/libc
diff --git a/usr/src/lib/libc/capabilities/sun4v-hwcap1/Makefile.com b/usr/src/lib/libc/capabilities/sun4v-hwcap1/Makefile.com
index 858fd7dda5..0b1a16be84 100644
--- a/usr/src/lib/libc/capabilities/sun4v-hwcap1/Makefile.com
+++ b/usr/src/lib/libc/capabilities/sun4v-hwcap1/Makefile.com
@@ -28,13 +28,14 @@ GEN_PLATFORM =	sun4v
 
 include		../../Makefile.com
 
-OBJECTS =	memcpy.o memset.o
+OBJECTS =	memcpy.o memset.o smt_pause.o
 
 include		$(SRC)/lib/Makefile.lib
 
 IFLAGS =	-I$(SRC)/uts/$(GEN_PLATFORM) \
 		-I$(ROOT)/usr/include/v9
 
+
 AS_CPPFLAGS +=	-D__STDC__ -D_ASM -DPIC -D_REENTRANT -D$(MACH) $(IFLAGS) \
 		-DNIAGARA_IMPL
 ASFLAGS =	-P -K pic
diff --git a/usr/src/lib/libc/capabilities/sun4v-hwcap1/common/mapfile-cap b/usr/src/lib/libc/capabilities/sun4v-hwcap1/common/mapfile-cap
index b4dc200a22..2b80c8474f 100644
--- a/usr/src/lib/libc/capabilities/sun4v-hwcap1/common/mapfile-cap
+++ b/usr/src/lib/libc/capabilities/sun4v-hwcap1/common/mapfile-cap
@@ -37,6 +37,7 @@ SYMBOL_SCOPE {
 		memcpy;
 		memmove;
 		memset;
+		smt_pause;
 	local:
 		*;
 };
diff --git a/usr/src/lib/libc/capabilities/sun4v-hwcap1/sparc/Makefile b/usr/src/lib/libc/capabilities/sun4v-hwcap1/sparc/Makefile
index b96dc6ffc0..5f9092b14b 100644
--- a/usr/src/lib/libc/capabilities/sun4v-hwcap1/sparc/Makefile
+++ b/usr/src/lib/libc/capabilities/sun4v-hwcap1/sparc/Makefile
@@ -25,10 +25,15 @@
 
 include		../Makefile.com
 
-ASFLAGS +=	-xarch=v8plus
+ASFLAGS +=	-xarch=v8plus -I$(LIBCBASE)/$(MACH)
 
 .KEEP_STATE:
 
-all install:	$(SYMCAP)
+all install:	$(LIBCBASE)/$(MACH)/assym.h $(SYMCAP)
+
+# We need assym.h, but it won't build directly. smt_pause.o depends on
+# assym.h, so building it forces assym.h to be built.
+$(LIBCBASE)/$(MACH)/assym.h:
+	@cd $(LIBCBASE)/$(MACH); $(MAKE) pics/smt_pause.o
 
 include		../../Makefile.targ
diff --git a/usr/src/lib/libc/capabilities/sun4v-hwcap1/sparcv9/Makefile b/usr/src/lib/libc/capabilities/sun4v-hwcap1/sparcv9/Makefile
index 16234a0700..f552f601b8 100644
--- a/usr/src/lib/libc/capabilities/sun4v-hwcap1/sparcv9/Makefile
+++ b/usr/src/lib/libc/capabilities/sun4v-hwcap1/sparcv9/Makefile
@@ -27,8 +27,13 @@ include		../Makefile.com
 include		$(SRC)/lib/Makefile.lib.64
 
 AS_CPPFLAGS +=	-D__sparcv9
-ASFLAGS +=	-xarch=v9
+ASFLAGS +=	-xarch=v9 -I$(LIBCBASE)/$(MACH64)
 
-all install:	$(SYMCAP)
+all install:	$(LIBCBASE)/$(MACH64)/assym.h $(SYMCAP)
+
+# We need assym.h, but it won't build directly. smt_pause.o depends on
+# assym.h, so building it forces assym.h to be built.
+$(LIBCBASE)/$(MACH64)/assym.h:
+	@cd $(LIBCBASE)/$(MACH64); $(MAKE) pics/smt_pause.o
 
 include		../../Makefile.targ
diff --git a/usr/src/lib/libc/capabilities/sun4v-hwcap2/Makefile.com b/usr/src/lib/libc/capabilities/sun4v-hwcap2/Makefile.com
index de1d0a297d..af1273234c 100644
--- a/usr/src/lib/libc/capabilities/sun4v-hwcap2/Makefile.com
+++ b/usr/src/lib/libc/capabilities/sun4v-hwcap2/Makefile.com
@@ -28,7 +28,7 @@ GEN_PLATFORM =	sun4v
 
 include		../../Makefile.com
 
-OBJECTS =	memcpy.o
+OBJECTS =	memcpy.o smt_pause.o
 
 include		$(SRC)/lib/Makefile.lib
 
diff --git a/usr/src/lib/libc/capabilities/sun4v-hwcap2/common/mapfile-cap b/usr/src/lib/libc/capabilities/sun4v-hwcap2/common/mapfile-cap
index 9b7d9fa961..3f5eb7b73e 100644
--- a/usr/src/lib/libc/capabilities/sun4v-hwcap2/common/mapfile-cap
+++ b/usr/src/lib/libc/capabilities/sun4v-hwcap2/common/mapfile-cap
@@ -35,6 +35,7 @@ SYMBOL_SCOPE {
 		_memmove;
 		memcpy;
 		memmove;
+		smt_pause;
 	local:
 		*;
 };
diff --git a/usr/src/lib/libc/capabilities/sun4v/common/smt_pause.s b/usr/src/lib/libc/capabilities/sun4v/common/smt_pause.s
new file mode 100644
index 0000000000..406f48fa83
--- /dev/null
+++ b/usr/src/lib/libc/capabilities/sun4v/common/smt_pause.s
@@ -0,0 +1,67 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ */
+
+	.file	"smt_pause.s"
+
+/*
+ * void smt_pause(void)
+ *
+ * Do nothing efficiently.
+ *
+ */
+
+#include <sys/asm_linkage.h>
+#include <sys/niagaraasi.h>
+#include <sys/asi.h>
+#include <sys/stack.h>
+
+#if defined(NIAGARA_IMPL)
+#include "assym.h"
+#endif
+
+	ENTRY(smt_pause)
+#if defined(NIAGARA2_IMPL) || defined(VFALLS_IMPL) || defined(KT_IMPL)
+	rd	%ccr, %g0
+	rd	%ccr, %g0
+	retl
+	rd	%ccr, %g0 
+#else	/* NIAGARA2_IMPL */
+#define	BANKS	(4 * 64)	/* 4 cachelines, cover all banks */
+	save %sp, -SA(MINFRAME+BANKS), %sp
+
+	ld      [%g7 + UL_LWPID], %i3   ! curthread->ul_lwpid
+	add	%fp, STACK_BIAS-BANKS, %i0
+	and     %i3, 0x3, %i2           ! save last 2 bits
+	sllx    %i2, 0x6, %i1           ! pick a slot
+	add	%i1, %i0, %o0
+
+	casx	[%o0], %g0, %g0
+	casx    [%o0], %g0, %g0
+	casx    [%o0], %g0, %g0
+	ret
+	restore
+#endif	/* NIAGARA2_IMPL */
+	SET_SIZE(smt_pause)
+
diff --git a/usr/src/lib/libc/i386/threads/machdep.c b/usr/src/lib/libc/i386/threads/machdep.c
index d13dd75480..c9c38e369b 100644
--- a/usr/src/lib/libc/i386/threads/machdep.c
+++ b/usr/src/lib/libc/i386/threads/machdep.c
@@ -20,8 +20,7 @@
  */
 
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #include "thr_uberdata.h"
@@ -256,3 +255,9 @@ __csigsetjmp(greg_t cs, greg_t ss, greg_t gs,
 
 	return (0);
 }
+
+void
+smt_pause(void)
+{
+	SMT_PAUSE();
+}
diff --git a/usr/src/lib/libc/inc/thr_uberdata.h b/usr/src/lib/libc/inc/thr_uberdata.h
index 140bdcd251..ca53ccf701 100644
--- a/usr/src/lib/libc/inc/thr_uberdata.h
+++ b/usr/src/lib/libc/inc/thr_uberdata.h
@@ -20,8 +20,7 @@
  */
 
 /*
- * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #ifndef _THR_UBERDATA_H
@@ -255,7 +254,7 @@ extern	void	ht_pause(void);		/* "pause" instruction */
 extern	void	SMT_PAUSE_FUNCTION(void);
 #define	SMT_PAUSE()	SMT_PAUSE_FUNCTION()
 #else
-#define	SMT_PAUSE()
+#define	SMT_PAUSE()	smt_pause()
 #endif	/* __x86 */
 
 /*
diff --git a/usr/src/lib/libc/port/mapfile-vers b/usr/src/lib/libc/port/mapfile-vers
index ed3f10dd05..3ceea9bac4 100644
--- a/usr/src/lib/libc/port/mapfile-vers
+++ b/usr/src/lib/libc/port/mapfile-vers
@@ -219,6 +219,7 @@ $endif
 	sigqueue;
 	sigtimedwait;
 	sigwaitinfo;
+	smt_pause;
 	strnlen;
 	strsep;
 	symlinkat;
diff --git a/usr/src/lib/libc/sparc/Makefile.com b/usr/src/lib/libc/sparc/Makefile.com
index 84c888d0fc..fd8b30caa4 100644
--- a/usr/src/lib/libc/sparc/Makefile.com
+++ b/usr/src/lib/libc/sparc/Makefile.com
@@ -149,6 +149,7 @@ GENOBJS=			\
 	setjmp.o		\
 	siginfolst.o		\
 	siglongjmp.o		\
+	smt_pause.o		\
 	sparc_data.o		\
 	strcasecmp.o		\
 	strchr.o		\
@@ -1241,6 +1242,7 @@ ASSYMDEP_OBJS=			\
 	_stack_grow.o		\
 	asm_subr.o		\
 	setjmp.o		\
+	smt_pause.o		\
 	tls_get_addr.o		\
 	unwind_frame.o		\
 	vforkx.o
diff --git a/usr/src/lib/libc/sparc/gen/smt_pause.s b/usr/src/lib/libc/sparc/gen/smt_pause.s
new file mode 100644
index 0000000000..93739d0196
--- /dev/null
+++ b/usr/src/lib/libc/sparc/gen/smt_pause.s
@@ -0,0 +1,60 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ */
+
+	.file	"smt_pause.s"
+
+#include <sys/asm_linkage.h>
+#include <../assym.h>
+#include <sys/stack.h>
+
+/*
+ * void smt_pause(void)
+ *
+ * Do nothing efficiently.
+ * We do the dance with the lwpid so that the actual address is spread
+ * across cache banks thus avoiding hot spots.
+ * Casx arguments are a no-op, but they force access to L2 cache, which
+ * takes lots of cycles.
+ */
+
+#ifdef lint
+void
+smt_pause(void)
+{
+}
+#else
+#define	BANKS	(4 * 64)	/* covers 4 cachelines, all banks */
+	ENTRY(smt_pause)
+	save	%sp, -SA(MINFRAME+BANKS), %sp
+	ld      [%g7 + UL_LWPID], %i5
+	add	%fp, STACK_BIAS-BANKS, %i3
+	and     %i5, 0x3, %i4           ! save last 2 bits
+	sll     %i4, 0x6, %i2           ! pick a slot
+	add     %i2, %i3, %o0
+	casx    [%o0], %g0, %g0
+	casx    [%o0], %g0, %g0
+	ret
+	restore
+	SET_SIZE(smt_pause)
+#endif
diff --git a/usr/src/lib/libc/sparc/offsets.in b/usr/src/lib/libc/sparc/offsets.in
index a052cfc67f..c56bbe1df2 100644
--- a/usr/src/lib/libc/sparc/offsets.in
+++ b/usr/src/lib/libc/sparc/offsets.in
@@ -48,6 +48,7 @@ ulwp_t
 	ul_siglink
 	ul_sigmask
 	ul_unwind_ret
+	ul_lwpid
 
 sigjmp_struct_t
 	sjs_sp
diff --git a/usr/src/lib/libc/sparcv9/Makefile.com b/usr/src/lib/libc/sparcv9/Makefile.com
index 897e294342..f96202fec0 100644
--- a/usr/src/lib/libc/sparcv9/Makefile.com
+++ b/usr/src/lib/libc/sparcv9/Makefile.com
@@ -144,6 +144,7 @@ GENOBJS=			\
 	setjmp.o		\
 	siginfolst.o		\
 	siglongjmp.o		\
+	smt_pause.o		\
 	sparc_data.o		\
 	strcasecmp.o		\
 	strchr.o		\
@@ -1167,6 +1168,7 @@ ASSYMDEP_OBJS=			\
 	_stack_grow.o		\
 	asm_subr.o		\
 	setjmp.o		\
+	smt_pause.o		\
 	tls_get_addr.o		\
 	unwind_frame.o		\
 	vforkx.o
diff --git a/usr/src/lib/libc/sparcv9/gen/smt_pause.s b/usr/src/lib/libc/sparcv9/gen/smt_pause.s
new file mode 100644
index 0000000000..dadbb0c569
--- /dev/null
+++ b/usr/src/lib/libc/sparcv9/gen/smt_pause.s
@@ -0,0 +1,55 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
+ */
+
+	.file	"smt_pause.s"
+
+#include <sys/asm_linkage.h>
+#include <../assym.h>
+
+/*
+ * void smt_pause(void)
+ *
+ * Do nothing efficiently.
+ */
+
+#ifdef lint
+void
+smt_pause(void)
+{
+}
+#else
+#define	BANKS	(4 * 64)
+	ENTRY(smt_pause)
+	save %sp, -SA(MINFRAME+BANKS), %sp
+	ld      [%g7 + UL_LWPID], %i3   ! curthread->ul_lwpid
+	add	%fp, STACK_BIAS-BANKS, %i0
+	and     %i3, 0x3, %i2           ! save last 2 bits
+	sllx    %i2, 0x6, %i1           ! pick a slot
+	add     %i1, %i0, %o0
+	casx    [%o0], %g0, %g0
+	casx    [%o0], %g0, %g0
+	ret
+	restore
+	SET_SIZE(smt_pause)
+#endif