1 files changed, 67 insertions, 93 deletions

diff --git a/usr/src/uts/common/sys/cpuvar.h b/usr/src/uts/common/sys/cpuvar.h
index 8565ca053e..2cfe5116d9 100644
--- a/usr/src/uts/common/sys/cpuvar.h
+++ b/usr/src/uts/common/sys/cpuvar.h
@@ -23,6 +23,7 @@
  * Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012 by Delphix. All rights reserved.
  * Copyright 2014 Igor Kozhukhov <ikozhukhov@gmail.com>.
+ * Copyright 2018 Joyent, Inc.
  * Copyright 2017 RackTop Systems.
  */
 
@@ -389,7 +390,6 @@ extern cpu_core_t cpu_core[];
 #define	CPU_DISP_HALTED		0x02	/* CPU halted waiting for interrupt */
 
 /* Note: inside ifdef: _KERNEL || _KMEMUSER || _BOOT */
-#if defined(_MACHDEP)
 
 /*
  * Macros for manipulating sets of CPUs as a bitmap.  Note that this
@@ -405,34 +405,60 @@ extern cpu_core_t cpu_core[];
 #define	CPUSET_WORDS	BT_BITOUL(NCPU)
 #define	CPUSET_NOTINSET	((uint_t)-1)
 
-#if	CPUSET_WORDS > 1
-
-typedef struct cpuset {
+#if defined(_MACHDEP)
+struct cpuset {
 	ulong_t	cpub[CPUSET_WORDS];
-} cpuset_t;
+};
+#else
+struct cpuset;
+#endif
+
+typedef struct cpuset cpuset_t;
+
+extern cpuset_t	*cpuset_alloc(int);
+extern void	cpuset_free(cpuset_t *);
 
 /*
- * Private functions for manipulating cpusets that do not fit in a
- * single word.  These should not be used directly; instead the
- * CPUSET_* macros should be used so the code will be portable
- * across different definitions of NCPU.
+ * Functions for manipulating cpusets.  These were previously considered
+ * private when some cpuset_t handling was performed in the CPUSET_* macros.
+ * They are now acceptable to use in non-_MACHDEP code.
  */
 extern	void	cpuset_all(cpuset_t *);
-extern	void	cpuset_all_but(cpuset_t *, uint_t);
+extern	void	cpuset_all_but(cpuset_t *, const uint_t);
 extern	int	cpuset_isnull(cpuset_t *);
-extern	int	cpuset_cmp(cpuset_t *, cpuset_t *);
-extern	void	cpuset_only(cpuset_t *, uint_t);
+extern	int	cpuset_isequal(cpuset_t *, cpuset_t *);
+extern	void	cpuset_only(cpuset_t *, const uint_t);
+extern	long	cpu_in_set(cpuset_t *, const uint_t);
+extern	void	cpuset_add(cpuset_t *, const uint_t);
+extern	void	cpuset_del(cpuset_t *, const uint_t);
 extern	uint_t	cpuset_find(cpuset_t *);
 extern	void	cpuset_bounds(cpuset_t *, uint_t *, uint_t *);
+extern	void	cpuset_atomic_del(cpuset_t *, const uint_t);
+extern	void	cpuset_atomic_add(cpuset_t *, const uint_t);
+extern	long	cpuset_atomic_xadd(cpuset_t *, const uint_t);
+extern	long	cpuset_atomic_xdel(cpuset_t *, const uint_t);
+extern	void	cpuset_or(cpuset_t *, cpuset_t *);
+extern	void	cpuset_xor(cpuset_t *, cpuset_t *);
+extern	void	cpuset_and(cpuset_t *, cpuset_t *);
+extern	void	cpuset_zero(cpuset_t *);
+
+
+#if defined(_MACHDEP)
+
+/*
+ * Prior to the cpuset_t restructuring, the CPUSET_* macros contained
+ * significant logic, rather than directly invoking the backend functions.
+ * They are maintained here so that existing _MACHDEP code can use them.
+ */
 
 #define	CPUSET_ALL(set)			cpuset_all(&(set))
 #define	CPUSET_ALL_BUT(set, cpu)	cpuset_all_but(&(set), cpu)
 #define	CPUSET_ONLY(set, cpu)		cpuset_only(&(set), cpu)
-#define	CPU_IN_SET(set, cpu)		BT_TEST((set).cpub, cpu)
-#define	CPUSET_ADD(set, cpu)		BT_SET((set).cpub, cpu)
-#define	CPUSET_DEL(set, cpu)		BT_CLEAR((set).cpub, cpu)
+#define	CPU_IN_SET(set, cpu)		cpu_in_set(&(set), cpu)
+#define	CPUSET_ADD(set, cpu)		cpuset_add(&(set), cpu)
+#define	CPUSET_DEL(set, cpu)		cpuset_del(&(set), cpu)
 #define	CPUSET_ISNULL(set)		cpuset_isnull(&(set))
-#define	CPUSET_ISEQUAL(set1, set2)	cpuset_cmp(&(set1), &(set2))
+#define	CPUSET_ISEQUAL(set1, set2)	cpuset_isequal(&(set1), &(set2))
 
 /*
  * Find one CPU in the cpuset.
@@ -460,86 +486,24 @@ extern	void	cpuset_bounds(cpuset_t *, uint_t *, uint_t *);
  * deleting a cpu that's not in the cpuset)
  */
 
-#define	CPUSET_ATOMIC_DEL(set, cpu)	BT_ATOMIC_CLEAR((set).cpub, (cpu))
-#define	CPUSET_ATOMIC_ADD(set, cpu)	BT_ATOMIC_SET((set).cpub, (cpu))
-
-#define	CPUSET_ATOMIC_XADD(set, cpu, result) \
-	BT_ATOMIC_SET_EXCL((set).cpub, cpu, result)
-
-#define	CPUSET_ATOMIC_XDEL(set, cpu, result) \
-	BT_ATOMIC_CLEAR_EXCL((set).cpub, cpu, result)
-
-
-#define	CPUSET_OR(set1, set2)		{		\
-	int _i;						\
-	for (_i = 0; _i < CPUSET_WORDS; _i++)		\
-		(set1).cpub[_i] |= (set2).cpub[_i];	\
-}
-
-#define	CPUSET_XOR(set1, set2)		{		\
-	int _i;						\
-	for (_i = 0; _i < CPUSET_WORDS; _i++)		\
-		(set1).cpub[_i] ^= (set2).cpub[_i];	\
-}
-
-#define	CPUSET_AND(set1, set2)		{		\
-	int _i;						\
-	for (_i = 0; _i < CPUSET_WORDS; _i++)		\
-		(set1).cpub[_i] &= (set2).cpub[_i];	\
-}
-
-#define	CPUSET_ZERO(set)		{		\
-	int _i;						\
-	for (_i = 0; _i < CPUSET_WORDS; _i++)		\
-		(set).cpub[_i] = 0;			\
-}
-
-#elif	CPUSET_WORDS == 1
-
-typedef	ulong_t	cpuset_t;	/* a set of CPUs */
-
-#define	CPUSET(cpu)			(1UL << (cpu))
-
-#define	CPUSET_ALL(set)			((void)((set) = ~0UL))
-#define	CPUSET_ALL_BUT(set, cpu)	((void)((set) = ~CPUSET(cpu)))
-#define	CPUSET_ONLY(set, cpu)		((void)((set) = CPUSET(cpu)))
-#define	CPU_IN_SET(set, cpu)		((set) & CPUSET(cpu))
-#define	CPUSET_ADD(set, cpu)		((void)((set) |= CPUSET(cpu)))
-#define	CPUSET_DEL(set, cpu)		((void)((set) &= ~CPUSET(cpu)))
-#define	CPUSET_ISNULL(set)		((set) == 0)
-#define	CPUSET_ISEQUAL(set1, set2)	((set1) == (set2))
-#define	CPUSET_OR(set1, set2)		((void)((set1) |= (set2)))
-#define	CPUSET_XOR(set1, set2)		((void)((set1) ^= (set2)))
-#define	CPUSET_AND(set1, set2)		((void)((set1) &= (set2)))
-#define	CPUSET_ZERO(set)		((void)((set) = 0))
-
-#define	CPUSET_FIND(set, cpu)		{		\
-	cpu = (uint_t)(lowbit(set) - 1);				\
-}
-
-#define	CPUSET_BOUNDS(set, smallest, largest)	{	\
-	smallest = (uint_t)(lowbit(set) - 1);		\
-	largest = (uint_t)(highbit(set) - 1);		\
-}
+#define	CPUSET_ATOMIC_DEL(set, cpu)	cpuset_atomic_del(&(set), cpu)
+#define	CPUSET_ATOMIC_ADD(set, cpu)	cpuset_atomic_add(&(set), cpu)
 
-#define	CPUSET_ATOMIC_DEL(set, cpu)	atomic_and_ulong(&(set), ~CPUSET(cpu))
-#define	CPUSET_ATOMIC_ADD(set, cpu)	atomic_or_ulong(&(set), CPUSET(cpu))
+#define	CPUSET_ATOMIC_XADD(set, cpu, result)	\
+	(result) = cpuset_atomic_xadd(&(set), cpu)
 
-#define	CPUSET_ATOMIC_XADD(set, cpu, result) \
-	{ result = atomic_set_long_excl(&(set), (cpu)); }
+#define	CPUSET_ATOMIC_XDEL(set, cpu, result)	\
+	(result) = cpuset_atomic_xdel(&(set), cpu)
 
-#define	CPUSET_ATOMIC_XDEL(set, cpu, result) \
-	{ result = atomic_clear_long_excl(&(set), (cpu)); }
+#define	CPUSET_OR(set1, set2)	cpuset_or(&(set1), &(set2))
 
-#else	/* CPUSET_WORDS <= 0 */
+#define	CPUSET_XOR(set1, set2)	cpuset_xor(&(set1), &(set2))
 
-#error NCPU is undefined or invalid
+#define	CPUSET_AND(set1, set2)	cpuset_and(&(set1), &(set2))
 
-#endif	/* CPUSET_WORDS	*/
-
-extern cpuset_t cpu_seqid_inuse;
+#define	CPUSET_ZERO(set)	cpuset_zero(&(set))
 
-#endif	/* _MACHDEP */
+#endif /* _MACHDEP */
 #endif /* _KERNEL || _KMEMUSER || _BOOT */
 
 #define	CPU_CPR_OFFLINE		0x0
@@ -550,10 +514,14 @@ extern cpuset_t cpu_seqid_inuse;
 
 #if defined(_KERNEL) || defined(_KMEMUSER)
 
+extern cpuset_t cpu_seqid_inuse;
+
 extern struct cpu	*cpu[];		/* indexed by CPU number */
 extern struct cpu	**cpu_seq;	/* indexed by sequential CPU id */
 extern cpu_t		*cpu_list;	/* list of CPUs */
 extern cpu_t		*cpu_active;	/* list of active CPUs */
+extern cpuset_t		cpu_active_set;	/* cached set of active CPUs */
+extern cpuset_t		cpu_available;	/* cached set of available CPUs */
 extern int		ncpus;		/* number of CPUs present */
 extern int		ncpus_online;	/* number of CPUs not quiesced */
 extern int		max_ncpus;	/* max present before ncpus is known */
@@ -572,13 +540,19 @@ extern struct cpu *curcpup(void);
 #endif
 
 /*
- * CPU_CURRENT indicates to thread_affinity_set to use CPU->cpu_id
- * as the target and to grab cpu_lock instead of requiring the caller
- * to grab it.
+ * CPU_CURRENT indicates to thread_affinity_set() to use whatever curthread's
+ * current CPU is; holding cpu_lock is not required.
  */
 #define	CPU_CURRENT	-3
 
 /*
+ * CPU_BEST can be used by thread_affinity_set() callers to set affinity to a
+ * good CPU (in particular, an ht_acquire()-friendly choice); holding cpu_lock
+ * is not required.
+ */
+#define	CPU_BEST	-4
+
+/*
  * Per-CPU statistics
  *
  * cpu_stats_t contains numerous system and VM-related statistics, in the form
@@ -613,7 +587,7 @@ extern struct cpu *curcpup(void);
  */
 #define	CPU_NEW_GENERATION(cp)	((cp)->cpu_generation++)
 
-#endif /* _KERNEL || _KMEMUSER */
+#endif /* defined(_KERNEL) || defined(_KMEMUSER) */
 
 /*
  * CPU support routines (not for genassym.c)