OS-7498 __ctype_mask[EOF] has been working by accident

OS-7511 undefined behavior in lx_cap_update_priv() turns gcc7 demonic
OS-7517 -faggressive-loop-optimizations is too aggressive
OS-7518 array over-read in has_saved_fp()
OS-7535 allow platform builds on base-64 image
OS-7536 stop shipping poold(1M) and its cohorts
Reviewed by: Robert Mustacchi <rm@joyent.com>
Reviewed by: John Levon <john.levon@joyent.com>
Approved by: Robert Mustacchi <rm@joyent.com>

10 files changed, 66 insertions, 37 deletions

diff --git a/usr/src/Makefile.master b/usr/src/Makefile.master
index 1b3fcfd7bf..65b5e220a7 100644
--- a/usr/src/Makefile.master
+++ b/usr/src/Makefile.master
@@ -381,6 +381,17 @@ CCNOREORDER=	\
 	-_gcc7=-fno-reorder-functions \
 	-_gcc8=-fno-reorder-functions
 
+#
+# gcc has a rather aggressive optimization on by default that infers loop
+# bounds based on undefined behavior (!!).  This can lead to some VERY
+# surprising optimizations -- ones that may be technically correct in the
+# strictest sense but also result in incorrect program behavior.  We turn
+# this optimization off, with extreme prejudice.
+#
+CCNOAGGRESSIVELOOPS= \
+	-_gcc7=-fno-aggressive-loop-optimizations \
+	-_gcc8=-fno-aggressive-loop-optimizations
+
 # One optimization the compiler might perform is to turn this:
 #	#pragma weak foo
 #	extern int foo;
@@ -618,10 +629,12 @@ $(SRCDBGBLD)CCSOURCEDEBUGFLAGS	= $(CCGDEBUG) -xs
 CFLAGS=         $(COPTFLAG) $($(MACH)_CFLAGS) $(SPACEFLAG) $(CCMODE) \
 		$(ILDOFF) $(CERRWARN) $(CSTD) $(CCUNBOUND) $(IROPTFLAG) \
 		$(CGLOBALSTATIC) $(CCNOAUTOINLINE) $(CCNOREORDER) \
+		$(CCNOAGGRESSIVELOOPS) \
 		$(CSOURCEDEBUGFLAGS) $(CUSERFLAGS)
 CFLAGS64=       $(COPTFLAG64) $($(MACH64)_CFLAGS) $(SPACEFLAG64) $(CCMODE64) \
 		$(ILDOFF) $(CERRWARN) $(CSTD) $(CCUNBOUND) $(IROPTFLAG64) \
 		$(CGLOBALSTATIC) $(CCNOAUTOINLINE) $(CCNOREORDER) \
+		$(CCNOAGGRESSIVELOOPS) \
 		$(CSOURCEDEBUGFLAGS) $(CUSERFLAGS64)
 #
 # Flags that are used to build parts of the code that are subsequently
diff --git a/usr/src/cmd/Makefile b/usr/src/cmd/Makefile
index 0f7d4ad1ad..12f0dc9df4 100644
--- a/usr/src/cmd/Makefile
+++ b/usr/src/cmd/Makefile
@@ -21,7 +21,7 @@
 
 #
 # Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
-# Copyright (c) 2018, Joyent, Inc.
+# Copyright (c) 2019, Joyent, Inc.
 # Copyright (c) 2012 by Delphix. All rights reserved.
 # Copyright (c) 2013 DEY Storage Systems, Inc. All rights reserved.
 # Copyright 2014 Garrett D'Amore <garrett@damore.org>
@@ -312,7 +312,6 @@ COMMON_SUBDIRS=		\
 	picl		\
 	plimit		\
 	policykit	\
-	pools		\
 	power		\
 	powertop	\
 	ppgsz		\
@@ -653,7 +652,6 @@ MSGSUBDIRS=		\
 	pg		\
 	pgrep		\
 	picl            \
-	pools		\
 	power		\
 	pr		\
 	praudit		\
diff --git a/usr/src/lib/brand/lx/lx_brand/common/capabilities.c b/usr/src/lib/brand/lx/lx_brand/common/capabilities.c
index ba6b587a92..e034560452 100644
--- a/usr/src/lib/brand/lx/lx_brand/common/capabilities.c
+++ b/usr/src/lib/brand/lx/lx_brand/common/capabilities.c
@@ -10,7 +10,7 @@
  */
 
 /*
- * Copyright 2016 Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  */
 
 /*
@@ -220,7 +220,7 @@ lx_cap_update_priv(priv_set_t *priv, const uint32_t cap[])
 	boolean_t updated = B_FALSE;
 	for (i = 0; i <= LX_CAP_MAX_CHECK; i++) {
 		cap_set = LX_CAP_CAPISSET(i, cap);
-		if (lx_cap_mapping[i] == NULL || i > LX_CAP_MAX_VALID) {
+		if (i > LX_CAP_MAX_VALID || lx_cap_mapping[i] == NULL) {
 			/* don't allow setting unsupported caps */
 			if (cap_set) {
 				/*
diff --git a/usr/src/lib/libc/port/locale/table.c b/usr/src/lib/libc/port/locale/table.c
index 3c0ce09901..0ccb3d9292 100644
--- a/usr/src/lib/libc/port/locale/table.c
+++ b/usr/src/lib/libc/port/locale/table.c
@@ -1,6 +1,7 @@
 /*
  * Copyright 2013 Garrett D'Amore <garrett@damore.org>
  * Copyright 2017 Nexenta Systems, Inc.
+ * Copyright 2019 Joyent, Inc.
  * Copyright (c) 1993
  *	The Regents of the University of California.  All rights reserved.
  *
@@ -39,7 +40,7 @@
 #include "mblocal.h"
 #include "_ctype.h"
 
-#define	_DEFRUNETYPE { \
+#define	_DEFRUNETYPE \
 	/* 00 */ \
 	_CTYPE_C, \
 	_CTYPE_C, \
@@ -183,10 +184,9 @@
 	_CTYPE_P|_CTYPE_R|_CTYPE_G, \
 	_CTYPE_P|_CTYPE_R|_CTYPE_G, \
 	_CTYPE_P|_CTYPE_R|_CTYPE_G, \
-	_CTYPE_C, \
-}
+	_CTYPE_C
 
-#define	_DEFMAPLOWER { \
+#define	_DEFMAPLOWER \
 	0x00,	0x01,	0x02,	0x03,	0x04,	0x05,	0x06,	0x07, \
 	0x08,	0x09,	0x0a,	0x0b,	0x0c,	0x0d,	0x0e,	0x0f, \
 	0x10,	0x11,	0x12,	0x13,	0x14,	0x15,	0x16,	0x17, \
@@ -218,10 +218,9 @@
 	0xe0,	0xe1,	0xe2,	0xe3,	0xe4,	0xe5,	0xe6,	0xe7, \
 	0xe8,	0xe9,	0xea,	0xeb,	0xec,	0xed,	0xee,	0xef, \
 	0xf0,	0xf1,	0xf2,	0xf3,	0xf4,	0xf5,	0xf6,	0xf7, \
-	0xf8,	0xf9,	0xfa,	0xfb,	0xfc,	0xfd,	0xfe,	0xff, \
-}
+	0xf8,	0xf9,	0xfa,	0xfb,	0xfc,	0xfd,	0xfe,	0xff
 
-#define	_DEFMAPUPPER { \
+#define	_DEFMAPUPPER \
 	0x00,	0x01,	0x02,	0x03,	0x04,	0x05,	0x06,	0x07, \
 	0x08,	0x09,	0x0a,	0x0b,	0x0c,	0x0d,	0x0e,	0x0f, \
 	0x10,	0x11,	0x12,	0x13,	0x14,	0x15,	0x16,	0x17, \
@@ -253,27 +252,32 @@
 	0xe0,	0xe1,	0xe2,	0xe3,	0xe4,	0xe5,	0xe6,	0xe7, \
 	0xe8,	0xe9,	0xea,	0xeb,	0xec,	0xed,	0xee,	0xef, \
 	0xf0,	0xf1,	0xf2,	0xf3,	0xf4,	0xf5,	0xf6,	0xf7, \
-	0xf8,	0xf9,	0xfa,	0xfb,	0xfc,	0xfd,	0xfe,	0xff, \
-}
+	0xf8,	0xf9,	0xfa,	0xfb,	0xfc,	0xfd,	0xfe,	0xff,
 
 _RuneLocale _DefaultRuneLocale = {
 	_RUNE_MAGIC_1,
 	"NONE",
-	_DEFRUNETYPE,
-	_DEFMAPLOWER,
-	_DEFMAPUPPER,
+	{ _DEFRUNETYPE },
+	{ _DEFMAPLOWER },
+	{ _DEFMAPUPPER },
 };
 
 /*
  * __ctype_mask, __trans_lower, and __trans_upper come from former _ctype.c and
  * have to stay pointers for binary compatibility, so we provide separate
- * storage for them, initialized to "C" locale contents by default.
+ * storage for them, initialized to "C" locale contents by default.  Note that
+ * legacy code may dereference __ctype_mask[-1] when checking against EOF,
+ * relying on that value to be 0.  To allow this, ___ctype_mask is expanded by
+ * one value and prepended with a leading 0, with __ctype_mask being set to
+ * point to ___ctype_mask[1].  (__trans_lower and __trans_upper do not suffer
+ * from this as EOF access was prevented in legacy code by a check against
+ * isascii(), which always returned 0 for EOF.)
  */
-static unsigned int ___ctype_mask[_CACHED_RUNES] = _DEFRUNETYPE;
-unsigned int *__ctype_mask = ___ctype_mask;
+static unsigned int ___ctype_mask[_CACHED_RUNES + 1] = { 0, _DEFRUNETYPE };
+unsigned int *__ctype_mask = &___ctype_mask[1];
 
-static int ___trans_lower[_CACHED_RUNES] = _DEFMAPLOWER;
+static int ___trans_lower[_CACHED_RUNES] = { _DEFMAPLOWER };
 int *__trans_lower = ___trans_lower;
 
-static int ___trans_upper[_CACHED_RUNES] = _DEFMAPUPPER;
+static int ___trans_upper[_CACHED_RUNES] = { _DEFMAPUPPER };
 int *__trans_upper = ___trans_upper;
diff --git a/usr/src/lib/libsaveargs/amd64/saveargs.c b/usr/src/lib/libsaveargs/amd64/saveargs.c
index f07ef354c1..c8276ddebb 100644
--- a/usr/src/lib/libsaveargs/amd64/saveargs.c
+++ b/usr/src/lib/libsaveargs/amd64/saveargs.c
@@ -23,6 +23,9 @@
  * Use is subject to license terms.
  */
 
+/*
+ * Copyright 2019 Joyent, Inc.
+ */
 
 /*
  * The Sun Studio and GCC (patched for opensolaris/illumos) compilers
@@ -231,7 +234,7 @@ has_saved_fp(dis_handle_t *dhp, uint8_t *ins, int size)
 				continue;
 
 			n = INSTR1(ins, i);
-			for (j = 0; j <= NUM_FP_PUSHES; j++)
+			for (j = 0; j < NUM_FP_PUSHES; j++)
 				if (save_fp_pushes[j] == n) {
 					found_push = B_TRUE;
 					break;
@@ -240,7 +243,7 @@ has_saved_fp(dis_handle_t *dhp, uint8_t *ins, int size)
 			if (sz != 3)
 				continue;
 			n = INSTR3(ins, i);
-			for (j = 0; j <= NUM_FP_MOVS; j++)
+			for (j = 0; j < NUM_FP_MOVS; j++)
 				if (save_fp_movs[j] == n)
 					return (B_TRUE);
 		}
diff --git a/usr/src/tools/ctf/Makefile.ctf b/usr/src/tools/ctf/Makefile.ctf
index 830da0990f..64c220cd4a 100644
--- a/usr/src/tools/ctf/Makefile.ctf
+++ b/usr/src/tools/ctf/Makefile.ctf
@@ -22,7 +22,7 @@
 # Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
-# Copyright (c) 2018, Joyent, Inc.
+# Copyright (c) 2019, Joyent, Inc.
 
 include ../../../Makefile.tools
 
@@ -43,7 +43,7 @@ HDRDIRS= \
 	-I/usr/include \
 	-I$(SRC)/uts/common \
 	-I$(SRC)/common/ctf \
-	-I$(NATIVE_ADJUNCT)/include
+	-I$(ADJUNCT_PROTO)/usr/include
 
 CPPFLAGS += $(HDRDIRS)
 CFLAGS += $(CCVERBOSE)
diff --git a/usr/src/tools/ctf/cvt/Makefile.com b/usr/src/tools/ctf/cvt/Makefile.com
index 052cf02cc4..df3696e633 100644
--- a/usr/src/tools/ctf/cvt/Makefile.com
+++ b/usr/src/tools/ctf/cvt/Makefile.com
@@ -22,6 +22,7 @@
 # Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 #
+# Copyright 2019 Joyent, Inc.
 
 include ../../Makefile.ctf
 
@@ -74,7 +75,7 @@ DWARFLDFLAGS	= \
 	-ldwarf
 DWARFCPPFLAGS	= -I$(SRC)/lib/libdwarf/common
 
-LDFLAGS		+= -L$(NATIVE_ADJUNCT)/lib
+LDFLAGS		+= -L$(ADJUNCT_PROTO)/lib
 LDLIBS		+= -lz -lelf
 CPPFLAGS	+= -D_REENTRANT
 CFLAGS		+= $(CTF_FLAGS)
diff --git a/usr/src/tools/mandoc/Makefile b/usr/src/tools/mandoc/Makefile
index 5b4d1ea764..c04ace16d4 100644
--- a/usr/src/tools/mandoc/Makefile
+++ b/usr/src/tools/mandoc/Makefile
@@ -11,7 +11,7 @@
 
 #
 # Copyright 2017 Nexenta Systems, Inc.
-# Copyright 2018 Joyent, Inc.
+# Copyright 2019 Joyent, Inc.
 #
 
 CMDDIR=		$(SRC)/cmd/mandoc
@@ -20,19 +20,16 @@ LCDIR=		$(SRC)/lib/libc/port
 include		$(SRC)/tools/Makefile.tools
 include		$(CMDDIR)/Makefile.common
 
-CPPFLAGS += -_gcc=-nostdinc -I/usr/include -I$(NATIVE_ADJUNCT)/include
-CPPFLAGS += -D_FILE_OFFSET_BITS=64
-LDFLAGS += -L$(NATIVE_ADJUNCT)/lib -R$(NATIVE_ADJUNCT)/lib
+CPPFLAGS +=	-_gcc=-nostdinc -I/usr/include -I$(ADJUNCT_PROTO)/usr/include
+CPPFLAGS +=	-I. -include fts.h
+CPPFLAGS +=	-D_FILE_OFFSET_BITS=64
+LDFLAGS +=	-L$(ADJUNCT_PROTO)/lib -R$(ADJUNCT_PROTO)/lib
+
 OBJS +=		fts.o \
 		reallocarray.o \
 		recallocarray.o \
 		strtonum.o
 
-CPPFLAGS +=	-_gcc=-nostdinc
-CPPFLAGS +=	-I. -include fts.h
-CPPFLAGS +=	-I$(NATIVE_ADJUNCT)/include
-LDFLAGS +=	-L$(NATIVE_ADJUNCT)/lib -R$(NATIVE_ADJUNCT)/lib
-
 # 3rd party code
 SMATCH=off
 
diff --git a/usr/src/tools/smatch/Makefile b/usr/src/tools/smatch/Makefile
index 7c18c6ce72..b876b8bb36 100644
--- a/usr/src/tools/smatch/Makefile
+++ b/usr/src/tools/smatch/Makefile
@@ -28,7 +28,11 @@ include ../Makefile.tools
 i386_CC = $(GNUC_ROOT)/bin/gcc
 sparc_CC = $(GNUC_ROOT)/bin/gcc
 
-CFLAGS = -O -D__sun -Wall -Wno-unknown-pragmas -std=gnu99 -nodefaultlibs
+#
+# Note that we always build smatch 64-bit, assuring that we get the same binary
+# regardless of the image of the build machine.
+#
+CFLAGS = -O -m64 -D__sun -Wall -Wno-unknown-pragmas -std=gnu99 -nodefaultlibs
 
 SMATCHDATADIR = $(ROOTONBLDSHARE)/smatch
 
@@ -38,7 +42,15 @@ CFLAGS += -DMULTIARCH_TRIPLET=NULL
 
 LDLIBS += -lsqlite3 -lcrypto -lm -lgcc -lc
 LDFLAGS = $(MAPFILE.NES:%=-Wl,-M%)
+
+#
+# To allow linking to $(NATIVE_ADJUNCT) libraries on both multiarch (where
+# 64-bit libraries will be found in $(NATIVE_ADJUNCT)/lib/$(MACH64)) and 64-bit
+# images (where 64-bit libraries will be found in $(NATIVE_ADJUNCT)/lib/), we
+# set our library and run paths to both locations.
+#
 LDFLAGS += -L$(NATIVE_ADJUNCT)/lib -R$(NATIVE_ADJUNCT)/lib
+LDFLAGS += -L$(NATIVE_ADJUNCT)/lib/$(MACH64) -R$(NATIVE_ADJUNCT)/lib/$(MACH64)
 
 CPPFLAGS += -nostdinc
 CPPFLAGS += -Isrc/
diff --git a/usr/src/uts/Makefile.uts b/usr/src/uts/Makefile.uts
index e1c0fd372b..e00d482fa3 100644
--- a/usr/src/uts/Makefile.uts
+++ b/usr/src/uts/Makefile.uts
@@ -248,6 +248,7 @@ CFLAGS_uts		+= $(CTF_FLAGS_$(CLASS))
 CFLAGS_uts		+= $(CERRWARN)
 CFLAGS_uts		+= $(CCNOAUTOINLINE)
 CFLAGS_uts		+= $(CCNOREORDER)
+CFLAGS_uts		+= $(CCNOAGGRESSIVELOOPS)
 CFLAGS_uts		+= $(CGLOBALSTATIC)
 CFLAGS_uts		+= $(EXTRA_CFLAGS)
 CFLAGS_uts		+= $(CSOURCEDEBUGFLAGS)