11530 badseg test creates cores

Reviewed by: Toomas Soome <tsoome@me.com>
Approved by: Dan McDonald <danmcd@joyent.com>

4 files changed, 61 insertions, 9 deletions

diff --git a/usr/src/pkg/manifests/system-test-ostest.mf b/usr/src/pkg/manifests/system-test-ostest.mf
index 78d46bdf6a..ed58094eda 100644
--- a/usr/src/pkg/manifests/system-test-ostest.mf
+++ b/usr/src/pkg/manifests/system-test-ostest.mf
@@ -46,6 +46,7 @@ file path=opt/os-tests/tests/file-locking/acquire-lock.64 mode=0555
 file path=opt/os-tests/tests/file-locking/runtests.32 mode=0555
 file path=opt/os-tests/tests/file-locking/runtests.64 mode=0555
 $(i386_ONLY)file path=opt/os-tests/tests/i386/badseg mode=0555
+$(i386_ONLY)file path=opt/os-tests/tests/i386/badseg_exec mode=0555
 $(i386_ONLY)file path=opt/os-tests/tests/i386/ldt mode=0555
 file path=opt/os-tests/tests/pf_key/acquire-compare mode=0555
 file path=opt/os-tests/tests/pf_key/acquire-spray mode=0555
diff --git a/usr/src/test/os-tests/tests/i386/Makefile b/usr/src/test/os-tests/tests/i386/Makefile
index 4933cf6e02..cbea8ec2b2 100644
--- a/usr/src/test/os-tests/tests/i386/Makefile
+++ b/usr/src/test/os-tests/tests/i386/Makefile
@@ -10,13 +10,13 @@
 #
 
 #
-# Copyright 2018 Joyent, Inc.
+# Copyright 2019 Joyent, Inc.
 #
 
 include $(SRC)/cmd/Makefile.cmd
 include $(SRC)/test/Makefile.com
 
-PROGS += ldt badseg
+PROGS += ldt badseg_exec badseg
 
 ROOTOPTPKG = $(ROOT)/opt/os-tests
 TESTDIR = $(ROOTOPTPKG)/tests/i386
@@ -24,15 +24,13 @@ ROOTOPTPROGS = $(PROGS:%=$(TESTDIR)/%)
 
 CSTD = $(CSTD_GNU99)
 
-# for badseg
+# for badseg_exec
 COPTFLAG =
 
 all: $(PROGS)
 
 install: all $(ROOTOPTPROGS)
 
-lint:
-
 clobber: clean
 	-$(RM) $(PROGS)
 
diff --git a/usr/src/test/os-tests/tests/i386/badseg.sh b/usr/src/test/os-tests/tests/i386/badseg.sh
new file mode 100644
index 0000000000..d21cb6bb17
--- /dev/null
+++ b/usr/src/test/os-tests/tests/i386/badseg.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# Copyright 2019 Joyent, Inc.
+#
+
+#
+# badseg intentionally core-dumps. It does a setrlimit(), but we need to
+# prevent global core dumps too: we'll do this by blocking the path for
+# badseg_exec, but let other processes core dump still just in case.
+#
+
+set -e
+set -x
+
+old_enabled=$(/usr/bin/svcprop -p config_params/global_enabled coreadm)
+old_pattern=$(/usr/bin/svcprop -p config_params/global_pattern coreadm)
+old_log=$(/usr/bin/svcprop -p config_params/global_log_enabled coreadm)
+
+mkfile 1m /var/cores/badseg_exec
+coreadm -e global -d log -g /var/cores/%f/%p
+# let it settle
+sleep 3
+
+$(dirname $0)/badseg_exec || true
+
+coreadm -g "$old_pattern"
+
+if [[ "$old_enabled" = "true" ]]; then
+       coreadm -e global
+fi
+
+if [[ "$old_log" = "true" ]]; then
+       coreadm -e log
+fi
+
+rm -f /var/cores/badseg_exec
diff --git a/usr/src/test/os-tests/tests/i386/badseg.c b/usr/src/test/os-tests/tests/i386/badseg_exec.c
index 51de1211cf..999d47c86b 100644
--- a/usr/src/test/os-tests/tests/i386/badseg.c
+++ b/usr/src/test/os-tests/tests/i386/badseg_exec.c
@@ -10,7 +10,7 @@
  */
 
 /*
- * Copyright 2018 Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  */
 
 #include <stdlib.h>
@@ -18,6 +18,8 @@
 #include <sys/wait.h>
 #include <unistd.h>
 #include <sys/regset.h>
+#include <sys/resource.h>
+#include <err.h>
 
 /*
  * Load a bunch of bad selectors into the seg regs: this will typically cause
@@ -117,9 +119,9 @@ inchild(void (*func)())
 	switch ((pid = fork())) {
 	case 0:
 		func();
-		exit(0);
+		exit(EXIT_SUCCESS);
 	case -1:
-		exit(1);
+		exit(EXIT_FAILURE);
 	default:
 		(void) waitpid(pid, NULL, 0);
 		return;
@@ -130,6 +132,12 @@ inchild(void (*func)())
 int
 main(int argc, char *argv[])
 {
+	struct rlimit rl = { 0, };
+
+	if (setrlimit(RLIMIT_CORE, &rl) != 0) {
+		err(EXIT_FAILURE, "failed to disable cores");
+	}
+
 	for (selector = 0; selector < 8194; selector++) {
 		inchild(resetcs);
 		inchild(resetds);
@@ -144,5 +152,5 @@ main(int argc, char *argv[])
 		inchild(badss);
 	}
 
-	exit(0);
+	exit(EXIT_SUCCESS);
 }