6961 64-bit octal printf may overflow internal buffer

Reviewed by: Toomas Soome <tsoome@me.com>
Reviewed by: Yuri Pankov <yuri.pankov@gmail.com>
Approved by: Dan McDonald <danmcd@kebe.com>

5 files changed, 122 insertions, 5 deletions

diff --git a/usr/src/lib/libc/port/print/print.h b/usr/src/lib/libc/port/print/print.h
index 427d32677a..bbc2c8ee7d 100644
--- a/usr/src/lib/libc/port/print/print.h
+++ b/usr/src/lib/libc/port/print/print.h
@@ -31,8 +31,6 @@
 #ifndef	_PRINT_H
 #define	_PRINT_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"	/* SVr4.0 1.9	*/
-
 #include "file64.h"
 #include <floatingpoint.h>
 #include <thread.h>
@@ -63,7 +61,7 @@ __qaconvert(long double *arg, int ndigits, int *exp, int *sign, char *buf);
 #define	MAXDIGS 11
 
 /* Maximum number of digits in any long long representation */
-#define	MAXLLDIGS 21
+#define	MAXLLDIGS 22
 
 /* Maximum total number of digits in E format */
 #define	MAXECVT (DECIMAL_STRING_LENGTH-1)
diff --git a/usr/src/pkg/manifests/system-test-libctest.mf b/usr/src/pkg/manifests/system-test-libctest.mf
index 16ec75f259..558dbdb622 100644
--- a/usr/src/pkg/manifests/system-test-libctest.mf
+++ b/usr/src/pkg/manifests/system-test-libctest.mf
@@ -93,6 +93,7 @@ file path=opt/libc-tests/tests/newlocale_test.$(ARCH64) mode=0555
 file path=opt/libc-tests/tests/nl_langinfo_test mode=0555
 file path=opt/libc-tests/tests/nl_langinfo_test.$(ARCH) mode=0555
 file path=opt/libc-tests/tests/nl_langinfo_test.$(ARCH64) mode=0555
+file path=opt/libc-tests/tests/printf-6961.64 mode=0555
 file path=opt/libc-tests/tests/priv_gettext mode=0555
 file path=opt/libc-tests/tests/pthread_attr_get_np mode=0555
 file path=opt/libc-tests/tests/quick_exit mode=0555
diff --git a/usr/src/test/libc-tests/runfiles/default.run b/usr/src/test/libc-tests/runfiles/default.run
index dfb4c9b161..0119248c7e 100644
--- a/usr/src/test/libc-tests/runfiles/default.run
+++ b/usr/src/test/libc-tests/runfiles/default.run
@@ -12,7 +12,7 @@
 #
 # Copyright (c) 2012 by Delphix. All rights reserved.
 # Copyright 2014 Garrett D'Amore <garrett@damore.org>
-# Copyright 2016 Joyent, Inc.
+# Copyright 2017 Joyent, Inc.
 #
 
 [DEFAULT]
@@ -74,6 +74,7 @@ timeout = 600
 [/opt/libc-tests/tests/endian.32]
 [/opt/libc-tests/tests/endian.64]
 [/opt/libc-tests/tests/quick_exit]
+[/opt/libc-tests/tests/printf-6961.64]
 [/opt/libc-tests/tests/priv_gettext]
 [/opt/libc-tests/tests/strerror]
 [/opt/libc-tests/tests/timespec_get.32]
diff --git a/usr/src/test/libc-tests/tests/Makefile b/usr/src/test/libc-tests/tests/Makefile
index 6c365aefb1..86d16c8d81 100644
--- a/usr/src/test/libc-tests/tests/Makefile
+++ b/usr/src/test/libc-tests/tests/Makefile
@@ -50,7 +50,9 @@ SCRIPTS = \
 CPPFLAGS += -D_REENTRANT
 
 PROGS32 = $(PROGS:%=%.32)
-PROGS64 = $(PROGS:%=%.64)
+PROGS64 = \
+	$(PROGS:%=%.64) \
+	printf-6961.64
 
 aligned_alloc.32 :=	LDLIBS += -lproc
 aligned_alloc.64 :=	LDLIBS64 += -lproc
diff --git a/usr/src/test/libc-tests/tests/printf-6961.c b/usr/src/test/libc-tests/tests/printf-6961.c
new file mode 100644
index 0000000000..5c60b96425
--- /dev/null
+++ b/usr/src/test/libc-tests/tests/printf-6961.c
@@ -0,0 +1,115 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2017 Joyent, Inc.
+ */
+
+/*
+ * Regression test for illumos #6961. We mistakenly zeroed out a character that
+ * we shouldn't have when dealing with a 64-bit libc.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <strings.h>
+#include <stdlib.h>
+
+static void
+print_diff(char *test, char *correct, char *wrong)
+{
+	int i;
+	printf("test failed: received incorrect octal for case %s\n", test);
+	for (i = 0; i < 32; i++) {
+		printf("byte %d: expected 0x%x, found 0x%x\n", i, correct[i],
+		    wrong[i]);
+	}
+}
+
+int
+main(void)
+{
+	int ret = 0;
+	char buf[32];
+
+	/* ~0L in octal */
+	char octal0[] = { 'r', 'r', 'r', 'r', '1', '7', '7', '7', '7', '7', '7',
+	    '7', '7', '7', '7', '7', '7', '7', '7', '7', '7', '7', '7', '7',
+	    '7', '7', '\0', 'r', 'r', 'r', 'r', 'r', 'r' };
+
+	char decimal0[] = { 'r', 'r', 'r', 'r', '-', '1', '\0', 'r', 'r', 'r',
+	    'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r',
+	    'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r' };
+
+	char hex0[] = { 'r', 'r', 'r', 'r', 'f', 'f', 'f', 'f', 'f', 'f',
+	    'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', 'f', '\0', 'r', 'r',
+	    'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r' };
+
+	/* 42 in octal */
+	char octal1[] = { 'r', 'r', 'r', 'r', '5', '2', '\0', 'r', 'r', 'r',
+	    'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r',
+	    'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r' };
+
+	/* 42 in decimal */
+	char decimal1[] = { 'r', 'r', 'r', 'r', '4', '2', '\0', 'r', 'r', 'r',
+	    'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r',
+	    'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r' };
+
+	/* 42 in hex */
+	char hex1[] = { 'r', 'r', 'r', 'r', '2', 'a', '\0', 'r', 'r', 'r', 'r',
+	    'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r',
+	    'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r', 'r' };
+
+
+	(void) memset(buf, 'r', sizeof (buf));
+	(void) snprintf(buf + 4, sizeof (buf), "%lo", ~0L);
+	if (bcmp(octal0, buf, sizeof (buf)) != 0) {
+		print_diff("~0 in Octal", octal0, buf);
+		ret++;
+	}
+
+	(void) memset(buf, 'r', sizeof (buf));
+	(void) snprintf(buf + 4, sizeof (buf), "%lo", 42L);
+	if (bcmp(octal1, buf, sizeof (buf)) != 0) {
+		print_diff("42 in Octal", octal1, buf);
+		ret++;
+	}
+
+	(void) memset(buf, 'r', sizeof (buf));
+	(void) snprintf(buf + 4, sizeof (buf), "%ld", ~0L);
+	if (bcmp(decimal0, buf, sizeof (buf)) != 0) {
+		print_diff("~0 in Decimal", decimal0, buf);
+		ret++;
+	}
+
+	(void) memset(buf, 'r', sizeof (buf));
+	(void) snprintf(buf + 4, sizeof (buf), "%ld", 42L);
+	if (bcmp(decimal1, buf, sizeof (buf)) != 0) {
+		print_diff("42 in Decimal", decimal1, buf);
+		ret++;
+	}
+
+	(void) memset(buf, 'r', sizeof (buf));
+	(void) snprintf(buf + 4, sizeof (buf), "%lx", ~0L);
+	if (bcmp(hex0, buf, sizeof (buf)) != 0) {
+		print_diff("~0 in Hex", hex0, buf);
+		ret++;
+	}
+
+	(void) memset(buf, 'r', sizeof (buf));
+	(void) snprintf(buf + 4, sizeof (buf), "%lx", 42L);
+	if (bcmp(hex1, buf, sizeof (buf)) != 0) {
+		print_diff("42 in Hex", hex1, buf);
+		ret++;
+	}
+
+	return (ret);
+}