mk: Add CTF infrastructure support.

Platform support is determined by _OPSYS_SUPPORTS_CTF from mk/platform, the
user enables support by setting PKGSRC_USE_CTF=yes, and packages can
explicitly disable support with CTF_SUPPORTED=no or skip certain files with
CTF_FILES_SKIP.

The path to ctfconvert is configured via TOOLS_PLATFORM.ctfconvert.

If all of the requisite variables are enabled, a compiler-specific debug flag
is passed via the wrappers to ensure we have DWARF information to convert,
_INSTALL_UNSTRIPPED is explicitly defined to avoid binaries being stripped
prior to conversion, and the conversion is performed during the install stage.

It is recommended that users who enable the feature also set STRIP_DEBUG=yes
to reduce the final binary size once the conversion has been performed.

This has been used for the past year in Joyent SmartOS builds.  FreeBSD is
marked as supported but is untested.

10 files changed, 95 insertions, 17 deletions

diff --git a/mk/bsd.prefs.mk b/mk/bsd.prefs.mk
index 9bd3ab7badc..237a37c22a6 100644
--- a/mk/bsd.prefs.mk
+++ b/mk/bsd.prefs.mk
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.prefs.mk,v 1.401 2018/10/29 01:50:00 sevan Exp $
+# $NetBSD: bsd.prefs.mk,v 1.402 2018/11/12 14:22:58 jperkin Exp $
 #
 # This file includes the mk.conf file, which contains the user settings.
 #
@@ -766,6 +766,20 @@ _PKGSRC_USE_STACK_CHECK=no
 _PKGSRC_USE_STACK_CHECK=yes
 .endif
 
+# Enable CTF conversion if the user requested it, the OPSYS supports it, there
+# is a tool for it, and the package supports it.  We also need to explicitly
+# turn on _INSTALL_UNSTRIPPED as conversion is impossible on stripped files.
+#
+.if ${PKGSRC_USE_CTF:Uno:tl} == "yes" && \
+    ${_OPSYS_SUPPORTS_CTF:Uno:tl} == "yes" && \
+    defined(TOOLS_PLATFORM.ctfconvert) && \
+    ${CTF_SUPPORTED:Uyes:tl} == "yes"
+_PKGSRC_USE_CTF=	yes
+_INSTALL_UNSTRIPPED=	# defined
+.else
+_PKGSRC_USE_CTF=	no
+.endif
+
 # Enable cwrappers if not building the wrappers themselves, and if the user has
 # explicitly requested them, or if they haven't but the compiler/platform is
 # known to support them.
diff --git a/mk/compiler.mk b/mk/compiler.mk
index 946de1bbdcf..19dc9cb26b6 100644
--- a/mk/compiler.mk
+++ b/mk/compiler.mk
@@ -1,4 +1,4 @@
-# $NetBSD: compiler.mk,v 1.89 2018/08/12 19:36:40 rillig Exp $
+# $NetBSD: compiler.mk,v 1.90 2018/11/12 14:22:58 jperkin Exp $
 #
 # This Makefile fragment implements handling for supported C/C++/Fortran
 # compilers.
@@ -205,6 +205,13 @@ CWRAPPERS_APPEND.cxx+=	${_SSP_CFLAGS}
 CWRAPPERS_APPEND.f77+=	${_SSP_CFLAGS}
 .endif
 
+# Add debug flags if the user has requested CTF and the compiler supports it.
+#
+.if ${_PKGSRC_USE_CTF} == "yes" && defined(_CTF_CFLAGS)
+_WRAP_EXTRA_ARGS.CC+=	${_CTF_CFLAGS}
+CWRAPPERS_APPEND.cc+=	${_CTF_CFLAGS}
+.endif
+
 # If the languages are not requested, force them not to be available
 # in the generated wrappers.
 #
diff --git a/mk/compiler/clang.mk b/mk/compiler/clang.mk
index a8eb69bc26e..c9684169ce2 100644
--- a/mk/compiler/clang.mk
+++ b/mk/compiler/clang.mk
@@ -1,4 +1,4 @@
-# $NetBSD: clang.mk,v 1.18 2017/07/11 19:41:20 joerg Exp $
+# $NetBSD: clang.mk,v 1.19 2018/11/12 14:22:58 jperkin Exp $
 #
 # This is the compiler definition for the clang compiler.
 #
@@ -53,6 +53,8 @@ _COMPILER_LD_FLAG=	-Wl,
 _LINKER_RPATH_FLAG=	-R
 _COMPILER_RPATH_FLAG=	${_COMPILER_LD_FLAG}${_LINKER_RPATH_FLAG}
 
+_CTF_CFLAGS=		-gdwarf-2
+
 # The user can choose the level of stack smashing protection.
 .if ${PKGSRC_USE_SSP} == "all"
 _SSP_CFLAGS=		-fstack-protector-all
diff --git a/mk/compiler/gcc.mk b/mk/compiler/gcc.mk
index 9a346c5a812..7c22078204e 100644
--- a/mk/compiler/gcc.mk
+++ b/mk/compiler/gcc.mk
@@ -1,4 +1,4 @@
-# $NetBSD: gcc.mk,v 1.197 2018/10/29 11:53:18 abs Exp $
+# $NetBSD: gcc.mk,v 1.198 2018/11/12 14:22:58 jperkin Exp $
 #
 # This is the compiler definition for the GNU Compiler Collection.
 #
@@ -395,6 +395,8 @@ _STACK_CHECK_CFLAGS=	-fstack-check
 _GCC_CFLAGS+=		${_STACK_CHECK_CFLAGS}
 .endif
 
+_CTF_CFLAGS=		-gdwarf-2
+
 # GCC has this annoying behaviour where it advocates in a multi-line
 # banner the use of "#include" over "#import" when including headers.
 # This generates a huge number of warnings when building practically all
diff --git a/mk/defaults/mk.conf b/mk/defaults/mk.conf
index 62b5f9d168f..5a7c74b465e 100644
--- a/mk/defaults/mk.conf
+++ b/mk/defaults/mk.conf
@@ -1,4 +1,4 @@
-# $NetBSD: mk.conf,v 1.296 2018/06/24 21:19:10 wiz Exp $
+# $NetBSD: mk.conf,v 1.297 2018/11/12 14:22:58 jperkin Exp $
 #
 
 # This file provides default values for variables that may be overridden
@@ -237,6 +237,13 @@ PKGSRC_MKREPRO?= no
 #
 # Keywords: reproducible
 
+PKGSRC_USE_CTF?= no
+# Turn on CTF conversion.
+# Possible: yes, no
+# Default: no
+#
+# Keywords: ctf compact type format
+
 PKGSRC_USE_FORTIFY?= strong
 # Turns on substitute wrappers for commonly used functions that do not bounds
 # checking regularly, but could in some cases. This is effectively in use only
diff --git a/mk/install/bsd.install-vars.mk b/mk/install/bsd.install-vars.mk
index 3655cb3a48c..51221ee3911 100644
--- a/mk/install/bsd.install-vars.mk
+++ b/mk/install/bsd.install-vars.mk
@@ -1,4 +1,4 @@
-# $NetBSD: bsd.install-vars.mk,v 1.8 2009/09/02 14:40:40 joerg Exp $
+# $NetBSD: bsd.install-vars.mk,v 1.9 2018/11/12 14:22:58 jperkin Exp $
 #
 # This Makefile fragment is included separately by bsd.pkg.mk and
 # defines some variables which must be defined earlier than where
@@ -6,6 +6,15 @@
 #
 # Package-settable variables:
 #
+# CTF_SUPPORTED
+#	If set to "no", CTF conversion will not be attempted and any
+#	necessary debug CFLAGS will not be added.
+#
+# CTF_FILES_SKIP
+#	A list of shell patterns (like lib/*) relative to ${PREFIX} that
+#	should be excluded from CTF conversion.  Note that a * in a pattern
+#	also matches a slash in a pathname.
+#
 # INSTALLATION_DIRS_FROM_PLIST
 #	If set to "yes", the static PLIST files of the package will
 #	be used to determine which directories need to be created before
@@ -46,6 +55,17 @@ _MANCOMPRESSED!=							\
 _MANZ=		${MANZ:Dyes:Uno}
 MAKEVARS+=	_MANCOMPRESSED _MANZ
 
+# The logic for setting _PKGSRC_USE_CTF is in bsd.prefs.mk, at this point we
+# are only concerned if we should set up the tools and variables or not.
+#
+.if ${_PKGSRC_USE_CTF} == "yes"
+TOOLS_CREATE+=		ctfconvert
+TOOLS_PATH.ctfconvert=	${TOOLS_PLATFORM.ctfconvert}
+TOOLS_ARGS.ctfconvert?=	-i
+CTFCONVERT?=		ctfconvert
+CTF_FILES_SKIP?=	# none
+.endif
+
 STRIP_DEBUG?=	no
 
 .if !empty(_MANCOMPRESSED:M[yY][eE][sS]) && empty(_MANZ:M[yY][eE][sS])
diff --git a/mk/install/install.mk b/mk/install/install.mk
index dc5e710729a..a1c5590587c 100644
--- a/mk/install/install.mk
+++ b/mk/install/install.mk
@@ -1,4 +1,4 @@
-# $NetBSD: install.mk,v 1.74 2018/11/12 13:47:32 jperkin Exp $
+# $NetBSD: install.mk,v 1.75 2018/11/12 14:22:58 jperkin Exp $
 #
 # This file provides the code for the "install" phase.
 #
@@ -183,6 +183,9 @@ _INSTALL_ALL_TARGETS+=		pre-install
 _INSTALL_ALL_TARGETS+=		do-install
 _INSTALL_ALL_TARGETS+=		post-install
 _INSTALL_ALL_TARGETS+=		plist
+.if ${_PKGSRC_USE_CTF} == "yes"
+_INSTALL_ALL_TARGETS+=		install-ctf
+.endif
 .if ${STRIP_DEBUG:Uno:tl} == "yes" && ${STRIP_DEBUG_SUPPORTED:Uyes:tl} == "yes"
 _INSTALL_ALL_TARGETS+=		install-strip-debug
 .endif
@@ -337,6 +340,30 @@ post-install:
 .endif
 
 ######################################################################
+### install-ctf (PRIVATE)
+######################################################################
+### install-ctf creates CTF information from debug binaries.
+###
+.PHONY: install-ctf
+install-ctf: plist
+	@${STEP_MSG} "Generating CTF data"
+	${RUN}cd ${DESTDIR:Q}${PREFIX:Q};				\
+	${CAT} ${_PLIST_NOKEYWORDS} | while read f; do			\
+		[ ! -h "$${f}" ] || continue;				\
+		case "$${f}" in						\
+		${CTF_FILES_SKIP:@p@${p}) continue ;;@}			\
+		*) ;;							\
+		esac;							\
+		tmp_f="$${f}.XXX";					\
+		if ${CTFCONVERT} -o "$${tmp_f}" "$${f}" 2>/dev/null; then \
+			if [ -f "$${tmp_f}" -a -f "$${f}" ]; then	\
+				${MV} "$${tmp_f}" "$${f}";		\
+			fi;						\
+		fi;							\
+		${RM} -f "$${tmp_f}";					\
+	done
+
+######################################################################
 ### install-strip-debug (PRIVATE)
 ######################################################################
 ### install-strip-debug tries to strip debug information from
diff --git a/mk/platform/FreeBSD.mk b/mk/platform/FreeBSD.mk
index fac7fc22486..973b6ed645c 100644
--- a/mk/platform/FreeBSD.mk
+++ b/mk/platform/FreeBSD.mk
@@ -1,4 +1,4 @@
-# $NetBSD: FreeBSD.mk,v 1.36 2017/10/03 13:18:00 jperkin Exp $
+# $NetBSD: FreeBSD.mk,v 1.37 2018/11/12 14:22:58 jperkin Exp $
 #
 # Variable definitions for the FreeBSD operating system.
 
@@ -77,6 +77,7 @@ PKG_HAVE_KQUEUE=	# defined
 _OPSYS_SUPPORTS_SSP=	yes
 .endif
 
+_OPSYS_SUPPORTS_CTF=		yes # Compact Type Format conversion.
 _OPSYS_SUPPORTS_CWRAPPERS=	yes
 _OPSYS_CAN_CHECK_SHLIBS=	yes # use readelf in check/bsd.check-vars.mk
 _OPSYS_CAN_CHECK_SSP=		no  # only supports libssp at this time
diff --git a/mk/platform/SunOS.mk b/mk/platform/SunOS.mk
index c7cd4837fca..bd7f920bbd1 100644
--- a/mk/platform/SunOS.mk
+++ b/mk/platform/SunOS.mk
@@ -1,4 +1,4 @@
-# $NetBSD: SunOS.mk,v 1.78 2018/11/05 19:26:38 schmonz Exp $
+# $NetBSD: SunOS.mk,v 1.79 2018/11/12 14:22:58 jperkin Exp $
 #
 # Variable definitions for the SunOS/Solaris operating system.
 
@@ -127,13 +127,10 @@ _OPSYS_INCLUDE_DIRS?=	/usr/include
 _OPSYS_SUPPORTS_CWRAPPERS=	yes
 .endif
 
-# support FORTIFY (with GCC)
-_OPSYS_SUPPORTS_FORTIFY=yes
-
-# support stack protection (with GCC)
-_OPSYS_SUPPORTS_SSP?=	yes
-
-_OPSYS_CAN_CHECK_SHLIBS=	yes # requires readelf
+_OPSYS_SUPPORTS_CTF=		yes # Compact Type Format conversion.
+_OPSYS_SUPPORTS_FORTIFY=	yes # Requires GCC
+_OPSYS_SUPPORTS_SSP?=		yes # Requires GCC
+_OPSYS_CAN_CHECK_SHLIBS=	yes # Requires readelf
 
 # check for maximum command line length and set it in configure's environment,
 # to avoid a test required by the libtool script that takes forever.
diff --git a/mk/tools/tools.FreeBSD.mk b/mk/tools/tools.FreeBSD.mk
index 222b2aff04d..53d31ff0771 100644
--- a/mk/tools/tools.FreeBSD.mk
+++ b/mk/tools/tools.FreeBSD.mk
@@ -1,4 +1,4 @@
-# $NetBSD: tools.FreeBSD.mk,v 1.46 2018/11/12 13:34:39 jperkin Exp $
+# $NetBSD: tools.FreeBSD.mk,v 1.47 2018/11/12 14:22:58 jperkin Exp $
 #
 # System-supplied tools for the FreeBSD operating system.
 
@@ -18,6 +18,7 @@ TOOLS_PLATFORM.chown?=		/usr/sbin/chown
 TOOLS_PLATFORM.cmp?=		/usr/bin/cmp
 TOOLS_PLATFORM.cp?=		/bin/cp
 TOOLS_PLATFORM.csh?=		/bin/csh
+TOOLS_PLATFORM.ctfconvert?=	/usr/bin/ctfconvert
 TOOLS_PLATFORM.cut?=		/usr/bin/cut
 TOOLS_PLATFORM.date?=		/bin/date
 TOOLS_PLATFORM.diff?=		/usr/bin/diff