6885 CTF Everywhere Part 1

6886 Want ctfdiff
6887 ctfdump should be written in terms of libctf
6888 ctfmerge should be implemented in terms of libctf
6889 ctfconvert should be implemented in terms of libctf
6890 Want general workq
6891 Want general mergeq
6892 ctf_add_encoded assigns() incorrect byte size to types
6893 ctf_add_{struct,union,enum} can reuse forwards
6894 ctf_add_{struct,union,enum} occasionally forget to dirty the ctf_file_t
6895 ctf_add_member could better handle bitfields
6896 ctf_type_size() reports wrong size for forwards
6897 Want libctf ctf_kind_name() function
6898 Want libctf function to set struct/union size
Portions contributed by: John Levon <john.levon@joyent.com>
Portions contributed by: Richard Lowe <richlowe@richlowe.net>
Reviewed by: John Levon <john.levon@joyent.com>
Reviewed by: Andy Fiddaman <andy@omniosce.org>
Reviewed by: Gergő Doma <domag02@gmail.com>
Approved by: Dan McDonald <danmcd@joyent.com>

1 files changed, 279 insertions, 17 deletions

diff --git a/usr/src/lib/libctf/common/ctf_lib.c b/usr/src/lib/libctf/common/ctf_lib.c
index e71ebc6d9d..e183b15f52 100644
--- a/usr/src/lib/libctf/common/ctf_lib.c
+++ b/usr/src/lib/libctf/common/ctf_lib.c
@@ -23,6 +23,9 @@
  * Copyright 2003 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright (c) 2015, Joyent, Inc.
+ */
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -33,6 +36,8 @@
 #include <errno.h>
 #include <dlfcn.h>
 #include <gelf.h>
+#include <zlib.h>
+#include <sys/debug.h>
 
 #ifdef _LP64
 static const char *_libctf_zlib = "/usr/lib/64/libz.so.1";
@@ -42,6 +47,9 @@ static const char *_libctf_zlib = "/usr/lib/libz.so.1";
 
 static struct {
 	int (*z_uncompress)(uchar_t *, ulong_t *, const uchar_t *, ulong_t);
+	int (*z_initcomp)(z_stream *, int, const char *, int);
+	int (*z_compress)(z_stream *, int);
+	int (*z_finicomp)(z_stream *);
 	const char *(*z_error)(int);
 	void *z_dlp;
 } zlib;
@@ -49,6 +57,18 @@ static struct {
 static size_t _PAGESIZE;
 static size_t _PAGEMASK;
 
+static uint64_t ctf_phase = 0;
+
+#define	CTF_COMPRESS_CHUNK	(64*1024)
+
+typedef struct ctf_zdata {
+	void		*czd_buf;
+	void		*czd_next;
+	ctf_file_t	*czd_ctfp;
+	size_t		czd_allocsz;
+	z_stream	czd_zstr;
+} ctf_zdata_t;
+
 #pragma init(_libctf_init)
 void
 _libctf_init(void)
@@ -84,9 +104,14 @@ ctf_zopen(int *errp)
 		return (ctf_set_open_errno(errp, ECTF_ZINIT));
 
 	zlib.z_uncompress = (int (*)()) dlsym(zlib.z_dlp, "uncompress");
+	zlib.z_initcomp = (int (*)()) dlsym(zlib.z_dlp, "deflateInit_");
+	zlib.z_compress = (int (*)()) dlsym(zlib.z_dlp, "deflate");
+	zlib.z_finicomp = (int (*)()) dlsym(zlib.z_dlp, "deflateEnd");
 	zlib.z_error = (const char *(*)()) dlsym(zlib.z_dlp, "zError");
 
-	if (zlib.z_uncompress == NULL || zlib.z_error == NULL) {
+	if (zlib.z_uncompress == NULL || zlib.z_error == NULL ||
+	    zlib.z_initcomp == NULL|| zlib.z_compress == NULL ||
+	    zlib.z_finicomp == NULL) {
 		(void) dlclose(zlib.z_dlp);
 		bzero(&zlib, sizeof (zlib));
 		return (ctf_set_open_errno(errp, ECTF_ZINIT));
@@ -111,6 +136,206 @@ z_strerror(int err)
 	return (zlib.z_error(err));
 }
 
+static int
+ctf_zdata_init(ctf_zdata_t *czd, ctf_file_t *fp)
+{
+	ctf_header_t *cthp;
+
+	bzero(czd, sizeof (ctf_zdata_t));
+
+	czd->czd_allocsz = fp->ctf_size;
+	czd->czd_buf = ctf_data_alloc(czd->czd_allocsz);
+	if (czd->czd_buf == MAP_FAILED)
+		return (ctf_set_errno(fp, ENOMEM));
+
+	bcopy(fp->ctf_base, czd->czd_buf, sizeof (ctf_header_t));
+	czd->czd_ctfp = fp;
+	cthp = czd->czd_buf;
+	cthp->cth_flags |= CTF_F_COMPRESS;
+	czd->czd_next = (void *)((uintptr_t)czd->czd_buf +
+	    sizeof (ctf_header_t));
+
+	if (zlib.z_initcomp(&czd->czd_zstr, Z_BEST_COMPRESSION,
+	    ZLIB_VERSION, sizeof (z_stream)) != Z_OK)
+		return (ctf_set_errno(fp, ECTF_ZLIB));
+
+	return (0);
+}
+
+static int
+ctf_zdata_grow(ctf_zdata_t *czd)
+{
+	size_t off;
+	size_t newsz;
+	void *ndata;
+
+	off = (uintptr_t)czd->czd_next - (uintptr_t)czd->czd_buf;
+	newsz = czd->czd_allocsz + CTF_COMPRESS_CHUNK;
+	ndata = ctf_data_alloc(newsz);
+	if (ndata == MAP_FAILED) {
+		return (ctf_set_errno(czd->czd_ctfp, ENOMEM));
+	}
+
+	bcopy(czd->czd_buf, ndata, off);
+	ctf_data_free(czd->czd_buf, czd->czd_allocsz);
+	czd->czd_allocsz = newsz;
+	czd->czd_buf = ndata;
+	czd->czd_next = (void *)((uintptr_t)ndata + off);
+
+	czd->czd_zstr.next_out = (Bytef *)czd->czd_next;
+	czd->czd_zstr.avail_out = CTF_COMPRESS_CHUNK;
+	return (0);
+}
+
+static int
+ctf_zdata_compress_buffer(ctf_zdata_t *czd, const void *buf, size_t bufsize)
+{
+	int err;
+
+	czd->czd_zstr.next_out = czd->czd_next;
+	czd->czd_zstr.avail_out = czd->czd_allocsz -
+	    ((uintptr_t)czd->czd_next - (uintptr_t)czd->czd_buf);
+	czd->czd_zstr.next_in = (Bytef *)buf;
+	czd->czd_zstr.avail_in = bufsize;
+
+	while (czd->czd_zstr.avail_in != 0) {
+		if (czd->czd_zstr.avail_out == 0) {
+			czd->czd_next = czd->czd_zstr.next_out;
+			if ((err = ctf_zdata_grow(czd)) != 0) {
+				return (err);
+			}
+		}
+
+		if ((err = zlib.z_compress(&czd->czd_zstr, Z_NO_FLUSH)) != Z_OK)
+			return (ctf_set_errno(czd->czd_ctfp, ECTF_ZLIB));
+	}
+	czd->czd_next = czd->czd_zstr.next_out;
+
+	return (0);
+}
+
+static int
+ctf_zdata_flush(ctf_zdata_t *czd, boolean_t finish)
+{
+	int err;
+	int flag = finish == B_TRUE ? Z_FINISH : Z_FULL_FLUSH;
+	int bret = finish == B_TRUE ? Z_STREAM_END : Z_BUF_ERROR;
+
+	for (;;) {
+		if (czd->czd_zstr.avail_out == 0) {
+			czd->czd_next = czd->czd_zstr.next_out;
+			if ((err = ctf_zdata_grow(czd)) != 0) {
+				return (err);
+			}
+		}
+
+		err = zlib.z_compress(&czd->czd_zstr, flag);
+		if (err == bret) {
+			break;
+		}
+		if (err != Z_OK)
+			return (ctf_set_errno(czd->czd_ctfp, ECTF_ZLIB));
+
+	}
+
+	czd->czd_next = czd->czd_zstr.next_out;
+
+	return (0);
+}
+
+static int
+ctf_zdata_end(ctf_zdata_t *czd)
+{
+	int ret;
+
+	if ((ret = ctf_zdata_flush(czd, B_TRUE)) != 0)
+		return (ret);
+
+	if ((ret = zlib.z_finicomp(&czd->czd_zstr)) != 0)
+		return (ctf_set_errno(czd->czd_ctfp, ECTF_ZLIB));
+
+	return (0);
+}
+
+static void
+ctf_zdata_cleanup(ctf_zdata_t *czd)
+{
+	ctf_data_free(czd->czd_buf, czd->czd_allocsz);
+	(void) zlib.z_finicomp(&czd->czd_zstr);
+}
+
+/*
+ * Compress our CTF data and return both the size of the compressed data and the
+ * size of the allocation. These may be different due to the nature of
+ * compression.
+ *
+ * In addition, we flush the compression between our two phases such that we
+ * maintain a different dictionary between the CTF data and the string section.
+ */
+int
+ctf_compress(ctf_file_t *fp, void **buf, size_t *allocsz, size_t *elfsize)
+{
+	int err;
+	ctf_zdata_t czd;
+	ctf_header_t *cthp = (ctf_header_t *)fp->ctf_base;
+
+	if ((err = ctf_zdata_init(&czd, fp)) != 0)
+		return (err);
+
+	if ((err = ctf_zdata_compress_buffer(&czd, fp->ctf_buf,
+	    cthp->cth_stroff)) != 0) {
+		ctf_zdata_cleanup(&czd);
+		return (err);
+	}
+
+	if ((err = ctf_zdata_flush(&czd, B_FALSE)) != 0) {
+		ctf_zdata_cleanup(&czd);
+		return (err);
+	}
+
+	if ((err = ctf_zdata_compress_buffer(&czd,
+	    fp->ctf_buf + cthp->cth_stroff, cthp->cth_strlen)) != 0) {
+		ctf_zdata_cleanup(&czd);
+		return (err);
+	}
+
+	if ((err = ctf_zdata_end(&czd)) != 0) {
+		ctf_zdata_cleanup(&czd);
+		return (err);
+	}
+
+	*buf = czd.czd_buf;
+	*allocsz = czd.czd_allocsz;
+	*elfsize = (uintptr_t)czd.czd_next - (uintptr_t)czd.czd_buf;
+
+	return (0);
+}
+
+int
+z_compress(void *dst, size_t *dstlen, const void *src, size_t srclen)
+{
+	z_stream zs;
+	int err;
+
+	bzero(&zs, sizeof (z_stream));
+	zs.next_in = (uchar_t *)src;
+	zs.avail_in = srclen;
+	zs.next_out = dst;
+	zs.avail_out = *dstlen;
+
+	if ((err = zlib.z_initcomp(&zs, Z_BEST_COMPRESSION, ZLIB_VERSION,
+	    sizeof (z_stream))) != Z_OK)
+		return (err);
+
+	if ((err = zlib.z_compress(&zs, Z_FINISH)) != Z_STREAM_END) {
+		(void) zlib.z_finicomp(&zs);
+		return (err == Z_OK ? Z_BUF_ERROR : err);
+	}
+
+	*dstlen = zs.total_out;
+	return (zlib.z_finicomp(&zs));
+}
+
 /*
  * Convert a 32-bit ELF file header into GElf.
  */
@@ -189,7 +414,7 @@ ctf_sect_munmap(const ctf_sect_t *sp)
  * responsible for closing the file descriptor when it is no longer needed.
  */
 ctf_file_t *
-ctf_fdopen(int fd, int *errp)
+ctf_fdcreate_int(int fd, int *errp, ctf_sect_t *ctfp)
 {
 	ctf_sect_t ctfsect, symsect, strsect;
 	ctf_file_t *fp = NULL;
@@ -221,6 +446,9 @@ ctf_fdopen(int fd, int *errp)
 	 */
 	if (nbytes >= sizeof (ctf_preamble_t) &&
 	    hdr.ctf.ctp_magic == CTF_MAGIC) {
+		if (ctfp != NULL)
+			return (ctf_set_open_errno(errp, EINVAL));
+
 		if (hdr.ctf.ctp_version > CTF_VERSION)
 			return (ctf_set_open_errno(errp, ECTF_CTFVERS));
 
@@ -370,7 +598,8 @@ ctf_fdopen(int fd, int *errp)
 				continue; /* corrupt sh_name field */
 
 			if (shp->sh_type == SHT_PROGBITS &&
-			    strcmp(strs + shp->sh_name, _CTF_SECTION) == 0) {
+			    strcmp(strs + shp->sh_name, _CTF_SECTION) == 0 &&
+			    ctfp == NULL) {
 				ctfsect.cts_name = strs + shp->sh_name;
 				ctfsect.cts_type = shp->sh_type;
 				ctfsect.cts_flags = shp->sh_flags;
@@ -397,18 +626,22 @@ ctf_fdopen(int fd, int *errp)
 
 		free(sp); /* free section header array */
 
-		if (ctfsect.cts_type == SHT_NULL) {
-			(void) munmap(strs_map, strs_mapsz);
-			return (ctf_set_open_errno(errp, ECTF_NOCTFDATA));
-		}
+		if (ctfp == NULL) {
+			if (ctfsect.cts_type == SHT_NULL && ctfp == NULL) {
+				(void) munmap(strs_map, strs_mapsz);
+				return (ctf_set_open_errno(errp,
+				    ECTF_NOCTFDATA));
+			}
 
-		/*
-		 * Now mmap the CTF data, symtab, and strtab sections and
-		 * call ctf_bufopen() to do the rest of the work.
-		 */
-		if (ctf_sect_mmap(&ctfsect, fd) == MAP_FAILED) {
-			(void) munmap(strs_map, strs_mapsz);
-			return (ctf_set_open_errno(errp, ECTF_MMAP));
+			/*
+			 * Now mmap the CTF data, symtab, and strtab sections
+			 * and call ctf_bufopen() to do the rest of the work.
+			 */
+			if (ctf_sect_mmap(&ctfsect, fd) == MAP_FAILED) {
+				(void) munmap(strs_map, strs_mapsz);
+				return (ctf_set_open_errno(errp, ECTF_MMAP));
+			}
+			ctfp = &ctfsect;
 		}
 
 		if (symsect.cts_type != SHT_NULL &&
@@ -418,12 +651,13 @@ ctf_fdopen(int fd, int *errp)
 				(void) ctf_set_open_errno(errp, ECTF_MMAP);
 				goto bad; /* unmap all and abort */
 			}
-			fp = ctf_bufopen(&ctfsect, &symsect, &strsect, errp);
+			fp = ctf_bufopen(ctfp, &symsect, &strsect, errp);
 		} else
-			fp = ctf_bufopen(&ctfsect, NULL, NULL, errp);
+			fp = ctf_bufopen(ctfp, NULL, NULL, errp);
 bad:
 		if (fp == NULL) {
-			ctf_sect_munmap(&ctfsect);
+			if (ctfp == NULL)
+				ctf_sect_munmap(&ctfsect);
 			ctf_sect_munmap(&symsect);
 			ctf_sect_munmap(&strsect);
 		} else
@@ -436,6 +670,12 @@ bad:
 	return (ctf_set_open_errno(errp, ECTF_FMT));
 }
 
+ctf_file_t *
+ctf_fdopen(int fd, int *errp)
+{
+	return (ctf_fdcreate_int(fd, errp, NULL));
+}
+
 /*
  * Open the specified file and return a pointer to a CTF container.  The file
  * can be either an ELF file or raw CTF file.  This is just a convenient
@@ -502,3 +742,25 @@ ctf_version(int version)
 
 	return (_libctf_version);
 }
+
+/*
+ * A utility function for folks debugging CTF conversion and merging.
+ */
+void
+ctf_phase_dump(ctf_file_t *fp, const char *phase)
+{
+	int fd;
+	static char *base;
+	char path[MAXPATHLEN];
+
+	if (base == NULL && (base = getenv("LIBCTF_WRITE_PHASES")) == NULL)
+		return;
+
+	(void) snprintf(path, sizeof (path), "%s/libctf.%s.%d.ctf", base,
+	    phase != NULL ? phase : "",
+	    ctf_phase);
+	if ((fd = open(path, O_CREAT | O_TRUNC | O_RDWR, 0777)) < 0)
+		return;
+	(void) ctf_write(fp, fd);
+	(void) close(fd);
+}