pbulk-base-0.52:

Move the tree iteration logic from the master to the client. This
matters primarily when using the additional package list in the top
level makefile and ensures that the client configuration is used
consistently.

5 files changed, 147 insertions, 18 deletions

diff --git a/pkgtools/pbulk-base/Makefile b/pkgtools/pbulk-base/Makefile
index f1be8a3db54..4e134ffb52b 100644
--- a/pkgtools/pbulk-base/Makefile
+++ b/pkgtools/pbulk-base/Makefile
@@ -1,6 +1,6 @@
-# $NetBSD: Makefile,v 1.22 2016/03/10 09:25:42 jperkin Exp $
+# $NetBSD: Makefile,v 1.23 2016/06/20 17:54:43 joerg Exp $
 
-DISTNAME=	pbulk-base-0.51
+DISTNAME=	pbulk-base-0.52
 COMMENT=	Core components of the modular bulk build framework
 
 .include "../../pkgtools/pbulk/Makefile.common"
diff --git a/pkgtools/pbulk/files/pbulk/pscan/client.c b/pkgtools/pbulk/files/pbulk/pscan/client.c
index 62db90ebf0d..9f28c9cff57 100644
--- a/pkgtools/pbulk/files/pbulk/pscan/client.c
+++ b/pkgtools/pbulk/files/pbulk/pscan/client.c
@@ -1,4 +1,4 @@
-/* $NetBSD: client.c,v 1.5 2015/12/07 16:52:40 joerg Exp $ */
+/* $NetBSD: client.c,v 1.6 2016/06/20 17:54:43 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -48,8 +48,104 @@
 #include "pbulk.h"
 #include "pscan.h"
 
+static void
+find_full_tree_client(int fd, const char *bmake_path)
+{
+	const char * extract_subdir[] = {
+		bmake_path,
+		"show-subdir-var",
+		"VARNAME=SUBDIR",
+		NULL
+	};
+	char **pkgs;
+	size_t i, allocated_pkgs, len_pkgs, len_pkgs_data;
+	char *cat_path;
+	char *buf, *buf_orig, *cat, *cat_orig;
+	size_t buf_len, cat_len;
+	uint32_t net_len_pkgs_data;
+	ssize_t sent_bytes;
+
+	buf = read_from_child(pkgsrc_tree, bmake_path, extract_subdir);
+
+	if (buf == NULL)
+		err(1, "Cannot extract categories");
+
+	cat = cat_orig = buf;
+	allocated_pkgs = len_pkgs = 0;
+	len_pkgs_data = 0;
+	pkgs = NULL;
+	for (;;) {
+		cat += strspn(cat, " \t\n");
+		cat_len = strcspn(cat, " \t\n");
+		if (cat_len == 0)
+			break;
+
+		cat_path = xasprintf("%s/%.*s", pkgsrc_tree, (int)cat_len, cat);
+		buf_orig = buf = read_from_child(cat_path, bmake_path, extract_subdir);
+		free(cat_path);
+		if (buf == NULL) {
+			warnx("Cannot extract subdirectories for %.*s", (int)cat_len, cat);
+			cat += cat_len;
+			continue;
+		}
+
+		for (;;) {
+			buf += strspn(buf, " \t\n");
+			buf_len = strcspn(buf, " \t\n");
+			if (buf_len == 0)
+				break;
+			if (len_pkgs == allocated_pkgs) {
+				if (allocated_pkgs == 0) {
+					allocated_pkgs = 1024;
+					pkgs = xmalloc(sizeof(*pkgs) *
+					               allocated_pkgs);
+				} else {
+					allocated_pkgs *= 2;
+					pkgs = xrealloc(pkgs,
+					    sizeof(*pkgs) * allocated_pkgs);
+				}
+			}
+			pkgs[len_pkgs] = xasprintf("%.*s/%.*s", (int)cat_len,
+			    cat, (int)buf_len, buf);
+			len_pkgs_data += strlen(pkgs[len_pkgs]) + 1;
+			++len_pkgs;
+			buf += buf_len;
+		}
+		free(buf_orig);
+
+		cat += cat_len;
+	}
+
+	free(cat_orig);
+
+	if (len_pkgs_data > 0xfffffffful)
+		errx(1, "Directory list too large");
+	net_len_pkgs_data = ntohl((uint32_t)len_pkgs_data);
+
+	sent_bytes = atomic_write(fd, &net_len_pkgs_data, 4);
+	if (sent_bytes == -1)
+		err(1, "Could not write to socket");
+	if (sent_bytes != 4)
+		errx(1, "Premature end of stream while writing to socket");
+	for (i = 0; i < len_pkgs; ++i) {
+		size_t l = strlen(pkgs[i]);
+		sent_bytes = atomic_write(fd, pkgs[i], l);
+		if (sent_bytes == -1)
+			err(1, "Could not write to socket");
+		if ((size_t)sent_bytes != l)
+			errx(1, "Premature end of stream while writing to socket");
+		sent_bytes = atomic_write(fd, "\n", 1);
+		if (sent_bytes == -1)
+			err(1, "Could not write to socket");
+		if (sent_bytes != 1)
+			errx(1, "Premature end of stream while writing to socket");
+		free(pkgs[i]);
+	}
+	free(pkgs);
+}
+
 void
-client_mode(const char *client_port)
+client_mode(const char *client_port, const char *bmake_path)
 {
 	uint16_t path_len;
 	uint32_t net_output_len;
@@ -82,6 +178,13 @@ loop:
 		err(1, "Could not read from socket");
 	if (recv_bytes != path_len || strlen(path) != path_len)
 		errx(1, "Premature end of stream while reading path from socket");
+
+	if (strcmp(path, "find_full_tree") == 0) {
+		free(path);
+		find_full_tree_client(fd, bmake_path);
+		goto loop;
+	}
+
 	if (path[0] == '/' ||
 	    strchr(path, '/') == NULL ||
 	    strchr(path, '/') != strrchr(path, '/') ||
@@ -104,12 +207,12 @@ loop:
 		errx(1, "Output too large");
 	net_output_len = htonl((uint32_t)output_len);
 
-	sent_bytes = write(fd, &net_output_len, 4);
+	sent_bytes = atomic_write(fd, &net_output_len, 4);
 	if (sent_bytes == -1)
 		err(1, "Could not write to socket");
 	if (sent_bytes != 4)
 		errx(1, "Premature end of stream while writing to socket");
-	sent_bytes = write(fd, output, output_len);
+	sent_bytes = atomic_write(fd, output, output_len);
 	if (sent_bytes == -1)
 		err(1, "Could not write to socket");
 	if ((size_t)sent_bytes != output_len)
diff --git a/pkgtools/pbulk/files/pbulk/pscan/jobs.c b/pkgtools/pbulk/files/pbulk/pscan/jobs.c
index 1ae250e2d9c..fa6284e6ead 100644
--- a/pkgtools/pbulk/files/pbulk/pscan/jobs.c
+++ b/pkgtools/pbulk/files/pbulk/pscan/jobs.c
@@ -1,4 +1,4 @@
-/* $NetBSD: jobs.c,v 1.8 2015/11/03 19:06:48 joerg Exp $ */
+/* $NetBSD: jobs.c,v 1.9 2016/06/20 17:54:43 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -342,11 +342,32 @@ get_job(void)
 	return NULL;
 }
 
+static void
+parse_full_tree(char *data) {
+	char *eol;
+
+	while (*data) {
+		eol = strchr(data, '\n');
+		if (eol == NULL)
+			err(1, "Incomplete line in full tree list");
+		if (data == eol)
+			continue;
+		*eol = '\0';
+		add_job_full(data);
+		data = eol + 1;
+	}
+}
+
 void
 process_job(struct scan_job *job, enum job_state state)
 {
 	job->state = state;
 
+	if (state == JOB_DONE &&
+	    strcmp(job->pkg_location, "find_full_tree") == 0) {
+		parse_full_tree(job->scan_output);
+	}
+
 	for (; first_undone_job < len_jobs; ++first_undone_job) {
 		if (jobs[first_undone_job].state != JOB_DONE)
 			break;
@@ -511,6 +532,8 @@ write_jobs(const char *output_file)
 			warnx("%s was not processed", jobs[i].pkg_location);
 			continue;
 		}
+		if (strcmp(jobs[i].pkg_location, "find_full_tree") == 0)
+			continue;
 		if (jobs[i].scan_output == NULL) {
 			warnx("Scan failed for %s", jobs[i].pkg_location);
 			continue;
diff --git a/pkgtools/pbulk/files/pbulk/pscan/pscan.c b/pkgtools/pbulk/files/pbulk/pscan/pscan.c
index 393868a4380..191fb637e94 100644
--- a/pkgtools/pbulk/files/pbulk/pscan/pscan.c
+++ b/pkgtools/pbulk/files/pbulk/pscan/pscan.c
@@ -1,4 +1,4 @@
-/* $NetBSD: pscan.c,v 1.9 2015/11/03 19:06:48 joerg Exp $ */
+/* $NetBSD: pscan.c,v 1.10 2016/06/20 17:54:43 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -136,7 +136,7 @@ main(int argc, char **argv)
 			usage();
 		pkgsrc_tree = argv[0];
 
-		client_mode(client_port);
+		client_mode(client_port, bmake_path);
 	}
 
 	if (argc != 2)
@@ -145,15 +145,18 @@ main(int argc, char **argv)
 	pkgsrc_tree = argv[0];
 	output_file = argv[1];
 
-	if (limited_scan == 0)
-		find_full_tree();
-	else
+	if (limited_scan != 0)
 		read_limited_list();
 
-	if (has_job()) {
-		if (master_port != NULL)
+	if (master_port != NULL) {
+		if (limited_scan == 0)
+			add_job_full("find_full_tree");
+		if (has_job())
 			master_mode(master_port, start_script);
-		else
+	} else {
+		if (limited_scan == 0)
+			find_full_tree();
+		if (has_job())
 			standalone_mode();
 	}
 
diff --git a/pkgtools/pbulk/files/pbulk/pscan/pscan.h b/pkgtools/pbulk/files/pbulk/pscan/pscan.h
index 61ad5381213..dd6f2156da2 100644
--- a/pkgtools/pbulk/files/pbulk/pscan/pscan.h
+++ b/pkgtools/pbulk/files/pbulk/pscan/pscan.h
@@ -1,4 +1,4 @@
-/* $NetBSD: pscan.h,v 1.4 2015/11/03 19:06:48 joerg Exp $ */
+/* $NetBSD: pscan.h,v 1.5 2016/06/20 17:54:43 joerg Exp $ */
 
 /*-
  * Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@@ -52,7 +52,7 @@ extern const char *pkgsrc_tree;
 
 char		*scan_pkglocation(const char *);
 
-void		 client_mode(const char *);
+void		 client_mode(const char *, const char *);
 void		 master_mode(const char *, const char *);
 
 void		 add_job(const char *, size_t, const char *, size_t);
@@ -62,4 +62,4 @@ void		 process_job(struct scan_job *, enum job_state);
 void		 write_jobs(const char *);
 int		 has_job(void);
 
-void		read_old_scan(const char *);
+void		 read_old_scan(const char *);