libfetch-2.26:

- Add support to aggressively cache directory listenings, useful for
HTTP
- Avoid leaking memory in error cases. From Xavier from Arch Linux.

8 files changed, 169 insertions, 30 deletions

diff --git a/net/libfetch/Makefile b/net/libfetch/Makefile
index 66561e2b28e..ee4851adeab 100644
--- a/net/libfetch/Makefile
+++ b/net/libfetch/Makefile
@@ -1,7 +1,7 @@
-# $NetBSD: Makefile,v 1.32 2009/08/16 20:31:29 joerg Exp $
+# $NetBSD: Makefile,v 1.33 2009/10/15 12:36:57 joerg Exp $
 #
 
-DISTNAME=	libfetch-2.25
+DISTNAME=	libfetch-2.26
 CATEGORIES=	net
 MASTER_SITES=	# empty
 DISTFILES=	# empty
diff --git a/net/libfetch/files/common.c b/net/libfetch/files/common.c
index 39e3adb1cf3..000444d3414 100644
--- a/net/libfetch/files/common.c
+++ b/net/libfetch/files/common.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: common.c,v 1.20 2009/08/16 20:31:29 joerg Exp $	*/
+/*	$NetBSD: common.c,v 1.21 2009/10/15 12:36:57 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
@@ -753,6 +753,40 @@ fetchInitURLList(struct url_list *ue)
 	ue->urls = NULL;
 }
 
+int
+fetchAppendURLList(struct url_list *dst, const struct url_list *src)
+{
+	size_t i, j, len;
+
+	len = dst->length + src->length;
+	if (len > dst->alloc_size) {
+		struct url *tmp;
+
+		tmp = realloc(dst->urls, len * sizeof(*tmp));
+		if (tmp == NULL) {
+			errno = ENOMEM;
+			fetch_syserr();
+			return (-1);
+		}
+		dst->alloc_size = len;
+		dst->urls = tmp;
+	}
+
+	for (i = 0, j = dst->length; i < src->length; ++i, ++j) {
+		dst->urls[j] = src->urls[i];
+		dst->urls[j].doc = strdup(src->urls[i].doc);
+		if (dst->urls[j].doc == NULL) {
+			while (i-- > 0)
+				free(dst->urls[j].doc);
+			fetch_syserr();
+			return -1;
+		}
+	}
+	dst->length = len;
+
+	return 0;
+}
+
 void
 fetchFreeURLList(struct url_list *ue)
 {
diff --git a/net/libfetch/files/fetch.3 b/net/libfetch/files/fetch.3
index 01b21d425f6..1f3138f5ab3 100644
--- a/net/libfetch/files/fetch.3
+++ b/net/libfetch/files/fetch.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD: fetch.3,v 1.64 2007/12/18 11:03:26 des Exp $
-.\" $NetBSD: fetch.3,v 1.12 2009/03/16 18:11:39 joerg Exp $
+.\" $NetBSD: fetch.3,v 1.13 2009/10/15 12:36:57 joerg Exp $
 .\"
 .Dd February 4, 2009
 .Dt FETCH 3
@@ -131,6 +131,8 @@
 .Fn fetchListFTP "struct url_list *list" "struct url *u" "const char *flags"
 .Ft void
 .Fn fetchInitURLList "struct url_list *ul"
+.Ft int
+.Fn fetchAppendURLList "struct url_list *dst" "const struct url_list *src"
 .Ft void
 .Fn fetchFreeURLList "struct url_list *ul"
 .Ft char *
@@ -281,6 +283,13 @@ The list should be initialized by calling
 .Fn fetchInitURLList
 and the entries be freed by calling
 .Fn fetchFreeURLList .
+The function
+.Fn fetchAppendURLList
+can be used to append one URL lists to another.
+If the
+.Ql c
+(cache result) flag is specified, the library is allowed to internally
+cache the result.
 .Pp
 .Fn fetchStringifyURL
 returns the URL as string.
diff --git a/net/libfetch/files/fetch.cat3 b/net/libfetch/files/fetch.cat3
index 0e10929d612..5ee7e388c0f 100644
--- a/net/libfetch/files/fetch.cat3
+++ b/net/libfetch/files/fetch.cat3
@@ -108,6 +108,9 @@ SSYYNNOOPPSSIISS
      _v_o_i_d
      ffeettcchhIInniittUURRLLLLiisstt(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_u_l);
 
+     _i_n_t
+     ffeettcchhAAppppeennddUURRLLLLiisstt(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_d_s_t, _c_o_n_s_t _s_t_r_u_c_t _u_r_l___l_i_s_t _*_s_r_c);
+
      _v_o_i_d
      ffeettcchhFFrreeeeUURRLLLLiisstt(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_u_l);
 
@@ -204,7 +207,10 @@ DDEESSCCRRIIPPTTIIOONN
      };
 
      The list should be initialized by calling ffeettcchhIInniittUURRLLLLiisstt() and the
-     entries be freed by calling ffeettcchhFFrreeeeUURRLLLLiisstt().
+     entries be freed by calling ffeettcchhFFrreeeeUURRLLLLiisstt().  The functio
+     ffeettcchhAAppppeennddUURRLLLLiisstt() can be used to append one URL lists to another.  If
+     the `c' (cache result) flag is specified, the library is allowed to
+     internally cache the result.
 
      ffeettcchhSSttrriinnggiiffyyUURRLL() returns the URL as string.  ffeettcchhUUnnqquuootteePPaatthh()
      returns the path name part of the URL with any quoting undone.  Query
diff --git a/net/libfetch/files/fetch.h b/net/libfetch/files/fetch.h
index c045ddcd9fb..c2e73e7113f 100644
--- a/net/libfetch/files/fetch.h
+++ b/net/libfetch/files/fetch.h
@@ -1,4 +1,4 @@
-/*	$NetBSD: fetch.h,v 1.14 2009/02/05 22:45:25 joerg Exp $	*/
+/*	$NetBSD: fetch.h,v 1.15 2009/10/15 12:36:57 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * All rights reserved.
@@ -153,6 +153,7 @@ void		 fetchFreeURL(struct url *);
 
 /* URL listening */
 void		 fetchInitURLList(struct url_list *);
+int		 fetchAppendURLList(struct url_list *, const struct url_list *);
 void		 fetchFreeURLList(struct url_list *);
 char		*fetchUnquotePath(struct url *);
 char		*fetchUnquoteFilename(struct url *);
diff --git a/net/libfetch/files/file.c b/net/libfetch/files/file.c
index b41255f74b7..b092fdf375c 100644
--- a/net/libfetch/files/file.c
+++ b/net/libfetch/files/file.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: file.c,v 1.14 2009/03/10 00:33:38 joerg Exp $	*/
+/*	$NetBSD: file.c,v 1.15 2009/10/15 12:36:57 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
@@ -234,6 +234,7 @@ fetchListFile(struct url_list *ue, struct url *u, const char *pattern, const cha
 	char *path;
 	struct dirent *de;
 	DIR *dir;
+	int ret;
 
 	if ((path = fetchUnquotePath(u)) == NULL) {
 		fetch_syserr();
@@ -248,13 +249,17 @@ fetchListFile(struct url_list *ue, struct url *u, const char *pattern, const cha
 		return -1;
 	}
 
+	ret = 0;
+
 	while ((de = readdir(dir)) != NULL) {
 		if (pattern && fnmatch(pattern, de->d_name, 0) != 0)
 			continue;
-		fetch_add_entry(ue, u, de->d_name, 0);
+		ret = fetch_add_entry(ue, u, de->d_name, 0);
+		if (ret)
+			break;
 	}
 
 	closedir(dir);
 
-	return 0;
+	return ret;
 }
diff --git a/net/libfetch/files/ftp.c b/net/libfetch/files/ftp.c
index 499f0c0d450..4745b5e16c6 100644
--- a/net/libfetch/files/ftp.c
+++ b/net/libfetch/files/ftp.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: ftp.c,v 1.29 2009/08/16 20:31:29 joerg Exp $	*/
+/*	$NetBSD: ftp.c,v 1.30 2009/10/15 12:36:57 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
@@ -587,8 +587,10 @@ ftp_closefn(void *v)
 	io->dconn->is_active = 0;
 	io->dconn = NULL;
 	r = ftp_chkerr(io->cconn);
-	if (io->cconn == cached_connection && io->cconn->ref == 1)
+	if (io->cconn == cached_connection && io->cconn->ref == 1) {
+		free(cached_host.doc);
 		cached_connection = NULL;
+	}
 	fetch_close(io->cconn);
 	free(io);
 	return;
@@ -1038,8 +1040,11 @@ static void
 ftp_disconnect(conn_t *conn)
 {
 	(void)ftp_cmd(conn, "QUIT");
-	if (conn == cached_connection && conn->ref == 1)
+	if (conn == cached_connection && conn->ref == 1) {
+		free(cached_host.doc);
+		cached_host.doc = NULL;
 		cached_connection = NULL;
+	}
 	fetch_close(conn);
 }
 
@@ -1063,6 +1068,7 @@ ftp_isconnected(struct url *url)
 static conn_t *
 ftp_cached_connect(struct url *url, struct url *purl, const char *flags)
 {
+	char *doc;
 	conn_t *conn;
 	int e;
 
@@ -1080,10 +1086,14 @@ ftp_cached_connect(struct url *url, struct url *purl, const char *flags)
 	/* connect to server */
 	if ((conn = ftp_connect(url, purl, flags)) == NULL)
 		return (NULL);
-	if (cached_connection)
-		ftp_disconnect(cached_connection);
-	cached_connection = fetch_ref(conn);
-	memcpy(&cached_host, url, sizeof(*url));
+	doc = strdup(url->doc);
+	if (doc != NULL) {
+		if (cached_connection)
+			ftp_disconnect(cached_connection);
+		cached_connection = fetch_ref(conn);
+		memcpy(&cached_host, url, sizeof(*url));
+		cached_host.doc = doc;
+	}
 	return (conn);
 }
 
@@ -1176,14 +1186,17 @@ ftp_request(struct url *url, const char *op, const char *op_arg,
 
 	if (if_modified_since && url->last_modified > 0 &&
 	    url->last_modified >= us->mtime) {
+		free(path);
 		fetchLastErrCode = FETCH_UNCHANGED;
 		snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged");
 		return NULL;
 	}
 
 	/* just a stat */
-	if (strcmp(op, "STAT") == 0)
+	if (strcmp(op, "STAT") == 0) {
+		free(path);
 		return fetchIO_unopen(NULL, NULL, NULL, NULL);
+	}
 	if (strcmp(op, "STOR") == 0 || strcmp(op, "APPE") == 0)
 		oflag = O_WRONLY;
 	else
@@ -1248,6 +1261,7 @@ fetchListFTP(struct url_list *ue, struct url *url, const char *pattern, const ch
 	char buf[2 * PATH_MAX], *eol, *eos;
 	ssize_t len;
 	size_t cur_off;
+	int ret;
 
 	/* XXX What about proxies? */
 	if (pattern == NULL || strcmp(pattern, "*") == 0)
@@ -1257,6 +1271,8 @@ fetchListFTP(struct url_list *ue, struct url *url, const char *pattern, const ch
 		return -1;
 
 	cur_off = 0;
+	ret = 0;
+
 	while ((len = fetchIO_read(f, buf + cur_off, sizeof(buf) - cur_off)) > 0) {
 		cur_off += len;
 		while ((eol = memchr(buf, '\n', cur_off)) != NULL) {
@@ -1268,11 +1284,15 @@ fetchListFTP(struct url_list *ue, struct url *url, const char *pattern, const ch
 				else
 					eos = eol;
 				*eos = '\0';
-				fetch_add_entry(ue, url, buf, 0);
+				ret = fetch_add_entry(ue, url, buf, 0);
+				if (ret)
+					break;
 				cur_off -= eol - buf + 1;
 				memmove(buf, eol + 1, cur_off);
 			}
 		}
+		if (ret)
+			break;
 	}
 	if (cur_off != 0 || len < 0) {
 		/* Not RFC conform, bail out. */
@@ -1280,5 +1300,5 @@ fetchListFTP(struct url_list *ue, struct url *url, const char *pattern, const ch
 		return -1;
 	}
 	fetchIO_close(f);
-	return 0;
+	return ret;
 }
diff --git a/net/libfetch/files/http.c b/net/libfetch/files/http.c
index 618e167e833..f2b73c5243a 100644
--- a/net/libfetch/files/http.c
+++ b/net/libfetch/files/http.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: http.c,v 1.24 2009/03/05 19:07:03 abs Exp $	*/
+/*	$NetBSD: http.c,v 1.25 2009/10/15 12:36:57 joerg Exp $	*/
 /*-
  * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav
  * Copyright (c) 2003 Thomas Klausner <wiz@NetBSD.org>
@@ -1122,7 +1122,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 
 	if (conn->err == HTTP_NOT_MODIFIED) {
 		http_seterr(HTTP_NOT_MODIFIED);
-		return (NULL);
+		goto ouch;
 	}
 
 	/* wrap it up in a fetchIO */
@@ -1223,7 +1223,7 @@ struct index_parser {
 	enum http_states state;
 };
 
-static size_t
+static ssize_t
 parse_index(struct index_parser *parser, const char *buf, size_t len)
 {
 	char *end_attr, p = *buf;
@@ -1352,12 +1352,21 @@ parse_index(struct index_parser *parser, const char *buf, size_t len)
 			return 0;
 		*end_attr = '\0';
 		parser->state = ST_TAGA;
-		fetch_add_entry(parser->ue, parser->url, buf, 1);
+		if (fetch_add_entry(parser->ue, parser->url, buf, 1))
+			return -1;
 		return end_attr + 1 - buf;
 	}
 	abort();
 }
 
+struct http_index_cache {
+	struct http_index_cache *next;
+	struct url *location;
+	struct url_list ue;
+};
+
+static struct http_index_cache *index_cache;
+
 /*
  * List a directory
  */
@@ -1366,17 +1375,53 @@ fetchListHTTP(struct url_list *ue, struct url *url, const char *pattern, const c
 {
 	fetchIO *f;
 	char buf[2 * PATH_MAX];
-	size_t buf_len, processed, sum_processed;
-	ssize_t read_len;
+	size_t buf_len, sum_processed;
+	ssize_t read_len, processed;
 	struct index_parser state;
+	struct http_index_cache *cache = NULL;
+	int do_cache, ret;
+
+	do_cache = CHECK_FLAG('c');
+
+	if (do_cache) {
+		for (cache = index_cache; cache != NULL; cache = cache->next) {
+			if (strcmp(cache->location->scheme, url->scheme))
+				continue;
+			if (strcmp(cache->location->user, url->user))
+				continue;
+			if (strcmp(cache->location->pwd, url->pwd))
+				continue;
+			if (strcmp(cache->location->host, url->host))
+				continue;
+			if (cache->location->port != url->port)
+				continue;
+			if (strcmp(cache->location->doc, url->doc))
+				continue;
+			return fetchAppendURLList(ue, &cache->ue);
+		}
 
-	state.url = url;
-	state.state = ST_NONE;
-	state.ue = ue;
+		cache = malloc(sizeof(*cache));
+		fetchInitURLList(&cache->ue);
+		cache->location = fetchCopyURL(url);
+	}
 
 	f = fetchGetHTTP(url, flags);
-	if (f == NULL)
+	if (f == NULL) {
+		if (do_cache) {
+			fetchFreeURLList(&cache->ue);
+			fetchFreeURL(cache->location);
+			free(cache);
+		}
 		return -1;
+	}
+
+	state.url = url;
+	state.state = ST_NONE;
+	if (do_cache) {
+		state.ue = &cache->ue;
+	} else {
+		state.ue = ue;
+	}
 
 	buf_len = 0;
 
@@ -1385,12 +1430,31 @@ fetchListHTTP(struct url_list *ue, struct url *url, const char *pattern, const c
 		sum_processed = 0;
 		do {
 			processed = parse_index(&state, buf + sum_processed, buf_len);
+			if (processed == -1)
+				break;
 			buf_len -= processed;
 			sum_processed += processed;
 		} while (processed != 0 && buf_len > 0);
+		if (processed == -1) {
+			read_len = -1;
+			break;
+		}
 		memmove(buf, buf + sum_processed, buf_len);
 	}
 
 	fetchIO_close(f);
-	return read_len < 0 ? -1 : 0;
+
+	ret = read_len < 0 ? -1 : 0;
+
+	if (do_cache) {
+		if (ret == 0) {
+			cache->next = index_cache;
+			index_cache = cache;
+		}
+
+		if (fetchAppendURLList(ue, &cache->ue))
+			ret = -1;
+	}
+
+	return ret;
 }