libfetch-2.6:

Change fetchList API to always return lists of full URLs.

11 files changed, 177 insertions, 261 deletions

diff --git a/net/libfetch/Makefile b/net/libfetch/Makefile
index d690dd9acd4..481b6137abb 100644
--- a/net/libfetch/Makefile
+++ b/net/libfetch/Makefile
@@ -1,7 +1,7 @@
-# $NetBSD: Makefile,v 1.8 2008/04/18 21:13:10 joerg Exp $
+# $NetBSD: Makefile,v 1.9 2008/04/19 14:49:23 joerg Exp $
 #
 
-DISTNAME=	libfetch-2.5
+DISTNAME=	libfetch-2.6
 CATEGORIES=	net
 MASTER_SITES=	# empty
 DISTFILES=	# empty
diff --git a/net/libfetch/buildlink3.mk b/net/libfetch/buildlink3.mk
index d3c764935e7..ea4178c3001 100644
--- a/net/libfetch/buildlink3.mk
+++ b/net/libfetch/buildlink3.mk
@@ -1,4 +1,4 @@
-# $NetBSD: buildlink3.mk,v 1.2 2008/04/02 15:33:14 joerg Exp $
+# $NetBSD: buildlink3.mk,v 1.3 2008/04/19 14:49:23 joerg Exp $
 
 BUILDLINK_DEPMETHOD.libfetch?=	build
 
@@ -14,7 +14,7 @@ BUILDLINK_PACKAGES+=	libfetch
 BUILDLINK_ORDER:=	${BUILDLINK_ORDER} ${BUILDLINK_DEPTH}libfetch
 
 .if ${LIBFETCH_BUILDLINK3_MK} == "+"
-BUILDLINK_API_DEPENDS.libfetch+=	libfetch>=2.1
+BUILDLINK_API_DEPENDS.libfetch+=	libfetch>=2.5
 BUILDLINK_PKGSRCDIR.libfetch?=	../../net/libfetch
 .endif	# LIBFETCH_BUILDLINK3_MK
 
diff --git a/net/libfetch/files/common.c b/net/libfetch/files/common.c
index 730e53fd036..fe56ab27d94 100644
--- a/net/libfetch/files/common.c
+++ b/net/libfetch/files/common.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: common.c,v 1.7 2008/04/17 19:04:12 joerg Exp $	*/
+/*	$NetBSD: common.c,v 1.8 2008/04/19 14:49:23 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
@@ -620,40 +620,75 @@ fetch_close(conn_t *conn)
 /*** Directory-related utility functions *************************************/
 
 int
-fetch_add_entry(struct url_ent **p, int *size, int *len,
-    const char *name, struct url_stat *us)
+fetch_add_entry(struct url_list *ue, struct url *base, const char *name)
 {
-	struct url_ent *tmp;
-
-	if (*p == NULL) {
-		*size = 0;
-		*len = 0;
+	struct url *tmp;
+	char *tmp_name;
+	size_t base_doc_len, name_len;
+
+	if (strchr(name, '/') != NULL ||
+	    strcmp(name, "..") == 0 ||
+	    strcmp(name, ".") == 0)
+		return 0;
+
+	base_doc_len = strlen(base->doc);
+	name_len = strlen(name);
+	tmp_name = malloc( base_doc_len + name_len + 2);
+	if (tmp_name == NULL) {
+		errno = ENOMEM;
+		fetch_syserr();
+		return (-1);
 	}
 
-	if (*len >= *size - 1) {
-		tmp = realloc(*p, (*size * 2 + 1) * sizeof(**p));
+	if (ue->length + 1 >= ue->alloc_size) {
+		tmp = realloc(ue->urls, (ue->alloc_size * 2 + 1) * sizeof(*tmp));
 		if (tmp == NULL) {
+			free(tmp_name);
 			errno = ENOMEM;
 			fetch_syserr();
 			return (-1);
 		}
-		*size = (*size * 2 + 1);
-		*p = tmp;
+		ue->alloc_size = ue->alloc_size * 2 + 1;
+		ue->urls = tmp;
 	}
 
-	tmp = *p + *len;
-	snprintf(tmp->name, PATH_MAX, "%s", name);
-	if (us)
-		memcpy(&tmp->stat, us, sizeof(*us));
-	else
-		memset(&tmp->stat, 0, sizeof(*us));
-
-	(*len)++;
-	(++tmp)->name[0] = 0;
+	tmp = ue->urls + ue->length;
+	strcpy(tmp->scheme, base->scheme);
+	strcpy(tmp->user, base->user);
+	strcpy(tmp->pwd, base->pwd);
+	strcpy(tmp->host, base->host);
+	tmp->port = base->port;
+	memcpy(tmp_name, base->doc, base_doc_len);
+	tmp_name[base_doc_len] = '/';
+	memcpy(tmp_name + base_doc_len + 1, name, name_len);
+	tmp_name[base_doc_len + name_len + 1] = '/';
+	tmp->doc = tmp_name;
+	tmp->offset = 0;
+	tmp->length = 0;
+
+	++ue->length;
 
 	return (0);
 }
 
+void
+fetch_init_url_list(struct url_list *ue)
+{
+	ue->length = ue->alloc_size = 0;
+	ue->urls = NULL;
+}
+
+void
+fetch_free_url_list(struct url_list *ue)
+{
+	size_t i;
+
+	for (i = 0; i < ue->length; ++i)
+		free(ue->urls[i].doc);
+	free(ue->urls);
+	ue->length = ue->alloc_size = 0;
+}
+
 
 /*** Authentication-related utility functions ********************************/
 
diff --git a/net/libfetch/files/common.h b/net/libfetch/files/common.h
index 0ae1f1a1400..f19230674a5 100644
--- a/net/libfetch/files/common.h
+++ b/net/libfetch/files/common.h
@@ -1,4 +1,4 @@
-/*	$NetBSD: common.h,v 1.5 2008/04/05 02:42:13 joerg Exp $	*/
+/*	$NetBSD: common.h,v 1.6 2008/04/19 14:49:23 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * All rights reserved.
@@ -93,8 +93,7 @@ ssize_t		 fetch_write(conn_t *, const char *, size_t);
 ssize_t		 fetch_writev(conn_t *, struct iovec *, int);
 int		 fetch_putln(conn_t *, const char *, size_t);
 int		 fetch_close(conn_t *);
-int		 fetch_add_entry(struct url_ent **, int *, int *,
-		     const char *, struct url_stat *);
+int		 fetch_add_entry(struct url_list *, struct url *, const char *);
 int		 fetch_netrc_auth(struct url *url);
 int		 fetch_no_proxy_match(const char *);
 
diff --git a/net/libfetch/files/fetch.3 b/net/libfetch/files/fetch.3
index b2ebe2e6f16..49800161836 100644
--- a/net/libfetch/files/fetch.3
+++ b/net/libfetch/files/fetch.3
@@ -24,9 +24,9 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD: fetch.3,v 1.64 2007/12/18 11:03:26 des Exp $
-.\" $NetBSD: fetch.3,v 1.4 2008/04/04 22:37:28 joerg Exp $
+.\" $NetBSD: fetch.3,v 1.5 2008/04/19 14:49:23 joerg Exp $
 .\"
-.Dd December 18, 2007
+.Dd April 19, 2008
 .Dt FETCH 3
 .Os
 .Sh NAME
@@ -37,31 +37,26 @@
 .Nm fetchGetURL ,
 .Nm fetchPutURL ,
 .Nm fetchStatURL ,
-.Nm fetchFilteredListURL ,
 .Nm fetchListURL ,
 .Nm fetchXGet ,
 .Nm fetchGet ,
 .Nm fetchPut ,
 .Nm fetchStat ,
-.Nm fetchFilteredList ,
 .Nm fetchList ,
 .Nm fetchXGetFile ,
 .Nm fetchGetFile ,
 .Nm fetchPutFile ,
 .Nm fetchStatFile ,
-.Nm fetchFilteredListFile ,
 .Nm fetchListFile ,
 .Nm fetchXGetHTTP ,
 .Nm fetchGetHTTP ,
 .Nm fetchPutHTTP ,
 .Nm fetchStatHTTP ,
-.Nm fetchFilteredListHTTP ,
 .Nm fetchListHTTP ,
 .Nm fetchXGetFTP ,
 .Nm fetchGetFTP ,
 .Nm fetchPutFTP ,
 .Nm fetchStatFTP ,
-.Nm fetchFilteredListFTP
 .Nm fetchListFTP
 .Nd file transfer functions
 .Sh LIBRARY
@@ -83,10 +78,8 @@
 .Fn fetchPutURL "const char *URL" "const char *flags"
 .Ft int
 .Fn fetchStatURL "const char *URL" "struct url_stat *us" "const char *flags"
-.Ft struct url_ent *
-.Fn fetchFilteredListURL "const char *URL" "const char *pattern" "const char *flags"
-.Ft struct url_ent *
-.Fn fetchListURL "const char *URL" "const char *flags"
+.Ft int
+.Fn fetchListURL "struct url_list *list" "const char *URL" "const char *flags"
 .Ft fetchIO *
 .Fn fetchXGet "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft fetchIO *
@@ -95,10 +88,8 @@
 .Fn fetchPut "struct url *u" "const char *flags"
 .Ft int
 .Fn fetchStat "struct url *u" "struct url_stat *us" "const char *flags"
-.Ft struct url_ent *
-.Fn fetchFilteredList "struct url *u" "const char *pattern" "const char *flags"
-.Ft struct url_ent *
-.Fn fetchList "struct url *u" "const char *flags"
+.Ft int
+.Fn fetchList "struct url_list *list" "struct url *u" "const char *flags"
 .Ft fetchIO *
 .Fn fetchXGetFile "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft fetchIO *
@@ -107,10 +98,8 @@
 .Fn fetchPutFile "struct url *u" "const char *flags"
 .Ft int
 .Fn fetchStatFile "struct url *u" "struct url_stat *us" "const char *flags"
-.Ft struct url_ent *
-.Fn fetchFilteredListFile "struct url *u" "const char *pattern" "const char *flags"
-.Ft struct url_ent *
-.Fn fetchListFile "struct url *u" "const char *flags"
+.Ft int
+.Fn fetchListFile "struct url_list *list" "struct url *u" "const char *flags"
 .Ft fetchIO *
 .Fn fetchXGetHTTP "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft fetchIO *
@@ -119,10 +108,8 @@
 .Fn fetchPutHTTP "struct url *u" "const char *flags"
 .Ft int
 .Fn fetchStatHTTP "struct url *u" "struct url_stat *us" "const char *flags"
-.Ft struct url_ent *
-.Fn fetchFilteredListHTTP "struct url *u" "const char *pattern" "const char *flags"
-.Ft struct url_ent *
-.Fn fetchListHTTP "struct url *u" "const char *flags"
+.Ft int
+.Fn fetchListHTTP "struct url_list *list" "struct url *u" "const char *flags"
 .Ft fetchIO *
 .Fn fetchXGetFTP "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft fetchIO *
@@ -131,10 +118,8 @@
 .Fn fetchPutFTP "struct url *u" "const char *flags"
 .Ft int
 .Fn fetchStatFTP "struct url *u" "struct url_stat *us" "const char *flags"
-.Ft struct url_ent *
-.Fn fetchFilteredListFTP "struct url *u" "const char *pattern" "const char *flags"
-.Ft struct url_ent *
-.Fn fetchListFTP "struct url *u" "const char *flags"
+.Ft int
+.Fn fetchListFTP "struct url_list *list" "struct url *u" "const char *flags"
 .Sh DESCRIPTION
 These functions implement a high-level library for retrieving and
 uploading files using Uniform Resource Locators (URLs).
@@ -241,34 +226,28 @@ If the access time could not be obtained from the server, the
 field is set to the modification time.
 .Pp
 .Fn fetchListURL
-attempts to list the contents of the directory pointed to by the URL
-provided.
-If successful, it returns a malloced array of
-.Vt url_ent
-structures.
+attempts to list the contents of the directory pointed to by the URL provided.
+The pattern can be a simple glob-like expression as hint.
+Callers should not depend on the server to filter names.
+If successful, it appends the list of entries to the 
+.Vt url_list
+structure.
 The
-.Vt url_ent
+.Vt url_list
 structure is defined as follows in
 .In fetch.h :
 .Bd -literal
-struct url_ent {
-    char         name[PATH_MAX];
-    struct url_stat stat;
+struct url_list {
+    size_t	length;
+    size_t	alloc_size;
+    struct url	*urls;
 };
 .Ed
 .Pp
-The list is terminated by an entry with an empty name.
-.Pp
-The pointer returned by
-.Fn fetchListURL
-should be freed using
-.Fn free .
-.Pp
-.Fn fetchFilteredListURL
-works like
-.Fn fetchListURL ,
-but filters the list according to the given glob pattern.
-Only * and ? should be used in the pattern.
+The list should be initialised by calling
+.Fn fetch_init_url_list
+and the entries be freed by calling
+.Fn fetch_free_url_list .
 .Pp
 .Fn fetchXGet ,
 .Fn fetchGet ,
@@ -674,9 +653,7 @@ This manual page was written by
 Some parts of the library are not yet implemented.
 The most notable
 examples of this are
-.Fn fetchPutHTTP ,
-.Fn fetchListHTTP ,
-.Fn fetchListFTP
+.Fn fetchPutHTTP
 and FTP proxy support.
 .Pp
 There is no way to select a proxy at run-time other than setting the
diff --git a/net/libfetch/files/fetch.c b/net/libfetch/files/fetch.c
index 508d52f356d..d17227196d6 100644
--- a/net/libfetch/files/fetch.c
+++ b/net/libfetch/files/fetch.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: fetch.c,v 1.3 2008/04/04 22:37:28 joerg Exp $	*/
+/*	$NetBSD: fetch.c,v 1.4 2008/04/19 14:49:23 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * All rights reserved.
@@ -153,44 +153,23 @@ fetchStat(struct url *URL, struct url_stat *us, const char *flags)
  * Select the appropriate protocol for the URL scheme, and return a
  * list of files in the directory pointed to by the URL.
  */
-struct url_ent *
-fetchList(struct url *URL, const char *flags)
+int
+fetchList(struct url_list *ue, struct url *URL, const char *pattern,
+    const char *flags)
 {
 	int direct;
 
 	direct = CHECK_FLAG('d');
 	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
-		return (fetchListFile(URL, flags));
+		return (fetchListFile(ue, URL, pattern, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
-		return (fetchListFTP(URL, flags));
+		return (fetchListFTP(ue, URL, pattern, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
-		return (fetchListHTTP(URL, flags));
+		return (fetchListHTTP(ue, URL, pattern, flags));
 	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
-		return (fetchListHTTP(URL, flags));
+		return (fetchListHTTP(ue, URL, pattern, flags));
 	url_seterr(URL_BAD_SCHEME);
-	return (NULL);
-}
-
-/*
- * Select the appropriate protocol for the URL scheme, and return a
- * list of files in the directory pointed to by the URL.
- */
-struct url_ent *
-fetchFilteredList(struct url *URL, const char *pattern, const char *flags)
-{
-	int direct;
-
-	direct = CHECK_FLAG('d');
-	if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
-		return (fetchFilteredListFile(URL, pattern, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
-		return (fetchFilteredListFTP(URL, pattern, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
-		return (fetchFilteredListHTTP(URL, pattern, flags));
-	else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
-		return (fetchFilteredListHTTP(URL, pattern, flags));
-	url_seterr(URL_BAD_SCHEME);
-	return (NULL);
+	return -1;
 }
 
 /*
@@ -259,37 +238,20 @@ fetchStatURL(const char *URL, struct url_stat *us, const char *flags)
 /*
  * Attempt to parse the given URL; if successful, call fetchList().
  */
-struct url_ent *
-fetchListURL(const char *URL, const char *flags)
-{
-	struct url *u;
-	struct url_ent *ue;
-
-	if ((u = fetchParseURL(URL)) == NULL)
-		return (NULL);
-
-	ue = fetchList(u, flags);
-
-	fetchFreeURL(u);
-	return (ue);
-}
-
-/*
- * Attempt to parse the given URL; if successful, call fetchList().
- */
-struct url_ent *
-fetchFilteredListURL(const char *URL, const char *pattern, const char *flags)
+int
+fetchListURL(struct url_list *ue, const char *URL, const char *pattern,
+    const char *flags)
 {
 	struct url *u;
-	struct url_ent *ue;
+	int rv;
 
 	if ((u = fetchParseURL(URL)) == NULL)
-		return (NULL);
+		return -1;
 
-	ue = fetchFilteredList(u, pattern, flags);
+	rv = fetchList(ue, u, pattern, flags);
 
 	fetchFreeURL(u);
-	return (ue);
+	return rv;
 }
 
 /*
diff --git a/net/libfetch/files/fetch.cat3 b/net/libfetch/files/fetch.cat3
index 230d950a9ce..48fd3d92c2b 100644
--- a/net/libfetch/files/fetch.cat3
+++ b/net/libfetch/files/fetch.cat3
@@ -2,13 +2,11 @@ FETCH(3)                NetBSD Library Functions Manual               FETCH(3)
 
 NNAAMMEE
      ffeettcchhMMaakkeeUURRLL, ffeettcchhPPaarrsseeUURRLL, ffeettcchhFFrreeeeUURRLL, ffeettcchhXXGGeettUURRLL, ffeettcchhGGeettUURRLL,
-     ffeettcchhPPuuttUURRLL, ffeettcchhSSttaattUURRLL, ffeettcchhFFiilltteerreeddLLiissttUURRLL, ffeettcchhLLiissttUURRLL, ffeettcchhXXGGeett,
-     ffeettcchhGGeett, ffeettcchhPPuutt, ffeettcchhSSttaatt, ffeettcchhFFiilltteerreeddLLiisstt, ffeettcchhLLiisstt,
-     ffeettcchhXXGGeettFFiillee, ffeettcchhGGeettFFiillee, ffeettcchhPPuuttFFiillee, ffeettcchhSSttaattFFiillee,
-     ffeettcchhFFiilltteerreeddLLiissttFFiillee, ffeettcchhLLiissttFFiillee, ffeettcchhXXGGeettHHTTTTPP, ffeettcchhGGeettHHTTTTPP,
-     ffeettcchhPPuuttHHTTTTPP, ffeettcchhSSttaattHHTTTTPP, ffeettcchhFFiilltteerreeddLLiissttHHTTTTPP, ffeettcchhLLiissttHHTTTTPP,
-     ffeettcchhXXGGeettFFTTPP, ffeettcchhGGeettFFTTPP, ffeettcchhPPuuttFFTTPP, ffeettcchhSSttaattFFTTPP,
-     ffeettcchhFFiilltteerreeddLLiissttFFTTPP ffeettcchhLLiissttFFTTPP -- file transfer functions
+     ffeettcchhPPuuttUURRLL, ffeettcchhSSttaattUURRLL, ffeettcchhLLiissttUURRLL, ffeettcchhXXGGeett, ffeettcchhGGeett, ffeettcchhPPuutt,
+     ffeettcchhSSttaatt, ffeettcchhLLiisstt, ffeettcchhXXGGeettFFiillee, ffeettcchhGGeettFFiillee, ffeettcchhPPuuttFFiillee,
+     ffeettcchhSSttaattFFiillee, ffeettcchhLLiissttFFiillee, ffeettcchhXXGGeettHHTTTTPP, ffeettcchhGGeettHHTTTTPP, ffeettcchhPPuuttHHTTTTPP,
+     ffeettcchhSSttaattHHTTTTPP, ffeettcchhLLiissttHHTTTTPP, ffeettcchhXXGGeettFFTTPP, ffeettcchhGGeettFFTTPP, ffeettcchhPPuuttFFTTPP,
+     ffeettcchhSSttaattFFTTPP, ffeettcchhLLiissttFFTTPP -- file transfer functions
 
 LLIIBBRRAARRYY
      library ``libfetch''
@@ -39,12 +37,8 @@ SSYYNNOOPPSSIISS
      _i_n_t
      ffeettcchhSSttaattUURRLL(_c_o_n_s_t _c_h_a_r _*_U_R_L, _s_t_r_u_c_t _u_r_l___s_t_a_t _*_u_s, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
 
-     _s_t_r_u_c_t _u_r_l___e_n_t _*
-     ffeettcchhFFiilltteerreeddLLiissttUURRLL(_c_o_n_s_t _c_h_a_r _*_U_R_L, _c_o_n_s_t _c_h_a_r _*_p_a_t_t_e_r_n,
-         _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
-
-     _s_t_r_u_c_t _u_r_l___e_n_t _*
-     ffeettcchhLLiissttUURRLL(_c_o_n_s_t _c_h_a_r _*_U_R_L, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
+     _i_n_t
+     ffeettcchhLLiissttUURRLL(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_l_i_s_t, _c_o_n_s_t _c_h_a_r _*_U_R_L, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
 
      _f_e_t_c_h_I_O _*
      ffeettcchhXXGGeett(_s_t_r_u_c_t _u_r_l _*_u, _s_t_r_u_c_t _u_r_l___s_t_a_t _*_u_s, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
@@ -58,11 +52,8 @@ SSYYNNOOPPSSIISS
      _i_n_t
      ffeettcchhSSttaatt(_s_t_r_u_c_t _u_r_l _*_u, _s_t_r_u_c_t _u_r_l___s_t_a_t _*_u_s, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
 
-     _s_t_r_u_c_t _u_r_l___e_n_t _*
-     ffeettcchhFFiilltteerreeddLLiisstt(_s_t_r_u_c_t _u_r_l _*_u, _c_o_n_s_t _c_h_a_r _*_p_a_t_t_e_r_n, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
-
-     _s_t_r_u_c_t _u_r_l___e_n_t _*
-     ffeettcchhLLiisstt(_s_t_r_u_c_t _u_r_l _*_u, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
+     _i_n_t
+     ffeettcchhLLiisstt(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_l_i_s_t, _s_t_r_u_c_t _u_r_l _*_u, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
 
      _f_e_t_c_h_I_O _*
      ffeettcchhXXGGeettFFiillee(_s_t_r_u_c_t _u_r_l _*_u, _s_t_r_u_c_t _u_r_l___s_t_a_t _*_u_s, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
@@ -76,12 +67,8 @@ SSYYNNOOPPSSIISS
      _i_n_t
      ffeettcchhSSttaattFFiillee(_s_t_r_u_c_t _u_r_l _*_u, _s_t_r_u_c_t _u_r_l___s_t_a_t _*_u_s, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
 
-     _s_t_r_u_c_t _u_r_l___e_n_t _*
-     ffeettcchhFFiilltteerreeddLLiissttFFiillee(_s_t_r_u_c_t _u_r_l _*_u, _c_o_n_s_t _c_h_a_r _*_p_a_t_t_e_r_n,
-         _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
-
-     _s_t_r_u_c_t _u_r_l___e_n_t _*
-     ffeettcchhLLiissttFFiillee(_s_t_r_u_c_t _u_r_l _*_u, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
+     _i_n_t
+     ffeettcchhLLiissttFFiillee(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_l_i_s_t, _s_t_r_u_c_t _u_r_l _*_u, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
 
      _f_e_t_c_h_I_O _*
      ffeettcchhXXGGeettHHTTTTPP(_s_t_r_u_c_t _u_r_l _*_u, _s_t_r_u_c_t _u_r_l___s_t_a_t _*_u_s, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
@@ -95,12 +82,8 @@ SSYYNNOOPPSSIISS
      _i_n_t
      ffeettcchhSSttaattHHTTTTPP(_s_t_r_u_c_t _u_r_l _*_u, _s_t_r_u_c_t _u_r_l___s_t_a_t _*_u_s, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
 
-     _s_t_r_u_c_t _u_r_l___e_n_t _*
-     ffeettcchhFFiilltteerreeddLLiissttHHTTTTPP(_s_t_r_u_c_t _u_r_l _*_u, _c_o_n_s_t _c_h_a_r _*_p_a_t_t_e_r_n,
-         _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
-
-     _s_t_r_u_c_t _u_r_l___e_n_t _*
-     ffeettcchhLLiissttHHTTTTPP(_s_t_r_u_c_t _u_r_l _*_u, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
+     _i_n_t
+     ffeettcchhLLiissttHHTTTTPP(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_l_i_s_t, _s_t_r_u_c_t _u_r_l _*_u, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
 
      _f_e_t_c_h_I_O _*
      ffeettcchhXXGGeettFFTTPP(_s_t_r_u_c_t _u_r_l _*_u, _s_t_r_u_c_t _u_r_l___s_t_a_t _*_u_s, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
@@ -114,12 +97,8 @@ SSYYNNOOPPSSIISS
      _i_n_t
      ffeettcchhSSttaattFFTTPP(_s_t_r_u_c_t _u_r_l _*_u, _s_t_r_u_c_t _u_r_l___s_t_a_t _*_u_s, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
 
-     _s_t_r_u_c_t _u_r_l___e_n_t _*
-     ffeettcchhFFiilltteerreeddLLiissttFFTTPP(_s_t_r_u_c_t _u_r_l _*_u, _c_o_n_s_t _c_h_a_r _*_p_a_t_t_e_r_n,
-         _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
-
-     _s_t_r_u_c_t _u_r_l___e_n_t _*
-     ffeettcchhLLiissttFFTTPP(_s_t_r_u_c_t _u_r_l _*_u, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
+     _i_n_t
+     ffeettcchhLLiissttFFTTPP(_s_t_r_u_c_t _u_r_l___l_i_s_t _*_l_i_s_t, _s_t_r_u_c_t _u_r_l _*_u, _c_o_n_s_t _c_h_a_r _*_f_l_a_g_s);
 
 DDEESSCCRRIIPPTTIIOONN
      These functions implement a high-level library for retrieving and upload-
@@ -191,22 +170,19 @@ DDEESSCCRRIIPPTTIIOONN
      time.
 
      ffeettcchhLLiissttUURRLL() attempts to list the contents of the directory pointed to
-     by the URL provided.  If successful, it returns a malloced array of
-     _u_r_l___e_n_t structures.  The _u_r_l___e_n_t structure is defined as follows in
-     <_f_e_t_c_h_._h>:
-
-     struct url_ent {
-         char         name[PATH_MAX];
-         struct url_stat stat;
+     by the URL provided.  The pattern can be a simple glob-like expression as
+     hint.  Callers should not depend on the server to filter names.  If suc-
+     cessful, it appends the list of entries to the _u_r_l___l_i_s_t structure.  The
+     _u_r_l___l_i_s_t structure is defined as follows in <_f_e_t_c_h_._h>:
+
+     struct url_list {
+         size_t      length;
+         size_t      alloc_size;
+         struct url  *urls;
      };
 
-     The list is terminated by an entry with an empty name.
-
-     The pointer returned by ffeettcchhLLiissttUURRLL() should be freed using ffrreeee().
-
-     ffeettcchhFFiilltteerreeddLLiissttUURRLL() works like ffeettcchhLLiissttUURRLL(), but filters the list
-     according to the given glob pattern.  Only * and ? should be used in the
-     pattern.
+     The list should be initialised by calling ffeettcchh__iinniitt__uurrll__lliisstt() and the
+     entries be freed by calling ffeettcchh__ffrreeee__uurrll__lliisstt().
 
      ffeettcchhXXGGeett(), ffeettcchhGGeett(), ffeettcchhPPuutt() and ffeettcchhSSttaatt() are similar to
      ffeettcchhXXGGeettUURRLL(), ffeettcchhGGeettUURRLL(), ffeettcchhPPuuttUURRLL() and ffeettcchhSSttaattUURRLL(), except
@@ -466,8 +442,7 @@ AAUUTTHHOORRSS
 
 BBUUGGSS
      Some parts of the library are not yet implemented.  The most notable
-     examples of this are ffeettcchhPPuuttHHTTTTPP(), ffeettcchhLLiissttHHTTTTPP(), ffeettcchhLLiissttFFTTPP() and
-     FTP proxy support.
+     examples of this are ffeettcchhPPuuttHHTTTTPP() and FTP proxy support.
 
      There is no way to select a proxy at run-time other than setting the
      HTTP_PROXY or FTP_PROXY environment variables as appropriate.
@@ -490,4 +465,4 @@ BBUUGGSS
 
      Some parts of the code are not fully reentrant.
 
-NetBSD 4.0                     December 18, 2007                    NetBSD 4.0
+NetBSD 4.0                      April 19, 2008                      NetBSD 4.0
diff --git a/net/libfetch/files/fetch.h b/net/libfetch/files/fetch.h
index 55378e303f1..86696eba7c2 100644
--- a/net/libfetch/files/fetch.h
+++ b/net/libfetch/files/fetch.h
@@ -1,4 +1,4 @@
-/*	$NetBSD: fetch.h,v 1.5 2008/04/04 22:37:28 joerg Exp $	*/
+/*	$NetBSD: fetch.h,v 1.6 2008/04/19 14:49:23 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * All rights reserved.
@@ -62,9 +62,10 @@ struct url_stat {
 	time_t		 mtime;
 };
 
-struct url_ent {
-	char		 name[PATH_MAX];
-	struct url_stat	 stat;
+struct url_list {
+	size_t		 length;
+	size_t		 alloc_size;
+	struct url	*urls;
 };
 
 /* Recognized schemes */
@@ -107,39 +108,38 @@ fetchIO		*fetchXGetFile(struct url *, struct url_stat *, const char *);
 fetchIO		*fetchGetFile(struct url *, const char *);
 fetchIO		*fetchPutFile(struct url *, const char *);
 int		 fetchStatFile(struct url *, struct url_stat *, const char *);
-struct url_ent	*fetchFilteredListFile(struct url *, const char *,
+int		 fetchListFile(struct url_list *, struct url *, const char *,
 		    const char *);
-struct url_ent	*fetchListFile(struct url *, const char *);
 
 /* HTTP-specific functions */
 fetchIO		*fetchXGetHTTP(struct url *, struct url_stat *, const char *);
 fetchIO		*fetchGetHTTP(struct url *, const char *);
 fetchIO		*fetchPutHTTP(struct url *, const char *);
 int		 fetchStatHTTP(struct url *, struct url_stat *, const char *);
-struct url_ent	*fetchFilteredListHTTP(struct url *,const char *, const char *);
-struct url_ent	*fetchListHTTP(struct url *, const char *);
+int		 fetchListHTTP(struct url_list *, struct url *, const char *,
+		    const char *);
 
 /* FTP-specific functions */
 fetchIO		*fetchXGetFTP(struct url *, struct url_stat *, const char *);
 fetchIO		*fetchGetFTP(struct url *, const char *);
 fetchIO		*fetchPutFTP(struct url *, const char *);
 int		 fetchStatFTP(struct url *, struct url_stat *, const char *);
-struct url_ent	*fetchFilteredListFTP(struct url *, const char *, const char *);
-struct url_ent	*fetchListFTP(struct url *, const char *);
+int		 fetchListFTP(struct url_list *, struct url *, const char *,
+		    const char *);
 
 /* Generic functions */
 fetchIO		*fetchXGetURL(const char *, struct url_stat *, const char *);
 fetchIO		*fetchGetURL(const char *, const char *);
 fetchIO		*fetchPutURL(const char *, const char *);
 int		 fetchStatURL(const char *, struct url_stat *, const char *);
-struct url_ent	*fetchFilteredListURL(const char *, const char *, const char *);
-struct url_ent	*fetchListURL(const char *, const char *);
+int		 fetchListURL(struct url_list *, const char *, const char *,
+		    const char *);
 fetchIO		*fetchXGet(struct url *, struct url_stat *, const char *);
 fetchIO		*fetchGet(struct url *, const char *);
 fetchIO		*fetchPut(struct url *, const char *);
 int		 fetchStat(struct url *, struct url_stat *, const char *);
-struct url_ent	*fetchFilteredList(struct url *, const char *, const char *);
-struct url_ent	*fetchList(struct url *, const char *);
+int		 fetchList(struct url_list *, struct url *, const char *,
+		    const char *);
 
 /* URL parsing */
 struct url	*fetchMakeURL(const char *, const char *, int,
@@ -147,6 +147,10 @@ struct url	*fetchMakeURL(const char *, const char *, int,
 struct url	*fetchParseURL(const char *);
 void		 fetchFreeURL(struct url *);
 
+/* URL listening */
+void		 fetch_init_url_list(struct url_list *);
+void		 fetch_free_url_list(struct url_list *);
+
 /* Authentication */
 typedef int (*auth_t)(struct url *);
 extern auth_t		 fetchAuthMethod;
diff --git a/net/libfetch/files/file.c b/net/libfetch/files/file.c
index ce5d08fa991..a580e48c16e 100644
--- a/net/libfetch/files/file.c
+++ b/net/libfetch/files/file.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: file.c,v 1.5 2008/04/04 23:19:16 joerg Exp $	*/
+/*	$NetBSD: file.c,v 1.6 2008/04/19 14:49:23 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * All rights reserved.
@@ -187,20 +187,17 @@ fetchStatFile(struct url *u, struct url_stat *us, const char *flags)
 	return (fetch_stat_file2(u->doc, us));
 }
 
-struct url_ent *
-fetchFilteredListFile(struct url *u, const char *pattern, const char *flags)
+int
+fetchListFile(struct url_list *ue, struct url *u, const char *pattern, const char *flags)
 {
 	struct dirent *de;
-	struct url_stat us;
-	struct url_ent *ue;
-	int size, len;
 	char fn[PATH_MAX], *p;
 	DIR *dir;
 	int l;
 
 	if ((dir = opendir(u->doc)) == NULL) {
 		fetch_syserr();
-		return (NULL);
+		return -1;
 	}
 
 	ue = NULL;
@@ -215,18 +212,8 @@ fetchFilteredListFile(struct url *u, const char *pattern, const char *flags)
 			continue;
 		strncpy(p, de->d_name, l - 1);
 		p[l - 1] = 0;
-		if (fetch_stat_file2(fn, &us) == -1) {
-			/* should I return a partial result, or abort? */
-			break;
-		}
-		fetch_add_entry(&ue, &size, &len, de->d_name, &us);
+		fetch_add_entry(ue, u, de->d_name);
 	}
 
-	return (ue);
-}
-
-struct url_ent *
-fetchListFile(struct url *u, const char *flags)
-{
-	return fetchFilteredListFile(u, "*", flags);
+	return 0;
 }
diff --git a/net/libfetch/files/ftp.c b/net/libfetch/files/ftp.c
index 15ba3906400..1a7c203ff62 100644
--- a/net/libfetch/files/ftp.c
+++ b/net/libfetch/files/ftp.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: ftp.c,v 1.17 2008/04/17 19:04:12 joerg Exp $	*/
+/*	$NetBSD: ftp.c,v 1.18 2008/04/19 14:49:23 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
@@ -1192,20 +1192,20 @@ fetchStatFTP(struct url *url, struct url_stat *us, const char *flags)
 /*
  * List a directory
  */
-struct url_ent *
-fetchFilteredListFTP(struct url *url, const char *pattern, const char *flags)
+int
+fetchListFTP(struct url_list *ue, struct url *url, const char *pattern, const char *flags)
 {
-	struct url_ent *ue;
 	fetchIO *f;
 	char buf[2 * PATH_MAX], *eol, *eos;
 	ssize_t len;
 	size_t cur_off;
-	int list_size, list_len;
 
 	/* XXX What about proxies? */
+	if (pattern == NULL || strcmp(pattern, "*") == 0)
+		pattern = "";
 	f = ftp_request(url, "NLST", pattern, NULL, ftp_get_proxy(url, flags), flags);
 	if (f == NULL)
-		return NULL;
+		return -1;
 
 	ue = NULL;
 	cur_off = 0;
@@ -1220,7 +1220,7 @@ fetchFilteredListFTP(struct url *url, const char *pattern, const char *flags)
 				else
 					eos = eol;
 				*eos = '\0';
-				fetch_add_entry(&ue, &list_size, &list_len, buf, NULL);
+				fetch_add_entry(ue, url, buf);
 				cur_off -= eol - buf + 1;
 				memmove(buf, eol + 1, cur_off);
 			}
@@ -1229,18 +1229,8 @@ fetchFilteredListFTP(struct url *url, const char *pattern, const char *flags)
 	if (cur_off != 0 || len < 0) {
 		/* Not RFC conform, bail out. */
 		fetchIO_close(f);
-		free(ue);
-		return NULL;
+		return -1;
 	}
 	fetchIO_close(f);
-	return ue;
-}
-
-/*
- * List a directory
- */
-struct url_ent *
-fetchListFTP(struct url *url, const char *flags)
-{
-	return fetchFilteredList(url, "", flags);
+	return 0;
 }
diff --git a/net/libfetch/files/http.c b/net/libfetch/files/http.c
index ba548f745ef..e821bc09817 100644
--- a/net/libfetch/files/http.c
+++ b/net/libfetch/files/http.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: http.c,v 1.14 2008/04/18 21:13:10 joerg Exp $	*/
+/*	$NetBSD: http.c,v 1.15 2008/04/19 14:49:24 joerg Exp $	*/
 /*-
  * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav
  * Copyright (c) 2003 Thomas Klausner <wiz@NetBSD.org>
@@ -1169,9 +1169,9 @@ enum http_states {
 };
 
 struct index_parser {
+	struct url_list *ue;
+	struct url *url;
 	enum http_states state;
-	struct url_ent *ue;
-	int list_size, list_len;
 };
 
 static size_t
@@ -1303,7 +1303,7 @@ parse_index(struct index_parser *parser, const char *buf, size_t len)
 			return 0;
 		*end_attr = '\0';
 		parser->state = ST_TAGA;
-		fetch_add_entry(&parser->ue, &parser->list_size, &parser->list_len, buf, NULL);
+		fetch_add_entry(parser->ue, parser->url, buf);
 		return end_attr + 1 - buf;
 	}
 	abort();
@@ -1312,8 +1312,8 @@ parse_index(struct index_parser *parser, const char *buf, size_t len)
 /*
  * List a directory
  */
-struct url_ent *
-fetchFilteredListHTTP(struct url *url, const char *pattern, const char *flags)
+int
+fetchListHTTP(struct url_list *ue, struct url *url, const char *pattern, const char *flags)
 {
 	fetchIO *f;
 	char buf[2 * PATH_MAX];
@@ -1321,13 +1321,13 @@ fetchFilteredListHTTP(struct url *url, const char *pattern, const char *flags)
 	ssize_t read_len;
 	struct index_parser state;
 
+	state.url = url;
 	state.state = ST_NONE;
-	state.ue = NULL;
-	state.list_size = state.list_len = 0;
+	state.ue = ue;
 
 	f = fetchGetHTTP(url, flags);
 	if (f == NULL)
-		return NULL;
+		return -1;
 
 	buf_len = 0;
 
@@ -1343,18 +1343,5 @@ fetchFilteredListHTTP(struct url *url, const char *pattern, const char *flags)
 	}
 
 	fetchIO_close(f);
-	if (read_len < 0) {
-		free(state.ue);
-		state.ue = NULL;
-	}
-	return state.ue;
-}
-
-/*
- * List a directory
- */
-struct url_ent *
-fetchListHTTP(struct url *url, const char *flags)
-{
-	return fetchFilteredList(url, "*", flags);
+	return read_len < 0 ? -1 : 0;
 }