libfetch-2.28:

Revamp FTP connection cache. Move it to the common layer to be later
shared with HTTP (for persistent connection). The application controls
how much caching is desired. Drop the reference counting on connections.
Add a callback when the cached connection is dropped due to LRU.
Over all, this allows more than one session cached per host and sessions
cached to different servers.

8 files changed, 181 insertions, 114 deletions

diff --git a/net/libfetch/Makefile b/net/libfetch/Makefile
index 86e49d4624b..27eab843918 100644
--- a/net/libfetch/Makefile
+++ b/net/libfetch/Makefile
@@ -1,7 +1,7 @@
-# $NetBSD: Makefile,v 1.34 2010/01/11 17:23:10 joerg Exp $
+# $NetBSD: Makefile,v 1.35 2010/01/22 13:21:09 joerg Exp $
 #
 
-DISTNAME=	libfetch-2.27
+DISTNAME=	libfetch-2.28
 CATEGORIES=	net
 MASTER_SITES=	# empty
 DISTFILES=	# empty
diff --git a/net/libfetch/buildlink3.mk b/net/libfetch/buildlink3.mk
index dbb90022876..5522d0cbfcb 100644
--- a/net/libfetch/buildlink3.mk
+++ b/net/libfetch/buildlink3.mk
@@ -1,4 +1,4 @@
-# $NetBSD: buildlink3.mk,v 1.7 2010/01/17 12:02:33 wiz Exp $
+# $NetBSD: buildlink3.mk,v 1.8 2010/01/22 13:21:09 joerg Exp $
 
 BUILDLINK_DEPMETHOD.libfetch?=	build
 
@@ -7,8 +7,7 @@ BUILDLINK_TREE+=	libfetch
 .if !defined(LIBFETCH_BUILDLINK3_MK)
 LIBFETCH_BUILDLINK3_MK:=
 
-BUILDLINK_API_DEPENDS.libfetch+=	libfetch>=2.21
-BUILDLINK_ABI_DEPENDS.libfetch?=	libfetch>=2.27
+BUILDLINK_API_DEPENDS.libfetch+=	libfetch>=2.28
 BUILDLINK_PKGSRCDIR.libfetch?=	../../net/libfetch
 
 pkgbase := libfetch
diff --git a/net/libfetch/files/common.c b/net/libfetch/files/common.c
index 000444d3414..1bb555374a1 100644
--- a/net/libfetch/files/common.c
+++ b/net/libfetch/files/common.c
@@ -1,7 +1,7 @@
-/*	$NetBSD: common.c,v 1.21 2009/10/15 12:36:57 joerg Exp $	*/
+/*	$NetBSD: common.c,v 1.22 2010/01/22 13:21:09 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
- * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
+ * Copyright (c) 2008, 2010 Joerg Sonnenberger <joerg@NetBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -234,23 +234,11 @@ fetch_reopen(int sd)
 	/* allocate and fill connection structure */
 	if ((conn = calloc(1, sizeof(*conn))) == NULL)
 		return (NULL);
+	conn->cache_url = NULL;
 	conn->next_buf = NULL;
 	conn->next_len = 0;
 	conn->sd = sd;
 	conn->is_active = 0;
-	++conn->ref;
-	return (conn);
-}
-
-
-/*
- * Bump a connection's reference count.
- */
-conn_t *
-fetch_ref(conn_t *conn)
-{
-
-	++conn->ref;
 	return (conn);
 }
 
@@ -281,7 +269,7 @@ fetch_bind(int sd, int af, const char *addr)
  * Establish a TCP connection to the specified port on the specified host.
  */
 conn_t *
-fetch_connect(const char *host, int port, int af, int verbose)
+fetch_connect(struct url *url, int af, int verbose)
 {
 	conn_t *conn;
 	char pbuf[10];
@@ -290,22 +278,22 @@ fetch_connect(const char *host, int port, int af, int verbose)
 	int sd, error;
 
 	if (verbose)
-		fetch_info("looking up %s", host);
+		fetch_info("looking up %s", url->host);
 
 	/* look up host name and set up socket address structure */
-	snprintf(pbuf, sizeof(pbuf), "%d", port);
+	snprintf(pbuf, sizeof(pbuf), "%d", url->port);
 	memset(&hints, 0, sizeof(hints));
 	hints.ai_family = af;
 	hints.ai_socktype = SOCK_STREAM;
 	hints.ai_protocol = 0;
-	if ((error = getaddrinfo(host, pbuf, &hints, &res0)) != 0) {
+	if ((error = getaddrinfo(url->host, pbuf, &hints, &res0)) != 0) {
 		netdb_seterr(error);
 		return (NULL);
 	}
 	bindaddr = getenv("FETCH_BIND_ADDRESS");
 
 	if (verbose)
-		fetch_info("connecting to %s:%d", host, port);
+		fetch_info("connecting to %s:%d", url->host, url->port);
 
 	/* try to connect */
 	for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) {
@@ -332,9 +320,114 @@ fetch_connect(const char *host, int port, int af, int verbose)
 		fetch_syserr();
 		close(sd);
 	}
+	conn->cache_url = fetchCopyURL(url);
+	conn->cache_af = af;
 	return (conn);
 }
 
+static conn_t *connection_cache;
+static int cache_global_limit = 0;
+static int cache_per_host_limit = 0;
+
+/*
+ * Initialise cache with the given limits.
+ */
+void
+fetchConnectionCacheInit(int global_limit, int per_host_limit)
+{
+
+	if (global_limit < 0)
+		cache_global_limit = INT_MAX;
+	else if (per_host_limit > global_limit)
+		cache_global_limit = per_host_limit;
+	else
+		cache_global_limit = global_limit;
+	if (per_host_limit < 0)
+		cache_per_host_limit = INT_MAX;
+	else
+		cache_per_host_limit = per_host_limit;
+}
+
+/*
+ * Flush cache and free all associated resources.
+ */
+void
+fetchConnectionCacheClose(void)
+{
+	conn_t *conn;
+
+	while ((conn = connection_cache) != NULL) {
+		connection_cache = conn->next_cached;
+		(*conn->cache_close)(conn);
+	}
+}
+
+/*
+ * Check connection cache for an existing entry matching
+ * protocol/host/port/user/password/family.
+ */
+conn_t *
+fetch_cache_get(const struct url *url, int af)
+{
+	conn_t *conn, *last_conn = NULL;
+
+	for (conn = connection_cache; conn; conn = conn->next_cached) {
+		if (conn->cache_url->port == url->port &&
+		    strcmp(conn->cache_url->scheme, url->scheme) == 0 &&
+		    strcmp(conn->cache_url->host, url->host) == 0 &&
+		    strcmp(conn->cache_url->user, url->user) == 0 &&
+		    strcmp(conn->cache_url->pwd, url->pwd) == 0 &&
+		    (conn->cache_af == AF_UNSPEC || af == AF_UNSPEC ||
+		     conn->cache_af == af)) {
+			if (last_conn != NULL)
+				last_conn->next_cached = conn->next_cached;
+			else
+				connection_cache = conn->next_cached;
+			return conn;
+		}
+	}
+
+	return NULL;
+}
+
+/*
+ * Put the connection back into the cache for reuse.
+ * If the connection is freed due to LRU or if the cache
+ * is explicitly closed, the given callback is called.
+ */
+void
+fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *))
+{
+	conn_t *iter, *last;
+	int global_count, host_count;
+
+	if (conn->cache_url == NULL || cache_global_limit == 0) {
+		(*closecb)(conn);
+		return;
+	}
+
+	global_count = host_count = 0;
+	last = NULL;
+	for (iter = connection_cache; iter;
+	    last = iter, iter = iter->next_cached) {
+		++global_count;
+		if (strcmp(conn->cache_url->host, iter->cache_url->host) == 0)
+			++host_count;
+		if (global_count < cache_global_limit &&
+		    host_count < cache_per_host_limit)
+			continue;
+		--global_count;
+		if (last != NULL)
+			last->next_cached = iter->next_cached;
+		else
+			connection_cache = iter->next_cached;
+		(*iter->cache_close)(iter);
+	}
+
+	conn->cache_close = closecb;
+	conn->next_cached = connection_cache;
+	connection_cache = conn;
+}
 
 /*
  * Enable SSL on a connection.
@@ -649,9 +742,9 @@ fetch_close(conn_t *conn)
 {
 	int ret;
 
-	if (--conn->ref > 0)
-		return (0);
 	ret = close(conn->sd);
+	if (conn->cache_url)
+		fetchFreeURL(conn->cache_url);
 	free(conn->buf);
 	free(conn);
 	return (ret);
diff --git a/net/libfetch/files/common.h b/net/libfetch/files/common.h
index 876d68f0afd..08ffd356ef2 100644
--- a/net/libfetch/files/common.h
+++ b/net/libfetch/files/common.h
@@ -1,4 +1,4 @@
-/*	$NetBSD: common.h,v 1.12 2009/08/16 20:31:29 joerg Exp $	*/
+/*	$NetBSD: common.h,v 1.13 2010/01/22 13:21:09 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * All rights reserved.
@@ -53,6 +53,7 @@
 
 /* Connection */
 typedef struct fetchconn conn_t;
+
 struct fetchconn {
 	int		 sd;		/* socket descriptor */
 	char		*buf;		/* buffer */
@@ -71,8 +72,12 @@ struct fetchconn {
 	const SSL_METHOD *ssl_meth;	/* SSL method */
 #  endif
 #endif
-	int		 ref;		/* reference count */
 	int		 is_active;
+
+	struct url	*cache_url;
+	int		cache_af;
+	int		(*cache_close)(conn_t *);
+	conn_t		*next_cached;
 };
 
 /* Structure used for error message lists */
@@ -91,9 +96,10 @@ void		 fetch_info(const char *, ...);
 int		 fetch_default_port(const char *);
 int		 fetch_default_proxy_port(const char *);
 int		 fetch_bind(int, int, const char *);
-conn_t		*fetch_connect(const char *, int, int, int);
+conn_t		*fetch_cache_get(const struct url *, int);
+void		 fetch_cache_put(conn_t *, int (*)(conn_t *));
+conn_t		*fetch_connect(struct url *, int, int);
 conn_t		*fetch_reopen(int);
-conn_t		*fetch_ref(conn_t *);
 int		 fetch_ssl(conn_t *, int);
 ssize_t		 fetch_read(conn_t *, char *, size_t);
 int		 fetch_getln(conn_t *);
diff --git a/net/libfetch/files/fetch.3 b/net/libfetch/files/fetch.3
index 1f3138f5ab3..86e97643ea5 100644
--- a/net/libfetch/files/fetch.3
+++ b/net/libfetch/files/fetch.3
@@ -1,5 +1,6 @@
 .\"-
 .\" Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
+.\" Copyright (c) 2010 Joerg Sonnenberger <joerg@NetBSD.org>
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -24,7 +25,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD: fetch.3,v 1.64 2007/12/18 11:03:26 des Exp $
-.\" $NetBSD: fetch.3,v 1.13 2009/10/15 12:36:57 joerg Exp $
+.\" $NetBSD: fetch.3,v 1.14 2010/01/22 13:21:09 joerg Exp $
 .\"
 .Dd February 4, 2009
 .Dt FETCH 3
@@ -64,6 +65,8 @@
 .Nm fetchUnquotePath ,
 .Nm fetchUnquoteFilename ,
 .Nm fetchStringifyURL ,
+.Nm fetchConnecitonCacheInit ,
+.Nm fetchConnecitonCacheClose ,
 .Nm fetch
 .Nd file transfer functions
 .Sh LIBRARY
@@ -141,6 +144,10 @@
 .Fn fetchUnquoteFilename "struct url *u"
 .Ft char *
 .Fn fetchStringifyURL "const struct url *u"
+.Ft void
+.Fn fetchConnectionCacheInit "int global" "int per_host"
+.Ft void
+.Fn fetchConnectionCacheClose "void"
 .Sh DESCRIPTION
 These functions implement a high-level library for retrieving and
 uploading files using Uniform Resource Locators (URLs).
@@ -307,6 +314,15 @@ return a string that should be deallocated with
 .Fn free
 after use.
 .Pp
+.Fn fetchConnectionCacheInit
+enables the connection cache.
+The first argument specifies the global limit on cached connections.
+The second argument specifies the host limit.
+Entries are considered to specify the same host, if the host name
+from the URL is identical, indepent of the address or address family.
+.Fn fetchConnectionCacheClose
+flushed the connection cache and closes all cached connections.
+.Pp
 .Fn fetchXGet ,
 .Fn fetchGet ,
 .Fn fetchPut ,
diff --git a/net/libfetch/files/fetch.h b/net/libfetch/files/fetch.h
index c2e73e7113f..30754864360 100644
--- a/net/libfetch/files/fetch.h
+++ b/net/libfetch/files/fetch.h
@@ -1,4 +1,4 @@
-/*	$NetBSD: fetch.h,v 1.15 2009/10/15 12:36:57 joerg Exp $	*/
+/*	$NetBSD: fetch.h,v 1.16 2010/01/22 13:21:09 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * All rights reserved.
@@ -158,6 +158,10 @@ void		 fetchFreeURLList(struct url_list *);
 char		*fetchUnquotePath(struct url *);
 char		*fetchUnquoteFilename(struct url *);
 
+/* Connection caching */
+void		 fetchConnectionCacheInit(int, int);
+void		 fetchConnectionCacheClose(void);
+
 /* Authentication */
 typedef int (*auth_t)(struct url *);
 extern auth_t		 fetchAuthMethod;
diff --git a/net/libfetch/files/ftp.c b/net/libfetch/files/ftp.c
index 669f44cad3f..a84a22c98f2 100644
--- a/net/libfetch/files/ftp.c
+++ b/net/libfetch/files/ftp.c
@@ -1,7 +1,7 @@
-/*	$NetBSD: ftp.c,v 1.31 2010/01/11 17:23:10 joerg Exp $	*/
+/*	$NetBSD: ftp.c,v 1.32 2010/01/22 13:21:09 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
- * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
+ * Copyright (c) 2008, 2009, 2010 Joerg Sonnenberger <joerg@NetBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -120,9 +120,6 @@
 #define FTP_SYNTAX_ERROR		500
 #define FTP_PROTOCOL_ERROR		999
 
-static struct url cached_host;
-static conn_t	*cached_connection;
-
 #define isftpreply(foo)				\
 	(isdigit((unsigned char)foo[0]) &&	\
 	    isdigit((unsigned char)foo[1]) &&	\
@@ -565,6 +562,13 @@ ftp_writefn(void *v, const void *buf, size_t len)
 	return (-1);
 }
 
+static int
+ftp_disconnect(conn_t *conn)
+{
+	ftp_cmd(conn, "QUIT");
+	return fetch_close(conn);
+}
+
 static void
 ftp_closefn(void *v)
 {
@@ -583,15 +587,11 @@ ftp_closefn(void *v)
 		return;
 	}
 	fetch_close(io->dconn);
-	io->dir = -1;
-	io->dconn->is_active = 0;
 	io->dconn = NULL;
+	io->dir = -1;
+	io->cconn->is_active = 0;
 	r = ftp_chkerr(io->cconn);
-	if (io->cconn == cached_connection && io->cconn->ref == 1) {
-		free(cached_host.doc);
-		cached_connection = NULL;
-	}
-	fetch_close(io->cconn);
+	fetch_cache_put(io->cconn, ftp_disconnect);
 	free(io);
 	return;
 }
@@ -1001,10 +1001,23 @@ ftp_connect(struct url *url, struct url *purl, const char *flags)
 	/* check for proxy */
 	if (purl) {
 		/* XXX proxy authentication! */
-		conn = fetch_connect(purl->host, purl->port, af, verbose);
+		/* XXX connetion caching */
+		if (!purl->port)
+			purl->port = fetch_default_port(purl->scheme);
+
+		conn = fetch_connect(purl, af, verbose);
 	} else {
 		/* no proxy, go straight to target */
-		conn = fetch_connect(url->host, url->port, af, verbose);
+		if (!url->port)
+			url->port = fetch_default_port(url->scheme);
+
+		while ((conn = fetch_cache_get(url, af)) != NULL) {
+			e = ftp_cmd(conn, "NOOP");
+			if (e == FTP_OK)
+				return conn;
+			fetch_close(conn);
+		}
+		conn = fetch_connect(url, af, verbose);
 		purl = NULL;
 	}
 
@@ -1034,70 +1047,6 @@ fouch:
 }
 
 /*
- * Disconnect from server
- */
-static void
-ftp_disconnect(conn_t *conn)
-{
-	(void)ftp_cmd(conn, "QUIT");
-	if (conn == cached_connection && conn->ref == 1) {
-		free(cached_host.doc);
-		cached_host.doc = NULL;
-		cached_connection = NULL;
-	}
-	fetch_close(conn);
-}
-
-/*
- * Check if we're already connected
- */
-static int
-ftp_isconnected(struct url *url)
-{
-	return (cached_connection
-	    && (cached_connection->is_active == 0)
-	    && (strcmp(url->host, cached_host.host) == 0)
-	    && (strcmp(url->user, cached_host.user) == 0)
-	    && (strcmp(url->pwd, cached_host.pwd) == 0)
-	    && (url->port == cached_host.port));
-}
-
-/*
- * Check the cache, reconnect if no luck
- */
-static conn_t *
-ftp_cached_connect(struct url *url, struct url *purl, const char *flags)
-{
-	char *doc;
-	conn_t *conn;
-	int e;
-
-	/* set default port */
-	if (!url->port)
-		url->port = fetch_default_port(url->scheme);
-
-	/* try to use previously cached connection */
-	if (ftp_isconnected(url)) {
-		e = ftp_cmd(cached_connection, "NOOP");
-		if (e == FTP_OK || e == FTP_SYNTAX_ERROR)
-			return (fetch_ref(cached_connection));
-	}
-
-	/* connect to server */
-	if ((conn = ftp_connect(url, purl, flags)) == NULL)
-		return (NULL);
-	doc = strdup(url->doc);
-	if (doc != NULL) {
-		if (cached_connection && !cached_connection->is_active)
-			ftp_disconnect(cached_connection);
-		cached_connection = fetch_ref(conn);
-		memcpy(&cached_host, url, sizeof(*url));
-		cached_host.doc = doc;
-	}
-	return (conn);
-}
-
-/*
  * Check the proxy settings
  */
 static struct url *
@@ -1155,7 +1104,7 @@ ftp_request(struct url *url, const char *op, const char *op_arg,
 	}
 
 	/* connect to server */
-	conn = ftp_cached_connect(url, purl, flags);
+	conn = ftp_connect(url, purl, flags);
 	if (purl)
 		fetchFreeURL(purl);
 	if (conn == NULL)
diff --git a/net/libfetch/files/http.c b/net/libfetch/files/http.c
index f2b73c5243a..95e7bc84c78 100644
--- a/net/libfetch/files/http.c
+++ b/net/libfetch/files/http.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: http.c,v 1.25 2009/10/15 12:36:57 joerg Exp $	*/
+/*	$NetBSD: http.c,v 1.26 2010/01/22 13:21:09 joerg Exp $	*/
 /*-
  * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav
  * Copyright (c) 2003 Thomas Klausner <wiz@NetBSD.org>
@@ -706,7 +706,7 @@ http_connect(struct url *URL, struct url *purl, const char *flags)
 		return (NULL);
 	}
 
-	if ((conn = fetch_connect(URL->host, URL->port, af, verbose)) == NULL)
+	if ((conn = fetch_connect(URL, af, verbose)) == NULL)
 		/* fetch_connect() has already set an error code */
 		return (NULL);
 	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 &&