diff options
author | joerg <joerg> | 2010-01-22 13:21:09 +0000 |
---|---|---|
committer | joerg <joerg> | 2010-01-22 13:21:09 +0000 |
commit | c2e2869275210846263b82e67513ca985cf872b8 (patch) | |
tree | ea62e80250b8c8ebe9d979ccf2d116b5ce1f6bbb | |
parent | 2655b44ef4dee346cec91925c1df301dd659c338 (diff) | |
download | pkgsrc-c2e2869275210846263b82e67513ca985cf872b8.tar.gz |
libfetch-2.28:
Revamp FTP connection cache. Move it to the common layer to be later
shared with HTTP (for persistent connection). The application controls
how much caching is desired. Drop the reference counting on connections.
Add a callback when the cached connection is dropped due to LRU.
Over all, this allows more than one session cached per host and sessions
cached to different servers.
-rw-r--r-- | net/libfetch/Makefile | 4 | ||||
-rw-r--r-- | net/libfetch/buildlink3.mk | 5 | ||||
-rw-r--r-- | net/libfetch/files/common.c | 137 | ||||
-rw-r--r-- | net/libfetch/files/common.h | 14 | ||||
-rw-r--r-- | net/libfetch/files/fetch.3 | 18 | ||||
-rw-r--r-- | net/libfetch/files/fetch.h | 6 | ||||
-rw-r--r-- | net/libfetch/files/ftp.c | 107 | ||||
-rw-r--r-- | net/libfetch/files/http.c | 4 |
8 files changed, 181 insertions, 114 deletions
diff --git a/net/libfetch/Makefile b/net/libfetch/Makefile index 86e49d4624b..27eab843918 100644 --- a/net/libfetch/Makefile +++ b/net/libfetch/Makefile @@ -1,7 +1,7 @@ -# $NetBSD: Makefile,v 1.34 2010/01/11 17:23:10 joerg Exp $ +# $NetBSD: Makefile,v 1.35 2010/01/22 13:21:09 joerg Exp $ # -DISTNAME= libfetch-2.27 +DISTNAME= libfetch-2.28 CATEGORIES= net MASTER_SITES= # empty DISTFILES= # empty diff --git a/net/libfetch/buildlink3.mk b/net/libfetch/buildlink3.mk index dbb90022876..5522d0cbfcb 100644 --- a/net/libfetch/buildlink3.mk +++ b/net/libfetch/buildlink3.mk @@ -1,4 +1,4 @@ -# $NetBSD: buildlink3.mk,v 1.7 2010/01/17 12:02:33 wiz Exp $ +# $NetBSD: buildlink3.mk,v 1.8 2010/01/22 13:21:09 joerg Exp $ BUILDLINK_DEPMETHOD.libfetch?= build @@ -7,8 +7,7 @@ BUILDLINK_TREE+= libfetch .if !defined(LIBFETCH_BUILDLINK3_MK) LIBFETCH_BUILDLINK3_MK:= -BUILDLINK_API_DEPENDS.libfetch+= libfetch>=2.21 -BUILDLINK_ABI_DEPENDS.libfetch?= libfetch>=2.27 +BUILDLINK_API_DEPENDS.libfetch+= libfetch>=2.28 BUILDLINK_PKGSRCDIR.libfetch?= ../../net/libfetch pkgbase := libfetch diff --git a/net/libfetch/files/common.c b/net/libfetch/files/common.c index 000444d3414..1bb555374a1 100644 --- a/net/libfetch/files/common.c +++ b/net/libfetch/files/common.c @@ -1,7 +1,7 @@ -/* $NetBSD: common.c,v 1.21 2009/10/15 12:36:57 joerg Exp $ */ +/* $NetBSD: common.c,v 1.22 2010/01/22 13:21:09 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav - * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org> + * Copyright (c) 2008, 2010 Joerg Sonnenberger <joerg@NetBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -234,23 +234,11 @@ fetch_reopen(int sd) /* allocate and fill connection structure */ if ((conn = calloc(1, sizeof(*conn))) == NULL) return (NULL); + conn->cache_url = NULL; conn->next_buf = NULL; conn->next_len = 0; conn->sd = sd; conn->is_active = 0; - ++conn->ref; - return (conn); -} - - -/* - * Bump a connection's reference count. - */ -conn_t * -fetch_ref(conn_t *conn) -{ - - ++conn->ref; return (conn); } @@ -281,7 +269,7 @@ fetch_bind(int sd, int af, const char *addr) * Establish a TCP connection to the specified port on the specified host. */ conn_t * -fetch_connect(const char *host, int port, int af, int verbose) +fetch_connect(struct url *url, int af, int verbose) { conn_t *conn; char pbuf[10]; @@ -290,22 +278,22 @@ fetch_connect(const char *host, int port, int af, int verbose) int sd, error; if (verbose) - fetch_info("looking up %s", host); + fetch_info("looking up %s", url->host); /* look up host name and set up socket address structure */ - snprintf(pbuf, sizeof(pbuf), "%d", port); + snprintf(pbuf, sizeof(pbuf), "%d", url->port); memset(&hints, 0, sizeof(hints)); hints.ai_family = af; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = 0; - if ((error = getaddrinfo(host, pbuf, &hints, &res0)) != 0) { + if ((error = getaddrinfo(url->host, pbuf, &hints, &res0)) != 0) { netdb_seterr(error); return (NULL); } bindaddr = getenv("FETCH_BIND_ADDRESS"); if (verbose) - fetch_info("connecting to %s:%d", host, port); + fetch_info("connecting to %s:%d", url->host, url->port); /* try to connect */ for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) { @@ -332,9 +320,114 @@ fetch_connect(const char *host, int port, int af, int verbose) fetch_syserr(); close(sd); } + conn->cache_url = fetchCopyURL(url); + conn->cache_af = af; return (conn); } +static conn_t *connection_cache; +static int cache_global_limit = 0; +static int cache_per_host_limit = 0; + +/* + * Initialise cache with the given limits. + */ +void +fetchConnectionCacheInit(int global_limit, int per_host_limit) +{ + + if (global_limit < 0) + cache_global_limit = INT_MAX; + else if (per_host_limit > global_limit) + cache_global_limit = per_host_limit; + else + cache_global_limit = global_limit; + if (per_host_limit < 0) + cache_per_host_limit = INT_MAX; + else + cache_per_host_limit = per_host_limit; +} + +/* + * Flush cache and free all associated resources. + */ +void +fetchConnectionCacheClose(void) +{ + conn_t *conn; + + while ((conn = connection_cache) != NULL) { + connection_cache = conn->next_cached; + (*conn->cache_close)(conn); + } +} + +/* + * Check connection cache for an existing entry matching + * protocol/host/port/user/password/family. + */ +conn_t * +fetch_cache_get(const struct url *url, int af) +{ + conn_t *conn, *last_conn = NULL; + + for (conn = connection_cache; conn; conn = conn->next_cached) { + if (conn->cache_url->port == url->port && + strcmp(conn->cache_url->scheme, url->scheme) == 0 && + strcmp(conn->cache_url->host, url->host) == 0 && + strcmp(conn->cache_url->user, url->user) == 0 && + strcmp(conn->cache_url->pwd, url->pwd) == 0 && + (conn->cache_af == AF_UNSPEC || af == AF_UNSPEC || + conn->cache_af == af)) { + if (last_conn != NULL) + last_conn->next_cached = conn->next_cached; + else + connection_cache = conn->next_cached; + return conn; + } + } + + return NULL; +} + +/* + * Put the connection back into the cache for reuse. + * If the connection is freed due to LRU or if the cache + * is explicitly closed, the given callback is called. + */ +void +fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *)) +{ + conn_t *iter, *last; + int global_count, host_count; + + if (conn->cache_url == NULL || cache_global_limit == 0) { + (*closecb)(conn); + return; + } + + global_count = host_count = 0; + last = NULL; + for (iter = connection_cache; iter; + last = iter, iter = iter->next_cached) { + ++global_count; + if (strcmp(conn->cache_url->host, iter->cache_url->host) == 0) + ++host_count; + if (global_count < cache_global_limit && + host_count < cache_per_host_limit) + continue; + --global_count; + if (last != NULL) + last->next_cached = iter->next_cached; + else + connection_cache = iter->next_cached; + (*iter->cache_close)(iter); + } + + conn->cache_close = closecb; + conn->next_cached = connection_cache; + connection_cache = conn; +} /* * Enable SSL on a connection. @@ -649,9 +742,9 @@ fetch_close(conn_t *conn) { int ret; - if (--conn->ref > 0) - return (0); ret = close(conn->sd); + if (conn->cache_url) + fetchFreeURL(conn->cache_url); free(conn->buf); free(conn); return (ret); diff --git a/net/libfetch/files/common.h b/net/libfetch/files/common.h index 876d68f0afd..08ffd356ef2 100644 --- a/net/libfetch/files/common.h +++ b/net/libfetch/files/common.h @@ -1,4 +1,4 @@ -/* $NetBSD: common.h,v 1.12 2009/08/16 20:31:29 joerg Exp $ */ +/* $NetBSD: common.h,v 1.13 2010/01/22 13:21:09 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav * All rights reserved. @@ -53,6 +53,7 @@ /* Connection */ typedef struct fetchconn conn_t; + struct fetchconn { int sd; /* socket descriptor */ char *buf; /* buffer */ @@ -71,8 +72,12 @@ struct fetchconn { const SSL_METHOD *ssl_meth; /* SSL method */ # endif #endif - int ref; /* reference count */ int is_active; + + struct url *cache_url; + int cache_af; + int (*cache_close)(conn_t *); + conn_t *next_cached; }; /* Structure used for error message lists */ @@ -91,9 +96,10 @@ void fetch_info(const char *, ...); int fetch_default_port(const char *); int fetch_default_proxy_port(const char *); int fetch_bind(int, int, const char *); -conn_t *fetch_connect(const char *, int, int, int); +conn_t *fetch_cache_get(const struct url *, int); +void fetch_cache_put(conn_t *, int (*)(conn_t *)); +conn_t *fetch_connect(struct url *, int, int); conn_t *fetch_reopen(int); -conn_t *fetch_ref(conn_t *); int fetch_ssl(conn_t *, int); ssize_t fetch_read(conn_t *, char *, size_t); int fetch_getln(conn_t *); diff --git a/net/libfetch/files/fetch.3 b/net/libfetch/files/fetch.3 index 1f3138f5ab3..86e97643ea5 100644 --- a/net/libfetch/files/fetch.3 +++ b/net/libfetch/files/fetch.3 @@ -1,5 +1,6 @@ .\"- .\" Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav +.\" Copyright (c) 2010 Joerg Sonnenberger <joerg@NetBSD.org> .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -24,7 +25,7 @@ .\" SUCH DAMAGE. .\" .\" $FreeBSD: fetch.3,v 1.64 2007/12/18 11:03:26 des Exp $ -.\" $NetBSD: fetch.3,v 1.13 2009/10/15 12:36:57 joerg Exp $ +.\" $NetBSD: fetch.3,v 1.14 2010/01/22 13:21:09 joerg Exp $ .\" .Dd February 4, 2009 .Dt FETCH 3 @@ -64,6 +65,8 @@ .Nm fetchUnquotePath , .Nm fetchUnquoteFilename , .Nm fetchStringifyURL , +.Nm fetchConnecitonCacheInit , +.Nm fetchConnecitonCacheClose , .Nm fetch .Nd file transfer functions .Sh LIBRARY @@ -141,6 +144,10 @@ .Fn fetchUnquoteFilename "struct url *u" .Ft char * .Fn fetchStringifyURL "const struct url *u" +.Ft void +.Fn fetchConnectionCacheInit "int global" "int per_host" +.Ft void +.Fn fetchConnectionCacheClose "void" .Sh DESCRIPTION These functions implement a high-level library for retrieving and uploading files using Uniform Resource Locators (URLs). @@ -307,6 +314,15 @@ return a string that should be deallocated with .Fn free after use. .Pp +.Fn fetchConnectionCacheInit +enables the connection cache. +The first argument specifies the global limit on cached connections. +The second argument specifies the host limit. +Entries are considered to specify the same host, if the host name +from the URL is identical, indepent of the address or address family. +.Fn fetchConnectionCacheClose +flushed the connection cache and closes all cached connections. +.Pp .Fn fetchXGet , .Fn fetchGet , .Fn fetchPut , diff --git a/net/libfetch/files/fetch.h b/net/libfetch/files/fetch.h index c2e73e7113f..30754864360 100644 --- a/net/libfetch/files/fetch.h +++ b/net/libfetch/files/fetch.h @@ -1,4 +1,4 @@ -/* $NetBSD: fetch.h,v 1.15 2009/10/15 12:36:57 joerg Exp $ */ +/* $NetBSD: fetch.h,v 1.16 2010/01/22 13:21:09 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav * All rights reserved. @@ -158,6 +158,10 @@ void fetchFreeURLList(struct url_list *); char *fetchUnquotePath(struct url *); char *fetchUnquoteFilename(struct url *); +/* Connection caching */ +void fetchConnectionCacheInit(int, int); +void fetchConnectionCacheClose(void); + /* Authentication */ typedef int (*auth_t)(struct url *); extern auth_t fetchAuthMethod; diff --git a/net/libfetch/files/ftp.c b/net/libfetch/files/ftp.c index 669f44cad3f..a84a22c98f2 100644 --- a/net/libfetch/files/ftp.c +++ b/net/libfetch/files/ftp.c @@ -1,7 +1,7 @@ -/* $NetBSD: ftp.c,v 1.31 2010/01/11 17:23:10 joerg Exp $ */ +/* $NetBSD: ftp.c,v 1.32 2010/01/22 13:21:09 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav - * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org> + * Copyright (c) 2008, 2009, 2010 Joerg Sonnenberger <joerg@NetBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -120,9 +120,6 @@ #define FTP_SYNTAX_ERROR 500 #define FTP_PROTOCOL_ERROR 999 -static struct url cached_host; -static conn_t *cached_connection; - #define isftpreply(foo) \ (isdigit((unsigned char)foo[0]) && \ isdigit((unsigned char)foo[1]) && \ @@ -565,6 +562,13 @@ ftp_writefn(void *v, const void *buf, size_t len) return (-1); } +static int +ftp_disconnect(conn_t *conn) +{ + ftp_cmd(conn, "QUIT"); + return fetch_close(conn); +} + static void ftp_closefn(void *v) { @@ -583,15 +587,11 @@ ftp_closefn(void *v) return; } fetch_close(io->dconn); - io->dir = -1; - io->dconn->is_active = 0; io->dconn = NULL; + io->dir = -1; + io->cconn->is_active = 0; r = ftp_chkerr(io->cconn); - if (io->cconn == cached_connection && io->cconn->ref == 1) { - free(cached_host.doc); - cached_connection = NULL; - } - fetch_close(io->cconn); + fetch_cache_put(io->cconn, ftp_disconnect); free(io); return; } @@ -1001,10 +1001,23 @@ ftp_connect(struct url *url, struct url *purl, const char *flags) /* check for proxy */ if (purl) { /* XXX proxy authentication! */ - conn = fetch_connect(purl->host, purl->port, af, verbose); + /* XXX connetion caching */ + if (!purl->port) + purl->port = fetch_default_port(purl->scheme); + + conn = fetch_connect(purl, af, verbose); } else { /* no proxy, go straight to target */ - conn = fetch_connect(url->host, url->port, af, verbose); + if (!url->port) + url->port = fetch_default_port(url->scheme); + + while ((conn = fetch_cache_get(url, af)) != NULL) { + e = ftp_cmd(conn, "NOOP"); + if (e == FTP_OK) + return conn; + fetch_close(conn); + } + conn = fetch_connect(url, af, verbose); purl = NULL; } @@ -1034,70 +1047,6 @@ fouch: } /* - * Disconnect from server - */ -static void -ftp_disconnect(conn_t *conn) -{ - (void)ftp_cmd(conn, "QUIT"); - if (conn == cached_connection && conn->ref == 1) { - free(cached_host.doc); - cached_host.doc = NULL; - cached_connection = NULL; - } - fetch_close(conn); -} - -/* - * Check if we're already connected - */ -static int -ftp_isconnected(struct url *url) -{ - return (cached_connection - && (cached_connection->is_active == 0) - && (strcmp(url->host, cached_host.host) == 0) - && (strcmp(url->user, cached_host.user) == 0) - && (strcmp(url->pwd, cached_host.pwd) == 0) - && (url->port == cached_host.port)); -} - -/* - * Check the cache, reconnect if no luck - */ -static conn_t * -ftp_cached_connect(struct url *url, struct url *purl, const char *flags) -{ - char *doc; - conn_t *conn; - int e; - - /* set default port */ - if (!url->port) - url->port = fetch_default_port(url->scheme); - - /* try to use previously cached connection */ - if (ftp_isconnected(url)) { - e = ftp_cmd(cached_connection, "NOOP"); - if (e == FTP_OK || e == FTP_SYNTAX_ERROR) - return (fetch_ref(cached_connection)); - } - - /* connect to server */ - if ((conn = ftp_connect(url, purl, flags)) == NULL) - return (NULL); - doc = strdup(url->doc); - if (doc != NULL) { - if (cached_connection && !cached_connection->is_active) - ftp_disconnect(cached_connection); - cached_connection = fetch_ref(conn); - memcpy(&cached_host, url, sizeof(*url)); - cached_host.doc = doc; - } - return (conn); -} - -/* * Check the proxy settings */ static struct url * @@ -1155,7 +1104,7 @@ ftp_request(struct url *url, const char *op, const char *op_arg, } /* connect to server */ - conn = ftp_cached_connect(url, purl, flags); + conn = ftp_connect(url, purl, flags); if (purl) fetchFreeURL(purl); if (conn == NULL) diff --git a/net/libfetch/files/http.c b/net/libfetch/files/http.c index f2b73c5243a..95e7bc84c78 100644 --- a/net/libfetch/files/http.c +++ b/net/libfetch/files/http.c @@ -1,4 +1,4 @@ -/* $NetBSD: http.c,v 1.25 2009/10/15 12:36:57 joerg Exp $ */ +/* $NetBSD: http.c,v 1.26 2010/01/22 13:21:09 joerg Exp $ */ /*- * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav * Copyright (c) 2003 Thomas Klausner <wiz@NetBSD.org> @@ -706,7 +706,7 @@ http_connect(struct url *URL, struct url *purl, const char *flags) return (NULL); } - if ((conn = fetch_connect(URL->host, URL->port, af, verbose)) == NULL) + if ((conn = fetch_connect(URL, af, verbose)) == NULL) /* fetch_connect() has already set an error code */ return (NULL); if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && |