diff options
-rw-r--r-- | net/libfetch/Makefile | 4 | ||||
-rw-r--r-- | net/libfetch/buildlink3.mk | 5 | ||||
-rw-r--r-- | net/libfetch/files/common.c | 137 | ||||
-rw-r--r-- | net/libfetch/files/common.h | 14 | ||||
-rw-r--r-- | net/libfetch/files/fetch.3 | 18 | ||||
-rw-r--r-- | net/libfetch/files/fetch.h | 6 | ||||
-rw-r--r-- | net/libfetch/files/ftp.c | 107 | ||||
-rw-r--r-- | net/libfetch/files/http.c | 4 |
8 files changed, 181 insertions, 114 deletions
diff --git a/net/libfetch/Makefile b/net/libfetch/Makefile index 86e49d4624b..27eab843918 100644 --- a/net/libfetch/Makefile +++ b/net/libfetch/Makefile @@ -1,7 +1,7 @@ -# $NetBSD: Makefile,v 1.34 2010/01/11 17:23:10 joerg Exp $ +# $NetBSD: Makefile,v 1.35 2010/01/22 13:21:09 joerg Exp $ # -DISTNAME= libfetch-2.27 +DISTNAME= libfetch-2.28 CATEGORIES= net MASTER_SITES= # empty DISTFILES= # empty diff --git a/net/libfetch/buildlink3.mk b/net/libfetch/buildlink3.mk index dbb90022876..5522d0cbfcb 100644 --- a/net/libfetch/buildlink3.mk +++ b/net/libfetch/buildlink3.mk @@ -1,4 +1,4 @@ -# $NetBSD: buildlink3.mk,v 1.7 2010/01/17 12:02:33 wiz Exp $ +# $NetBSD: buildlink3.mk,v 1.8 2010/01/22 13:21:09 joerg Exp $ BUILDLINK_DEPMETHOD.libfetch?= build @@ -7,8 +7,7 @@ BUILDLINK_TREE+= libfetch .if !defined(LIBFETCH_BUILDLINK3_MK) LIBFETCH_BUILDLINK3_MK:= -BUILDLINK_API_DEPENDS.libfetch+= libfetch>=2.21 -BUILDLINK_ABI_DEPENDS.libfetch?= libfetch>=2.27 +BUILDLINK_API_DEPENDS.libfetch+= libfetch>=2.28 BUILDLINK_PKGSRCDIR.libfetch?= ../../net/libfetch pkgbase := libfetch diff --git a/net/libfetch/files/common.c b/net/libfetch/files/common.c index 000444d3414..1bb555374a1 100644 --- a/net/libfetch/files/common.c +++ b/net/libfetch/files/common.c @@ -1,7 +1,7 @@ -/* $NetBSD: common.c,v 1.21 2009/10/15 12:36:57 joerg Exp $ */ +/* $NetBSD: common.c,v 1.22 2010/01/22 13:21:09 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav - * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org> + * Copyright (c) 2008, 2010 Joerg Sonnenberger <joerg@NetBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -234,23 +234,11 @@ fetch_reopen(int sd) /* allocate and fill connection structure */ if ((conn = calloc(1, sizeof(*conn))) == NULL) return (NULL); + conn->cache_url = NULL; conn->next_buf = NULL; conn->next_len = 0; conn->sd = sd; conn->is_active = 0; - ++conn->ref; - return (conn); -} - - -/* - * Bump a connection's reference count. - */ -conn_t * -fetch_ref(conn_t *conn) -{ - - ++conn->ref; return (conn); } @@ -281,7 +269,7 @@ fetch_bind(int sd, int af, const char *addr) * Establish a TCP connection to the specified port on the specified host. */ conn_t * -fetch_connect(const char *host, int port, int af, int verbose) +fetch_connect(struct url *url, int af, int verbose) { conn_t *conn; char pbuf[10]; @@ -290,22 +278,22 @@ fetch_connect(const char *host, int port, int af, int verbose) int sd, error; if (verbose) - fetch_info("looking up %s", host); + fetch_info("looking up %s", url->host); /* look up host name and set up socket address structure */ - snprintf(pbuf, sizeof(pbuf), "%d", port); + snprintf(pbuf, sizeof(pbuf), "%d", url->port); memset(&hints, 0, sizeof(hints)); hints.ai_family = af; hints.ai_socktype = SOCK_STREAM; hints.ai_protocol = 0; - if ((error = getaddrinfo(host, pbuf, &hints, &res0)) != 0) { + if ((error = getaddrinfo(url->host, pbuf, &hints, &res0)) != 0) { netdb_seterr(error); return (NULL); } bindaddr = getenv("FETCH_BIND_ADDRESS"); if (verbose) - fetch_info("connecting to %s:%d", host, port); + fetch_info("connecting to %s:%d", url->host, url->port); /* try to connect */ for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) { @@ -332,9 +320,114 @@ fetch_connect(const char *host, int port, int af, int verbose) fetch_syserr(); close(sd); } + conn->cache_url = fetchCopyURL(url); + conn->cache_af = af; return (conn); } +static conn_t *connection_cache; +static int cache_global_limit = 0; +static int cache_per_host_limit = 0; + +/* + * Initialise cache with the given limits. + */ +void +fetchConnectionCacheInit(int global_limit, int per_host_limit) +{ + + if (global_limit < 0) + cache_global_limit = INT_MAX; + else if (per_host_limit > global_limit) + cache_global_limit = per_host_limit; + else + cache_global_limit = global_limit; + if (per_host_limit < 0) + cache_per_host_limit = INT_MAX; + else + cache_per_host_limit = per_host_limit; +} + +/* + * Flush cache and free all associated resources. + */ +void +fetchConnectionCacheClose(void) +{ + conn_t *conn; + + while ((conn = connection_cache) != NULL) { + connection_cache = conn->next_cached; + (*conn->cache_close)(conn); + } +} + +/* + * Check connection cache for an existing entry matching + * protocol/host/port/user/password/family. + */ +conn_t * +fetch_cache_get(const struct url *url, int af) +{ + conn_t *conn, *last_conn = NULL; + + for (conn = connection_cache; conn; conn = conn->next_cached) { + if (conn->cache_url->port == url->port && + strcmp(conn->cache_url->scheme, url->scheme) == 0 && + strcmp(conn->cache_url->host, url->host) == 0 && + strcmp(conn->cache_url->user, url->user) == 0 && + strcmp(conn->cache_url->pwd, url->pwd) == 0 && + (conn->cache_af == AF_UNSPEC || af == AF_UNSPEC || + conn->cache_af == af)) { + if (last_conn != NULL) + last_conn->next_cached = conn->next_cached; + else + connection_cache = conn->next_cached; + return conn; + } + } + + return NULL; +} + +/* + * Put the connection back into the cache for reuse. + * If the connection is freed due to LRU or if the cache + * is explicitly closed, the given callback is called. + */ +void +fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *)) +{ + conn_t *iter, *last; + int global_count, host_count; + + if (conn->cache_url == NULL || cache_global_limit == 0) { + (*closecb)(conn); + return; + } + + global_count = host_count = 0; + last = NULL; + for (iter = connection_cache; iter; + last = iter, iter = iter->next_cached) { + ++global_count; + if (strcmp(conn->cache_url->host, iter->cache_url->host) == 0) + ++host_count; + if (global_count < cache_global_limit && + host_count < cache_per_host_limit) + continue; + --global_count; + if (last != NULL) + last->next_cached = iter->next_cached; + else + connection_cache = iter->next_cached; + (*iter->cache_close)(iter); + } + + conn->cache_close = closecb; + conn->next_cached = connection_cache; + connection_cache = conn; +} /* * Enable SSL on a connection. @@ -649,9 +742,9 @@ fetch_close(conn_t *conn) { int ret; - if (--conn->ref > 0) - return (0); ret = close(conn->sd); + if (conn->cache_url) + fetchFreeURL(conn->cache_url); free(conn->buf); free(conn); return (ret); diff --git a/net/libfetch/files/common.h b/net/libfetch/files/common.h index 876d68f0afd..08ffd356ef2 100644 --- a/net/libfetch/files/common.h +++ b/net/libfetch/files/common.h @@ -1,4 +1,4 @@ -/* $NetBSD: common.h,v 1.12 2009/08/16 20:31:29 joerg Exp $ */ +/* $NetBSD: common.h,v 1.13 2010/01/22 13:21:09 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav * All rights reserved. @@ -53,6 +53,7 @@ /* Connection */ typedef struct fetchconn conn_t; + struct fetchconn { int sd; /* socket descriptor */ char *buf; /* buffer */ @@ -71,8 +72,12 @@ struct fetchconn { const SSL_METHOD *ssl_meth; /* SSL method */ # endif #endif - int ref; /* reference count */ int is_active; + + struct url *cache_url; + int cache_af; + int (*cache_close)(conn_t *); + conn_t *next_cached; }; /* Structure used for error message lists */ @@ -91,9 +96,10 @@ void fetch_info(const char *, ...); int fetch_default_port(const char *); int fetch_default_proxy_port(const char *); int fetch_bind(int, int, const char *); -conn_t *fetch_connect(const char *, int, int, int); +conn_t *fetch_cache_get(const struct url *, int); +void fetch_cache_put(conn_t *, int (*)(conn_t *)); +conn_t *fetch_connect(struct url *, int, int); conn_t *fetch_reopen(int); -conn_t *fetch_ref(conn_t *); int fetch_ssl(conn_t *, int); ssize_t fetch_read(conn_t *, char *, size_t); int fetch_getln(conn_t *); diff --git a/net/libfetch/files/fetch.3 b/net/libfetch/files/fetch.3 index 1f3138f5ab3..86e97643ea5 100644 --- a/net/libfetch/files/fetch.3 +++ b/net/libfetch/files/fetch.3 @@ -1,5 +1,6 @@ .\"- .\" Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav +.\" Copyright (c) 2010 Joerg Sonnenberger <joerg@NetBSD.org> .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without @@ -24,7 +25,7 @@ .\" SUCH DAMAGE. .\" .\" $FreeBSD: fetch.3,v 1.64 2007/12/18 11:03:26 des Exp $ -.\" $NetBSD: fetch.3,v 1.13 2009/10/15 12:36:57 joerg Exp $ +.\" $NetBSD: fetch.3,v 1.14 2010/01/22 13:21:09 joerg Exp $ .\" .Dd February 4, 2009 .Dt FETCH 3 @@ -64,6 +65,8 @@ .Nm fetchUnquotePath , .Nm fetchUnquoteFilename , .Nm fetchStringifyURL , +.Nm fetchConnecitonCacheInit , +.Nm fetchConnecitonCacheClose , .Nm fetch .Nd file transfer functions .Sh LIBRARY @@ -141,6 +144,10 @@ .Fn fetchUnquoteFilename "struct url *u" .Ft char * .Fn fetchStringifyURL "const struct url *u" +.Ft void +.Fn fetchConnectionCacheInit "int global" "int per_host" +.Ft void +.Fn fetchConnectionCacheClose "void" .Sh DESCRIPTION These functions implement a high-level library for retrieving and uploading files using Uniform Resource Locators (URLs). @@ -307,6 +314,15 @@ return a string that should be deallocated with .Fn free after use. .Pp +.Fn fetchConnectionCacheInit +enables the connection cache. +The first argument specifies the global limit on cached connections. +The second argument specifies the host limit. +Entries are considered to specify the same host, if the host name +from the URL is identical, indepent of the address or address family. +.Fn fetchConnectionCacheClose +flushed the connection cache and closes all cached connections. +.Pp .Fn fetchXGet , .Fn fetchGet , .Fn fetchPut , diff --git a/net/libfetch/files/fetch.h b/net/libfetch/files/fetch.h index c2e73e7113f..30754864360 100644 --- a/net/libfetch/files/fetch.h +++ b/net/libfetch/files/fetch.h @@ -1,4 +1,4 @@ -/* $NetBSD: fetch.h,v 1.15 2009/10/15 12:36:57 joerg Exp $ */ +/* $NetBSD: fetch.h,v 1.16 2010/01/22 13:21:09 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav * All rights reserved. @@ -158,6 +158,10 @@ void fetchFreeURLList(struct url_list *); char *fetchUnquotePath(struct url *); char *fetchUnquoteFilename(struct url *); +/* Connection caching */ +void fetchConnectionCacheInit(int, int); +void fetchConnectionCacheClose(void); + /* Authentication */ typedef int (*auth_t)(struct url *); extern auth_t fetchAuthMethod; diff --git a/net/libfetch/files/ftp.c b/net/libfetch/files/ftp.c index 669f44cad3f..a84a22c98f2 100644 --- a/net/libfetch/files/ftp.c +++ b/net/libfetch/files/ftp.c @@ -1,7 +1,7 @@ -/* $NetBSD: ftp.c,v 1.31 2010/01/11 17:23:10 joerg Exp $ */ +/* $NetBSD: ftp.c,v 1.32 2010/01/22 13:21:09 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav - * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org> + * Copyright (c) 2008, 2009, 2010 Joerg Sonnenberger <joerg@NetBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -120,9 +120,6 @@ #define FTP_SYNTAX_ERROR 500 #define FTP_PROTOCOL_ERROR 999 -static struct url cached_host; -static conn_t *cached_connection; - #define isftpreply(foo) \ (isdigit((unsigned char)foo[0]) && \ isdigit((unsigned char)foo[1]) && \ @@ -565,6 +562,13 @@ ftp_writefn(void *v, const void *buf, size_t len) return (-1); } +static int +ftp_disconnect(conn_t *conn) +{ + ftp_cmd(conn, "QUIT"); + return fetch_close(conn); +} + static void ftp_closefn(void *v) { @@ -583,15 +587,11 @@ ftp_closefn(void *v) return; } fetch_close(io->dconn); - io->dir = -1; - io->dconn->is_active = 0; io->dconn = NULL; + io->dir = -1; + io->cconn->is_active = 0; r = ftp_chkerr(io->cconn); - if (io->cconn == cached_connection && io->cconn->ref == 1) { - free(cached_host.doc); - cached_connection = NULL; - } - fetch_close(io->cconn); + fetch_cache_put(io->cconn, ftp_disconnect); free(io); return; } @@ -1001,10 +1001,23 @@ ftp_connect(struct url *url, struct url *purl, const char *flags) /* check for proxy */ if (purl) { /* XXX proxy authentication! */ - conn = fetch_connect(purl->host, purl->port, af, verbose); + /* XXX connetion caching */ + if (!purl->port) + purl->port = fetch_default_port(purl->scheme); + + conn = fetch_connect(purl, af, verbose); } else { /* no proxy, go straight to target */ - conn = fetch_connect(url->host, url->port, af, verbose); + if (!url->port) + url->port = fetch_default_port(url->scheme); + + while ((conn = fetch_cache_get(url, af)) != NULL) { + e = ftp_cmd(conn, "NOOP"); + if (e == FTP_OK) + return conn; + fetch_close(conn); + } + conn = fetch_connect(url, af, verbose); purl = NULL; } @@ -1034,70 +1047,6 @@ fouch: } /* - * Disconnect from server - */ -static void -ftp_disconnect(conn_t *conn) -{ - (void)ftp_cmd(conn, "QUIT"); - if (conn == cached_connection && conn->ref == 1) { - free(cached_host.doc); - cached_host.doc = NULL; - cached_connection = NULL; - } - fetch_close(conn); -} - -/* - * Check if we're already connected - */ -static int -ftp_isconnected(struct url *url) -{ - return (cached_connection - && (cached_connection->is_active == 0) - && (strcmp(url->host, cached_host.host) == 0) - && (strcmp(url->user, cached_host.user) == 0) - && (strcmp(url->pwd, cached_host.pwd) == 0) - && (url->port == cached_host.port)); -} - -/* - * Check the cache, reconnect if no luck - */ -static conn_t * -ftp_cached_connect(struct url *url, struct url *purl, const char *flags) -{ - char *doc; - conn_t *conn; - int e; - - /* set default port */ - if (!url->port) - url->port = fetch_default_port(url->scheme); - - /* try to use previously cached connection */ - if (ftp_isconnected(url)) { - e = ftp_cmd(cached_connection, "NOOP"); - if (e == FTP_OK || e == FTP_SYNTAX_ERROR) - return (fetch_ref(cached_connection)); - } - - /* connect to server */ - if ((conn = ftp_connect(url, purl, flags)) == NULL) - return (NULL); - doc = strdup(url->doc); - if (doc != NULL) { - if (cached_connection && !cached_connection->is_active) - ftp_disconnect(cached_connection); - cached_connection = fetch_ref(conn); - memcpy(&cached_host, url, sizeof(*url)); - cached_host.doc = doc; - } - return (conn); -} - -/* * Check the proxy settings */ static struct url * @@ -1155,7 +1104,7 @@ ftp_request(struct url *url, const char *op, const char *op_arg, } /* connect to server */ - conn = ftp_cached_connect(url, purl, flags); + conn = ftp_connect(url, purl, flags); if (purl) fetchFreeURL(purl); if (conn == NULL) diff --git a/net/libfetch/files/http.c b/net/libfetch/files/http.c index f2b73c5243a..95e7bc84c78 100644 --- a/net/libfetch/files/http.c +++ b/net/libfetch/files/http.c @@ -1,4 +1,4 @@ -/* $NetBSD: http.c,v 1.25 2009/10/15 12:36:57 joerg Exp $ */ +/* $NetBSD: http.c,v 1.26 2010/01/22 13:21:09 joerg Exp $ */ /*- * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav * Copyright (c) 2003 Thomas Klausner <wiz@NetBSD.org> @@ -706,7 +706,7 @@ http_connect(struct url *URL, struct url *purl, const char *flags) return (NULL); } - if ((conn = fetch_connect(URL->host, URL->port, af, verbose)) == NULL) + if ((conn = fetch_connect(URL, af, verbose)) == NULL) /* fetch_connect() has already set an error code */ return (NULL); if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && |