diff options
author | joerg <joerg> | 2009-02-05 16:59:45 +0000 |
---|---|---|
committer | joerg <joerg> | 2009-02-05 16:59:45 +0000 |
commit | 37c2dfa1930c0cc8e5b72d143ff1d92fcf5c7a06 (patch) | |
tree | 1f0bacab7917ad0349c944bb74c1d007b8912e0e /net | |
parent | 70d772c283917c3749dee5e1ccec8241d179ba68 (diff) | |
download | pkgsrc-37c2dfa1930c0cc8e5b72d143ff1d92fcf5c7a06.tar.gz |
libfetch-2.21:
Add support for conditional GET using the 'i' flag. Inspired by
the HTTP support for the same feature in FreeBSD by Murray Stokely, but
mostly rewritten and extended to work for all protocols.
Diffstat (limited to 'net')
-rw-r--r-- | net/libfetch/Makefile | 4 | ||||
-rw-r--r-- | net/libfetch/buildlink3.mk | 4 | ||||
-rw-r--r-- | net/libfetch/files/common.c | 3 | ||||
-rw-r--r-- | net/libfetch/files/fetch.3 | 21 | ||||
-rw-r--r-- | net/libfetch/files/fetch.cat3 | 12 | ||||
-rw-r--r-- | net/libfetch/files/fetch.h | 4 | ||||
-rw-r--r-- | net/libfetch/files/file.c | 19 | ||||
-rw-r--r-- | net/libfetch/files/ftp.c | 18 | ||||
-rw-r--r-- | net/libfetch/files/http.c | 37 | ||||
-rw-r--r-- | net/libfetch/files/http.errors | 4 |
10 files changed, 104 insertions, 22 deletions
diff --git a/net/libfetch/Makefile b/net/libfetch/Makefile index 870399468ed..337ed723529 100644 --- a/net/libfetch/Makefile +++ b/net/libfetch/Makefile @@ -1,7 +1,7 @@ -# $NetBSD: Makefile,v 1.23 2008/11/04 16:14:24 joerg Exp $ +# $NetBSD: Makefile,v 1.24 2009/02/05 16:59:45 joerg Exp $ # -DISTNAME= libfetch-2.20 +DISTNAME= libfetch-2.21 CATEGORIES= net MASTER_SITES= # empty DISTFILES= # empty diff --git a/net/libfetch/buildlink3.mk b/net/libfetch/buildlink3.mk index 817efac3334..5de5816a9c2 100644 --- a/net/libfetch/buildlink3.mk +++ b/net/libfetch/buildlink3.mk @@ -1,4 +1,4 @@ -# $NetBSD: buildlink3.mk,v 1.4 2008/10/06 22:03:43 joerg Exp $ +# $NetBSD: buildlink3.mk,v 1.5 2009/02/05 16:59:45 joerg Exp $ BUILDLINK_DEPMETHOD.libfetch?= build @@ -14,7 +14,7 @@ BUILDLINK_PACKAGES+= libfetch BUILDLINK_ORDER:= ${BUILDLINK_ORDER} ${BUILDLINK_DEPTH}libfetch .if ${LIBFETCH_BUILDLINK3_MK} == "+" -BUILDLINK_API_DEPENDS.libfetch+= libfetch>=2.16 +BUILDLINK_API_DEPENDS.libfetch+= libfetch>=2.21 BUILDLINK_PKGSRCDIR.libfetch?= ../../net/libfetch .endif # LIBFETCH_BUILDLINK3_MK diff --git a/net/libfetch/files/common.c b/net/libfetch/files/common.c index 8498cc1eaf4..9a4325acc1d 100644 --- a/net/libfetch/files/common.c +++ b/net/libfetch/files/common.c @@ -1,4 +1,4 @@ -/* $NetBSD: common.c,v 1.16 2008/10/07 15:50:00 joerg Exp $ */ +/* $NetBSD: common.c,v 1.17 2009/02/05 16:59:45 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org> @@ -728,6 +728,7 @@ fetch_add_entry(struct url_list *ue, struct url *base, const char *name, tmp->offset = 0; tmp->length = 0; + tmp->last_modified = -1; ++ue->length; diff --git a/net/libfetch/files/fetch.3 b/net/libfetch/files/fetch.3 index 2c87a21731e..3544f7cae26 100644 --- a/net/libfetch/files/fetch.3 +++ b/net/libfetch/files/fetch.3 @@ -24,9 +24,9 @@ .\" SUCH DAMAGE. .\" .\" $FreeBSD: fetch.3,v 1.64 2007/12/18 11:03:26 des Exp $ -.\" $NetBSD: fetch.3,v 1.10 2008/10/08 15:11:21 joerg Exp $ +.\" $NetBSD: fetch.3,v 1.11 2009/02/05 16:59:45 joerg Exp $ .\" -.Dd April 25, 2008 +.Dd February 4, 2009 .Dt FETCH 3 .Os .Sh NAME @@ -194,6 +194,7 @@ struct url { char *doc; off_t offset; size_t length; + time_t last_modified; }; .Ed .Pp @@ -204,6 +205,9 @@ and .Fn fetchParseURL should be freed using .Fn fetchFreeURL . +The size of +.Vt struct URL +is not part of the ABI. .Pp .Fn fetchXGetURL , .Fn fetchGetURL , @@ -326,6 +330,19 @@ or functions is read-only, and that a stream returned by one of the .Fn fetchPutXXX functions is write-only. +.Sh PROTOCOL INDEPENDENT FLAGS +If the +.Ql i +(if-modified-since) flag is specified, the library will try to fetch +the content only if it is newer than +.Va last_modified . +For HTTP an +.Li If-Modified-Since +HTTP header is sent. +For FTP a +.Li MTDM +command is sent first and compared locally. +For FILE the source file is compared. .Sh FILE SCHEME .Fn fetchXGetFile , .Fn fetchGetFile , diff --git a/net/libfetch/files/fetch.cat3 b/net/libfetch/files/fetch.cat3 index 63156b90ad8..8057a34e717 100644 --- a/net/libfetch/files/fetch.cat3 +++ b/net/libfetch/files/fetch.cat3 @@ -157,10 +157,12 @@ DDEESSCCRRIIPPTTIIOONN char *doc; off_t offset; size_t length; + time_t last_modified; }; The pointer returned by ffeettcchhMMaakkeeUURRLL(), ffeettcchhCCooppyyUURRLL(), and - ffeettcchhPPaarrsseeUURRLL() should be freed using ffeettcchhFFrreeeeUURRLL(). + ffeettcchhPPaarrsseeUURRLL() should be freed using ffeettcchhFFrreeeeUURRLL(). The size of _s_t_r_u_c_t + _U_R_L is not part of the ABI. ffeettcchhXXGGeettUURRLL(), ffeettcchhGGeettUURRLL(), and ffeettcchhPPuuttUURRLL() constitute the recom- mended interface to the ffeettcchh library. They examine the URL passed to @@ -225,6 +227,12 @@ DDEESSCCRRIIPPTTIIOONN ffeettcchhGGeettXXXXXX() functions is read-only, and that a stream returned by one of the ffeettcchhPPuuttXXXXXX() functions is write-only. +PPRROOTTOOCCOOLL IINNDDEEPPEENNDDEENNTT FFLLAAGGSS + If the `i' (if-modified-since) flag is specified, the library will try to + fetch the content only if it is newer than _l_a_s_t___m_o_d_i_f_i_e_d. For HTTP an + If-Modified-Since HTTP header is sent. For FTP a MTDM command is sent + first and compared locally. For FILE the source file is compared. + FFIILLEE SSCCHHEEMMEE ffeettcchhXXGGeettFFiillee(), ffeettcchhGGeettFFiillee(), and ffeettcchhPPuuttFFiillee() provide access to documents which are files in a locally mounted file system. Only the @@ -495,4 +503,4 @@ BBUUGGSS Some parts of the code are not fully reentrant. -NetBSD 4.0 April 25, 2008 NetBSD 4.0 +NetBSD 5.0 February 4, 2009 NetBSD 5.0 diff --git a/net/libfetch/files/fetch.h b/net/libfetch/files/fetch.h index 9772306f9e4..6ad8063afbd 100644 --- a/net/libfetch/files/fetch.h +++ b/net/libfetch/files/fetch.h @@ -1,4 +1,4 @@ -/* $NetBSD: fetch.h,v 1.12 2008/10/06 12:58:29 joerg Exp $ */ +/* $NetBSD: fetch.h,v 1.13 2009/02/05 16:59:45 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav * All rights reserved. @@ -54,6 +54,7 @@ struct url { char *doc; off_t offset; size_t length; + time_t last_modified; }; struct url_stat { @@ -94,6 +95,7 @@ struct url_list { #define FETCH_UNKNOWN 17 #define FETCH_URL 18 #define FETCH_VERBOSE 19 +#define FETCH_UNCHANGED 20 #if defined(__cplusplus) extern "C" { diff --git a/net/libfetch/files/file.c b/net/libfetch/files/file.c index d99e08b42f0..fb858803349 100644 --- a/net/libfetch/files/file.c +++ b/net/libfetch/files/file.c @@ -1,7 +1,7 @@ -/* $NetBSD: file.c,v 1.12 2008/10/06 12:58:29 joerg Exp $ */ +/* $NetBSD: file.c,v 1.13 2009/02/05 16:59:45 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav - * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org> + * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -79,7 +79,12 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags) { char *path; fetchIO *f; - int fd, *cookie; + struct url_stat local_us; + int if_modified_since, fd, *cookie; + + if_modified_since = CHECK_FLAG('i'); + if (if_modified_since && us == NULL) + us = &local_us; if ((path = fetchUnquotePath(u)) == NULL) { fetch_syserr(); @@ -95,6 +100,14 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags) if (us && fetch_stat_file(fd, us) == -1) { close(fd); + fetch_syserr(); + return NULL; + } + + if (if_modified_since && u->last_modified > 0 && + u->last_modified >= us->mtime) { + fetchLastErrCode = FETCH_UNCHANGED; + snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged"); return NULL; } diff --git a/net/libfetch/files/ftp.c b/net/libfetch/files/ftp.c index be5fee7498e..ee286faf81b 100644 --- a/net/libfetch/files/ftp.c +++ b/net/libfetch/files/ftp.c @@ -1,7 +1,7 @@ -/* $NetBSD: ftp.c,v 1.25 2008/12/02 16:59:03 joerg Exp $ */ +/* $NetBSD: ftp.c,v 1.26 2009/02/05 16:59:45 joerg Exp $ */ /*- * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav - * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org> + * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -1127,7 +1127,8 @@ ftp_request(struct url *url, const char *op, const char *op_arg, fetchIO *f; char *path; conn_t *conn; - int oflag; + int if_modified_since, oflag; + struct url_stat local_us; /* check if we should use HTTP instead */ if (purl && strcasecmp(purl->scheme, SCHEME_HTTP) == 0) { @@ -1159,6 +1160,10 @@ ftp_request(struct url *url, const char *op, const char *op_arg, return (NULL); } + if_modified_since = CHECK_FLAG('i'); + if (if_modified_since && us == NULL) + us = &local_us; + /* stat file */ if (us && ftp_stat(conn, path, us) == -1 && fetchLastErrCode != FETCH_PROTO @@ -1167,6 +1172,13 @@ ftp_request(struct url *url, const char *op, const char *op_arg, return (NULL); } + if (if_modified_since && url->last_modified > 0 && + url->last_modified >= us->mtime) { + fetchLastErrCode = FETCH_UNCHANGED; + snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged"); + return NULL; + } + /* just a stat */ if (strcmp(op, "STAT") == 0) return fetchIO_unopen(NULL, NULL, NULL, NULL); diff --git a/net/libfetch/files/http.c b/net/libfetch/files/http.c index c1df4b8f1ea..b1c5745b092 100644 --- a/net/libfetch/files/http.c +++ b/net/libfetch/files/http.c @@ -1,8 +1,8 @@ -/* $NetBSD: http.c,v 1.20 2008/10/06 12:58:29 joerg Exp $ */ +/* $NetBSD: http.c,v 1.21 2009/02/05 16:59:45 joerg Exp $ */ /*- * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav * Copyright (c) 2003 Thomas Klausner <wiz@NetBSD.org> - * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org> + * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -112,6 +112,7 @@ #define HTTP_MOVED_PERM 301 #define HTTP_MOVED_TEMP 302 #define HTTP_SEE_OTHER 303 +#define HTTP_NOT_MODIFIED 304 #define HTTP_TEMP_REDIRECT 307 #define HTTP_NEED_AUTH 401 #define HTTP_NEED_PROXY_AUTH 407 @@ -745,6 +746,21 @@ http_get_proxy(struct url * url, const char *flags) return (NULL); } +static void +set_if_modified_since(conn_t *conn, time_t last_modified) +{ + static const char weekdays[] = "SunMonTueWedThuFriSat"; + static const char months[] = "JanFebMarAprMayJunJulAugSepOctNovDec"; + struct tm tm; + char buf[80]; + gmtime_r(&last_modified, &tm); + snprintf(buf, sizeof(buf), "%.3s, %02d %.3s %4d %02d:%02d:%02d GMT", + weekdays + tm.tm_wday * 3, tm.tm_mday, months + tm.tm_mon * 3, + tm.tm_year + 1900, tm.tm_hour, tm.tm_min, tm.tm_sec); + http_cmd(conn, "If-Modified-Since: %s", buf); +} + + /***************************************************************************** * Core */ @@ -761,7 +777,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us, { conn_t *conn; struct url *url, *new; - int chunked, direct, need_auth, noredirect, verbose; + int chunked, direct, if_modified_since, need_auth, noredirect, verbose; int e, i, n, val; off_t offset, clength, length, size; time_t mtime; @@ -773,6 +789,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us, direct = CHECK_FLAG('d'); noredirect = CHECK_FLAG('A'); verbose = CHECK_FLAG('v'); + if_modified_since = CHECK_FLAG('i'); if (direct && purl) { fetchFreeURL(purl); @@ -841,6 +858,9 @@ http_request(struct url *URL, const char *op, struct url_stat *us, op, url->doc); } + if (if_modified_since && url->last_modified > 0) + set_if_modified_since(conn, url->last_modified); + /* virtual host */ http_cmd(conn, "Host: %s", host); @@ -904,6 +924,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us, switch (http_get_reply(conn)) { case HTTP_OK: case HTTP_PARTIAL: + case HTTP_NOT_MODIFIED: /* fine */ break; case HTTP_MOVED_PERM: @@ -1036,7 +1057,10 @@ http_request(struct url *URL, const char *op, struct url_stat *us, } /* we have a hit or an error */ - if (conn->err == HTTP_OK || conn->err == HTTP_PARTIAL || HTTP_ERROR(conn->err)) + if (conn->err == HTTP_OK || + conn->err == HTTP_PARTIAL || + conn->err == HTTP_NOT_MODIFIED || + HTTP_ERROR(conn->err)) break; /* all other cases: we got a redirect */ @@ -1089,6 +1113,11 @@ http_request(struct url *URL, const char *op, struct url_stat *us, URL->offset = offset; URL->length = clength; + if (conn->err == HTTP_NOT_MODIFIED) { + http_seterr(HTTP_NOT_MODIFIED); + return (NULL); + } + /* wrap it up in a fetchIO */ if ((f = http_funopen(conn, chunked)) == NULL) { fetch_syserr(); diff --git a/net/libfetch/files/http.errors b/net/libfetch/files/http.errors index bdf51d0e335..004aac27b8d 100644 --- a/net/libfetch/files/http.errors +++ b/net/libfetch/files/http.errors @@ -1,5 +1,5 @@ # $FreeBSD: http.errors,v 1.5 2001/05/23 18:52:02 des Exp $ -# $NetBSD: http.errors,v 1.2 2008/10/06 12:58:29 joerg Exp $ +# $NetBSD: http.errors,v 1.3 2009/02/05 16:59:45 joerg Exp $ # # This list is taken from RFC 2068. # @@ -16,7 +16,7 @@ 301 MOVED Moved Permanently 302 MOVED Moved Temporarily 303 MOVED See Other -304 OK Not Modified +304 UNCHANGED Not Modified 305 INFO Use Proxy 307 MOVED Temporary Redirect 400 PROTO Bad Request |