libfetch-2.21:

Add support for conditional GET using the 'i' flag. Inspired by
the HTTP support for the same feature in FreeBSD by Murray Stokely, but
mostly rewritten and extended to work for all protocols.

10 files changed, 104 insertions, 22 deletions

diff --git a/net/libfetch/Makefile b/net/libfetch/Makefile
index 870399468ed..337ed723529 100644
--- a/net/libfetch/Makefile
+++ b/net/libfetch/Makefile
@@ -1,7 +1,7 @@
-# $NetBSD: Makefile,v 1.23 2008/11/04 16:14:24 joerg Exp $
+# $NetBSD: Makefile,v 1.24 2009/02/05 16:59:45 joerg Exp $
 #
 
-DISTNAME=	libfetch-2.20
+DISTNAME=	libfetch-2.21
 CATEGORIES=	net
 MASTER_SITES=	# empty
 DISTFILES=	# empty
diff --git a/net/libfetch/buildlink3.mk b/net/libfetch/buildlink3.mk
index 817efac3334..5de5816a9c2 100644
--- a/net/libfetch/buildlink3.mk
+++ b/net/libfetch/buildlink3.mk
@@ -1,4 +1,4 @@
-# $NetBSD: buildlink3.mk,v 1.4 2008/10/06 22:03:43 joerg Exp $
+# $NetBSD: buildlink3.mk,v 1.5 2009/02/05 16:59:45 joerg Exp $
 
 BUILDLINK_DEPMETHOD.libfetch?=	build
 
@@ -14,7 +14,7 @@ BUILDLINK_PACKAGES+=	libfetch
 BUILDLINK_ORDER:=	${BUILDLINK_ORDER} ${BUILDLINK_DEPTH}libfetch
 
 .if ${LIBFETCH_BUILDLINK3_MK} == "+"
-BUILDLINK_API_DEPENDS.libfetch+=	libfetch>=2.16
+BUILDLINK_API_DEPENDS.libfetch+=	libfetch>=2.21
 BUILDLINK_PKGSRCDIR.libfetch?=	../../net/libfetch
 .endif	# LIBFETCH_BUILDLINK3_MK
 
diff --git a/net/libfetch/files/common.c b/net/libfetch/files/common.c
index 8498cc1eaf4..9a4325acc1d 100644
--- a/net/libfetch/files/common.c
+++ b/net/libfetch/files/common.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: common.c,v 1.16 2008/10/07 15:50:00 joerg Exp $	*/
+/*	$NetBSD: common.c,v 1.17 2009/02/05 16:59:45 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
@@ -728,6 +728,7 @@ fetch_add_entry(struct url_list *ue, struct url *base, const char *name,
 
 	tmp->offset = 0;
 	tmp->length = 0;
+	tmp->last_modified = -1;
 
 	++ue->length;
 
diff --git a/net/libfetch/files/fetch.3 b/net/libfetch/files/fetch.3
index 2c87a21731e..3544f7cae26 100644
--- a/net/libfetch/files/fetch.3
+++ b/net/libfetch/files/fetch.3
@@ -24,9 +24,9 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD: fetch.3,v 1.64 2007/12/18 11:03:26 des Exp $
-.\" $NetBSD: fetch.3,v 1.10 2008/10/08 15:11:21 joerg Exp $
+.\" $NetBSD: fetch.3,v 1.11 2009/02/05 16:59:45 joerg Exp $
 .\"
-.Dd April 25, 2008
+.Dd February 4, 2009
 .Dt FETCH 3
 .Os
 .Sh NAME
@@ -194,6 +194,7 @@ struct url {
     char	*doc;
     off_t	 offset;
     size_t	 length;
+    time_t	 last_modified;
 };
 .Ed
 .Pp
@@ -204,6 +205,9 @@ and
 .Fn fetchParseURL
 should be freed using
 .Fn fetchFreeURL .
+The size of
+.Vt struct URL
+is not part of the ABI.
 .Pp
 .Fn fetchXGetURL ,
 .Fn fetchGetURL ,
@@ -326,6 +330,19 @@ or
 functions is read-only, and that a stream returned by one of the
 .Fn fetchPutXXX
 functions is write-only.
+.Sh PROTOCOL INDEPENDENT FLAGS
+If the
+.Ql i
+(if-modified-since) flag is specified, the library will try to fetch
+the content only if it is newer than
+.Va last_modified .
+For HTTP an
+.Li If-Modified-Since
+HTTP header is sent.
+For FTP a
+.Li MTDM
+command is sent first and compared locally.
+For FILE the source file is compared.
 .Sh FILE SCHEME
 .Fn fetchXGetFile ,
 .Fn fetchGetFile ,
diff --git a/net/libfetch/files/fetch.cat3 b/net/libfetch/files/fetch.cat3
index 63156b90ad8..8057a34e717 100644
--- a/net/libfetch/files/fetch.cat3
+++ b/net/libfetch/files/fetch.cat3
@@ -157,10 +157,12 @@ DDEESSCCRRIIPPTTIIOONN
          char        *doc;
          off_t        offset;
          size_t       length;
+         time_t       last_modified;
      };
 
      The pointer returned by ffeettcchhMMaakkeeUURRLL(), ffeettcchhCCooppyyUURRLL(), and
-     ffeettcchhPPaarrsseeUURRLL() should be freed using ffeettcchhFFrreeeeUURRLL().
+     ffeettcchhPPaarrsseeUURRLL() should be freed using ffeettcchhFFrreeeeUURRLL().  The size of _s_t_r_u_c_t
+     _U_R_L is not part of the ABI.
 
      ffeettcchhXXGGeettUURRLL(), ffeettcchhGGeettUURRLL(), and ffeettcchhPPuuttUURRLL() constitute the recom-
      mended interface to the ffeettcchh library.  They examine the URL passed to
@@ -225,6 +227,12 @@ DDEESSCCRRIIPPTTIIOONN
      ffeettcchhGGeettXXXXXX() functions is read-only, and that a stream returned by one
      of the ffeettcchhPPuuttXXXXXX() functions is write-only.
 
+PPRROOTTOOCCOOLL IINNDDEEPPEENNDDEENNTT FFLLAAGGSS
+     If the `i' (if-modified-since) flag is specified, the library will try to
+     fetch the content only if it is newer than _l_a_s_t___m_o_d_i_f_i_e_d.  For HTTP an
+     If-Modified-Since HTTP header is sent.  For FTP a MTDM command is sent
+     first and compared locally.  For FILE the source file is compared.
+
 FFIILLEE SSCCHHEEMMEE
      ffeettcchhXXGGeettFFiillee(), ffeettcchhGGeettFFiillee(), and ffeettcchhPPuuttFFiillee() provide access to
      documents which are files in a locally mounted file system.  Only the
@@ -495,4 +503,4 @@ BBUUGGSS
 
      Some parts of the code are not fully reentrant.
 
-NetBSD 4.0                      April 25, 2008                      NetBSD 4.0
+NetBSD 5.0                     February 4, 2009                     NetBSD 5.0
diff --git a/net/libfetch/files/fetch.h b/net/libfetch/files/fetch.h
index 9772306f9e4..6ad8063afbd 100644
--- a/net/libfetch/files/fetch.h
+++ b/net/libfetch/files/fetch.h
@@ -1,4 +1,4 @@
-/*	$NetBSD: fetch.h,v 1.12 2008/10/06 12:58:29 joerg Exp $	*/
+/*	$NetBSD: fetch.h,v 1.13 2009/02/05 16:59:45 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
  * All rights reserved.
@@ -54,6 +54,7 @@ struct url {
 	char		*doc;
 	off_t		 offset;
 	size_t		 length;
+	time_t		 last_modified;
 };
 
 struct url_stat {
@@ -94,6 +95,7 @@ struct url_list {
 #define	FETCH_UNKNOWN	17
 #define	FETCH_URL	18
 #define	FETCH_VERBOSE	19
+#define	FETCH_UNCHANGED	20
 
 #if defined(__cplusplus)
 extern "C" {
diff --git a/net/libfetch/files/file.c b/net/libfetch/files/file.c
index d99e08b42f0..fb858803349 100644
--- a/net/libfetch/files/file.c
+++ b/net/libfetch/files/file.c
@@ -1,7 +1,7 @@
-/*	$NetBSD: file.c,v 1.12 2008/10/06 12:58:29 joerg Exp $	*/
+/*	$NetBSD: file.c,v 1.13 2009/02/05 16:59:45 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
- * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
+ * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -79,7 +79,12 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
 {
 	char *path;
 	fetchIO *f;
-	int fd, *cookie;
+	struct url_stat local_us;
+	int if_modified_since, fd, *cookie;
+
+	if_modified_since = CHECK_FLAG('i');
+	if (if_modified_since && us == NULL)
+		us = &local_us;
 
 	if ((path = fetchUnquotePath(u)) == NULL) {
 		fetch_syserr();
@@ -95,6 +100,14 @@ fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
 
 	if (us && fetch_stat_file(fd, us) == -1) {
 		close(fd);
+		fetch_syserr();
+		return NULL;
+	}
+
+	if (if_modified_since && u->last_modified > 0 &&
+	    u->last_modified >= us->mtime) {
+		fetchLastErrCode = FETCH_UNCHANGED;
+		snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged");
 		return NULL;
 	}
 
diff --git a/net/libfetch/files/ftp.c b/net/libfetch/files/ftp.c
index be5fee7498e..ee286faf81b 100644
--- a/net/libfetch/files/ftp.c
+++ b/net/libfetch/files/ftp.c
@@ -1,7 +1,7 @@
-/*	$NetBSD: ftp.c,v 1.25 2008/12/02 16:59:03 joerg Exp $	*/
+/*	$NetBSD: ftp.c,v 1.26 2009/02/05 16:59:45 joerg Exp $	*/
 /*-
  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
- * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
+ * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -1127,7 +1127,8 @@ ftp_request(struct url *url, const char *op, const char *op_arg,
 	fetchIO *f;
 	char *path;
 	conn_t *conn;
-	int oflag;
+	int if_modified_since, oflag;
+	struct url_stat local_us;
 
 	/* check if we should use HTTP instead */
 	if (purl && strcasecmp(purl->scheme, SCHEME_HTTP) == 0) {
@@ -1159,6 +1160,10 @@ ftp_request(struct url *url, const char *op, const char *op_arg,
 		return (NULL);
 	}
 
+	if_modified_since = CHECK_FLAG('i');
+	if (if_modified_since && us == NULL)
+		us = &local_us;
+
 	/* stat file */
 	if (us && ftp_stat(conn, path, us) == -1
 	    && fetchLastErrCode != FETCH_PROTO
@@ -1167,6 +1172,13 @@ ftp_request(struct url *url, const char *op, const char *op_arg,
 		return (NULL);
 	}
 
+	if (if_modified_since && url->last_modified > 0 &&
+	    url->last_modified >= us->mtime) {
+		fetchLastErrCode = FETCH_UNCHANGED;
+		snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged");
+		return NULL;
+	}
+
 	/* just a stat */
 	if (strcmp(op, "STAT") == 0)
 		return fetchIO_unopen(NULL, NULL, NULL, NULL);
diff --git a/net/libfetch/files/http.c b/net/libfetch/files/http.c
index c1df4b8f1ea..b1c5745b092 100644
--- a/net/libfetch/files/http.c
+++ b/net/libfetch/files/http.c
@@ -1,8 +1,8 @@
-/*	$NetBSD: http.c,v 1.20 2008/10/06 12:58:29 joerg Exp $	*/
+/*	$NetBSD: http.c,v 1.21 2009/02/05 16:59:45 joerg Exp $	*/
 /*-
  * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav
  * Copyright (c) 2003 Thomas Klausner <wiz@NetBSD.org>
- * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
+ * Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -112,6 +112,7 @@
 #define HTTP_MOVED_PERM		301
 #define HTTP_MOVED_TEMP		302
 #define HTTP_SEE_OTHER		303
+#define HTTP_NOT_MODIFIED	304
 #define HTTP_TEMP_REDIRECT	307
 #define HTTP_NEED_AUTH		401
 #define HTTP_NEED_PROXY_AUTH	407
@@ -745,6 +746,21 @@ http_get_proxy(struct url * url, const char *flags)
 	return (NULL);
 }
 
+static void
+set_if_modified_since(conn_t *conn, time_t last_modified)
+{
+	static const char weekdays[] = "SunMonTueWedThuFriSat";
+	static const char months[] = "JanFebMarAprMayJunJulAugSepOctNovDec";
+	struct tm tm;
+	char buf[80];
+	gmtime_r(&last_modified, &tm);
+	snprintf(buf, sizeof(buf), "%.3s, %02d %.3s %4d %02d:%02d:%02d GMT",
+	    weekdays + tm.tm_wday * 3, tm.tm_mday, months + tm.tm_mon * 3,
+	    tm.tm_year + 1900, tm.tm_hour, tm.tm_min, tm.tm_sec);
+	http_cmd(conn, "If-Modified-Since: %s", buf);
+}
+
+
 /*****************************************************************************
  * Core
  */
@@ -761,7 +777,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 {
 	conn_t *conn;
 	struct url *url, *new;
-	int chunked, direct, need_auth, noredirect, verbose;
+	int chunked, direct, if_modified_since, need_auth, noredirect, verbose;
 	int e, i, n, val;
 	off_t offset, clength, length, size;
 	time_t mtime;
@@ -773,6 +789,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 	direct = CHECK_FLAG('d');
 	noredirect = CHECK_FLAG('A');
 	verbose = CHECK_FLAG('v');
+	if_modified_since = CHECK_FLAG('i');
 
 	if (direct && purl) {
 		fetchFreeURL(purl);
@@ -841,6 +858,9 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 			    op, url->doc);
 		}
 
+		if (if_modified_since && url->last_modified > 0)
+			set_if_modified_since(conn, url->last_modified);
+
 		/* virtual host */
 		http_cmd(conn, "Host: %s", host);
 
@@ -904,6 +924,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		switch (http_get_reply(conn)) {
 		case HTTP_OK:
 		case HTTP_PARTIAL:
+		case HTTP_NOT_MODIFIED:
 			/* fine */
 			break;
 		case HTTP_MOVED_PERM:
@@ -1036,7 +1057,10 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		}
 
 		/* we have a hit or an error */
-		if (conn->err == HTTP_OK || conn->err == HTTP_PARTIAL || HTTP_ERROR(conn->err))
+		if (conn->err == HTTP_OK ||
+		    conn->err == HTTP_PARTIAL ||
+		    conn->err == HTTP_NOT_MODIFIED ||
+		    HTTP_ERROR(conn->err))
 			break;
 
 		/* all other cases: we got a redirect */
@@ -1089,6 +1113,11 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 	URL->offset = offset;
 	URL->length = clength;
 
+	if (conn->err == HTTP_NOT_MODIFIED) {
+		http_seterr(HTTP_NOT_MODIFIED);
+		return (NULL);
+	}
+
 	/* wrap it up in a fetchIO */
 	if ((f = http_funopen(conn, chunked)) == NULL) {
 		fetch_syserr();
diff --git a/net/libfetch/files/http.errors b/net/libfetch/files/http.errors
index bdf51d0e335..004aac27b8d 100644
--- a/net/libfetch/files/http.errors
+++ b/net/libfetch/files/http.errors
@@ -1,5 +1,5 @@
 # $FreeBSD: http.errors,v 1.5 2001/05/23 18:52:02 des Exp $
-# $NetBSD: http.errors,v 1.2 2008/10/06 12:58:29 joerg Exp $
+# $NetBSD: http.errors,v 1.3 2009/02/05 16:59:45 joerg Exp $
 #
 # This list is taken from RFC 2068.
 #
@@ -16,7 +16,7 @@
 301 MOVED	Moved Permanently
 302 MOVED	Moved Temporarily
 303 MOVED	See Other
-304 OK		Not Modified
+304 UNCHANGED	Not Modified
 305 INFO	Use Proxy
 307 MOVED	Temporary Redirect
 400 PROTO	Bad Request