Sync pkgtools/pkg_install with src/usr.sbin/pkg_install from 20031003.

New features include fetching binary packages using HTTP (thanks wiz!)
and some minor speed optimizations for pkg_delete.

12 files changed, 867 insertions, 485 deletions

diff --git a/pkgtools/pkg_install/files/README b/pkgtools/pkg_install/files/README
index a25c06f1d74..4ab6524ee98 100644
--- a/pkgtools/pkg_install/files/README
+++ b/pkgtools/pkg_install/files/README
@@ -1,4 +1,4 @@
-# $NetBSD: README,v 1.3 2003/09/23 07:13:45 grant Exp $
+# $NetBSD: README,v 1.4 2003/10/29 23:00:27 jlam Exp $
 # Original from FreeBSD, no rcs id.
 
 This is the pkg_install suite of tools for doing maintainance of
@@ -12,7 +12,7 @@ of dozen features on top.  Whee! :-)
 
 In another round of enhancements, NetBSD changes were added by
 Alistair Crooks, Hubert Feyrer, Thorsten Frueauf, Christian E. Hopps,
-Johnny Lam and Matthias Scheler.
+Thomas Klausner, Johnny Lam, and Matthias Scheler.
 
 When making snapshots, please modify PKGTOOLS_VERSION in lib/version.h
 to that day's date.
diff --git a/pkgtools/pkg_install/files/add/futil.c b/pkgtools/pkg_install/files/add/futil.c
index 28802b01ea2..b15874ff500 100644
--- a/pkgtools/pkg_install/files/add/futil.c
+++ b/pkgtools/pkg_install/files/add/futil.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: futil.c,v 1.5 2003/09/23 13:22:38 grant Exp $	*/
+/*	$NetBSD: futil.c,v 1.6 2003/10/29 23:00:28 jlam Exp $	*/
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -11,7 +11,7 @@
 #if 0
 static const char *rcsid = "from FreeBSD Id: futil.c,v 1.7 1997/10/08 07:45:39 charnier Exp";
 #else
-__RCSID("$NetBSD: futil.c,v 1.5 2003/09/23 13:22:38 grant Exp $");
+__RCSID("$NetBSD: futil.c,v 1.6 2003/10/29 23:00:28 jlam Exp $");
 #endif
 #endif
 
@@ -95,8 +95,8 @@ apply_perms(char *dir, char *arg)
 	if (Owner != NULL && Group != NULL) {
 		if (snprintf(owner_group, sizeof(owner_group),
 			     "%s:%s", Owner, Group) > sizeof(owner_group)) {
-			warnx("'%s:%s' is too long (%d max)",
-			      Owner, Group, sizeof(owner_group));
+			warnx("'%s:%s' is too long (%lu max)",
+			      Owner, Group, (unsigned long) sizeof(owner_group));
 			return;
 		}
 		if (fcexec(cd_to, CHOWN_CMD, "-R", owner_group, arg))
diff --git a/pkgtools/pkg_install/files/add/pkg_add.1 b/pkgtools/pkg_install/files/add/pkg_add.1
index 6c816a8a283..8d76d763a73 100644
--- a/pkgtools/pkg_install/files/add/pkg_add.1
+++ b/pkgtools/pkg_install/files/add/pkg_add.1
@@ -1,4 +1,4 @@
-.\" $NetBSD: pkg_add.1,v 1.5 2003/09/23 07:13:47 grant Exp $
+.\" $NetBSD: pkg_add.1,v 1.6 2003/10/29 23:00:28 jlam Exp $
 .\"
 .\" FreeBSD install - a package for the installation and maintenance
 .\" of non-core utilities.
@@ -17,7 +17,7 @@
 .\"
 .\"     @(#)pkg_add.1
 .\"
-.Dd September 8, 2003
+.Dd October 3, 2003
 .Dt PKG_ADD 1
 .Os
 .Sh NAME
@@ -44,7 +44,7 @@
 .Bk -words
 .Op Fl w Ar view
 .Ek
-.Ar \fR[ftp://[\fIuser\fR[:\fIpassword]\fR@]\fIhost\fR[:\fIport\fR]][/\fIpath/\fR]pkg-name ...
+.Ar \fR[[ftp|http]://[\fIuser\fR[:\fIpassword]\fR@]\fIhost\fR[:\fIport\fR]][/\fIpath/\fR]pkg-name ...
 .Sh DESCRIPTION
 The
 .Nm
@@ -56,7 +56,7 @@ Packages are prepared collections of pre-built binaries, documentation,
 configurations, installation instructions and/or other files.
 .Nm
 can recursively install other packages that the current package
-depends on or requires from both local disk and via FTP.
+depends on or requires from both local disk and via FTP or HTTP.
 .Sh WARNING
 .Bf -emphasis
 Since the
@@ -273,15 +273,17 @@ arguments may be specified, each being either a file containing the
 package (these usually ending with the ``.tgz'' suffix) or a
 URL pointing at a file available on an ftp or web site.
 Thus you may extract files directly from their anonymous ftp or WWW
-locations (e.g.
+locations (e.g.,
 .Nm
-ftp://ftp.NetBSD.org/pub/NetBSD/packages/1.5/i386/shells/bash-2.04.tgz).
-Note:  If you wish to use
+ftp://ftp.NetBSD.org/pub/NetBSD/packages/1.5/i386/shells/bash-2.04.tgz
+or
+.Nm
+http://www.example.org/packages/screen-4.0.tbz).
+Note:  For ftp transfers, if you wish to use
 .Bf -emphasis
 passive mode
 .Ef
-ftp in such transfers, set
-the variable
+ftp in such transfers, set the variable
 .Bf -emphasis
 FTP_PASSIVE_MODE
 .Ef
@@ -558,6 +560,8 @@ refinements.
 .It "Hubert Feyrer"
 .Nx
 wildcard dependency processing, pkgdb, upgrading, etc.
+.It Thomas Klausner
+HTTP support.
 .El
 .Sh BUGS
 Hard links between files in a distribution are only preserved if either
diff --git a/pkgtools/pkg_install/files/add/pkg_add.cat1 b/pkgtools/pkg_install/files/add/pkg_add.cat1
index e0e4b3f5009..e466a6f7d15 100644
--- a/pkgtools/pkg_install/files/add/pkg_add.cat1
+++ b/pkgtools/pkg_install/files/add/pkg_add.cat1
@@ -1,4 +1,4 @@
-PKG_ADD(1)              NetBSD General Commands Manual              PKG_ADD(1)
+PKG_ADD(1)                  NetBSD Reference Manual                 PKG_ADD(1)
 
 NNAAMMEE
      ppkkgg__aadddd - a utility for installing and upgrading software package distri-
@@ -7,7 +7,7 @@ NNAAMMEE
 SSYYNNOOPPSSIISS
      ppkkgg__aadddd [--ffIILLMMnnRRSSuuVVvv] [--KK _p_k_g___d_b_d_i_r] [--pp _p_r_e_f_i_x] [--ss _v_e_r_i_f_i_c_a_t_i_o_n_-_t_y_p_e]
              [--tt _t_e_m_p_l_a_t_e] [--WW _v_i_e_w_b_a_s_e] [--ww _v_i_e_w]
-             [ftp://[_u_s_e_r[:_p_a_s_s_w_o_r_d_]@]_h_o_s_t[:_p_o_r_t]][/_p_a_t_h_/]pkg-name ...
+             [[ftp|http]://[_u_s_e_r[:_p_a_s_s_w_o_r_d_]@]_h_o_s_t[:_p_o_r_t]][/_p_a_t_h_/]pkg-name ...
 
 DDEESSCCRRIIPPTTIIOONN
      The ppkkgg__aadddd command is used to extract and upgrade packages that have
@@ -15,7 +15,7 @@ DDEESSCCRRIIPPTTIIOONN
      prepared collections of pre-built binaries, documentation, configura-
      tions, installation instructions and/or other files.  ppkkgg__aadddd can recur-
      sively install other packages that the current package depends on or re-
-     quires from both local disk and via FTP.
+     quires from both local disk and via FTP or HTTP.
 
 WWAARRNNIINNGG
      _S_i_n_c_e _t_h_e ppkkgg__aadddd _c_o_m_m_a_n_d _m_a_y _e_x_e_c_u_t_e _s_c_r_i_p_t_s _o_r _p_r_o_g_r_a_m_s _c_o_n_t_a_i_n_e_d _w_i_t_h_-
@@ -146,14 +146,15 @@ OOPPTTIIOONNSS
      One or more _p_k_g_-_n_a_m_e arguments may be specified, each being either a file
      containing the package (these usually ending with the ``.tgz'' suffix) or
      a URL pointing at a file available on an ftp or web site.  Thus you may
-     extract files directly from their anonymous ftp or WWW locations (e.g.
+     extract files directly from their anonymous ftp or WWW locations (e.g.,
      ppkkgg__aadddd ftp://ftp.NetBSD.org/pub/NetBSD/pack-
-     ages/1.5/i386/shells/bash-2.04.tgz).  Note:  If you wish to use _p_a_s_s_i_v_e
-     _m_o_d_e ftp in such transfers, set the variable _F_T_P___P_A_S_S_I_V_E___M_O_D_E to some
-     value in your environment.  Otherwise, the more standard ACTIVE mode may
-     be used.  If ppkkgg__aadddd consistently fails to fetch a package from a site
-     known to work, it may be because you have a firewall that demands the us-
-     age of _p_a_s_s_i_v_e _m_o_d_e ftp.
+     ages/1.5/i386/shells/bash-2.04.tgz or ppkkgg__aadddd http://www.exam-
+     ple.org/packages/screen-4.0.tbz).  Note:  For ftp transfers, if you wish
+     to use _p_a_s_s_i_v_e _m_o_d_e  ftp in such transfers, set the variable _F_T_P___P_A_S_-
+     _S_I_V_E___M_O_D_E  to some value in your environment.  Otherwise, the more stan-
+     dard ACTIVE mode may be used.  If ppkkgg__aadddd consistently fails to fetch a
+     package from a site known to work, it may be because you have a firewall
+     that demands the usage of _p_a_s_s_i_v_e _m_o_d_e  ftp.
 
 TTEECCHHNNIICCAALL DDEETTAAIILLSS
      ppkkgg__aadddd extracts each package's "packing list" into a special staging di-
@@ -320,6 +321,8 @@ AAUUTTHHOORRSS
              NetBSD refinements.
      Hubert Feyrer
              NetBSD wildcard dependency processing, pkgdb, upgrading, etc.
+     Thomas Klausner
+             HTTP support.
 
 BBUUGGSS
      Hard links between files in a distribution are only preserved if either
@@ -334,4 +337,4 @@ BBUUGGSS
 
      Sure to be others.
 
-NetBSD 1.6                     September 8, 2003                    NetBSD 1.6
+NetBSD 1.6.1                    October 3, 2003                              6
diff --git a/pkgtools/pkg_install/files/create/pl.c b/pkgtools/pkg_install/files/create/pl.c
index 85ca376d8d3..3e69e52ba82 100644
--- a/pkgtools/pkg_install/files/create/pl.c
+++ b/pkgtools/pkg_install/files/create/pl.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: pl.c,v 1.5 2003/09/23 07:13:50 grant Exp $	*/
+/*	$NetBSD: pl.c,v 1.6 2003/10/29 23:00:28 jlam Exp $	*/
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -11,7 +11,7 @@
 #if 0
 static const char *rcsid = "from FreeBSD Id: pl.c,v 1.11 1997/10/08 07:46:35 charnier Exp";
 #else
-__RCSID("$NetBSD: pl.c,v 1.5 2003/09/23 07:13:50 grant Exp $");
+__RCSID("$NetBSD: pl.c,v 1.6 2003/10/29 23:00:28 jlam Exp $");
 #endif
 #endif
 
@@ -63,10 +63,10 @@ CheckSymlink(char *name, char *prefix, size_t prefixcc)
 			for (slashc = 0, slash = &name[prefixcc + 1]; (slash = strchr(slash, '/')) != (char *) NULL; slash++, slashc++) {
 			}
 			for (cc = i = 0; i < slashc; i++) {
-				strnncpy(&newtgt[cc], sizeof(newtgt) - cc, "../", 3);
+				strlcpy(&newtgt[cc], "../", sizeof(newtgt) - cc);
 				cc += 3;
 			}
-			strnncpy(&newtgt[cc], sizeof(newtgt) - cc, &oldtgt[prefixcc + 1], strlen(&oldtgt[prefixcc + 1]));
+			strlcpy(&newtgt[cc], &oldtgt[prefixcc + 1], sizeof(newtgt) - cc);
 			(void) fprintf(stderr, "Full pathname symlink `%s' is target of `%s' - adjusting to `%s'\n", oldtgt, name, newtgt);
 			if (unlink(name) != 0) {
 				warn("can't unlink `%s'", name);
diff --git a/pkgtools/pkg_install/files/delete/perform.c b/pkgtools/pkg_install/files/delete/perform.c
index c0a963d2c50..8ce28d54c0e 100644
--- a/pkgtools/pkg_install/files/delete/perform.c
+++ b/pkgtools/pkg_install/files/delete/perform.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: perform.c,v 1.9 2003/09/23 13:22:39 grant Exp $	*/
+/*	$NetBSD: perform.c,v 1.10 2003/10/29 23:00:28 jlam Exp $	*/
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -11,7 +11,7 @@
 #if 0
 static const char *rcsid = "from FreeBSD Id: perform.c,v 1.15 1997/10/13 15:03:52 jkh Exp";
 #else
-__RCSID("$NetBSD: perform.c,v 1.9 2003/09/23 13:22:39 grant Exp $");
+__RCSID("$NetBSD: perform.c,v 1.10 2003/10/29 23:00:28 jlam Exp $");
 #endif
 #endif
 
@@ -628,7 +628,7 @@ pkg_do(char *pkg)
 		TAILQ_INIT(&trypkgs);
 		snprintf(try, FILENAME_MAX, "%s-[0-9]*", pkg);
 		if (findmatchingname(_pkgdb_getPKGDB_DIR(), try,
-			add_to_list_fn, &trypkgs) == NULL) {
+			add_to_list_fn, &trypkgs) == 0) {
 			warnx("package '%s' not installed", pkg);
 			return 1;
 		}
@@ -836,7 +836,7 @@ pkg_do(char *pkg)
 	if (!Fake) {
 		/* Finally nuke the +-files and the pkgdb-dir (/var/db/pkg/foo) */
 		if (is_depoted_pkg) {
-			(void) vsystem("%s %s/+*", REMOVE_CMD, LogDir);
+			(void) remove_files(LogDir, "+*");
 			if (isemptydir(LogDir))
 				(void) fexec(RMDIR_CMD, LogDir, NULL);
 			else
diff --git a/pkgtools/pkg_install/files/lib/fexec.c b/pkgtools/pkg_install/files/lib/fexec.c
index 008db2c7004..94810825656 100644
--- a/pkgtools/pkg_install/files/lib/fexec.c
+++ b/pkgtools/pkg_install/files/lib/fexec.c
@@ -65,7 +65,7 @@
 #include "lib.h"
 
 #ifndef lint
-__RCSID("$NetBSD: fexec.c,v 1.4 2003/09/23 13:22:41 grant Exp $");
+__RCSID("$NetBSD: fexec.c,v 1.5 2003/10/29 23:00:28 jlam Exp $");
 #endif
 
 static int	vfcexec(const char *, int, const char *, va_list);
@@ -82,7 +82,7 @@ vfcexec(const char *path, int skipempty, const char *arg, va_list ap)
 	if (argv == NULL) {
 		argv = malloc(max * sizeof(const char *));
 		if (argv == NULL) {
-			warnx("vfcexec can't alloc arg space");
+			warn("vfcexec: Can't alloc arg space");
 			return -1;
 		}
 	}
@@ -98,8 +98,10 @@ vfcexec(const char *path, int skipempty, const char *arg, va_list ap)
 			new = max * 2;
 			ptr = realloc(argv, new * sizeof(const char *));
 			if (ptr == NULL) {
-				warnx("vfcexec can't alloc arg space");
+				warn("vfcexec: Can't alloc arg space");
 				free(argv);
+				argv = NULL;
+				max = 4;
 				return -1;
 			}
 			argv = ptr;
diff --git a/pkgtools/pkg_install/files/lib/file.c b/pkgtools/pkg_install/files/lib/file.c
index 18a3d74593a..1da29045f37 100644
--- a/pkgtools/pkg_install/files/lib/file.c
+++ b/pkgtools/pkg_install/files/lib/file.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: file.c,v 1.8 2003/09/23 13:22:41 grant Exp $	*/
+/*	$NetBSD: file.c,v 1.9 2003/10/29 23:00:28 jlam Exp $	*/
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -11,7 +11,7 @@
 #if 0
 static const char *rcsid = "from FreeBSD Id: file.c,v 1.29 1997/10/08 07:47:54 charnier Exp";
 #else
-__RCSID("$NetBSD: file.c,v 1.8 2003/09/23 13:22:41 grant Exp $");
+__RCSID("$NetBSD: file.c,v 1.9 2003/10/29 23:00:28 jlam Exp $");
 #endif
 #endif
 
@@ -37,12 +37,19 @@ __RCSID("$NetBSD: file.c,v 1.8 2003/09/23 13:22:41 grant Exp $");
 
 #include "lib.h"
 
+#if HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
+
 #if HAVE_ASSERT_H
 #include <assert.h>
 #endif
 #if HAVE_ERR_H
 #include <err.h>
 #endif
+#if HAVE_GLOB_H
+#include <glob.h>
+#endif
 #if HAVE_NETDB_H
 #include <netdb.h>
 #endif
@@ -535,6 +542,40 @@ move_file(char *dir, char *fname, char *to)
 	}
 }
 
+void
+remove_files(const char *path, const char *pattern)
+{
+	char	fpath[FILENAME_MAX];
+	glob_t	globbed;
+	int	i;
+
+	(void) snprintf(fpath, sizeof(fpath), "%s/%s", path, pattern);
+	if ((i=glob(fpath, GLOB_NOSORT, NULL, &globbed)) != 0) {
+		switch(i) {
+		case GLOB_NOMATCH:
+			warn("no files matching ``%s'' found", fpath);
+			break;
+		case GLOB_ABORTED:
+			warn("globbing aborted");
+			break;
+		case GLOB_NOSPACE:
+			warn("out-of-memory during globbing");
+			break;
+		default:
+			warn("unknown error during globbing");
+			break;
+		}
+		return;
+	}
+
+	/* deleting globbed files */
+	for (i=0; i<globbed.gl_pathc; i++)
+		if (unlink(globbed.gl_pathv[i]) < 0)
+			warn("can't delete ``%s''", globbed.gl_pathv[i]);
+
+	return;
+}
+
 /*
  * Unpack a tar file
  */
@@ -659,12 +700,12 @@ format_cmd(char *buf, size_t size, char *fmt, char *dir, char *name)
 			}
 			switch (*fmt) {
 			case 'F':
-				strnncpy(bufp, size - (int) (bufp - buf), name, strlen(name));
+				strlcpy(bufp, name, size - (int) (bufp - buf));
 				bufp += strlen(bufp);
 				break;
 
 			case 'D':
-				strnncpy(bufp, size - (int) (bufp - buf), dir, strlen(dir));
+				strlcpy(bufp, dir, size - (int) (bufp - buf));
 				bufp += strlen(bufp);
 				break;
 
@@ -673,7 +714,8 @@ format_cmd(char *buf, size_t size, char *fmt, char *dir, char *name)
 				if ((cp = strrchr(scratch, '/')) == (char *) NULL) {
 					cp = scratch;
 				}
-				strnncpy(bufp, size - (int) (bufp - buf), scratch, (size_t) (cp - scratch));
+				*cp = '\0';
+				strlcpy(bufp, scratch, size - (int) (bufp - buf));
 				bufp += strlen(bufp);
 				break;
 
@@ -684,7 +726,7 @@ format_cmd(char *buf, size_t size, char *fmt, char *dir, char *name)
 				} else {
 					cp++;
 				}
-				strnncpy(bufp, size - (int) (bufp - buf), cp, strlen(cp));
+				strlcpy(bufp, cp, size - (int) (bufp - buf));
 				bufp += strlen(bufp);
 				break;
 
diff --git a/pkgtools/pkg_install/files/lib/ftpio.c b/pkgtools/pkg_install/files/lib/ftpio.c
index b298185bdd7..f8544801eb6 100644
--- a/pkgtools/pkg_install/files/lib/ftpio.c
+++ b/pkgtools/pkg_install/files/lib/ftpio.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: ftpio.c,v 1.6 2003/09/23 13:22:42 grant Exp $	*/
+/*	$NetBSD: ftpio.c,v 1.7 2003/10/29 23:00:28 jlam Exp $	*/
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -8,9 +8,45 @@
 #include <sys/cdefs.h>
 #endif
 #ifndef lint
-__RCSID("$NetBSD: ftpio.c,v 1.6 2003/09/23 13:22:42 grant Exp $");
+__RCSID("$NetBSD: ftpio.c,v 1.7 2003/10/29 23:00:28 jlam Exp $");
 #endif
 
+/*-
+ * Copyright (c) 2003 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Thomas Klausner.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
 /*
  * Copyright (c) 1999 Hubert Feyrer.  All rights reserved.
  *
@@ -50,6 +86,9 @@ __RCSID("$NetBSD: ftpio.c,v 1.6 2003/09/23 13:22:42 grant Exp $");
 #if HAVE_SYS_POLL_H
 #include <sys/poll.h>
 #endif
+#if HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
 #if HAVE_SIGNAL_H
 #include <signal.h>
 #endif
@@ -116,9 +155,6 @@ typedef struct {
 #if EXPECT_DEBUG
 static int	 expect_debug = 1;
 #endif /* EXPECT_DEBUG */
-#ifdef STANDALONE
-int		Verbose=1;
-#endif
 static int	 needclose=0;
 static int	 ftp_started=0;
 static fds	 ftpio;
@@ -127,6 +163,12 @@ static char	 term[1024];
 static char	 bold_on[1024];
 static char	 bold_off[1024];
 
+static char *ftp_expand_URL(const char *, char *);
+static int hexvalue(char);
+static char *http_expand_URL(const char *, char *);
+static int http_extract_fn(char *, char *, size_t);
+static void URL_decode(char *);
+
 /*
  * expect "str" (a regular expression) on file descriptor "fd", storing
  * the FTP return code of the command in the integer "ftprc". The "str"
@@ -136,126 +178,126 @@ static char	 bold_off[1024];
 static int
 expect(int fd, const char *str, int *ftprc)
 {
-    int rc;
-    char buf[90];  
+	int rc;
+	char buf[90];  
 #if EXPECT_DEBUG
-    char *vstr;
+	char *vstr;
 #endif /* EXPECT_DEBUG */
-    regex_t rstr;
-    int done;
-    struct pollfd set[1];
-    int retval;
-    regmatch_t match;
-    int verbose_expect=0;
+	regex_t rstr;
+	int done;
+	struct pollfd set[1];
+	int retval;
+	regmatch_t match;
+	int verbose_expect=0;
 
 #if EXPECT_DEBUG
-    vstr=malloc(2*sizeof(buf));
-    if (vstr == NULL)
-	err(EXIT_FAILURE, "expect: malloc() failed");
-    strvis(vstr, str, VIS_NL|VIS_SAFE|VIS_CSTYLE);
+	vstr=malloc(2*sizeof(buf));
+	if (vstr == NULL)
+		err(EXIT_FAILURE, "expect: malloc() failed");
+	strvis(vstr, str, VIS_NL|VIS_SAFE|VIS_CSTYLE);
 #endif /* EXPECT_DEBUG */
 	    
-    if (regcomp(&rstr, str, REG_EXTENDED) != 0)
-	err(EXIT_FAILURE, "expect: regcomp() failed");
+	if (regcomp(&rstr, str, REG_EXTENDED) != 0)
+		err(EXIT_FAILURE, "expect: regcomp() failed");
 
 #if EXPECT_DEBUG
-    if (expect_debug)
-	printf("expecting \"%s\" on fd %d ...\n", vstr, fd);
+	if (expect_debug)
+		printf("expecting \"%s\" on fd %d ...\n", vstr, fd);
 #endif /* EXPECT_DEBUG */
 
-    if(0) setbuf(stdout, NULL);
-
-    memset(buf, '\n', sizeof(buf));
-
-    done=0;
-    retval=0;
-    set[0].fd = fd;
-    set[0].events = POLLIN;
-    while(!done) {
-	rc = poll(set, 1, 10*60*1000);    /* seconds until next message from tar */
-	switch (rc) {
-	case -1:
-	    if (errno == EINTR)
-		break;
-	    warn("expect: poll() failed (probably ftp died because of bad args)");
-	    done = 1;
-	    retval = -1;
-	    break;
-	case 0:
-	    warnx("expect: poll() timeout");
-	    /* need to send ftp coprocess SIGINT to make it stop
-	     * downloading into dir that we'll blow away in a second */
-	    kill(ftp_pid, SIGINT);
-
-	    /* Wait until ftp coprocess is responsive again 
-	     * XXX Entering recursion here!
-	     */
-	    rc = ftp_cmd("cd .\n", "\n(550|250).*\n");
-	    if (rc != 250) {
-		    /* now we have a really good reason to bail out ;) */
-	    }
-	    /* ftp is at command prompt again, and will wait for our
-	     * next command. If we were downloading, we can now safely
-	     * continue and remove the dir that the tar command was
-	     * expanding to */
+	if(0) setbuf(stdout, NULL);
+
+	memset(buf, '\n', sizeof(buf));
+
+	done=0;
+	retval=0;
+	set[0].fd = fd;
+	set[0].events = POLLIN;
+	while(!done) {
+		rc = poll(set, 1, 10*60*1000);    /* seconds until next message from tar */
+		switch (rc) {
+		case -1:
+			if (errno == EINTR)
+				break;
+			warn("expect: poll() failed (probably ftp died because of bad args)");
+			done = 1;
+			retval = -1;
+			break;
+		case 0:
+			warnx("expect: poll() timeout");
+			/* need to send ftp coprocess SIGINT to make it stop
+			 * downloading into dir that we'll blow away in a second */
+			kill(ftp_pid, SIGINT);
+
+			/* Wait until ftp coprocess is responsive again 
+			 * XXX Entering recursion here!
+			 */
+			rc = ftp_cmd("cd .\n", "\n(550|250).*\n");
+			if (rc != 250) {
+				/* now we have a really good reason to bail out ;) */
+			}
+			/* ftp is at command prompt again, and will wait for our
+			 * next command. If we were downloading, we can now safely
+			 * continue and remove the dir that the tar command was
+			 * expanding to */
 		    
-	    done = 1;	/* hope that's ok */
-	    retval = -1;
-	    break;
-	default:
-	    if (set[0].revents & POLLHUP) {
-		done = 1;
-		retval = -1;
-		break;
-	    }
+			done = 1;	/* hope that's ok */
+			retval = -1;
+			break;
+		default:
+			if (set[0].revents & POLLHUP) {
+				done = 1;
+				retval = -1;
+				break;
+			}
 
-	    rc = read(fd, &buf[sizeof(buf) - 1], 1);
-	    if (rc <= 0) {
-		done = 1;
-		retval = -1;
-		break;
-	    }
+			rc = read(fd, &buf[sizeof(buf) - 1], 1);
+			if (rc <= 0) {
+				done = 1;
+				retval = -1;
+				break;
+			}
 
-	    if (verbose_expect)
-		putchar(buf[sizeof(buf)-1]);
+			if (verbose_expect)
+				putchar(buf[sizeof(buf)-1]);
 
 #if EXPECT_DEBUG
-	    {
-		    char *v=malloc(2*sizeof(buf));
-		    strvis(v, buf, VIS_NL|VIS_SAFE|VIS_CSTYLE);
-		    if (expect_debug)
-			    printf("expect=<%s>, buf=<%*s>\n", vstr, strlen(v), v);
-		    free(v);
-	    }
+			{
+				char *v=malloc(2*sizeof(buf));
+				strvis(v, buf, VIS_NL|VIS_SAFE|VIS_CSTYLE);
+				if (expect_debug)
+					printf("expect=<%s>, buf=<%*s>\n", vstr, strlen(v), v);
+				free(v);
+			}
 #endif /* EXPECT_DEBUG */
 
-	    if (regexec(&rstr, buf, 1, &match, 0) == 0) {
+			if (regexec(&rstr, buf, 1, &match, 0) == 0) {
 #if EXPECT_DEBUG
-		if (expect_debug)
-		    printf("Gotcha -> %s!\n", buf+match.rm_so+1);
-		fflush(stdout);
+				if (expect_debug)
+					printf("Gotcha -> %s!\n", buf+match.rm_so+1);
+				fflush(stdout);
 #endif /* EXPECT_DEBUG */
 
-		if (ftprc && isdigit(buf[match.rm_so+1])) 
-		    *ftprc = atoi(buf+match.rm_so+1);
+				if (ftprc && isdigit(buf[match.rm_so+1])) 
+					*ftprc = atoi(buf+match.rm_so+1);
 
-		done=1;
-		retval=0;
-	    }
+				done=1;
+				retval=0;
+			}
 	    
-	    memmove(buf, buf+1, sizeof(buf)-1); /* yes, this is non-performant */
-	    break;
+			memmove(buf, buf+1, sizeof(buf)-1); /* yes, this is non-performant */
+			break;
+		}
 	}
-    }
 
 #if EXPECT_DEBUG
-    printf("done.\n");
+	printf("done.\n");
 
-    if (str)
-	free(vstr);
+	if (str)
+		free(vstr);
 #endif /* EXPECT_DEBUG */
 
-    return retval;
+	return retval;
 }
 
 /*
@@ -266,29 +308,29 @@ expect(int fd, const char *str, int *ftprc)
 int
 ftp_cmd(const char *cmd, const char *expectstr)
 {
-    int rc=0, verbose_ftp=0;
-    int len;
+	int rc=0, verbose_ftp=0;
+	int len;
 
-    if (Verbose)
-	    verbose_ftp=1;
+	if (Verbose)
+		verbose_ftp=1;
     
-    if (verbose_ftp)
-	    fprintf(stderr, "\n%sftp> %s%s", bold_on, cmd, bold_off);
+	if (verbose_ftp)
+		fprintf(stderr, "\n%sftp> %s%s", bold_on, cmd, bold_off);
     
-    fflush(stdout);
-    len = write(ftpio.command, cmd, strlen(cmd));
-    if (len == strlen(cmd)) {
-	    if (expectstr) {
-		    /* set "rc" to the FTP error code: */
-		    if (expect(ftpio.answer, expectstr, &rc) == -1)
-			    rc = -1;	/* some error occurred */
-	    }
-    } else {	
-	    if (Verbose)
-		    warn("short write");
-    }
-
-    return rc;
+	fflush(stdout);
+	len = write(ftpio.command, cmd, strlen(cmd));
+	if (len == strlen(cmd)) {
+		if (expectstr) {
+			/* set "rc" to the FTP error code: */
+			if (expect(ftpio.answer, expectstr, &rc) == -1)
+				rc = -1;	/* some error occurred */
+		}
+	} else {	
+		if (Verbose)
+			warn("short write");
+	}
+
+	return rc;
 }
 
 
@@ -298,85 +340,85 @@ ftp_cmd(const char *cmd, const char *expectstr)
 static int
 setupCoproc(const char *base)
 {
-    int command_pipe[2];
-    int answer_pipe[2];
-    int rc1, rc2;
-    char buf[20];
-    char *argv0 = strrchr(FTP_CMD, '/');
-    if (argv0 == NULL)
-	argv0 = FTP_CMD;
-    else
-	argv0++;
-
-    rc1 = pipe(command_pipe);
-    rc2 = pipe(answer_pipe);
-
-    if(rc1==-1 || rc2==-1) {
-	warn("setupCoproc: pipe() failed");
-	return -1;
-    }
-
-    if (command_pipe[0] == -1 || command_pipe[1] == -1 ||
-	answer_pipe[0] == -1  || answer_pipe[1] == -1 ) {
-	warn("setupCoproc: pipe() returned bogus descriptor");
-	return -1;
-    }
-
-    rc1 = fork();
-    switch (rc1) {
-    case -1:
-	    /* Error */
+	int command_pipe[2];
+	int answer_pipe[2];
+	int rc1, rc2;
+	char buf[20];
+	char *argv0 = strrchr(FTP_CMD, '/');
+	if (argv0 == NULL)
+		argv0 = FTP_CMD;
+	else
+		argv0++;
+
+	rc1 = pipe(command_pipe);
+	rc2 = pipe(answer_pipe);
+
+	if(rc1==-1 || rc2==-1) {
+		warn("setupCoproc: pipe() failed");
+		return -1;
+	}
+
+	if (command_pipe[0] == -1 || command_pipe[1] == -1 ||
+	    answer_pipe[0] == -1  || answer_pipe[1] == -1 ) {
+		warn("setupCoproc: pipe() returned bogus descriptor");
+		return -1;
+	}
+
+	rc1 = fork();
+	switch (rc1) {
+	case -1:
+		/* Error */
 	    
-	    warn("setupCoproc: fork() failed");
-	    return -1;
-	    break;
+		warn("setupCoproc: fork() failed");
+		return -1;
+		break;
 
-    case 0:
-	    /* Child */
+	case 0:
+		/* Child */
 	    
-	    (void) close(command_pipe[1]);
-	    rc1 = dup2(command_pipe[0], 0);
-            if (rc1 == -1) {
-                    err(EXIT_FAILURE, "setupCoproc: dup2 failed (command_pipe[0])");
-            }
-	    (void) close(command_pipe[0]);
+		(void) close(command_pipe[1]);
+		rc1 = dup2(command_pipe[0], 0);
+		if (rc1 == -1) {
+			err(EXIT_FAILURE, "setupCoproc: dup2 failed (command_pipe[0])");
+		}
+		(void) close(command_pipe[0]);
 	    
-	    (void) close(answer_pipe[0]);
-	    rc1 = dup2(answer_pipe[1], 1);
-            if (rc1 == -1) {
-                    err(EXIT_FAILURE, "setupCoproc: dup2 failed (answer_pipe[1])");
-            }
-	    (void) close(answer_pipe[1]);
+		(void) close(answer_pipe[0]);
+		rc1 = dup2(answer_pipe[1], 1);
+		if (rc1 == -1) {
+			err(EXIT_FAILURE, "setupCoproc: dup2 failed (answer_pipe[1])");
+		}
+		(void) close(answer_pipe[1]);
 	    
-	    setbuf(stdout, NULL);
+		setbuf(stdout, NULL);
 	    
-	    if (Verbose)
-		    fprintf(stderr, "%sftp -detv %s%s\n", bold_on, base, bold_off);
-	    rc1 = execlp(FTP_CMD, argv0, "-detv", base, NULL);
-	    warn("setupCoproc: execlp() failed");
-	    exit(1);
-	    break;
-    default: 
-	    /* Parent */
-	    (void) close(command_pipe[0]);
-	    (void) close(answer_pipe[1]);
+		if (Verbose)
+			fprintf(stderr, "%sftp -detv %s%s\n", bold_on, base, bold_off);
+		rc1 = execlp(FTP_CMD, argv0, "-detv", base, NULL);
+		warn("setupCoproc: execlp() failed");
+		exit(1);
+		break;
+	default: 
+		/* Parent */
+		(void) close(command_pipe[0]);
+		(void) close(answer_pipe[1]);
 	    
-	    (void) snprintf(buf, sizeof(buf), "%d", command_pipe[1]);
-	    setenv(PKG_FTPIO_COMMAND, buf, 1);
-	    (void) snprintf(buf, sizeof(buf), "%d", answer_pipe[0]);
-	    setenv(PKG_FTPIO_ANSWER, buf, 1);
+		(void) snprintf(buf, sizeof(buf), "%d", command_pipe[1]);
+		setenv(PKG_FTPIO_COMMAND, buf, 1);
+		(void) snprintf(buf, sizeof(buf), "%d", answer_pipe[0]);
+		setenv(PKG_FTPIO_ANSWER, buf, 1);
 	    
-	    ftpio.command = command_pipe[1];
-	    ftpio.answer  = answer_pipe[0];
-	    ftp_pid = rc1;			/* to ^C transfers */
+		ftpio.command = command_pipe[1];
+		ftpio.answer  = answer_pipe[0];
+		ftp_pid = rc1;			/* to ^C transfers */
 	    
-	    fcntl(ftpio.command, F_SETFL, O_NONBLOCK);
-	    fcntl(ftpio.answer , F_SETFL, O_NONBLOCK);
+		fcntl(ftpio.command, F_SETFL, O_NONBLOCK);
+		fcntl(ftpio.answer , F_SETFL, O_NONBLOCK);
 	    
-	    break;
-    }
+		break;
+	}
 
-    return 0;
+	return 0;
 }
 
 
@@ -399,10 +441,10 @@ sigchld_handler (int n)
 static void
 sigpipe_handler(int n)
 {
-    /* aparently our ftp companion died */
-    if (Verbose)
-	    fprintf(stderr, "SIGPIPE!\n");
-    needclose = 0;
+	/* aparently our ftp companion died */
+	if (Verbose)
+		fprintf(stderr, "SIGPIPE!\n");
+	needclose = 0;
 }
 
 
@@ -451,7 +493,7 @@ ftp_stop(void)
  * coprocess is currently at, close first. 
  */
 int
-ftp_start(char *base)
+ftp_start(const char *base)
 {
 	const char *tmp1, *tmp2;
 	char *p;
@@ -462,7 +504,6 @@ ftp_start(char *base)
 	const char *currentDir=getenv(PKG_FTPIO_CURRENTDIR);
 	int urllen;
 
-#if HAVE_TGETENT
 	/* talk to termcap for bold on/off escape sequences */
 	if (getenv("TERM") != NULL && tgetent(term, getenv("TERM")) > 0) {
 		p = bold_on;  tgetstr("md", &p);
@@ -471,10 +512,6 @@ ftp_start(char *base)
 		bold_on[0]  = '\0';
 		bold_off[0] = '\0';
 	}
-#else
-	bold_on[0]  = '\0';
-	bold_off[0] = '\0';
-#endif
 	
 	fileURLHost(base, newHost, sizeof(newHost));
 	urllen = URLlength(base);
@@ -523,7 +560,7 @@ ftp_start(char *base)
 			return -1;
 		}
 		
-		/* lukemftp now issues a CWD for each part of the path
+		/* nbftp now issues a CWD for each part of the path
 		 * and will return a code for each of them. No idea how to
 		 * deal with that other than to issue a 'prompt off' to
 		 * get something that we can wait for and that does NOT
@@ -570,71 +607,98 @@ ftp_start(char *base)
 int
 expandURL(char *expandedurl, const char *wildcardurl)
 {
-    char *pkg;
-    int rc;
-    char base[FILENAME_MAX];
-
-    pkg=strrchr(wildcardurl, '/');
-    if (pkg == NULL){
-	warnx("expandURL: no '/' in URL %s?!", wildcardurl);
-	return -1;
-    }
-    (void) snprintf(base, sizeof(base), "%*.*s/", (int)(pkg-wildcardurl),
-             (int)(pkg-wildcardurl), wildcardurl);
-    pkg++;
-
-    rc = ftp_start(base);
-    if (rc == -1) {
-	    warnx("ftp_start() failed");
-	    return -1; /* error */
-    }
-
-    /* for a given wildcard URL, find the best matching pkg */
-    {
+	char *pattern;
+	char *bestmatch;
+	char base[FILENAME_MAX];
+
+	pattern=strrchr(wildcardurl, '/');
+	if (pattern == NULL){
+		warnx("expandURL: no '/' in URL %s?!", wildcardurl);
+		return -1;
+	}
+	if (pattern-strchr(wildcardurl, '/') < 2) {
+		/* only one or two slashes in total */
+		warnx("expandURL: not enough '/' in URL %s", wildcardurl);
+		return -1;
+	}
+	(void) snprintf(base, sizeof(base), "%.*s/",
+			(int)(pattern-wildcardurl), wildcardurl);
+	pattern++;
+
+	if (strncmp(wildcardurl, "ftp://", 6) == 0)
+		bestmatch=ftp_expand_URL(base, pattern);
+	else if (strncmp(wildcardurl, "http://", 7) == 0)
+		bestmatch=http_expand_URL(base, pattern);
+	else {
+		warnx("expandURL: unknown protocol in URL `%s'", wildcardurl);
+		return -1;
+	}
+
+	/* no match found */
+	if (bestmatch == NULL)
+		return -1;
+
+	snprintf(expandedurl, FILENAME_MAX, "%s%s", base, bestmatch);
+	if (Verbose)
+		printf("best match: '%s'\n", expandedurl);
+
+	return 0;
+}
+
+/* for a given wildcard ftp:// URL, find the best matching pkg */
+static char *
+ftp_expand_URL(const char *base, char *pattern)
+{
 	char *s, buf[FILENAME_MAX];
 	char tmpname[FILENAME_MAX];
 	char best[FILENAME_MAX];
-	int tfd;
+	int rc, tfd;
+
+	rc = ftp_start(base);
+	if (rc == -1) {
+		warnx("ftp_start() failed");
+		return NULL;
+	}
 
 	strlcpy(tmpname, "/var/tmp/pkg.XXXXXX", sizeof(tmpname));
 	tfd=mkstemp(tmpname);
 	if (tfd == -1) {
 		warnx("Cannot generate temp file for ftp(1)'s nlist output");
-		return -1; /* error */
+		return NULL;
 	}
 	close(tfd); /* We don't need the file descriptor, but will use 
 		       the file in a second */
 
-	s=strpbrk(pkg, "<>[]?*{"); /* Could leave out "[]?*" here;
-				    * ftp(1) is not that stupid */
+	s=strpbrk(pattern, "<>[]?*{"); /* Could leave out "[]?*" here;
+					* ftp(1) is not that stupid */
 	if (!s) {
 		/* This should only happen when getting here with (only) a package
 		 * name specified to pkg_add, and PKG_PATH containing some URL.
 		 */
-		(void) snprintf(buf, sizeof(buf), "nlist %s %s\n", pkg, tmpname);
+		(void) snprintf(buf, sizeof(buf), "nlist %s %s\n", pattern, tmpname);
 	} else {
 		/* replace possible version(wildcard) given with "-*".
 		 * we can't use the pkg wildcards here as dewey compare
 		 * and alternates won't be handled by ftp(1); sort
 		 * out later, using pmatch() */
-		(void) snprintf(buf,  sizeof(buf), "nlist %*.*s*.t[bg]z %s\n",
-                         (int)(s-pkg), (int)(s-pkg), pkg, tmpname);
+		(void) snprintf(buf,  sizeof(buf), "nlist %.*s*.t[bg]z %s\n",
+				(int)(s-pattern), pattern, tmpname);
 	}
-	
+
 	rc = ftp_cmd(buf, "\n(550|226).*\n"); /* catch errors */
 	if (rc != 226) {
-	    if (Verbose)
-		    warnx("nlist failed!");
-	    unlink(tmpname);	/* remove clutter */
-	    return -1;
+		if (Verbose)
+			warnx("nlist failed!");
+		unlink(tmpname);	/* remove clutter */
+		return NULL;
 	}
 
 	/* Sync - don't remove */
 	rc = ftp_cmd("cd .\n", "\n(550|250).*\n");
 	if (rc != 250) {
-	    warnx("chdir failed!");
-	    unlink(tmpname);	/* remove clutter */
-	    return -1;
+		warnx("chdir failed!");
+		unlink(tmpname);	/* remove clutter */
+		return NULL;
 	}
 	
 	best[0]='\0';
@@ -642,12 +706,12 @@ expandURL(char *expandedurl, const char *wildcardurl)
 		int matches;
 		FILE *f;
 		char filename[FILENAME_MAX];
-		
+
 		f=fopen(tmpname, "r");
 		if (f == NULL) {
-		    warn("fopen");
-	            unlink(tmpname);	/* remove clutter */
-		    return -1;
+			warn("fopen");
+			unlink(tmpname);	/* remove clutter */
+			return NULL;
 		}
 		matches=0;
 		/* The following loop is basically the same as the readdir() loop
@@ -655,19 +719,19 @@ expandURL(char *expandedurl, const char *wildcardurl)
 		while (fgets(filename, sizeof(filename), f)) {
 
 			/*
-			 * We need to stripp of any .t[bg]z etc.
+			 * We need to strip off any .t[bg]z etc.
 			 * suffix here
 			 */
 
 			char s_filename[FILENAME_MAX];
-			char s_pkg[FILENAME_MAX];
-			
+			char s_pattern[FILENAME_MAX];
+	    
 			filename[strlen(filename)-1] = '\0';
 
 			strip_txz(s_filename, NULL, filename);
-			strip_txz(s_pkg, NULL, pkg);
-			
-			if (pmatch(s_pkg, s_filename)) {
+			strip_txz(s_pattern, NULL, pattern);
+
+			if (pmatch(s_pattern, s_filename)) {
 				matches++;
 
 				/* compare findbestmatchingname() */
@@ -675,26 +739,426 @@ expandURL(char *expandedurl, const char *wildcardurl)
 			}
 		}
 		(void) fclose(f);
-		
+
 		if (matches == 0 && Verbose)
 			warnx("nothing appropriate found");
 	}
 
 	unlink(tmpname);
 
-	if (best[0] != '\0') {
+	if (best[0] == '\0')
+		return NULL;
+
+	return strdup(best);
+}
+
+/* for a given wildcard http:// URL, find the best matching pkg */
+static char *
+http_expand_URL(const char *base, char *pattern)
+{
+	char    best[FILENAME_MAX];
+	char    line[BUFSIZ];
+	char    filename[FILENAME_MAX];
+	FILE   *fp;
+	int	pipefds[2];
+	int     state;
+	pid_t   pid;
+
+	*best = '\0';
+
+	/* Set up a pipe for getting the file list */
+	if (pipe(pipefds) == -1) {
+		warnx("cannot create pipe");
+		return NULL;
+	}
+	if ((pid = fork()) == -1) {
+		warnx("cannot fork ftp process");
+		return NULL;
+	}
+	if (pid == 0) {		/* The child */
+		if (dup2(pipefds[1], STDOUT_FILENO) == -1) {
+			warnx("dup2 failed before starting ftp");
+			_exit(2);
+		}
+		close(pipefds[0]);
+		close(pipefds[1]);
+		/* get URL contents to stdout and thus to parent,
+		 * silently */
+		execlp("ftp", "ftp", "-V", "-o", "-", base, NULL);
+		warnx("failed to execute ftp");
+		_exit(2);
+	}
+
+	/* parent */
+	close(pipefds[1]);
+
+	if ((fp=fdopen(pipefds[0], "r")) == NULL)
+		warn("can't fdopen pipe end");
+	else {
+		char s_pattern[FILENAME_MAX];
+		int len, offset;
+
+		/* strip of .t[bg]z for comparison */
+		strip_txz(s_pattern, NULL, pattern);
+
+		/* initialize http_extract_fn internal state */
+		http_extract_fn(NULL, NULL, 0);
+
+		/* read line from HTTP output and extract filenames */
+		while (fgets(line, sizeof(line), fp) != NULL) {
+			len = offset = 0;
+			while ((len=http_extract_fn(line+offset, filename,
+						      sizeof(filename))) > 0) {
+				char s_filename[FILENAME_MAX];
+
+				offset += len;
+				strip_txz(s_filename, NULL, filename);
+
+				if (pmatch(s_pattern, s_filename)) {
+					/* compare findbestmatchingname() */
+					findbestmatchingname_fn(filename,
+								best);
+				}
+			}
+		}
+
+	}
+
+	/* wait for child to exit */
+	if (waitpid(pid, &state, 0) < 0) {
+		/* error has been reported by child */
+		return NULL;
+	}
+
+	if (best[0] == '\0') {
 		if (Verbose)
-			printf("best match: '%s%s'\n", base, best);
-		snprintf(expandedurl, FILENAME_MAX, "%s%s", base, best);
+			warnx("nothing appropriate found");
+		return NULL;
+	}
+
+	return strdup(best);
+
+}
+
+enum http_states {
+	ST_NONE,
+	ST_LT, ST_LTA, ST_TAGA, ST_H, ST_R, ST_E, ST_F, ST_HREF,
+	ST_TAG, ST_TAGAX
+};
+
+/* return any hrefs found */
+static int
+http_extract_fn(char *input, char *outbuf, size_t outbuflen)
+{
+	/* partial copied hrefs from previous calls are saved here */
+	static char tempbuf[FILENAME_MAX];
+	/* fill state of tempbuf */
+	static int tempbuffill = 0;
+	/* parsing state information */
+	static enum http_states state;
+	/* currently in double quotes (in parsing) */
+	static int dqflag;
+	char p;
+	int offset, found;
+
+	if (outbuf == NULL) {
+		/* init */
+		dqflag = tempbuffill = 0;
+		state = ST_NONE;
+		return 0;
+	}
+
+	offset = 0;
+	found = 0;
+	while ((p=input[offset++]) != '\0') {
+		/* handle anything that's inside double quotes */
+		if (dqflag) {
+			/* incomplete href */
+			if (state == ST_HREF) {
+				/* check if space left in output
+				 * buffer */
+				if (tempbuffill >= sizeof(tempbuf)) {
+					warnx("href starting with `%.*s'"
+					      " too long", 60, tempbuf);
+					/* ignore remainder */
+					tempbuffill = 0;
+					/* need space before "href"
+					 * can start again (invalidly,
+					 * of course, but we don't
+					 * care) */
+					state = ST_TAGAX;
+				}
+
+				/* href complete */
+				if (p == '\"') {
+					/* complete */
+					dqflag = 0;
+					tempbuf[tempbuffill++] = '\0';
+					/* need space before "href"
+					 * can start again (invalidly,
+					 * of course, but we don't
+					 * care) */
+					state = ST_TAGAX;
+					found = 1;
+					break;
+				} else {
+					/* copy one more char */
+					tempbuf[tempbuffill++] = p;
+				}
+			} else {
+				/* leaving double quotes */
+				if (p == '\"')
+					dqflag = 0;
+			}
+			continue;
+		}
+
+		/*
+		 * entering double quotes? (only relevant inside a tag)
+		 */
+		if (state != ST_NONE && p == '\"') {
+			dqflag = 1;
+			continue;
+		}
+
+		/* other cases */
+		switch (state) {
+		case ST_NONE:
+			/* plain text, not in markup */
+			if (p == '<')
+				state = ST_LT;
+			break;
+		case ST_LT:
+			/* in tag -- "<" already found */
+			if (p == '>')
+				state = ST_NONE;
+			else if (p == 'a' || p == 'A')
+				state = ST_LTA;
+			else if (!isspace(p))
+				state = ST_TAG;
+			break;
+		case ST_LTA:
+			/* in tag -- "<a" already found */
+			if (p == '>')
+				state = ST_NONE;
+			else if (isspace(p))
+				state = ST_TAGA;
+			else
+				state = ST_TAG;
+			break;
+		case ST_TAG:
+			/* in tag, but not "<a" -- disregard */
+			if (p == '>')
+				state = ST_NONE;
+			break;
+		case ST_TAGA:
+			/* in a-tag -- "<a " already found */
+			if (p == '>')
+				state = ST_NONE;
+			else if (p == 'h' || p == 'H')
+				state = ST_H;
+			else if (!isspace(p))
+				state = ST_TAGAX;
+			break;
+		case ST_TAGAX:
+			/* in unknown keyword in a-tag */
+			if (p == '>')
+				state = ST_NONE;
+			else if (isspace(p))
+				state = ST_TAGA;
+			break;
+		case ST_H:
+			/* in a-tag -- "<a h" already found */
+			if (p == '>')
+				state = ST_NONE;
+			else if (p == 'r' || p == 'R')
+				state = ST_R;
+			else if (isspace(p))
+				state = ST_TAGA;
+			else
+				state = ST_TAGAX;
+			break;
+		case ST_R:
+			/* in a-tag -- "<a hr" already found */
+			if (p == '>')
+				state = ST_NONE;
+			else if (p == 'e' || p == 'E')
+				state = ST_E;
+			else if (isspace(p))
+				state = ST_TAGA;
+			else
+				state = ST_TAGAX;
+			break;
+		case ST_E:
+			/* in a-tag -- "<a hre" already found */
+			if (p == '>')
+				state = ST_NONE;
+			else if (p == 'f' || p == 'F')
+				state = ST_F;
+			else if (isspace(p))
+				state = ST_TAGA;
+			else
+				state = ST_TAGAX;
+			break;
+		case ST_F:
+			/* in a-tag -- "<a href" already found */
+			if (p == '>')
+				state = ST_NONE;
+			else if (p == '=')
+				state = ST_HREF;
+			else if (!isspace(p))
+				state = ST_TAGAX;
+			break;
+		case ST_HREF:
+			/* in a-tag -- "<a href=" already found */
+			/* XXX: handle missing double quotes? */
+			if (p == '>')
+				state = ST_NONE;
+			/* skip spaces before URL */
+			else if (!isspace(p))
+				state = ST_TAGA;
+			break;
+			/* no default case by purpose */
+		}
+	}
+
+	if (p == '\0')
+		return -1;
+
+	if (found) {
+		char *q;
+
+		URL_decode(tempbuf);
+
+		/* strip path (XXX) */
+		if ((q=strrchr(tempbuf, '/')) == NULL)
+			q = tempbuf;
+			
+		(void)strlcpy(outbuf, q, outbuflen);
+		tempbuffill = 0;
 	}
+		
+	return offset;
+}
+
+
+static int
+hexvalue(char p)
+{
+	if (p >= '0' && p <= '9')
+		return (p-'0');
+	else if (p >= 'a' && p <= 'f')
+		return (p-'a'+10);
+	else if (p >= 'A' && p <= 'F')
+		return (p-'A'+10);
 	else
 		return -1;
-    }
+}
+
+/* fetch and extract URL url into directory path */
+static int
+http_fetch(const char *url, const char *path)
+{
+	int     pipefds[2];
+	int	stateftp, state;
+	pid_t   pidftp, pid;
+
+	/* Set up a pipe for passing the fetched contents. */
+	if (pipe(pipefds) == -1) {
+		warn("cannot create pipe");
+		return -1;
+	}
+	/* fork ftp child */
+	if ((pidftp = fork()) == -1) {
+		warn("cannot fork process for ftp");
+		return -1;
+	}
+	if (pidftp == 0) {
+		/* child */
+		if (dup2(pipefds[1], STDOUT_FILENO) == -1) {
+			warn("dup2 failed before executing ftp");
+			_exit(2);
+		}
+		close(pipefds[0]);
+		close(pipefds[1]);
+		execlp(FTP_CMD, FTP_CMD, "-o", "-", url, NULL);
+		warnx("failed to execute ftp");
+		_exit(2);
+	}
+
+	/* fork unpack child */
+	if ((pid = fork()) == -1) {
+		warn("cannot fork unpack process");
+		return -1;
+	}
+	if (pid == 0) {
+		/* child */
+		if (dup2(pipefds[0], STDIN_FILENO) == -1) {
+			warn("dup2 failed before unpack");
+			_exit(2);
+		}
+		close(pipefds[0]);
+		close(pipefds[1]);
+		if ((path != NULL) && (chdir(path) < 0))
+			_exit(127);
+
+		if (unpack("-", Verbose ? "-vv" : NULL, NULL) != 0) {
+			warnx("unpack failed");
+			_exit(2);
+		}
+
+		_exit(0);
+	}
+
+	close(pipefds[0]);
+	close(pipefds[1]);
+
+	/* wait for unpack to exit */
+	while (waitpid(pid, &state, 0) < 0) {
+		if (errno != EINTR) {
+			(void)waitpid(pidftp, &stateftp, 0);
+			return -1;
+		}
+	}
+	while (waitpid(pidftp, &stateftp, 0) < 0) {
+		if (errno != EINTR) {
+			return -1;
+		}
+	}
+
+	if (!WIFEXITED(state) || !WIFEXITED(stateftp))
+		return -1;
 
-    return 0;
+	if (WEXITSTATUS(state) != 0 || WEXITSTATUS(stateftp) != 0)
+		return -1;
+
+	return 0;
 }
 
+static void
+URL_decode(char *URL)
+{
+	char *in, *out;
+
+	in = out = URL;
+
+	while (*in != '\0') {
+		if (in[0] == '%' && in[1] != '\0' && in[2] != '\0') {
+			/* URL-decode character */
+			if (hexvalue(in[1]) != -1 && hexvalue(in[2]) != -1) {
+				*out++ = hexvalue(in[1])*16+hexvalue(in[2]);
+			}
+			/* skip invalid encoded signs too */
+			in += 3;
+		}
+		else
+			*out++ = *in++;
+	}
+
+	*out = '\0';
 
+	return;
+}
 /*
  * extract the given (expanded) URL "url" to the given directory "dir"
  * return -1 on error, 0 else;
@@ -728,9 +1192,8 @@ unpackURL(const char *url, const char *dir)
 		warnx("unpackURL: no '/' in URL %s?!", url);
 		return -1;
 	}
-	(void) snprintf(base, sizeof(base), "%*.*s/", (int)(pkg-url),
-                 (int)(pkg-url), url);
-	(void) snprintf(pkg_path, sizeof(pkg_path), "%*.*s", (int)(pkg-url),
+	(void) snprintf(base, sizeof(base), "%.*s/", (int)(pkg-url), url);
+	(void) snprintf(pkg_path, sizeof(pkg_path), "%.*s",
                  (int)(pkg-url), url); /* no trailing '/' */
 	pkg++;
 
@@ -742,7 +1205,10 @@ unpackURL(const char *url, const char *dir)
 #endif
 		printf("setenv PKG_PATH='%s'\n",pkg_path);
 	}
-	
+
+	if (strncmp(url, "http://", 7) == 0)
+	    return http_fetch(url, dir);
+
 	rc = ftp_start(base);
 	if (rc == -1) {
 		warnx("ftp_start() failed");
@@ -768,186 +1234,69 @@ unpackURL(const char *url, const char *dir)
 }
 
 
-#if 0
-/*
- * Some misc stuff not needed yet, but maybe later
- */
-int
-miscstuff(const char *url)
-{
-    char *pkg;
-    int rc;
-    char base[FILENAME_MAX];
-
-    pkg=strrchr(url, '/');
-    if (pkg == NULL){
-	warnx("miscstuff: no '/' in URL %s?!", url);
-	return -1;
-    }
-    (void) snprintf(base, sizeof(base), "%*.*s/", (int)(pkg-url), (int)(pkg-url),
-             url);
-    pkg++;
-
-    rc = ftp_start(base);
-    if (rc == -1) {
-	    warnx("ftp_start() failed");
-	    return -1; /* error */
-    }
-
-    /* basic operation */
-    if (0) {
-	    rc = ftp_cmd("cd ../All\n", "\n(550|250).*\n");
-	    if (rc != 250) {
-		    warnx("chdir failed!");
-		    return -1;
-	    }
-    }
-
-    /* get and extract a file to tmpdir */
-    if (0) {
-	char cmd[256];
-	char tmpdir[256];
-
-	(void) snprintf(tmpdir, sizeof(tmpdir), "/var/tmp/dir%s",
-		 (getenv(PKG_FTPIO_CNT))?getenv(PKG_FTPIO_CNT):"");
-
-	mkdir(tmpdir, 0755);
-
-	/* yes, this is gross, but needed for borken ftp(1) */
-	(void) snprintf(cmd, sizeof(cmd), "get xpmroot-1.01.tgz \"| ( cd %s ; gunzip 2>/dev/null | tar -vvx -f - )\"\n", tmpdir);
-	rc = ftp_cmd(cmd, "\n(226|550).*\n");
-	if (rc != 226) {
-	    warnx("Cannot fetch file (%d != 226)!", rc);
-	    return -1;
-	}
-    }
-    
-    /* check if one more file(s) exist */
-    if (0) {
-	char buf[FILENAME_MAX];
-	(void) snprintf(buf, sizeof(buf), "nlist %s /var/tmp/xxx\n", pkg);
-	rc = ftp_cmd(buf, "\n(226|550).*\n"); /* catch errors */
-	if (rc != 226) {
-	    if (Verbose)
-		warnx("nlist failed!");
-	    return -1;
-	}
-
-	/* Sync - don't remove */
-	rc = ftp_cmd("cd .\n", "\n(550|250).*\n");
-	if (rc != 250) {
-	    warnx("chdir failed!");
-	    return -1;
-	}
-	
-	if (access("/var/tmp/xxx", R_OK)==0) {
-	    fexec("cat", "/var/tmp/xxx", NULL);
-	    
-	    {
-		/* count lines - >0 -> fexists() == true */
-		int len, count;
-		FILE *f;
-		
-		f=fopen("/var/tmp/xxx", "r");
-		if (f == NULL) {
-		    warn("fopen");
-		    return -1;
-		}
-		count=0;
-		while (fgetln(f, &len))
-		    count++;
-		(void) fclose(f);
-		
-		printf("#lines = %d\n", count);
-	    }
-	} else
-	    printf("NO MATCH\n");
-	
-	unlink("/var/tmp/xxx");
-    }
-
-    /* for a given wildcard URL, find the best matching pkg */
-    /* spawn child - like pkg_add'ing another package */
-    /* Here's where the connection caching kicks in */
-#if 0
-    if (0) {
-	char *s, buf[FILENAME_MAX];
-
-	if ((s=getenv(PKG_FTPIO_CNT)) && atoi(s)>0){
-	    (void) snprintf(buf, sizeof(buf),"%d", atoi(s)-1);
-	    setenv(PKG_FTPIO_CNT, buf, 1);
-
-	    (void) snprintf(buf, sizeof(buf), "%s/%s", url, pkg);
-	    printf("%s>>> %s %s\n", s, argv0, buf);
-	    fexec(argv0, buf, NULL);
-	}
-    }
-#endif 
-
-    return 0;
-
-}
-#endif
-
-
 #ifdef STANDALONE
 static void
 usage(void)
 {
-  errx(EXIT_FAILURE, "Usage: foo [-v] ftp://-pattern");
+	errx(EXIT_FAILURE, "Usage: foo [-v] ftp://-pattern");
 }
 
 
 int
 main(int argc, char *argv[])
 {
-    int rc, ch;
-    char *argv0 = argv[0];
-
-    while ((ch = getopt(argc, argv, "v")) != -1) {
-      switch (ch) {
-      case 'v':
-	Verbose=1;
-	break;
-      default:
-	usage();
-      }
-    }
-    argc -= optind;
-    argv += optind;
-
-    if (argc<1)
-	usage();
-
-    while(argv[0] != NULL) {
-	    char newurl[FILENAME_MAX];
+	int rc, ch;
+	char *argv0 = argv[0];
+
+	while ((ch = getopt(argc, argv, "v")) != -1) {
+		switch (ch) {
+		case 'v':
+			Verbose=1;
+			break;
+		default:
+			usage();
+		}
+	}
+	argc -= optind;
+	argv += optind;
+
+	if (argc<1)
+		usage();
+
+	while(argv[0] != NULL) {
+		char newurl[FILENAME_MAX];
 	    
-	    printf("Expand %s:\n", argv[0]);
-	    rc = expandURL(newurl, argv[0]);
-	    if (rc==-1)
-		    warnx("Cannot expand %s", argv[0]);
-	    else
-		    printf("Expanded URL: %s\n", newurl);
-
-	    /* test out connection caching */
-	    if (1) {
-		    char *s, buf[FILENAME_MAX];
+		printf("Expand %s:\n", argv[0]);
+		rc = expandURL(newurl, argv[0]);
+		if (rc==-1)
+			warnx("Cannot expand %s", argv[0]);
+		else
+			printf("Expanded URL: %s\n", newurl);
+
+		/* test out connection caching */
+		if (1) {
+			char *s, buf[FILENAME_MAX];
 		    
-		    if ((s=getenv(PKG_FTPIO_CNT)) && atoi(s)>0){
-			    (void) snprintf(buf, sizeof(buf),"%d", atoi(s)-1);
-			    setenv(PKG_FTPIO_CNT, buf, 1);
+			if ((s=getenv(PKG_FTPIO_CNT)) && atoi(s)>0){
+				(void) snprintf(buf, sizeof(buf),"%d", atoi(s)-1);
+				setenv(PKG_FTPIO_CNT, buf, 1);
 			    
-			    printf("%s>>> %s -v %s\n", s, argv0, argv[0]);
-			    fexec(argv0, "-v", argv[0], NULL);
-		    }
-	    }
+				printf("%s>>> %s -v %s\n", s, argv0, argv[0]);
+				fexec(argv0, "-v", argv[0], NULL);
+			}
+		}
 	    
-	    printf("\n\n\n");
-	    argv++;
-    }
+		printf("\n\n\n");
+		argv++;
+	}
 
-    ftp_stop();
+	ftp_stop();
 
-    return 0;
+	return 0;
+}
+
+void
+cleanup(int i)
+{
 }
 #endif /* STANDALONE */
diff --git a/pkgtools/pkg_install/files/lib/lib.h b/pkgtools/pkg_install/files/lib/lib.h
index 7a686bd9bec..83f7ac6ccbc 100644
--- a/pkgtools/pkg_install/files/lib/lib.h
+++ b/pkgtools/pkg_install/files/lib/lib.h
@@ -1,4 +1,4 @@
-/* $NetBSD: lib.h,v 1.5 2003/09/23 13:22:42 grant Exp $ */
+/* $NetBSD: lib.h,v 1.6 2003/10/29 23:00:28 jlam Exp $ */
 
 /* from FreeBSD Id: lib.h,v 1.25 1997/10/08 07:48:03 charnier Exp */
 
@@ -226,8 +226,6 @@ typedef int (*matchfn) (const char *, void *);
 
 /* Prototypes */
 /* Misc */
-int     vsystem(const char *,...)
-	__attribute__((__format__(__printf__, 1, 2)));
 void    cleanup(int);
 char   *make_playpen(char *, size_t, size_t);
 char   *where_playpen(void);
@@ -250,7 +248,6 @@ int     pmatch(const char *, const char *);
 int     findmatchingname(const char *, const char *, matchfn, void *); /* doesn't really belong to "strings" */
 char   *findbestmatchingname(const char *, const char *);	/* neither */
 int     ispkgpattern(const char *);
-char   *strnncpy(char *, size_t, char *, size_t);
 void	strip_txz(char *, char *, const char *);
 
 /* callback functions for findmatchingname */
@@ -277,6 +274,7 @@ Boolean make_preserve_name(char *, size_t, char *, char *);
 void    write_file(char *, char *);
 void    copy_file(char *, char *, char *);
 void    move_file(char *, char *, char *);
+void    remove_files(const char *, const char *);
 int     delete_hierarchy(char *, Boolean, Boolean);
 int     unpack(const char *, const char *, const char *);
 void    format_cmd(char *, size_t, char *, char *, char *);
@@ -285,7 +283,7 @@ void    format_cmd(char *, size_t, char *, char *, char *);
 int	expandURL(char *, const char *);
 int	unpackURL(const char *, const char *);
 int	ftp_cmd(const char *, const char *);
-int	ftp_start(char *);
+int	ftp_start(const char *);
 void	ftp_stop(void);
 
 /* Packing list */
diff --git a/pkgtools/pkg_install/files/lib/str.c b/pkgtools/pkg_install/files/lib/str.c
index 41200fcb4f6..0a53862223f 100644
--- a/pkgtools/pkg_install/files/lib/str.c
+++ b/pkgtools/pkg_install/files/lib/str.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: str.c,v 1.6 2003/09/23 07:13:54 grant Exp $	*/
+/*	$NetBSD: str.c,v 1.7 2003/10/29 23:00:28 jlam Exp $	*/
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -11,7 +11,7 @@
 #if 0
 static const char *rcsid = "Id: str.c,v 1.5 1997/10/08 07:48:21 charnier Exp";
 #else
-__RCSID("$NetBSD: str.c,v 1.6 2003/09/23 07:13:54 grant Exp $");
+__RCSID("$NetBSD: str.c,v 1.7 2003/10/29 23:00:28 jlam Exp $");
 #endif
 #endif
 
@@ -576,22 +576,6 @@ findbestmatchingname(const char *dir, const char *pattern)
 }
 
 /*
- * Bounds-checking strncpy()
- */
-char   *
-strnncpy(char *to, size_t tosize, char *from, size_t cc)
-{
-	size_t  len;
-
-	if ((len = cc) >= tosize - 1) {
-		len = tosize - 1;
-	}
-	(void) strncpy(to, from, len);
-	to[len] = 0;
-	return to;
-}
-
-/*
  * Strip off any .tgz, .tbz or .t[bg]z suffix from fname,
  * and copy into buffer "buf", the suffix is stored in "sfx"
  * if "sfx" is not NULL. If no suffix is found, "sfx" is set
diff --git a/pkgtools/pkg_install/files/lib/version.h b/pkgtools/pkg_install/files/lib/version.h
index be41f9cb779..49d620f68f5 100644
--- a/pkgtools/pkg_install/files/lib/version.h
+++ b/pkgtools/pkg_install/files/lib/version.h
@@ -1,4 +1,4 @@
-/*	$NetBSD: version.h,v 1.16 2003/09/23 07:13:54 grant Exp $	*/
+/*	$NetBSD: version.h,v 1.17 2003/10/29 23:00:28 jlam Exp $	*/
 
 /*
  * Copyright (c) 2001 Thomas Klausner.  All rights reserved.
@@ -33,6 +33,6 @@
 #ifndef _INST_LIB_VERSION_H_
 #define _INST_LIB_VERSION_H_
 
-#define PKGTOOLS_VERSION "20030923"
+#define PKGTOOLS_VERSION "20031003"
 
 #endif /* _INST_LIB_VERSION_H_ */