diff options
author | joerg <joerg@pkgsrc.org> | 2008-10-08 15:28:30 +0000 |
---|---|---|
committer | joerg <joerg@pkgsrc.org> | 2008-10-08 15:28:30 +0000 |
commit | a900d504d59dd04f1202692d8ac63671d71562c3 (patch) | |
tree | 4e50f5762c0c395704387c5c45185b03aed98936 /net/fetch | |
parent | 33cbc91d9638e7a16d3fce7ceb093295934b6f5c (diff) | |
download | pkgsrc-a900d504d59dd04f1202692d8ac63671d71562c3.tar.gz |
Add fetch-1.0 from FreeBSD, the frontend for libfetch(3).
Diffstat (limited to 'net/fetch')
-rw-r--r-- | net/fetch/DESCR | 3 | ||||
-rw-r--r-- | net/fetch/Makefile | 28 | ||||
-rw-r--r-- | net/fetch/PLIST | 4 | ||||
-rw-r--r-- | net/fetch/files/Makefile | 14 | ||||
-rw-r--r-- | net/fetch/files/fetch.1 | 275 | ||||
-rw-r--r-- | net/fetch/files/fetch.c | 1016 | ||||
-rw-r--r-- | net/fetch/files/fetch.cat1 | 153 | ||||
-rw-r--r-- | net/fetch/options.mk | 15 |
8 files changed, 1508 insertions, 0 deletions
diff --git a/net/fetch/DESCR b/net/fetch/DESCR new file mode 100644 index 00000000000..7f989d7f51f --- /dev/null +++ b/net/fetch/DESCR @@ -0,0 +1,3 @@ +fetch is a command line for transferring files with URL syntax. +Supported schemas are FTP, HTTP, HTTPS and FILE. It includes Advanced +features like proxy connections and restarting of interrupted transfers. diff --git a/net/fetch/Makefile b/net/fetch/Makefile new file mode 100644 index 00000000000..0f56807a6fa --- /dev/null +++ b/net/fetch/Makefile @@ -0,0 +1,28 @@ +# $NetBSD: Makefile,v 1.1.1.1 2008/10/08 15:28:30 joerg Exp $ +# + +DISTNAME= fetch-1.0 +CATEGORIES= net +MASTER_SITES= # empty +DISTFILES= # empty + +MAINTAINER= joerg@NetBSD.org +HOMEPAGE= http://www.FreeBSD.org/ +COMMENT= Client to fetch URLs +#LICENSE= modified-bsd + +PKG_DESTDIR_SUPPORT= user-destdir + +USE_FEATURES= nbcompat + +USE_BSD_MAKEFILE= yes + +INSTALLATION_DIRS= bin ${PKGMANDIR}/cat1 ${PKGMANDIR}/man1 + +do-extract: + ${CP} -r ${FILESDIR} ${WRKSRC} + +.include "options.mk" + +.include "../../net/libfetch/buildlink3.mk" +.include "../../mk/bsd.pkg.mk" diff --git a/net/fetch/PLIST b/net/fetch/PLIST new file mode 100644 index 00000000000..514cd6efd46 --- /dev/null +++ b/net/fetch/PLIST @@ -0,0 +1,4 @@ +@comment $NetBSD: PLIST,v 1.1.1.1 2008/10/08 15:28:30 joerg Exp $ +bin/fetch +man/cat1/fetch.0 +man/man1/fetch.1 diff --git a/net/fetch/files/Makefile b/net/fetch/files/Makefile new file mode 100644 index 00000000000..90815d12f43 --- /dev/null +++ b/net/fetch/files/Makefile @@ -0,0 +1,14 @@ +# $FreeBSD$ + +.include <bsd.own.mk> + +PROG= fetch +LDADD= -lfetch + +FETCH_WITH_OPENSSL?= no + +.if ${FETCH_WITH_OPENSSL} != "no" +LDADD+= -lssl -lcrypto +.endif + +.include <bsd.prog.mk> diff --git a/net/fetch/files/fetch.1 b/net/fetch/files/fetch.1 new file mode 100644 index 00000000000..eaf6a297015 --- /dev/null +++ b/net/fetch/files/fetch.1 @@ -0,0 +1,275 @@ +.\"- +.\" Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav +.\" All rights reserved. +.\" Portions Copyright (c) 1999 Massachusetts Institute of Technology; used +.\" by permission. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer +.\" in this position and unchanged. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" 3. The name of the author may not be used to endorse or promote products +.\" derived from this software without specific prior written permission. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +.\" IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +.\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +.\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +.\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +.\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd March 11, 2003 +.Dt FETCH 1 +.Os +.Sh NAME +.Nm fetch +.Nd retrieve a file by Uniform Resource Locator +.Sh SYNOPSIS +.Nm +.Op Fl 146AFMRUadlmnqrsv +.Op Fl B Ar bytes +.Op Fl S Ar bytes +.Op Fl T Ar seconds +.Op Fl N Ar file +.Op Fl o Ar file +.Op Fl w Ar seconds +.Op Fl h Ar host +.Op Fl c Ar dir +.Op Fl f Ar file +.Op Ar URL ... +.Sh DESCRIPTION +The +.Nm +utility provides a command-line interface to the +.Xr fetch 3 +library. +Its purpose is to retrieve the file(s) pointed to by the URL(s) on the +command line. +.Pp +The following options are available: +.Bl -tag -width Fl +.It Fl \&1 +Stop and return exit code 0 at the first successfully retrieved file. +.It Fl 4 +Forces +.Nm +to use IPv4 addresses only. +.It Fl 6 +Forces +.Nm +to use IPv6 addresses only. +.It Fl A +Do not automatically follow ``temporary'' (302) redirects. +Some broken Web sites will return a redirect instead of a not-found +error when the requested object does not exist. +.It Fl a +Automatically retry the transfer upon soft failures. +.It Fl B Ar bytes +Specify the read buffer size in bytes. +The default is 4096 bytes. +Attempts to set a buffer size lower than this will be silently +ignored. +The number of reads actually performed is reported at verbosity level +two or higher (see the +.Fl v +flag). +.It Fl c Ar dir +The file to retrieve is in directory +.Ar dir +on the remote host. +This option is deprecated and is provided for backward compatibility +only. +.It Fl d +Use a direct connection even if a proxy is configured. +.It Fl F +In combination with the +.Fl r +flag, forces a restart even if the local and remote files have +different modification times. +Implies +.Fl R . +.It Fl f Ar file +The file to retrieve is named +.Ar file +on the remote host. +This option is deprecated and is provided for backward compatibility +only. +.It Fl h Ar host +The file to retrieve is located on the host +.Ar host . +This option is deprecated and is provided for backward compatibility +only. +.It Fl l +If the target is a file-scheme URL, make a symbolic link to the target +rather than trying to copy it. +.It Fl M +.It Fl m +Mirror mode: if the file already exists locally and has the same size +and modification time as the remote file, it will not be fetched. +Note that the +.Fl m +and +.Fl r +flags are mutually exclusive. +.It Fl N Ar file +Use +.Ar file +instead of +.Pa ~/.netrc +to look up login names and passwords for FTP sites. +See +.Xr ftp 1 +for a description of the file format. +This feature is experimental. +.It Fl n +Do not preserve the modification time of the transferred file. +.It Fl o Ar file +Set the output file name to +.Ar file . +By default, a ``pathname'' is extracted from the specified URI, and +its basename is used as the name of the output file. +A +.Ar file +argument of +.Sq Li \&- +indicates that results are to be directed to the standard output. +If the +.Ar file +argument is a directory, fetched file(s) will be placed within the +directory, with name(s) selected as in the default behaviour. +.It Fl q +Quiet mode. +.It Fl R +The output files are precious, and should not be deleted under any +circumstances, even if the transfer failed or was incomplete. +.It Fl r +Restart a previously interrupted transfer. +Note that the +.Fl m +and +.Fl r +flags are mutually exclusive. +.It Fl S Ar bytes +Require the file size reported by the server to match the specified +value. +If it does not, a message is printed and the file is not fetched. +If the server does not support reporting file sizes, this option is +ignored and the file is fetched unconditionally. +.It Fl s +Print the size in bytes of each requested file, without fetching it. +.It Fl T Ar seconds +Set timeout value to +.Ar seconds . +Overrides the environment variables +.Ev FTP_TIMEOUT +for FTP transfers or +.Ev HTTP_TIMEOUT +for HTTP transfers if set. +.It Fl U +When using passive FTP, allocate the port for the data connection from +the low (default) port range. +See +.Xr ip 4 +for details on how to specify which port range this corresponds to. +.It Fl v +Increase verbosity level. +.It Fl w Ar seconds +When the +.Fl a +flag is specified, wait this many seconds between successive retries. +.El +.Pp +If +.Nm +receives a +.Dv SIGINFO +signal (see the +.Cm status +argument for +.Xr stty 1 ) , +the current transfer rate statistics will be written to the +standard error output, in the same format as the standard completion +message. +.Sh ENVIRONMENT +.Bl -tag -width HTTP_TIMEOUT +.It Ev FTP_TIMEOUT +Maximum time, in seconds, to wait before aborting an FTP connection. +.It Ev HTTP_TIMEOUT +Maximum time, in seconds, to wait before aborting an HTTP connection. +.El +.Pp +See +.Xr fetch 3 +for a description of additional environment variables, including +.Ev FETCH_BIND_ADDRESS , +.Ev FTP_LOGIN , +.Ev FTP_PASSIVE_MODE , +.Ev FTP_PASSWORD , +.Ev FTP_PROXY , +.Ev ftp_proxy , +.Ev HTTP_AUTH , +.Ev HTTP_PROXY , +.Ev http_proxy , +.Ev HTTP_PROXY_AUTH , +.Ev HTTP_REFERER , +.Ev HTTP_USER_AGENT , +.Ev NETRC , +.Ev NO_PROXY and +.Ev no_proxy . +.Sh EXIT STATUS +The +.Nm +command returns zero on success, or one on failure. +If multiple URLs are listed on the command line, +.Nm +will attempt to retrieve each one of them in turn, and will return +zero only if they were all successfully retrieved. +.Sh SEE ALSO +.Xr fetch 3 +.Sh HISTORY +The +.Nm +command appeared in +.Fx 2.1.5 . +This implementation first appeared in +.Fx 4.1 . +.Sh AUTHORS +.An -nosplit +The original implementation of +.Nm +was done by +.An Jean-Marc Zucconi Aq jmz@FreeBSD.org . +It was extensively re-worked for +.Fx 2.2 +by +.An Garrett Wollman Aq wollman@FreeBSD.org , +and later completely rewritten to use the +.Xr fetch 3 +library by +.An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org . +.Sh NOTES +The +.Fl b +and +.Fl t +options are no longer supported and will generate warnings. +They were workarounds for bugs in other OSes which this implementation +does not trigger. +.Pp +One cannot both use the +.Fl h , +.Fl c +and +.Fl f +options and specify URLs on the command line. diff --git a/net/fetch/files/fetch.c b/net/fetch/files/fetch.c new file mode 100644 index 00000000000..c7f36a912ca --- /dev/null +++ b/net/fetch/files/fetch.c @@ -0,0 +1,1016 @@ +/*- + * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#if HAVE_CONFIG_H +#include "config.h" +#endif +#ifndef NETBSD +#include <nbcompat.h> +#endif + +#if HAVE_SYS_PARAM_H +#include <sys/param.h> +#endif +#if HAVE_SYS_SOCKET_H +#include <sys/socket.h> +#endif +#if HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif +#if HAVE_SYS_TIME_H +#include <sys/time.h> +#endif + +#include <ctype.h> +#if HAVE_ERR_H +#include <err.h> +#endif +#include <errno.h> +#include <signal.h> +#if HAVE_STDINT_H +#include <stdint.h> +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#if HAVE_TERMIOS_H +#include <termios.h> +#endif +#include <unistd.h> + +#include <fetch.h> + +#if HAVE_SYSEXITS_H +#include <sysexits.h> +#endif + +#ifndef EX_USAGE +#define EX_USAGE 64 +#endif + +#ifndef EX_IOERR +#define EX_IOERR 74 +#endif + +#define MINBUFSIZE 4096 + +/* Option flags */ +int A_flag; /* -A: do not follow 302 redirects */ +int a_flag; /* -a: auto retry */ +off_t B_size; /* -B: buffer size */ +int b_flag; /*! -b: workaround TCP bug */ +char *c_dirname; /* -c: remote directory */ +int d_flag; /* -d: direct connection */ +int F_flag; /* -F: restart without checking mtime */ +char *f_filename; /* -f: file to fetch */ +char *h_hostname; /* -h: host to fetch from */ +int l_flag; /* -l: link rather than copy file: URLs */ +int m_flag; /* -[Mm]: mirror mode */ +char *N_filename; /* -N: netrc file name */ +int n_flag; /* -n: do not preserve modification time */ +int o_flag; /* -o: specify output file */ +int o_directory; /* output file is a directory */ +char *o_filename; /* name of output file */ +int o_stdout; /* output file is stdout */ +int once_flag; /* -1: stop at first successful file */ +int R_flag; /* -R: don't delete partially transferred files */ +int r_flag; /* -r: restart previously interrupted transfer */ +off_t S_size; /* -S: require size to match */ +int s_flag; /* -s: show size, don't fetch */ +long T_secs = 120; /* -T: transfer timeout in seconds */ +int t_flag; /*! -t: workaround TCP bug */ +int U_flag; /* -U: do not use high ports */ +int v_level = 1; /* -v: verbosity level */ +int v_tty; /* stdout is a tty */ +pid_t pgrp; /* our process group */ +long w_secs; /* -w: retry delay */ +int family = PF_UNSPEC; /* -[46]: address family to use */ + +int sigalrm; /* SIGALRM received */ +int siginfo; /* SIGINFO received */ +int sigint; /* SIGINT received */ + +long ftp_timeout; /* default timeout for FTP transfers */ +long http_timeout; /* default timeout for HTTP transfers */ +char *buf; /* transfer buffer */ + + +/* + * Signal handler + */ +static void +sig_handler(int sig) +{ + switch (sig) { + case SIGALRM: + sigalrm = 1; + break; + case SIGINFO: + siginfo = 1; + break; + case SIGINT: + sigint = 1; + break; + } +} + +struct xferstat { + char name[64]; + struct timeval start; + struct timeval last; + off_t size; + off_t offset; + off_t rcvd; +}; + +/* + * Compute and display ETA + */ +static const char * +stat_eta(struct xferstat *xs) +{ + static char str[16]; + long elapsed, eta; + off_t received, expected; + + elapsed = xs->last.tv_sec - xs->start.tv_sec; + received = xs->rcvd - xs->offset; + expected = xs->size - xs->rcvd; + eta = (long)((double)elapsed * expected / received); + if (eta > 3600) + snprintf(str, sizeof str, "%02ldh%02ldm", + eta / 3600, (eta % 3600) / 60); + else + snprintf(str, sizeof str, "%02ldm%02lds", + eta / 60, eta % 60); + return (str); +} + +/* + * Format a number as "xxxx YB" where Y is ' ', 'k', 'M'... + */ +static const char *prefixes = " kMGTP"; +static const char * +stat_bytes(off_t bytes) +{ + static char str[16]; + const char *prefix = prefixes; + + while (bytes > 9999 && prefix[1] != '\0') { + bytes /= 1024; + prefix++; + } + snprintf(str, sizeof str, "%4jd %cB", (intmax_t)bytes, *prefix); + return (str); +} + +/* + * Compute and display transfer rate + */ +static const char * +stat_bps(struct xferstat *xs) +{ + static char str[16]; + double delta, bps; + + delta = (xs->last.tv_sec + (xs->last.tv_usec / 1.e6)) + - (xs->start.tv_sec + (xs->start.tv_usec / 1.e6)); + if (delta == 0.0) { + snprintf(str, sizeof str, "?? Bps"); + } else { + bps = (xs->rcvd - xs->offset) / delta; + snprintf(str, sizeof str, "%sps", stat_bytes((off_t)bps)); + } + return (str); +} + +/* + * Update the stats display + */ +static void +stat_display(struct xferstat *xs, int force) +{ + struct timeval now; + int ctty_pgrp; + + /* check if we're the foreground process */ + if (ioctl(STDERR_FILENO, TIOCGPGRP, &ctty_pgrp) == -1 || + (pid_t)ctty_pgrp != pgrp) + return; + + gettimeofday(&now, NULL); + if (!force && now.tv_sec <= xs->last.tv_sec) + return; + xs->last = now; + + fprintf(stderr, "\r%-46.46s", xs->name); + if (xs->size <= 0) { + setproctitle("%s [%s]", xs->name, stat_bytes(xs->rcvd)); + fprintf(stderr, " %s", stat_bytes(xs->rcvd)); + } else { + setproctitle("%s [%d%% of %s]", xs->name, + (int)((100.0 * xs->rcvd) / xs->size), + stat_bytes(xs->size)); + fprintf(stderr, "%3d%% of %s", + (int)((100.0 * xs->rcvd) / xs->size), + stat_bytes(xs->size)); + } + fprintf(stderr, " %s", stat_bps(xs)); + if (xs->size > 0 && xs->rcvd > 0 && + xs->last.tv_sec >= xs->start.tv_sec + 10) + fprintf(stderr, " %s", stat_eta(xs)); +} + +/* + * Initialize the transfer statistics + */ +static void +stat_start(struct xferstat *xs, const char *name, off_t size, off_t offset) +{ + snprintf(xs->name, sizeof xs->name, "%s", name); + gettimeofday(&xs->start, NULL); + xs->last.tv_sec = xs->last.tv_usec = 0; + xs->size = size; + xs->offset = offset; + xs->rcvd = offset; + if (v_tty && v_level > 0) + stat_display(xs, 1); + else if (v_level > 0) + fprintf(stderr, "%-46s", xs->name); +} + +/* + * Update the transfer statistics + */ +static void +stat_update(struct xferstat *xs, off_t rcvd) +{ + xs->rcvd = rcvd; + if (v_tty && v_level > 0) + stat_display(xs, 0); +} + +/* + * Finalize the transfer statistics + */ +static void +stat_end(struct xferstat *xs) +{ + gettimeofday(&xs->last, NULL); + if (v_tty && v_level > 0) { + stat_display(xs, 1); + putc('\n', stderr); + } else if (v_level > 0) { + fprintf(stderr, " %s %s\n", + stat_bytes(xs->size), stat_bps(xs)); + } +} + +/* + * Ask the user for authentication details + */ +static int +query_auth(struct url *URL) +{ + struct termios tios; + tcflag_t saved_flags; + int i, nopwd; + + fprintf(stderr, "Authentication required for <%s://%s:%d/>!\n", + URL->scheme, URL->host, URL->port); + + fprintf(stderr, "Login: "); + if (fgets(URL->user, sizeof URL->user, stdin) == NULL) + return (-1); + for (i = strlen(URL->user); i >= 0; --i) + if (URL->user[i] == '\r' || URL->user[i] == '\n') + URL->user[i] = '\0'; + + fprintf(stderr, "Password: "); + if (tcgetattr(STDIN_FILENO, &tios) == 0) { + saved_flags = tios.c_lflag; + tios.c_lflag &= ~ECHO; + tios.c_lflag |= ECHONL|ICANON; + tcsetattr(STDIN_FILENO, TCSAFLUSH|TCSASOFT, &tios); + nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); + tios.c_lflag = saved_flags; + tcsetattr(STDIN_FILENO, TCSANOW|TCSASOFT, &tios); + } else { + nopwd = (fgets(URL->pwd, sizeof URL->pwd, stdin) == NULL); + } + if (nopwd) + return (-1); + for (i = strlen(URL->pwd); i >= 0; --i) + if (URL->pwd[i] == '\r' || URL->pwd[i] == '\n') + URL->pwd[i] = '\0'; + + return (0); +} + +/* + * Fetch a file + */ +static int +fetch(char *URL, const char *path) +{ + struct url *url; + struct url_stat us; + struct stat sb, nsb; + struct xferstat xs; + FILE *of; + fetchIO *f; + size_t size, wr; + off_t count; + char flags[8]; + char *tmppath; + int r; + unsigned timeout; + char *ptr; + + f = NULL; + of = NULL; + tmppath = NULL; + + timeout = 0; + *flags = 0; + count = 0; + + /* set verbosity level */ + if (v_level > 1) + strcat(flags, "v"); + if (v_level > 2) + fetchDebug = 1; + + /* parse URL */ + if ((url = fetchParseURL(URL)) == NULL) { + warnx("%s: parse error", URL); + goto failure; + } + + /* if no scheme was specified, take a guess */ + if (!*url->scheme) { + if (!*url->host) + strcpy(url->scheme, SCHEME_FILE); + else if (strncasecmp(url->host, "ftp.", 4) == 0) + strcpy(url->scheme, SCHEME_FTP); + else if (strncasecmp(url->host, "www.", 4) == 0) + strcpy(url->scheme, SCHEME_HTTP); + } + + /* common flags */ + switch (family) { + case PF_INET: + strcat(flags, "4"); + break; + case PF_INET6: + strcat(flags, "6"); + break; + } + + /* FTP specific flags */ + if (strcmp(url->scheme, SCHEME_FTP) == 0) { + if (d_flag) + strcat(flags, "d"); + if (U_flag) + strcat(flags, "l"); + timeout = T_secs ? T_secs : ftp_timeout; + } + + /* HTTP specific flags */ + if (strcmp(url->scheme, SCHEME_HTTP) == 0) { + if (d_flag) + strcat(flags, "d"); + if (A_flag) + strcat(flags, "A"); + timeout = T_secs ? T_secs : http_timeout; + } + + /* set the protocol timeout. */ + fetchTimeout = timeout; + + /* just print size */ + if (s_flag) { + if (timeout) + alarm(timeout); + r = fetchStat(url, &us, flags); + if (timeout) + alarm(0); + if (sigalrm || sigint) + goto signal; + if (r == -1) { + warnx("%s", fetchLastErrString); + goto failure; + } + if (us.size == -1) + printf("Unknown\n"); + else + printf("%jd\n", (intmax_t)us.size); + goto success; + } + + /* + * If the -r flag was specified, we have to compare the local + * and remote files, so we should really do a fetchStat() + * first, but I know of at least one HTTP server that only + * sends the content size in response to GET requests, and + * leaves it out of replies to HEAD requests. Also, in the + * (frequent) case that the local and remote files match but + * the local file is truncated, we have sufficient information + * before the compare to issue a correct request. Therefore, + * we always issue a GET request as if we were sure the local + * file was a truncated copy of the remote file; we can drop + * the connection later if we change our minds. + */ + sb.st_size = -1; + if (!o_stdout) { + r = stat(path, &sb); + if (r == 0 && r_flag && S_ISREG(sb.st_mode)) { + url->offset = sb.st_size; + } else if (r == -1 || !S_ISREG(sb.st_mode)) { + /* + * Whatever value sb.st_size has now is either + * wrong (if stat(2) failed) or irrelevant (if the + * path does not refer to a regular file) + */ + sb.st_size = -1; + } + if (r == -1 && errno != ENOENT) { + warnx("%s: stat()", path); + goto failure; + } + } + + /* start the transfer */ + if (timeout) + alarm(timeout); + f = fetchXGet(url, &us, flags); + if (timeout) + alarm(0); + if (sigalrm || sigint) + goto signal; + if (f == NULL) { + warnx("%s: %s", URL, fetchLastErrString); + goto failure; + } + if (sigint) + goto signal; + + /* check that size is as expected */ + if (S_size) { + if (us.size == -1) { + warnx("%s: size unknown", URL); + } else if (us.size != S_size) { + warnx("%s: size mismatch: expected %jd, actual %jd", + URL, (intmax_t)S_size, (intmax_t)us.size); + goto failure; + } + } + + /* symlink instead of copy */ + if (l_flag && strcmp(url->scheme, "file") == 0 && !o_stdout) { + char *name = fetchUnquotePath(url); + if (name == NULL) { + warnx("Can't unquote URL"); + goto failure; + } + if (symlink(name, path) == -1) { + warn("%s: symlink()", path); + free(name); + goto failure; + } + free(name); + goto success; + } + + if (us.size == -1 && !o_stdout && v_level > 0) + warnx("%s: size of remote file is not known", URL); + if (v_level > 1) { + if (sb.st_size != -1) + fprintf(stderr, "local size / mtime: %jd / %ld\n", + (intmax_t)sb.st_size, (long)sb.st_mtime); + if (us.size != -1) + fprintf(stderr, "remote size / mtime: %jd / %ld\n", + (intmax_t)us.size, (long)us.mtime); + } + + /* open output file */ + if (o_stdout) { + /* output to stdout */ + of = stdout; + } else if (r_flag && sb.st_size != -1) { + /* resume mode, local file exists */ + if (!F_flag && us.mtime && sb.st_mtime != us.mtime) { + /* no match! have to refetch */ + fetchIO_close(f); + /* if precious, warn the user and give up */ + if (R_flag) { + warnx("%s: local modification time " + "does not match remote", path); + goto failure_keep; + } + } else if (us.size != -1) { + if (us.size == sb.st_size) + /* nothing to do */ + goto success; + if (sb.st_size > us.size) { + /* local file too long! */ + warnx("%s: local file (%jd bytes) is longer " + "than remote file (%jd bytes)", path, + (intmax_t)sb.st_size, (intmax_t)us.size); + goto failure; + } + /* we got it, open local file */ + if ((of = fopen(path, "a")) == NULL) { + warn("%s: fopen()", path); + goto failure; + } + /* check that it didn't move under our feet */ + if (fstat(fileno(of), &nsb) == -1) { + /* can't happen! */ + warn("%s: fstat()", path); + goto failure; + } + if (nsb.st_dev != sb.st_dev || + nsb.st_ino != nsb.st_ino || + nsb.st_size != sb.st_size) { + warnx("%s: file has changed", URL); + fclose(of); + of = NULL; + sb = nsb; + } + } + } else if (m_flag && sb.st_size != -1) { + /* mirror mode, local file exists */ + if (sb.st_size == us.size && sb.st_mtime == us.mtime) + goto success; + } + + if (of == NULL) { + /* + * We don't yet have an output file; either this is a + * vanilla run with no special flags, or the local and + * remote files didn't match. + */ + + if (url->offset > 0) { + /* + * We tried to restart a transfer, but for + * some reason gave up - so we have to restart + * from scratch if we want the whole file + */ + url->offset = 0; + if ((f = fetchXGet(url, &us, flags)) == NULL) { + warnx("%s: %s", URL, fetchLastErrString); + goto failure; + } + if (sigint) + goto signal; + } + + /* construct a temp file name */ + if (sb.st_size != -1 && S_ISREG(sb.st_mode)) { + asprintf(&tmppath, "%s.fetch.XXXXXX", path); + if (tmppath != NULL) { + int fd; + + fd = mkstemp(tmppath); + if (fd == -1) { + warn("%s: mkstemp failed"); + goto failure; + } + fchown(fd, sb.st_uid, sb.st_gid); + fchmod(fd, sb.st_mode & ALLPERMS); + of = fdopen(fd, "w"); + if (of == NULL) { + close(fd); + unlink(tmppath); + free(tmppath); + tmppath = NULL; + } + } + } + if (of == NULL) + of = fopen(path, "w"); + if (of == NULL) { + warn("%s: open()", path); + goto failure; + } + } + count = url->offset; + + /* start the counter */ + stat_start(&xs, path, us.size, count); + + sigalrm = siginfo = sigint = 0; + + /* suck in the data */ + signal(SIGINFO, sig_handler); + while (!sigint) { + if (us.size != -1 && us.size - count < B_size && + us.size - count >= 0) + size = us.size - count; + else + size = B_size; + if (siginfo) { + stat_end(&xs); + siginfo = 0; + } + if ((size = fetchIO_read(f, buf, B_size)) == 0) + break; + stat_update(&xs, count += size); + for (ptr = buf; size > 0; ptr += wr, size -= wr) + if ((wr = fwrite(ptr, 1, size, of)) < size) { + if (ferror(of) && errno == EINTR && !sigint) + clearerr(of); + else + break; + } + if (size != 0) + break; + } + if (!sigalrm) + sigalrm = 0; + signal(SIGINFO, SIG_DFL); + + stat_end(&xs); + + /* + * If the transfer timed out or was interrupted, we still want to + * set the mtime in case the file is not removed (-r or -R) and + * the user later restarts the transfer. + */ + signal: + /* set mtime of local file */ + if (!n_flag && us.mtime && !o_stdout && of != NULL && + (stat(path, &sb) != -1) && sb.st_mode & S_IFREG) { + struct timeval tv[2]; + + fflush(of); + tv[0].tv_sec = (long)(us.atime ? us.atime : us.mtime); + tv[1].tv_sec = (long)us.mtime; + tv[0].tv_usec = tv[1].tv_usec = 0; + if (utimes(tmppath ? tmppath : path, tv)) + warn("%s: utimes()", tmppath ? tmppath : path); + } + + /* timed out or interrupted? */ + if (sigalrm) + warnx("transfer timed out"); + if (sigint) { + warnx("transfer interrupted"); + goto failure; + } + + /* timeout / interrupt before connection completley established? */ + if (f == NULL) + goto failure; + + if (!sigalrm) { + /* check the status of our files */ + if (ferror(of)) + warn("%s", path); + if (ferror(of)) + goto failure; + } + + /* did the transfer complete normally? */ + if (us.size != -1 && count < us.size) { + warnx("%s appears to be truncated: %jd/%jd bytes", + path, (intmax_t)count, (intmax_t)us.size); + goto failure_keep; + } + + /* + * If the transfer timed out and we didn't know how much to + * expect, assume the worst (i.e. we didn't get all of it) + */ + if (sigalrm && us.size == -1) { + warnx("%s may be truncated", path); + goto failure_keep; + } + + success: + r = 0; + if (tmppath != NULL && rename(tmppath, path) == -1) { + warn("%s: rename()", path); + goto failure_keep; + } + goto done; + failure: + if (of && of != stdout && !R_flag && !r_flag) + if (stat(path, &sb) != -1 && (sb.st_mode & S_IFREG)) + unlink(tmppath ? tmppath : path); + if (R_flag && tmppath != NULL && sb.st_size == -1) + rename(tmppath, path); /* ignore errors here */ + failure_keep: + r = -1; + goto done; + done: + if (f) + fetchIO_close(f); + if (of && of != stdout) + fclose(of); + if (url) + fetchFreeURL(url); + if (tmppath != NULL) + free(tmppath); + return (r); +} + +static void +usage(void) +{ + fprintf(stderr, "%s\n%s\n%s\n", + "usage: fetch [-146AFMPRUadlmnpqrsv] [-N netrc] [-o outputfile]", + " [-S bytes] [-B bytes] [-T seconds] [-w seconds]", + " [-h host -f file [-c dir] | URL ...]"); +} + + +/* + * Entry point + */ +int +main(int argc, char *argv[]) +{ + struct stat sb; + struct sigaction sa; + const char *p, *s; + char *end, *q; + int c, e, r; + + while ((c = getopt(argc, argv, + "146AaB:bc:dFf:Hh:lMmN:no:qRrS:sT:tUvw:")) != -1) + switch (c) { + case '1': + once_flag = 1; + break; + case '4': + family = PF_INET; + break; + case '6': + family = PF_INET6; + break; + case 'A': + A_flag = 1; + break; + case 'a': + a_flag = 1; + break; + case 'B': + B_size = (off_t)strtol(optarg, &end, 10); + if (*optarg == '\0' || *end != '\0') + errx(1, "invalid buffer size (%s)", optarg); + break; + case 'b': + warnx("warning: the -b option is deprecated"); + b_flag = 1; + break; + case 'c': + c_dirname = optarg; + break; + case 'd': + d_flag = 1; + break; + case 'F': + F_flag = 1; + break; + case 'f': + f_filename = optarg; + break; + case 'H': + warnx("the -H option is now implicit, " + "use -U to disable"); + break; + case 'h': + h_hostname = optarg; + break; + case 'l': + l_flag = 1; + break; + case 'o': + o_flag = 1; + o_filename = optarg; + break; + case 'M': + case 'm': + if (r_flag) + errx(1, "the -m and -r flags " + "are mutually exclusive"); + m_flag = 1; + break; + case 'N': + N_filename = optarg; + break; + case 'n': + n_flag = 1; + break; + case 'q': + v_level = 0; + break; + case 'R': + R_flag = 1; + break; + case 'r': + if (m_flag) + errx(1, "the -m and -r flags " + "are mutually exclusive"); + r_flag = 1; + break; + case 'S': + S_size = (off_t)strtol(optarg, &end, 10); + if (*optarg == '\0' || *end != '\0') + errx(1, "invalid size (%s)", optarg); + break; + case 's': + s_flag = 1; + break; + case 'T': + T_secs = strtol(optarg, &end, 10); + if (*optarg == '\0' || *end != '\0') + errx(1, "invalid timeout (%s)", optarg); + break; + case 't': + t_flag = 1; + warnx("warning: the -t option is deprecated"); + break; + case 'U': + U_flag = 1; + break; + case 'v': + v_level++; + break; + case 'w': + a_flag = 1; + w_secs = strtol(optarg, &end, 10); + if (*optarg == '\0' || *end != '\0') + errx(1, "invalid delay (%s)", optarg); + break; + default: + usage(); + exit(EX_USAGE); + } + + argc -= optind; + argv += optind; + + if (h_hostname || f_filename || c_dirname) { + if (!h_hostname || !f_filename || argc) { + usage(); + exit(EX_USAGE); + } + /* XXX this is a hack. */ + if (strcspn(h_hostname, "@:/") != strlen(h_hostname)) + errx(1, "invalid hostname"); + if (asprintf(argv, "ftp://%s/%s/%s", h_hostname, + c_dirname ? c_dirname : "", f_filename) == -1) + errx(1, "%s", strerror(ENOMEM)); + argc++; + } + + if (!argc) { + usage(); + exit(EX_USAGE); + } + + /* allocate buffer */ + if (B_size < MINBUFSIZE) + B_size = MINBUFSIZE; + if ((buf = malloc(B_size)) == NULL) + errx(1, "%s", strerror(ENOMEM)); + + /* timeouts */ + if ((s = getenv("FTP_TIMEOUT")) != NULL) { + ftp_timeout = strtol(s, &end, 10); + if (*s == '\0' || *end != '\0' || ftp_timeout < 0) { + warnx("FTP_TIMEOUT (%s) is not a positive integer", s); + ftp_timeout = 0; + } + } + if ((s = getenv("HTTP_TIMEOUT")) != NULL) { + http_timeout = strtol(s, &end, 10); + if (*s == '\0' || *end != '\0' || http_timeout < 0) { + warnx("HTTP_TIMEOUT (%s) is not a positive integer", s); + http_timeout = 0; + } + } + + /* signal handling */ + sa.sa_flags = 0; + sa.sa_handler = sig_handler; + sigemptyset(&sa.sa_mask); + sigaction(SIGALRM, &sa, NULL); + sa.sa_flags = SA_RESETHAND; + sigaction(SIGINT, &sa, NULL); + fetchRestartCalls = 0; + + /* output file */ + if (o_flag) { + if (strcmp(o_filename, "-") == 0) { + o_stdout = 1; + } else if (stat(o_filename, &sb) == -1) { + if (errno == ENOENT) { + if (argc > 1) + errx(EX_USAGE, "%s is not a directory", + o_filename); + } else { + err(EX_IOERR, "%s", o_filename); + } + } else { + if (sb.st_mode & S_IFDIR) + o_directory = 1; + } + } + + /* check if output is to a tty (for progress report) */ + v_tty = isatty(STDERR_FILENO); + if (v_tty) + pgrp = getpgrp(); + + r = 0; + + /* authentication */ + if (v_tty) + fetchAuthMethod = query_auth; + if (N_filename != NULL) + setenv("NETRC", N_filename, 1); + + while (argc) { + if ((p = strrchr(*argv, '/')) == NULL) + p = *argv; + else + p++; + + if (!*p) + p = "fetch.out"; + + fetchLastErrCode = 0; + + if (o_flag) { + if (o_stdout) { + e = fetch(*argv, "-"); + } else if (o_directory) { + asprintf(&q, "%s/%s", o_filename, p); + e = fetch(*argv, q); + free(q); + } else { + e = fetch(*argv, o_filename); + } + } else { + e = fetch(*argv, p); + } + + if (sigint) + kill(getpid(), SIGINT); + + if (e == 0 && once_flag) + exit(0); + + if (e) { + r = 1; + if ((fetchLastErrCode + && fetchLastErrCode != FETCH_UNAVAIL + && fetchLastErrCode != FETCH_MOVED + && fetchLastErrCode != FETCH_URL + && fetchLastErrCode != FETCH_RESOLV + && fetchLastErrCode != FETCH_UNKNOWN)) { + if (w_secs && v_level) + fprintf(stderr, "Waiting %ld seconds " + "before retrying\n", w_secs); + if (w_secs) + sleep(w_secs); + if (a_flag) + continue; + } + } + + argc--, argv++; + } + + exit(r); +} diff --git a/net/fetch/files/fetch.cat1 b/net/fetch/files/fetch.cat1 new file mode 100644 index 00000000000..3aaa043269a --- /dev/null +++ b/net/fetch/files/fetch.cat1 @@ -0,0 +1,153 @@ +FETCH(1) NetBSD General Commands Manual FETCH(1) + +NNAAMMEE + ffeettcchh -- retrieve a file by Uniform Resource Locator + +SSYYNNOOPPSSIISS + ffeettcchh [--114466AAFFMMRRUUaaddllmmnnqqrrssvv] [--BB _b_y_t_e_s] [--SS _b_y_t_e_s] [--TT _s_e_c_o_n_d_s] [--NN _f_i_l_e] + [--oo _f_i_l_e] [--ww _s_e_c_o_n_d_s] [--hh _h_o_s_t] [--cc _d_i_r] [--ff _f_i_l_e] [_U_R_L _._._.] + +DDEESSCCRRIIPPTTIIOONN + The ffeettcchh utility provides a command-line interface to the fetch(3) + library. Its purpose is to retrieve the file(s) pointed to by the URL(s) + on the command line. + + The following options are available: + + --11 Stop and return exit code 0 at the first successfully + retrieved file. + + --44 Forces ffeettcchh to use IPv4 addresses only. + + --66 Forces ffeettcchh to use IPv6 addresses only. + + --AA Do not automatically follow ``temporary'' (302) redirects. + Some broken Web sites will return a redirect instead of a + not-found error when the requested object does not exist. + + --aa Automatically retry the transfer upon soft failures. + + --BB _b_y_t_e_s Specify the read buffer size in bytes. The default is 4096 + bytes. Attempts to set a buffer size lower than this will be + silently ignored. The number of reads actually performed is + reported at verbosity level two or higher (see the --vv flag). + + --cc _d_i_r The file to retrieve is in directory _d_i_r on the remote host. + This option is deprecated and is provided for backward com- + patibility only. + + --dd Use a direct connection even if a proxy is configured. + + --FF In combination with the --rr flag, forces a restart even if the + local and remote files have different modification times. + Implies --RR. + + --ff _f_i_l_e The file to retrieve is named _f_i_l_e on the remote host. This + option is deprecated and is provided for backward compatibil- + ity only. + + --hh _h_o_s_t The file to retrieve is located on the host _h_o_s_t. This + option is deprecated and is provided for backward compatibil- + ity only. + + --ll If the target is a file-scheme URL, make a symbolic link to + the target rather than trying to copy it. + + --MM + + --mm Mirror mode: if the file already exists locally and has the + same size and modification time as the remote file, it will + not be fetched. Note that the --mm and --rr flags are mutually + exclusive. + + --NN _f_i_l_e Use _f_i_l_e instead of _~_/_._n_e_t_r_c to look up login names and pass- + words for FTP sites. See ftp(1) for a description of the + file format. This feature is experimental. + + --nn Do not preserve the modification time of the transferred + file. + + --oo _f_i_l_e Set the output file name to _f_i_l_e. By default, a ``pathname'' + is extracted from the specified URI, and its basename is used + as the name of the output file. A _f_i_l_e argument of `-' indi- + cates that results are to be directed to the standard output. + If the _f_i_l_e argument is a directory, fetched file(s) will be + placed within the directory, with name(s) selected as in the + default behaviour. + + --qq Quiet mode. + + --RR The output files are precious, and should not be deleted + under any circumstances, even if the transfer failed or was + incomplete. + + --rr Restart a previously interrupted transfer. Note that the --mm + and --rr flags are mutually exclusive. + + --SS _b_y_t_e_s Require the file size reported by the server to match the + specified value. If it does not, a message is printed and + the file is not fetched. If the server does not support + reporting file sizes, this option is ignored and the file is + fetched unconditionally. + + --ss Print the size in bytes of each requested file, without + fetching it. + + --TT _s_e_c_o_n_d_s Set timeout value to _s_e_c_o_n_d_s. Overrides the environment + variables FTP_TIMEOUT for FTP transfers or HTTP_TIMEOUT for + HTTP transfers if set. + + --UU When using passive FTP, allocate the port for the data con- + nection from the low (default) port range. See ip(4) for + details on how to specify which port range this corresponds + to. + + --vv Increase verbosity level. + + --ww _s_e_c_o_n_d_s When the --aa flag is specified, wait this many seconds between + successive retries. + + If ffeettcchh receives a SIGINFO signal (see the ssttaattuuss argument for stty(1)), + the current transfer rate statistics will be written to the standard + error output, in the same format as the standard completion message. + +EENNVVIIRROONNMMEENNTT + FTP_TIMEOUT Maximum time, in seconds, to wait before aborting an FTP + connection. + + HTTP_TIMEOUT Maximum time, in seconds, to wait before aborting an HTTP + connection. + + See fetch(3) for a description of additional environment variables, + including FETCH_BIND_ADDRESS, FTP_LOGIN, FTP_PASSIVE_MODE, FTP_PASSWORD, + FTP_PROXY, ftp_proxy, HTTP_AUTH, HTTP_PROXY, http_proxy, HTTP_PROXY_AUTH, + HTTP_REFERER, HTTP_USER_AGENT, NETRC, NO_PROXY and no_proxy. + +EEXXIITT SSTTAATTUUSS + The ffeettcchh command returns zero on success, or one on failure. If multi- + ple URLs are listed on the command line, ffeettcchh will attempt to retrieve + each one of them in turn, and will return zero only if they were all suc- + cessfully retrieved. + +SSEEEE AALLSSOO + fetch(3) + +HHIISSTTOORRYY + The ffeettcchh command appeared in FreeBSD 2.1.5. This implementation first + appeared in FreeBSD 4.1. + +AAUUTTHHOORRSS + The original implementation of ffeettcchh was done by Jean-Marc Zucconi + <jmz@FreeBSD.org>. It was extensively re-worked for FreeBSD 2.2 by + Garrett Wollman <wollman@FreeBSD.org>, and later completely rewritten to + use the fetch(3) library by Dag-Erling Sm/orgrav <des@FreeBSD.org>. + +NNOOTTEESS + The --bb and --tt options are no longer supported and will generate warnings. + They were workarounds for bugs in other OSes which this implementation + does not trigger. + + One cannot both use the --hh, --cc and --ff options and specify URLs on the + command line. + +NetBSD 4.0 March 11, 2003 NetBSD 4.0 diff --git a/net/fetch/options.mk b/net/fetch/options.mk new file mode 100644 index 00000000000..97d03f118d3 --- /dev/null +++ b/net/fetch/options.mk @@ -0,0 +1,15 @@ +# $NetBSD: options.mk,v 1.1.1.1 2008/10/08 15:28:30 joerg Exp $ + +PKG_OPTIONS_VAR= PKG_OPTIONS.libfetch +PKG_SUPPORTED_OPTIONS= openssl +PKG_SUGGESTED_OPTIONS= openssl + +.include "../../mk/bsd.options.mk" + +.if !empty(PKG_OPTIONS:Mopenssl) +MAKE_ENV+= FETCH_WITH_OPENSSL=yes + +.include "../../security/openssl/buildlink3.mk" +.else +MAKE_ENV+= FETCH_WITH_OPENSSL=no +.endif |