diff options
| -rw-r--r-- | mk/bsd.pkg.mk | 495 | ||||
| -rw-r--r-- | mk/bsd.prefs.mk | 5 | ||||
| -rw-r--r-- | mk/extract/bsd.extract-vars.mk | 63 | ||||
| -rw-r--r-- | mk/extract/bsd.extract.mk | 15 | ||||
| -rwxr-xr-x | mk/extract/extract (renamed from mk/scripts/extract) | 2 | ||||
| -rw-r--r-- | mk/extract/extract.mk (renamed from mk/bsd.pkg.extract.mk) | 203 | ||||
| -rw-r--r-- | mk/fetch/bsd.fetch-vars.mk | 22 | ||||
| -rw-r--r-- | mk/fetch/bsd.fetch.mk | 9 | ||||
| -rw-r--r-- | mk/fetch/distclean.mk | 26 | ||||
| -rw-r--r-- | mk/fetch/fetch.mk | 453 | ||||
| -rw-r--r-- | mk/fetch/sites.mk (renamed from mk/bsd.sites.mk) | 11 |
11 files changed, 694 insertions, 610 deletions
diff --git a/mk/bsd.pkg.mk b/mk/bsd.pkg.mk index fd791ac382e..4976dc1f39b 100644 --- a/mk/bsd.pkg.mk +++ b/mk/bsd.pkg.mk @@ -1,4 +1,4 @@ -# $NetBSD: bsd.pkg.mk,v 1.1839 2006/06/05 22:49:44 jlam Exp $ +# $NetBSD: bsd.pkg.mk,v 1.1840 2006/06/06 03:05:48 jlam Exp $ # # This file is in the public domain. # @@ -89,7 +89,6 @@ PKGNAME_NOREV= ${PKGNAME} ##### Others -_DISTDIR?= ${DISTDIR}/${DIST_SUBDIR} BUILD_DEPENDS?= # empty BUILD_TARGET?= all COMMENT?= (no description) @@ -98,7 +97,6 @@ CONFIGURE_SCRIPT?= ./configure DEPENDS?= # empty DESCR_SRC?= ${PKGDIR}/DESCR DIGEST_ALGORITHMS?= SHA1 RMD160 -DISTFILES?= ${DISTNAME}${EXTRACT_SUFX} DISTINFO_FILE?= ${PKGDIR}/distinfo INTERACTIVE_STAGE?= none MAINTAINER?= pkgsrc-users@NetBSD.org @@ -430,40 +428,6 @@ ERROR_MSG?= ${ECHO_MSG} 1>&2 "ERROR:" # do something. DO_NADA?= ${TRUE} -# If this host is behind a filtering firewall, use passive ftp(1) -FETCH_BEFORE_ARGS+= ${PASSIVE_FETCH:D-p} - -# Include popular master sites -.include "../../mk/bsd.sites.mk" - -_MASTER_SITE_BACKUP= ${MASTER_SITE_BACKUP:=${DIST_SUBDIR}${DIST_SUBDIR:D/}} -_MASTER_SITE_OVERRIDE= ${MASTER_SITE_OVERRIDE:=${DIST_SUBDIR}${DIST_SUBDIR:D/}} - -# Where to put distfiles that don't have any other master site -MASTER_SITE_LOCAL?= ${MASTER_SITE_BACKUP:=LOCAL_PORTS/} - -ALLFILES?= ${DISTFILES} ${PATCHFILES} -CKSUMFILES?= ${ALLFILES} -.for __tmp__ in ${IGNOREFILES} -CKSUMFILES:= ${CKSUMFILES:N${__tmp__}} -.endfor - -# List of all files, with ${DIST_SUBDIR} in front. Used for fetch and checksum. -.if defined(DIST_SUBDIR) -_CKSUMFILES?= ${CKSUMFILES:S/^/${DIST_SUBDIR}\//} -_DISTFILES?= ${DISTFILES:S/^/${DIST_SUBDIR}\//} -_IGNOREFILES?= ${IGNOREFILES:S/^/${DIST_SUBDIR}\//} -_PATCHFILES?= ${PATCHFILES:S/^/${DIST_SUBDIR}\//} -.else -_CKSUMFILES?= ${CKSUMFILES} -_DISTFILES?= ${DISTFILES} -_IGNOREFILES?= ${IGNOREFILES} -_PATCHFILES?= ${PATCHFILES} -.endif -_ALLFILES?= ${_DISTFILES} ${_PATCHFILES} - -BUILD_DEFS+= _DISTFILES _PATCHFILES - .if defined(GNU_CONFIGURE) HAS_CONFIGURE= yes @@ -572,9 +536,6 @@ USE_TOOLS+= tee tsort USE_TOOLS+= shlock sleep .endif -# Extract -.include "../../mk/bsd.pkg.extract.mk" - # Patch .include "../../mk/bsd.pkg.patch.mk" @@ -806,300 +767,6 @@ _build: configure # adding pre-* or post-* targets/scripts, override these. ################################################################ -### -### _RESUME_TRANSFER: -### -### Macro to resume a previous transfer, needs to have defined -### the following options in mk.conf: -### -### PKG_RESUME_TRANSFERS -### FETCH_RESUME_ARGS (if FETCH_CMD != default) -### FETCH_OUTPUT_ARGS (if FETCH_CMD != default) -### -### For example if you want to use wget (pkgsrc/net/wget): -### -### FETCH_CMD=wget -### FETCH_RESUME_ARGS=-c -### FETCH_OUTPUT_ARGS=-O -### -### How does it work? -### -### FETCH_CMD downloads the file and saves it temporally into $$bfile.temp -### if the checksum match the correct one, $$bfile.temp is renamed to -### the original name. -### - -_RESUME_TRANSFER= \ - ofile="${DISTDIR}/${DIST_SUBDIR}/$$bfile"; \ - tfile="${DISTDIR}/${DIST_SUBDIR}/$$bfile.temp"; \ - tsize=`${AWK} '/^Size/ && $$2 == '"\"($$file)\""' { print $$4 }' ${DISTINFO_FILE}` || ${TRUE}; \ - osize=`${WC} -c < $$ofile`; \ - \ - case "$$tsize" in \ - "") ${ECHO_MSG} "No size in distinfo file (${DISTINFO_FILE})"; \ - break;; \ - esac; \ - \ - if [ "$$osize" -eq "$$tsize" ]; then \ - if [ -f "$$tfile" ]; then \ - ${RM} $$tfile; \ - fi; \ - need_fetch=no; \ - elif [ "$$osize" -lt "$$tsize" -a ! -f "$$tfile" ]; then \ - ${CP} $$ofile $$tfile; \ - dsize=`${WC} -c < $$tfile`; \ - need_fetch=yes; \ - elif [ -f "$$tfile" -a "$$dsize" -gt "$$ossize" ]; then \ - dsize=`${WC} -c < $$tfile`; \ - need_fetch=yes; \ - else \ - if [ -f "$$tfile" ]; then \ - dsize=`${WC} -c < $$tfile`; \ - fi; \ - need_fetch=yes; \ - fi; \ - if [ "$$need_fetch" = "no" ]; then \ - break; \ - elif [ -f "$$tfile" -a "$$dsize" -eq "$$tsize" ]; then \ - ${MV} $$tfile $$ofile; \ - break; \ - elif [ -n "$$ftp_proxy" -o -n "$$http_proxy" ]; then \ - ${ECHO_MSG} "===> Resume is not supported by ftp(1) using http/ftp proxies."; \ - break; \ - elif [ "$$need_fetch" = "yes" -a "$$dsize" -lt "$$tsize" ]; then \ - if [ "${FETCH_CMD:T}" != "ftp" -a -z "${FETCH_RESUME_ARGS}" ]; then \ - ${ECHO_MSG} "=> Resume transfers are not supported, FETCH_RESUME_ARGS is empty."; \ - break; \ - else \ - for res_site in $$sites; do \ - if [ -z "${FETCH_OUTPUT_ARGS}" ]; then \ - ${ECHO_MSG} "=> FETCH_OUTPUT_ARGS has to be defined."; \ - break; \ - fi; \ - ${ECHO_MSG} "=> $$bfile not completed, resuming:"; \ - ${ECHO_MSG} "=> Downloaded: $$dsize Total: $$tsize."; \ - ${ECHO_MSG}; \ - cd ${DISTDIR}; \ - ${FETCH_CMD} ${FETCH_BEFORE_ARGS} ${FETCH_RESUME_ARGS} \ - ${FETCH_OUTPUT_ARGS} $${bfile}.temp $${res_site}$${bfile}; \ - if [ $$? -eq 0 ]; then \ - ndsize=`${WC} -c < $$tfile`; \ - if [ "$$tsize" -eq "$$ndsize" ]; then \ - ${MV} $$tfile $$ofile; \ - fi; \ - break; \ - fi; \ - done; \ - fi; \ - elif [ "$$dsize" -gt "$$tsize" ]; then \ - ${ECHO_MSG} "==> Downloaded file larger than the recorded size."; \ - break; \ - fi - -# -# Define the elementary fetch macros. -# -_FETCH_FILE= \ - if [ ! -f $$file -a ! -f $$bfile -a ! -h $$bfile ]; then \ - ${ECHO_MSG} "=> $$bfile doesn't seem to exist on this system."; \ - if [ ! -w ${_DISTDIR}/. ]; then \ - ${ECHO_MSG} "=> Can't download to ${_DISTDIR} (permission denied?)."; \ - exit 1; \ - fi; \ - for site in $$sites; do \ - ${ECHO_MSG} "=> Attempting to fetch $$bfile from $${site}."; \ - if [ -f ${DISTINFO_FILE} ]; then \ - ${AWK} 'NF == 5 && $$1 == "Size" && $$2 == "('$$bfile')" { printf("=> [%s %s]\n", $$4, $$5) }' ${DISTINFO_FILE}; \ - fi; \ - if ${FETCH_CMD} ${FETCH_BEFORE_ARGS} $${site}$${bfile} ${FETCH_AFTER_ARGS}; then \ - if [ -n "${FAILOVER_FETCH}" -a -f ${DISTINFO_FILE} -a -f ${_DISTDIR}/$$bfile ]; then \ - alg=`${AWK} 'NF == 4 && $$2 == "('$$file')" && $$3 == "=" {print $$1; exit}' ${DISTINFO_FILE}`; \ - if [ -z "$$alg" ]; then \ - alg=${PATCH_DIGEST_ALGORITHM};\ - fi; \ - CKSUM=`${DIGEST} $$alg < ${_DISTDIR}/$$bfile`; \ - CKSUM2=`${AWK} '$$1 == "'$$alg'" && $$2 == "('$$file')" {print $$4; exit}' <${DISTINFO_FILE}`; \ - if [ "$$CKSUM" = "$$CKSUM2" -o "$$CKSUM2" = "IGNORE" ]; then \ - break; \ - else \ - ${ECHO_MSG} "=> Checksum failure - trying next site."; \ - fi; \ - elif [ ! -f ${_DISTDIR}/$$bfile ]; then \ - ${ECHO_MSG} "=> FTP didn't fetch expected file, trying next site." ; \ - else \ - break; \ - fi; \ - fi \ - done; \ - if [ ! -f ${_DISTDIR}/$$bfile ]; then \ - ${ECHO_MSG} "=> Couldn't fetch $$bfile - please try to retrieve this";\ - ${ECHO_MSG} "=> file manually into ${_DISTDIR} and try again."; \ - exit 1; \ - fi; \ - fi - -_CHECK_DIST_PATH= \ - if [ "X${DIST_PATH}" != "X" ]; then \ - for d in "" ${DIST_PATH:S/:/ /g}; do \ - case $$d in "" | ${DISTDIR}) continue;; esac; \ - if [ -f $$d/${DIST_SUBDIR}/$$bfile ]; then \ - ${ECHO} "Using $$d/${DIST_SUBDIR}/$$bfile"; \ - ${RM} -f $$bfile; \ - ${LN} -s $$d/${DIST_SUBDIR}/$$bfile $$bfile; \ - break; \ - fi; \ - done; \ - fi - -# -# Set up ORDERED_SITES to work out the exact list of sites for every file, -# using the dynamic sites script, or sorting according to the master site -# list or the patterns in MASTER_SORT or MASTER_SORT_REGEX as appropriate. -# No actual sorting is done until ORDERED_SITES is expanded. -# -.if defined(MASTER_SORT) || defined(MASTER_SORT_REGEX) -MASTER_SORT?= -MASTER_SORT_REGEX?= -MASTER_SORT_REGEX+= ${MASTER_SORT:S/./\\./g:C/.*/:\/\/[^\/]*&\//} - -MASTER_SORT_AWK= BEGIN { RS = " "; ORS = " "; IGNORECASE = 1 ; gl = "${MASTER_SORT_REGEX:S/\\/\\\\/g}"; } -. for srt in ${MASTER_SORT_REGEX} -MASTER_SORT_AWK+= /${srt:C/\//\\\//g}/ { good["${srt:S/\\/\\\\/g}"] = good["${srt:S/\\/\\\\/g}"] " " $$0 ; next; } -. endfor -MASTER_SORT_AWK+= { rest = rest " " $$0; } END { n=split(gl, gla); for(i=1;i<=n;i++) { print good[gla[i]]; } print rest; } - -SORT_SITES_CMD= ${ECHO} $$unsorted_sites | ${AWK} '${MASTER_SORT_AWK}' -ORDERED_SITES= ${_MASTER_SITE_OVERRIDE} `${SORT_SITES_CMD:S/\\/\\\\/g:C/"/\"/g}` -.else -ORDERED_SITES= ${_MASTER_SITE_OVERRIDE} $$unsorted_sites -.endif - -# -# Associate each file to fetch with the correct site(s). -# -.if defined(DYNAMIC_MASTER_SITES) -. for fetchfile in ${_ALLFILES} -SITES_${fetchfile:T:S/=/--/}?= `${SH} ${FILESDIR}/getsite.sh ${fetchfile:T}` -SITES.${fetchfile:T:S/=/--/}?= ${SITES_${fetchfile:T:S/=/--/}} -. endfor -.endif -.if !empty(_DISTFILES) -. for fetchfile in ${_DISTFILES} -SITES_${fetchfile:T:S/=/--/}?= ${MASTER_SITES} -SITES.${fetchfile:T:S/=/--/}?= ${SITES_${fetchfile:T:S/=/--/}} -. endfor -.endif -.if !empty(_PATCHFILES) -. for fetchfile in ${_PATCHFILES} -SITES_${fetchfile:T:S/=/--/}?= ${PATCH_SITES} -SITES.${fetchfile:T:S/=/--/}?= ${SITES_${fetchfile:T:S/=/--/}} -. endfor -.endif - -# This code is only called in a batch case, to check for the presence of -# the distfiles -.PHONY: batch-check-distfiles -batch-check-distfiles: - ${_PKG_SILENT}${_PKG_DEBUG} \ - gotfiles=yes; \ - for file in "" ${_ALLFILES}; do \ - case "$$file" in \ - "") continue ;; \ - *) if [ ! -f ${DISTDIR}/$$file ]; then \ - gotfiles=no; \ - fi ;; \ - esac; \ - done; \ - case "$$gotfiles" in \ - no) ${ERROR_MSG} "This package requires user intervention to download the distfiles"; \ - ${ERROR_MSG} "Please fetch the distfiles manually and place them in"; \ - ${ERROR_MSG} "${DISTDIR}"; \ - [ ! -z "${MASTER_SITES}" ] && \ - ${ERROR_MSG} "The distfiles are available from ${MASTER_SITES}"; \ - [ ! -z "${HOMEPAGE}" ] && \ - ${ERROR_MSG} "See ${HOMEPAGE} for more details"; \ - ${ECHO}; \ - ${TOUCH} ${_INTERACTIVE_COOKIE}; \ - ${FALSE} ;; \ - esac - -.PHONY: do-fetch -.if !target(do-fetch) -do-fetch: -. if !defined(ALLOW_VULNERABLE_PACKAGES) - ${_PKG_SILENT}${_PKG_DEBUG} \ - if [ -f ${PKGVULNDIR}/pkg-vulnerabilities ]; then \ - ${PHASE_MSG} "Checking for vulnerabilities in ${PKGNAME}"; \ - vul=`${MAKE} ${MAKEFLAGS} check-vulnerable`; \ - case "$$vul" in \ - "") ;; \ - *) ${ECHO} "$$vul"; \ - ${ECHO} "or define ALLOW_VULNERABLE_PACKAGES if this package is absolutely essential"; \ - ${FALSE} ;; \ - esac; \ - else \ - ${WARNING_MSG} "No ${PKGVULNDIR}/pkg-vulnerabilities file found,"; \ - ${WARNING_MSG} "skipping vulnerability checks. To fix, install"; \ - ${WARNING_MSG} "the pkgsrc/security/audit-packages package and run"; \ - ${WARNING_MSG} "'${LOCALBASE}/sbin/download-vulnerability-list'."; \ - fi -. endif -. if !empty(_ALLFILES) - ${_PKG_SILENT}${_PKG_DEBUG} \ - ${TEST} -d ${_DISTDIR} || ${MKDIR} ${_DISTDIR} -. if !empty(INTERACTIVE_STAGE:Mfetch) && defined(BATCH) - ${_PKG_SILENT}${_PKG_DEBUG} \ - ${MAKE} ${MAKEFLAGS} batch-check-distfiles -. else -. for fetchfile in ${_ALLFILES} -. if defined(FETCH_MESSAGE) && !empty(FETCH_MESSAGE) - ${_PKG_SILENT}${_PKG_DEBUG} set -e; \ - ${TEST} -f ${DISTDIR:Q}/${fetchfile:Q} || { \ - h="==============="; h="$$h$$h$$h$$h$$h"; \ - ${ECHO} "$$h"; ${ECHO} ""; \ - for l in ${FETCH_MESSAGE}; do ${ECHO} "$$l"; done; \ - ${ECHO} ""; ${ECHO} "$$h"; \ - exit 1; \ - } -. elif defined(_FETCH_MESSAGE) - ${_PKG_SILENT}${_PKG_DEBUG} \ - file="${fetchfile}"; \ - if [ ! -f ${DISTDIR}/$$file ]; then \ - ${_FETCH_MESSAGE}; \ - fi -. else - ${_PKG_SILENT}${_PKG_DEBUG} \ - cd ${_DISTDIR}; \ - file="${fetchfile}"; \ - bfile="${fetchfile:T}"; \ - unsorted_sites="${SITES.${fetchfile:T:S/=/--/}} ${_MASTER_SITE_BACKUP}"; \ - sites="${ORDERED_SITES}"; \ - ${_CHECK_DIST_PATH}; \ - if ${TEST} "${PKG_RESUME_TRANSFERS:M[Yy][Ee][Ss]}" ; then \ - ${_FETCH_FILE}; ${_RESUME_TRANSFER}; \ - else \ - ${_FETCH_FILE}; \ - fi -. endif # defined(_FETCH_MESSAGE) -. endfor -. endif # INTERACTIVE_STAGE == fetch -. endif # !empty(_ALLFILES) -.endif - -.PHONY: show-distfiles -.if !target(show-distfiles) -show-distfiles: -. if defined(PKG_FAIL_REASON) - ${_PKG_SILENT}${_PKG_DEBUG}${DO_NADA} -. else - ${_PKG_SILENT}${_PKG_DEBUG} \ - for file in "" ${_CKSUMFILES}; do \ - if [ "X$$file" = "X" ]; then continue; fi; \ - ${ECHO} $$file; \ - done -. endif -.endif - # acquire-lock, release-lock are .USE macro targets for acquiring and # release coarse-grained locks. # @@ -1482,6 +1149,12 @@ do-test: # Clean .include "../../mk/bsd.pkg.clean.mk" +# Fetch +.include "${PKGSRCDIR}/mk/fetch/bsd.fetch.mk" + +# Extract +.include "${PKGSRCDIR}/mk/extract/bsd.extract.mk" + # Install .include "${PKGSRCDIR}/mk/install/bsd.install.mk" @@ -1513,12 +1186,6 @@ release-build-lock: release-lock # call the necessary targets/scripts. ################################################################ -.PHONY: fetch -.if !target(fetch) -fetch: - @cd ${.CURDIR} && ${MAKE} ${MAKEFLAGS} real-fetch PKG_PHASE=fetch -.endif - .PHONY: tools .if !target(tools) tools: patch acquire-tools-lock ${_TOOLS_COOKIE} release-tools-lock @@ -1636,7 +1303,6 @@ build-cookie: test-cookie: ${_PKG_SILENT}${_PKG_DEBUG} ${TOUCH} ${TOUCH_FLAGS} ${_TEST_COOKIE} -.ORDER: pre-fetch do-fetch post-fetch .ORDER: tools-message tools-vars pre-tools do-tools post-tools tools-cookie .ORDER: wrapper-message wrapper-vars pre-wrapper do-wrapper post-wrapper wrapper-cookie .ORDER: configure-message configure-vars pre-configure pre-configure-override do-configure post-configure configure-cookie @@ -1646,10 +1312,8 @@ test-cookie: # Please note that the order of the following targets is important, and # should not be modified (.ORDER is not recognised by make(1) in a serial # make i.e. without -j n) -.PHONY: real-fetch .PHONY: real-tools real-wrapper .PHONY: real-configure real-build real-test -real-fetch: pre-fetch do-fetch post-fetch real-tools: tools-message tools-vars pre-tools do-tools post-tools tools-cookie real-wrapper: wrapper-message wrapper-vars pre-wrapper do-wrapper post-wrapper wrapper-cookie real-configure: configure-message configure-vars pre-configure pre-configure-override do-configure post-configure configure-cookie @@ -1682,7 +1346,7 @@ su-target: .USE # Empty pre-* and post-* targets -.for name in fetch tools wrapper configure build test +.for name in tools wrapper configure build test . if !target(pre-${name}) pre-${name}: @@ -1705,139 +1369,6 @@ post-${name}: lint: ${_PKG_SILENT}${_PKG_DEBUG}${LOCALBASE}/bin/pkglint -# This is for the use of sites which store distfiles which others may -# fetch - only fetch the distfile if it is allowed to be -# re-distributed freely -.PHONY: mirror-distfiles -mirror-distfiles: -.if !defined(NO_SRC_ON_FTP) - @${_PKG_SILENT}${_PKG_DEBUG}${MAKE} ${MAKEFLAGS} fetch NO_SKIP=yes -.endif - - -# Cleaning up - -.PHONY: pre-distclean -.if !target(pre-distclean) -pre-distclean: - @${DO_NADA} -.endif - -.PHONY: distclean -.if !target(distclean) -distclean: pre-distclean clean - @${PHASE_MSG} "Dist cleaning for ${PKGNAME}" - ${_PKG_SILENT}${_PKG_DEBUG}if [ -d ${_DISTDIR} ]; then \ - cd ${_DISTDIR} && \ - ${TEST} -z "${DISTFILES}" || ${RM} -f ${DISTFILES}; \ - if [ "${PKG_RESUME_TRANSFERS:M[Yy][Ee][Ss]}" ]; then \ - ${TEST} -z "${DISTFILES}.temp" || ${RM} -f ${DISTFILES}.temp; \ - fi; \ - ${TEST} -z "${PATCHFILES}" || ${RM} -f ${PATCHFILES}; \ - fi -. if defined(DIST_SUBDIR) && exists(DIST_SUBDIR) - -${_PKG_SILENT}${_PKG_DEBUG}${RMDIR} ${_DISTDIR} -. endif - -${_PKG_SILENT}${_PKG_DEBUG}${RM} -f README.html -.endif - -# Prints out a script to fetch all needed files (no checksumming). -.PHONY: fetch-list -.if !target(fetch-list) - -fetch-list: - @${ECHO} '#!/bin/sh' - @${ECHO} '#' - @${ECHO} '# This is an auto-generated script, the result of running' - @${ECHO} '# `${MAKE} fetch-list'"'"' in directory "'"`${PWD_CMD}`"'"' - @${ECHO} '# on host "'"`${UNAME} -n`"'" on "'"`date`"'".' - @${ECHO} '#' - @${MAKE} ${MAKEFLAGS} fetch-list-recursive -.endif # !target(fetch-list) - -.PHONY: fetch-list-recursive -.if !target(fetch-list-recursive) - -fetch-list-recursive: - ${_PKG_SILENT}${_PKG_DEBUG} \ - for dir in `${MAKE} ${MAKEFLAGS} show-all-depends-dirs`; do \ - (cd ../../$$dir && \ - ${MAKE} ${MAKEFLAGS} fetch-list-one-pkg \ - | ${AWK} ' \ - /^[^#]/ { FoundSomething = 1 } \ - /^unsorted/ { gsub(/[[:space:]]+/, " \\\n\t") } \ - /^echo/ { gsub(/;[[:space:]]+/, "\n") } \ - { block[line_c++] = $$0 } \ - END { if (FoundSomething) \ - for (line = 0; line < line_c; line++) \ - print block[line] } \ - ') \ - done -.endif # !target(fetch-list-recursive) - -.PHONY: fetch-list-one-pkg -.if !target(fetch-list-one-pkg) - -fetch-list-one-pkg: -. if !empty(_ALLFILES) - @${ECHO} - @${ECHO} '#' - @location=`${PWD_CMD} | ${AWK} -F / '{ print $$(NF-1) "/" $$NF }'`; \ - ${ECHO} '# Need additional files for ${PKGNAME} ('$$location')...' -. for fetchfile in ${_ALLFILES} -. if defined(_FETCH_MESSAGE) - @(if [ ! -f ${_DISTDIR}/${fetchfile:T} ]; then \ - ${ECHO}; \ - filesize=`${AWK} ' \ - /^Size/ && $$2 == "(${fetchfile})" { print $$4 } \ - ' ${DISTINFO_FILE}` || true; \ - ${ECHO} '# Prompt user to get ${fetchfile} ('$${filesize-???}' bytes) manually:'; \ - ${ECHO} '#'; \ - ${ECHO} ${_FETCH_MESSAGE:Q}; \ - fi) -. elif defined(DYNAMIC_MASTER_SITES) - @(if [ ! -f ${_DISTDIR}/${fetchfile:T} ]; then \ - ${ECHO}; \ - filesize=`${AWK} ' \ - /^Size/ && $$2 == "(${fetchfile})" { print $$4 } \ - ' ${DISTINFO_FILE}` || true; \ - ${ECHO} '# Fetch ${fetchfile} ('$${filesize-???}' bytes):'; \ - ${ECHO} '#'; \ - ${ECHO} '${SH} -s ${fetchfile:T} <<"EOF" |('; \ - ${CAT} ${FILESDIR}/getsite.sh; \ - ${ECHO} EOF; \ - ${ECHO} read unsorted_sites; \ - ${ECHO} 'unsorted_sites="$${unsorted_sites} ${_MASTER_SITE_BACKUP}"'; \ - ${ECHO} sites='"'${ORDERED_SITES:Q}'"'; \ - ${ECHO} "${MKDIR} ${_DISTDIR}"; \ - ${ECHO} 'cd ${_DISTDIR} && [ -f ${fetchfile} -o -f ${fetchfile:T} ] ||'; \ - ${ECHO} 'for site in $$sites; do'; \ - ${ECHO} ' ${FETCH_CMD} ${FETCH_BEFORE_ARGS} "$${site}${fetchfile:T}" ${FETCH_AFTER_ARGS} && break ||'; \ - ${ECHO} ' ${ECHO} ${fetchfile:T} not fetched'; \ - ${ECHO} done; \ - ${ECHO} ')'; \ - fi) -. else - @(if [ ! -f ${_DISTDIR}/${fetchfile:T} ]; then \ - ${ECHO}; \ - filesize=`${AWK} ' \ - /^Size/ && $$2 == "(${fetchfile})" { print $$4 } \ - ' ${DISTINFO_FILE}` || true; \ - ${ECHO} '# Fetch ${fetchfile} ('$${filesize-???}' bytes):'; \ - ${ECHO} '#'; \ - ${ECHO} 'unsorted_sites="${SITES.${fetchfile:T:S/=/--/}} ${_MASTER_SITE_BACKUP}"'; \ - ${ECHO} sites='"'${ORDERED_SITES:Q}'"'; \ - ${ECHO} "${MKDIR} ${_DISTDIR}"; \ - ${ECHO} 'cd ${_DISTDIR} && [ -f ${fetchfile} -o -f ${fetchfile:T} ] ||'; \ - ${ECHO} 'for site in $$sites; do'; \ - ${ECHO} ' ${FETCH_CMD} ${FETCH_BEFORE_ARGS} "$${site}${fetchfile:T}" ${FETCH_AFTER_ARGS} && break ||'; \ - ${ECHO} ' ${ECHO} ${fetchfile:T} not fetched'; \ - ${ECHO} done; \ - fi) -. endif # defined(_FETCH_MESSAGE) || defined(DYNAMIC_MASTER_SITES) -. endfor -. endif # !empty(_ALLFILES) -.endif # !target(fetch-list-one-pkg) # Checksumming utilities @@ -2456,14 +1987,13 @@ PKG_ERROR_HANDLER.${_class_}?= { \ # .for _phase_ in ${ALL_PHASES} ${_MAKEVARS_MK.${_phase_}}: ${WRKDIR} -. if !empty(PKG_PHASE:M${_phase_}) ${_PKG_SILENT}${_PKG_DEBUG}${RM} -f ${.TARGET}.tmp -. for _var_ in ${MAKEVARS:O:u} -. if defined(${_var_}) +. for _var_ in ${MAKEVARS:O:u} +. if defined(${_var_}) ${_PKG_SILENT}${_PKG_DEBUG} \ ${ECHO} ${_var_}"= "${${_var_}:Q} >> ${.TARGET}.tmp -. endif -. endfor +. endif +. endfor ${_PKG_SILENT}${_PKG_DEBUG} \ if ${TEST} -f ${.TARGET}.tmp; then \ ( ${ECHO} ".if !defined(_MAKEVARS_MK)"; \ @@ -2475,7 +2005,6 @@ ${_MAKEVARS_MK.${_phase_}}: ${WRKDIR} ) > ${.TARGET}; \ ${RM} -f ${.TARGET}.tmp; \ fi -. endif ${_PKG_SILENT}${_PKG_DEBUG}${TOUCH} ${TOUCH_FLAGS} ${.TARGET} .endfor .undef _phase_ diff --git a/mk/bsd.prefs.mk b/mk/bsd.prefs.mk index e8de8101463..f15e31c7fbe 100644 --- a/mk/bsd.prefs.mk +++ b/mk/bsd.prefs.mk @@ -1,4 +1,4 @@ -# $NetBSD: bsd.prefs.mk,v 1.222 2006/06/05 17:11:37 joerg Exp $ +# $NetBSD: bsd.prefs.mk,v 1.223 2006/06/06 03:05:48 jlam Exp $ # # Make file, included to get the site preferences, if any. Should # only be included by package Makefiles before any .if defined() @@ -502,7 +502,6 @@ MAKEFLAGS+= _PKGSRCDIR=${_PKGSRCDIR:Q} .endif PKGSRCDIR= ${_PKGSRCDIR} -DISTDIR?= ${PKGSRCDIR}/distfiles PACKAGES?= ${PKGSRCDIR}/packages TEMPLATES?= ${PKGSRCDIR}/templates @@ -559,6 +558,8 @@ PREPEND_PATH+= ${USE_X11:D${X11BASE}/bin} ${LOCALBASE}/bin .include "${PKGSRCDIR}/mk/flavor/bsd.flavor-vars.mk" .include "${PKGSRCDIR}/mk/check/bsd.check-vars.mk" .include "${PKGSRCDIR}/mk/depends/bsd.depends-vars.mk" +.include "${PKGSRCDIR}/mk/fetch/bsd.fetch-vars.mk" +.include "${PKGSRCDIR}/mk/extract/bsd.extract-vars.mk" .include "${PKGSRCDIR}/mk/install/bsd.install-vars.mk" USE_TOOLS+= awk:pkgsrc cut:pkgsrc echo:pkgsrc pwd:pkgsrc \ diff --git a/mk/extract/bsd.extract-vars.mk b/mk/extract/bsd.extract-vars.mk new file mode 100644 index 00000000000..3ff08de0e9b --- /dev/null +++ b/mk/extract/bsd.extract-vars.mk @@ -0,0 +1,63 @@ +# $NetBSD: bsd.extract-vars.mk,v 1.1 2006/06/06 03:05:48 jlam Exp $ +# +# This Makefile fragment is included to bsd.prefs.mk and defines some +# variables which must be defined earlier than where bsd.extract.mk +# is included. +# +# The following variables may be set by the package Makefile and +# specify how extraction happens: +# +# EXTRACT_ONLY is a list of distfiles relative to ${_DISTDIR} to +# extract and defaults to ${DISTFILES}. +# +# EXTRACT_SUFX is the suffix for the default distfile to be +# extracted. The default suffix is ".tar.gz". +# + +EXTRACT_ONLY?= ${DISTFILES} +EXTRACT_SUFX?= .tar.gz + +### +### Discover which tools we need based on the file extensions of the +### distfiles. +### +_EXTRACT_PATTERNS= ${EXTRACT_ONLY} ${EXTRACT_SUFX} + +.if !empty(_EXTRACT_PATTERNS:M*.tar) || \ + !empty(_EXTRACT_PATTERNS:M*.tar.*) || \ + !empty(_EXTRACT_PATTERNS:M*.tbz) || \ + !empty(_EXTRACT_PATTERNS:M*.tbz2) || \ + !empty(_EXTRACT_PATTERNS:M*.tgz) || \ + !empty(_EXTRACT_PATTERNS:M*-tar.gz) || \ + !empty(_EXTRACT_PATTERNS:M*_tar.gz) +. if !empty(EXTRACT_USING:Mgtar) +USE_TOOLS+= gtar +. elif !empty(EXTRACT_USING:Mnbtar) +USE_TOOLS+= tar +. else +USE_TOOLS+= pax +. endif +.endif +.if !empty(_EXTRACT_PATTERNS:M*.bz2) || \ + !empty(_EXTRACT_PATTERNS:M*.tbz) || \ + !empty(_EXTRACT_PATTERNS:M*.tbz2) +USE_TOOLS+= bzcat +.endif +.if !empty(_EXTRACT_PATTERNS:M*.zip) +USE_TOOLS+= unzip +.endif +.if !empty(_EXTRACT_PATTERNS:M*.lzh) || \ + !empty(_EXTRACT_PATTERNS:M*.lha) +USE_TOOLS+= lha +.endif +.if !empty(_EXTRACT_PATTERNS:M*.gz) || \ + !empty(_EXTRACT_PATTERNS:M*.tgz) || \ + !empty(_EXTRACT_PATTERNS:M*.Z) +USE_TOOLS+= gzcat +.endif +.if !empty(_EXTRACT_PATTERNS:M*.zoo) +USE_TOOLS+= unzoo +.endif +.if !empty(_EXTRACT_PATTERNS:M*.rar) +USE_TOOLS+= unrar +.endif diff --git a/mk/extract/bsd.extract.mk b/mk/extract/bsd.extract.mk new file mode 100644 index 00000000000..3c837c7afbe --- /dev/null +++ b/mk/extract/bsd.extract.mk @@ -0,0 +1,15 @@ +# $NetBSD: bsd.extract.mk,v 1.1 2006/06/06 03:05:48 jlam Exp $ +# +# This Makefile fragment is included to bsd.pkg.mk and defines the +# relevant variables and targets for the "extract" phase. +# +# The following are the "public" targets provided by this module: +# +# extract +# +# The following targets may be overridden in a package Makefile: +# +# pre-extract, do-extract, post-extract +# + +.include "${PKGSRCDIR}/mk/extract/extract.mk" diff --git a/mk/scripts/extract b/mk/extract/extract index 544d1f2283e..b8590dcb8ad 100755 --- a/mk/scripts/extract +++ b/mk/extract/extract @@ -1,6 +1,6 @@ #!/bin/sh # -# $NetBSD: extract,v 1.21 2006/05/30 23:51:38 wiz Exp $ +# $NetBSD: extract,v 1.1 2006/06/06 03:05:48 jlam Exp $ # # Copyright (c) 2006 The NetBSD Foundation, Inc. # All rights reserved. diff --git a/mk/bsd.pkg.extract.mk b/mk/extract/extract.mk index 19c393283cf..f4c7aac4947 100644 --- a/mk/bsd.pkg.extract.mk +++ b/mk/extract/extract.mk @@ -1,17 +1,8 @@ -# $NetBSD: bsd.pkg.extract.mk,v 1.25 2006/06/05 22:49:44 jlam Exp $ -# -# This Makefile fragment is included to bsd.pkg.mk and defines the -# relevant variables and targets for the "extract" phase. +# $NetBSD: extract.mk,v 1.1 2006/06/06 03:05:48 jlam Exp $ # # The following variables may be set by the package Makefile and # specify how extraction happens: # -# EXTRACT_ONLY is a list of distfiles relative to ${_DISTDIR} to -# extract and defaults to ${DISTFILES}. -# -# EXTRACT_SUFX is the suffix for the default distfile to be -# extracted. The default suffix is ".tar.gz". -# # EXTRACT_CMD is a shell command list that extracts the contents of # an archive named by the variable ${DOWNLOADED_DISTFILE} to the # current working directory. The default is ${EXTRACT_CMD_DEFAULT}. @@ -43,69 +34,97 @@ # # EXTRACT_CMD= ${TAIL} +25 ${DOWNLOADED_DISTFILE} > foo.pl # -# The following targets are defined by bsd.pkg.extract.mk: -# -# extract is the target that is invoked by the user to perform -# extraction. -# -# do-extract is the target that causes the actual extraction of -# the distfiles to occur during the "extract" phase. This target -# may be overridden in a package Makefile. -# -# {pre,post}-extract are the targets that are invoked before and after -# do-extract, and may be overridden in a package Makefile. -# -EXTRACT_ONLY?= ${DISTFILES} -EXTRACT_SUFX?= .tar.gz -EXTRACT_USING?= nbtar -EXTRACT_ELEMENTS?= # empty +_EXTRACT_COOKIE= ${WRKDIR}/.extract_done +###################################################################### +### extract (PUBLIC) +###################################################################### +### extract is a public target to perform extraction. ### -### Discover which tools we need based on the file extensions of the -### distfiles. -### -_EXTRACT_PATTERNS= ${EXTRACT_ONLY} ${EXTRACT_SUFX} - -.if !empty(_EXTRACT_PATTERNS:M*.tar) || \ - !empty(_EXTRACT_PATTERNS:M*.tar.*) || \ - !empty(_EXTRACT_PATTERNS:M*.tbz) || \ - !empty(_EXTRACT_PATTERNS:M*.tbz2) || \ - !empty(_EXTRACT_PATTERNS:M*.tgz) || \ - !empty(_EXTRACT_PATTERNS:M*-tar.gz) || \ - !empty(_EXTRACT_PATTERNS:M*_tar.gz) -. if !empty(EXTRACT_USING:Mgtar) -USE_TOOLS+= gtar -. elif !empty(EXTRACT_USING:Mnbtar) -USE_TOOLS+= tar -. else -USE_TOOLS+= pax -. endif -.endif -.if !empty(_EXTRACT_PATTERNS:M*.bz2) || \ - !empty(_EXTRACT_PATTERNS:M*.tbz) || \ - !empty(_EXTRACT_PATTERNS:M*.tbz2) -USE_TOOLS+= bzcat -.endif -.if !empty(_EXTRACT_PATTERNS:M*.zip) -USE_TOOLS+= unzip -.endif -.if !empty(_EXTRACT_PATTERNS:M*.lzh) || \ - !empty(_EXTRACT_PATTERNS:M*.lha) -USE_TOOLS+= lha -.endif -.if !empty(_EXTRACT_PATTERNS:M*.gz) || \ - !empty(_EXTRACT_PATTERNS:M*.tgz) || \ - !empty(_EXTRACT_PATTERNS:M*.Z) -USE_TOOLS+= gzcat +_EXTRACT_TARGETS+= checksum +_EXTRACT_TARGETS+= ${WRKDIR} +_EXTRACT_TARGETS+= depends +_EXTRACT_TARGETS+= acquire-extract-lock +_EXTRACT_TARGETS+= ${_EXTRACT_COOKIE} +_EXTRACT_TARGETS+= release-extract-lock + +.PHONY: extract +.if !target(extract) +extract: ${_EXTRACT_TARGETS} .endif -.if !empty(_EXTRACT_PATTERNS:M*.zoo) -USE_TOOLS+= unzoo + +.PHONY: acquire-extract-lock release-extract-lock +acquire-extract-lock: acquire-lock +release-extract-lock: release-lock + +.if !exists(${_EXTRACT_COOKIE}) +${_EXTRACT_COOKIE}: real-extract +.else +${_EXTRACT_COOKIE}: + @${DO_NADA} .endif -.if !empty(_EXTRACT_PATTERNS:M*.rar) -USE_TOOLS+= unrar + +###################################################################### +### real-extract (PRIVATE) +###################################################################### +### real-extract is a helper target onto which one can hook all of the +### targets that do the actual extraction work. +### +_REAL_EXTRACT_TARGETS+= extract-check-interactive +_REAL_EXTRACT_TARGETS+= extract-message +_REAL_EXTRACT_TARGETS+= extract-vars +_REAL_EXTRACT_TARGETS+= pre-extract +_REAL_EXTRACT_TARGETS+= do-extract +_REAL_EXTRACT_TARGETS+= post-extract +_REAL_EXTRACT_TARGETS+= extract-cookie + +.PHONY: real-extract +real-extract: ${_REAL_EXTRACT_TARGETS} + +.PHONY: extract-message +extract-message: + @${PHASE_MSG} "Extracting for ${PKGNAME}" + +###################################################################### +### extract-check-interactive (PRIVATE) +###################################################################### +### extract-check-interactive checks whether we can do an interactive +### extraction or not. +### +extract-check-interactive: +.if !empty(INTERACTIVE_STAGE:Mextract) && defined(BATCH) + @${ERROR_MSG} "The extract stage of this package requires user interaction" + @${ERROR_MSG} "Please extract manually with:" + @${ERROR_MSG} " \"cd ${PKGSRCDIR}/${PKGPATH} && ${MAKE} extract\"" + @${TOUCH} ${_INTERACTIVE_COOKIE} + @${FALSE} +.else + @${DO_NADA} .endif +###################################################################### +### extract-cookie (PRIVATE) +###################################################################### +### extract-cookie creates the "extract" cookie file. The contents +### are the name of the package. +### +.PHONY: extract-cookie +extract-cookie: + ${_PKG_SILENT}${_PKG_DEBUG}${MKDIR} ${_EXTRACT_COOKIE:H} + ${_PKG_SILENT}${_PKG_DEBUG}${ECHO} ${PKGNAME} >> ${_EXTRACT_COOKIE} + +###################################################################### +### pre-extract, do-extract, post-extract (PUBLIC, override) +###################################################################### +### {pre,do,post}-extract are the heart of the package-customizable +### extract targets, and may be overridden within a package Makefile. +### +.PHONY: pre-extract do-extract post-extract + +EXTRACT_USING?= nbtar +EXTRACT_ELEMENTS?= # empty + ### ### Build the default extraction command ### @@ -143,7 +162,7 @@ EXTRACT_OPTS+= ${TOOLS_PAX:D -t ${TOOLS_PAX}} EXTRACT_CMD_DEFAULT= \ ${SETENV} ${_EXTRACT_ENV} \ - ${SH} ${.CURDIR}/../../mk/scripts/extract \ + ${SH} ${PKGSRCDIR}//mk/extract/extract \ ${EXTRACT_OPTS} \ ${DOWNLOADED_DISTFILE} ${EXTRACT_ELEMENTS} @@ -151,7 +170,6 @@ EXTRACT_CMD?= ${EXTRACT_CMD_DEFAULT} DOWNLOADED_DISTFILE= $${extract_file} -.PHONY: do-extract .if !target(do-extract) do-extract: ${WRKDIR} . for __file__ in ${EXTRACT_ONLY} @@ -161,55 +179,6 @@ do-extract: ${WRKDIR} . endfor .endif -_EXTRACT_COOKIE= ${WRKDIR}/.extract_done - -_EXTRACT_TARGETS+= checksum -_EXTRACT_TARGETS+= ${WRKDIR} -_EXTRACT_TARGETS+= depends -_EXTRACT_TARGETS+= acquire-extract-lock -_EXTRACT_TARGETS+= ${_EXTRACT_COOKIE} -_EXTRACT_TARGETS+= release-extract-lock - -.ORDER: ${_EXTRACT_TARGETS} - -.PHONY: extract -extract: ${_EXTRACT_TARGETS} - -.PHONY: acquire-extract-lock release-extract-lock -acquire-extract-lock: acquire-lock -release-extract-lock: release-lock - -${_EXTRACT_COOKIE}: -.if !empty(INTERACTIVE_STAGE:Mextract) && defined(BATCH) - @${ECHO} "*** The extract stage of this package requires user interaction" - @${ECHO} "*** Please extract manually with \"cd ${PKGDIR} && ${MAKE} extract\"" - @${TOUCH} ${_INTERACTIVE_COOKIE} - @${FALSE} -.else - ${_PKG_SILENT}${_PKG_DEBUG}cd ${.CURDIR} && ${MAKE} ${MAKEFLAGS} real-extract DEPENDS_TARGET=${DEPENDS_TARGET:Q} PKG_PHASE=extract -.endif - -_REAL_EXTRACT_TARGETS+= extract-message -_REAL_EXTRACT_TARGETS+= extract-vars -_REAL_EXTRACT_TARGETS+= pre-extract -_REAL_EXTRACT_TARGETS+= do-extract -_REAL_EXTRACT_TARGETS+= post-extract -_REAL_EXTRACT_TARGETS+= extract-cookie - -.ORDER: ${_REAL_EXTRACT_TARGETS} - -.PHONY: extract-message -extract-message: - @${PHASE_MSG} "Extracting for ${PKGNAME}" - -.PHONY: extract-cookie -extract-cookie: - ${_PKG_SILENT}${_PKG_DEBUG}${ECHO} ${PKGNAME} >> ${_EXTRACT_COOKIE} - -.PHONY: real-extract -real-extract: ${_REAL_EXTRACT_TARGETS} - -.PHONY: pre-extract post-extract .if !target(pre-extract) pre-extract: @${DO_NADA} diff --git a/mk/fetch/bsd.fetch-vars.mk b/mk/fetch/bsd.fetch-vars.mk new file mode 100644 index 00000000000..b5b6897d707 --- /dev/null +++ b/mk/fetch/bsd.fetch-vars.mk @@ -0,0 +1,22 @@ +# $NetBSD: bsd.fetch-vars.mk,v 1.1 2006/06/06 03:05:48 jlam Exp $ +# +# This Makefile fragment is included to bsd.prefs.mk and defines some +# variables which must be defined earlier than where bsd.fetch.mk +# is included. +# +# The following variables may be set by the user: +# +# DISTDIR is the top-level directory into which all original +# distribution files are fetched. +# +# The following variables may be set in a package Makefile: +# +# DIST_SUBDIR is the subdirectory of ${DISTDIR} in which the original +# distribution files for the package are fetched. +# +# DISTFILES is the list of distribution files that are fetched. +# + +DISTDIR?= ${PKGSRCDIR}/distfiles +_DISTDIR= ${DISTDIR}/${DIST_SUBDIR} +DISTFILES?= ${DISTNAME}${EXTRACT_SUFX} diff --git a/mk/fetch/bsd.fetch.mk b/mk/fetch/bsd.fetch.mk new file mode 100644 index 00000000000..e311caea9bc --- /dev/null +++ b/mk/fetch/bsd.fetch.mk @@ -0,0 +1,9 @@ +# $NetBSD: bsd.fetch.mk,v 1.1 2006/06/06 03:05:48 jlam Exp $ +# +# This Makefile fragment is included by bsd.pkg.mk and defines the +# relevant variables and targets for the "fetch" step. +# + +.include "${PKGSRCDIR}/mk/fetch/sites.mk" +.include "${PKGSRCDIR}/mk/fetch/fetch.mk" +.include "${PKGSRCDIR}/mk/fetch/distclean.mk" diff --git a/mk/fetch/distclean.mk b/mk/fetch/distclean.mk new file mode 100644 index 00000000000..3140e1b286b --- /dev/null +++ b/mk/fetch/distclean.mk @@ -0,0 +1,26 @@ +# $NetBSD: distclean.mk,v 1.1 2006/06/06 03:05:48 jlam Exp $ + +.PHONY: pre-distclean +.if !target(pre-distclean) +pre-distclean: + @${DO_NADA} +.endif + +.PHONY: distclean +.if !target(distclean) +distclean: pre-distclean clean + @${PHASE_MSG} "Dist cleaning for ${PKGNAME}" + ${_PKG_SILENT}${_PKG_DEBUG}if [ -d ${_DISTDIR} ]; then \ + cd ${_DISTDIR} && \ + ${TEST} -z "${DISTFILES}" || ${RM} -f ${DISTFILES}; \ + if [ "${PKG_RESUME_TRANSFERS:M[Yy][Ee][Ss]}" ]; then \ + ${TEST} -z "${DISTFILES}.temp" || ${RM} -f ${DISTFILES}.temp; \ + fi; \ + ${TEST} -z "${PATCHFILES}" || ${RM} -f ${PATCHFILES}; \ + fi +. if defined(DIST_SUBDIR) + -${_PKG_SILENT}${_PKG_DEBUG} \ + ${TEST} ! -d ${_DISTDIR} || ${RMDIR} ${_DISTDIR} 2>/dev/null +. endif + ${_PKG_SILENT}${_PKG_DEBUG}${RM} -f README.html +.endif diff --git a/mk/fetch/fetch.mk b/mk/fetch/fetch.mk new file mode 100644 index 00000000000..7f9a0176844 --- /dev/null +++ b/mk/fetch/fetch.mk @@ -0,0 +1,453 @@ +# $NetBSD: fetch.mk,v 1.1 2006/06/06 03:05:48 jlam Exp $ + +###################################################################### +### fetch (PUBLIC) +###################################################################### +### fetch is a public target to fetch all of the package distribution +### files. +### +.PHONY: fetch +.if !target(fetch) +fetch: pre-fetch do-fetch post-fetch +.endif + +# If this host is behind a filtering firewall, use passive ftp(1) +FETCH_BEFORE_ARGS+= ${PASSIVE_FETCH:D-p} + +_MASTER_SITE_BACKUP= ${MASTER_SITE_BACKUP:=${DIST_SUBDIR}${DIST_SUBDIR:D/}} +_MASTER_SITE_OVERRIDE= ${MASTER_SITE_OVERRIDE:=${DIST_SUBDIR}${DIST_SUBDIR:D/}} + +# Where to put distfiles that don't have any other master site +MASTER_SITE_LOCAL?= ${MASTER_SITE_BACKUP:=LOCAL_PORTS/} + +ALLFILES?= ${DISTFILES} ${PATCHFILES} +CKSUMFILES?= ${ALLFILES} +.for __tmp__ in ${IGNOREFILES} +CKSUMFILES:= ${CKSUMFILES:N${__tmp__}} +.endfor + +# List of all files, with ${DIST_SUBDIR} in front. Used for fetch and checksum. +.if defined(DIST_SUBDIR) +_CKSUMFILES?= ${CKSUMFILES:S/^/${DIST_SUBDIR}\//} +_DISTFILES?= ${DISTFILES:S/^/${DIST_SUBDIR}\//} +_IGNOREFILES?= ${IGNOREFILES:S/^/${DIST_SUBDIR}\//} +_PATCHFILES?= ${PATCHFILES:S/^/${DIST_SUBDIR}\//} +.else +_CKSUMFILES?= ${CKSUMFILES} +_DISTFILES?= ${DISTFILES} +_IGNOREFILES?= ${IGNOREFILES} +_PATCHFILES?= ${PATCHFILES} +.endif +_ALLFILES?= ${_DISTFILES} ${_PATCHFILES} + +BUILD_DEFS+= _DISTFILES _PATCHFILES + +### +### _RESUME_TRANSFER: +### +### Macro to resume a previous transfer, needs to have defined +### the following options in mk.conf: +### +### PKG_RESUME_TRANSFERS +### FETCH_RESUME_ARGS (if FETCH_CMD != default) +### FETCH_OUTPUT_ARGS (if FETCH_CMD != default) +### +### For example if you want to use wget (pkgsrc/net/wget): +### +### FETCH_CMD=wget +### FETCH_RESUME_ARGS=-c +### FETCH_OUTPUT_ARGS=-O +### +### How does it work? +### +### FETCH_CMD downloads the file and saves it temporally into $$bfile.temp +### if the checksum match the correct one, $$bfile.temp is renamed to +### the original name. +### + +_RESUME_TRANSFER= \ + ofile="${DISTDIR}/${DIST_SUBDIR}/$$bfile"; \ + tfile="${DISTDIR}/${DIST_SUBDIR}/$$bfile.temp"; \ + tsize=`${AWK} '/^Size/ && $$2 == '"\"($$file)\""' { print $$4 }' ${DISTINFO_FILE}` || ${TRUE}; \ + osize=`${WC} -c < $$ofile`; \ + \ + case "$$tsize" in \ + "") ${ECHO_MSG} "No size in distinfo file (${DISTINFO_FILE})"; \ + break;; \ + esac; \ + \ + if [ "$$osize" -eq "$$tsize" ]; then \ + if [ -f "$$tfile" ]; then \ + ${RM} $$tfile; \ + fi; \ + need_fetch=no; \ + elif [ "$$osize" -lt "$$tsize" -a ! -f "$$tfile" ]; then \ + ${CP} $$ofile $$tfile; \ + dsize=`${WC} -c < $$tfile`; \ + need_fetch=yes; \ + elif [ -f "$$tfile" -a "$$dsize" -gt "$$ossize" ]; then \ + dsize=`${WC} -c < $$tfile`; \ + need_fetch=yes; \ + else \ + if [ -f "$$tfile" ]; then \ + dsize=`${WC} -c < $$tfile`; \ + fi; \ + need_fetch=yes; \ + fi; \ + if [ "$$need_fetch" = "no" ]; then \ + break; \ + elif [ -f "$$tfile" -a "$$dsize" -eq "$$tsize" ]; then \ + ${MV} $$tfile $$ofile; \ + break; \ + elif [ -n "$$ftp_proxy" -o -n "$$http_proxy" ]; then \ + ${ECHO_MSG} "===> Resume is not supported by ftp(1) using http/ftp proxies."; \ + break; \ + elif [ "$$need_fetch" = "yes" -a "$$dsize" -lt "$$tsize" ]; then \ + if [ "${FETCH_CMD:T}" != "ftp" -a -z "${FETCH_RESUME_ARGS}" ]; then \ + ${ECHO_MSG} "=> Resume transfers are not supported, FETCH_RESUME_ARGS is empty."; \ + break; \ + else \ + for res_site in $$sites; do \ + if [ -z "${FETCH_OUTPUT_ARGS}" ]; then \ + ${ECHO_MSG} "=> FETCH_OUTPUT_ARGS has to be defined."; \ + break; \ + fi; \ + ${ECHO_MSG} "=> $$bfile not completed, resuming:"; \ + ${ECHO_MSG} "=> Downloaded: $$dsize Total: $$tsize."; \ + ${ECHO_MSG}; \ + cd ${DISTDIR}; \ + ${FETCH_CMD} ${FETCH_BEFORE_ARGS} ${FETCH_RESUME_ARGS} \ + ${FETCH_OUTPUT_ARGS} $${bfile}.temp $${res_site}$${bfile}; \ + if [ $$? -eq 0 ]; then \ + ndsize=`${WC} -c < $$tfile`; \ + if [ "$$tsize" -eq "$$ndsize" ]; then \ + ${MV} $$tfile $$ofile; \ + fi; \ + break; \ + fi; \ + done; \ + fi; \ + elif [ "$$dsize" -gt "$$tsize" ]; then \ + ${ECHO_MSG} "==> Downloaded file larger than the recorded size."; \ + break; \ + fi + +# +# Define the elementary fetch macros. +# +_FETCH_FILE= \ + if [ ! -f $$file -a ! -f $$bfile -a ! -h $$bfile ]; then \ + ${ECHO_MSG} "=> $$bfile doesn't seem to exist on this system."; \ + if [ ! -w ${_DISTDIR}/. ]; then \ + ${ECHO_MSG} "=> Can't download to ${_DISTDIR} (permission denied?)."; \ + exit 1; \ + fi; \ + for site in $$sites; do \ + ${ECHO_MSG} "=> Attempting to fetch $$bfile from $${site}."; \ + if [ -f ${DISTINFO_FILE} ]; then \ + ${AWK} 'NF == 5 && $$1 == "Size" && $$2 == "('$$bfile')" { printf("=> [%s %s]\n", $$4, $$5) }' ${DISTINFO_FILE}; \ + fi; \ + if ${FETCH_CMD} ${FETCH_BEFORE_ARGS} $${site}$${bfile} ${FETCH_AFTER_ARGS}; then \ + if [ -n "${FAILOVER_FETCH}" -a -f ${DISTINFO_FILE} -a -f ${_DISTDIR}/$$bfile ]; then \ + alg=`${AWK} 'NF == 4 && $$2 == "('$$file')" && $$3 == "=" {print $$1; exit}' ${DISTINFO_FILE}`; \ + if [ -z "$$alg" ]; then \ + alg=${PATCH_DIGEST_ALGORITHM};\ + fi; \ + CKSUM=`${DIGEST} $$alg < ${_DISTDIR}/$$bfile`; \ + CKSUM2=`${AWK} '$$1 == "'$$alg'" && $$2 == "('$$file')" {print $$4; exit}' <${DISTINFO_FILE}`; \ + if [ "$$CKSUM" = "$$CKSUM2" -o "$$CKSUM2" = "IGNORE" ]; then \ + break; \ + else \ + ${ECHO_MSG} "=> Checksum failure - trying next site."; \ + fi; \ + elif [ ! -f ${_DISTDIR}/$$bfile ]; then \ + ${ECHO_MSG} "=> FTP didn't fetch expected file, trying next site." ; \ + else \ + break; \ + fi; \ + fi \ + done; \ + if [ ! -f ${_DISTDIR}/$$bfile ]; then \ + ${ECHO_MSG} "=> Couldn't fetch $$bfile - please try to retrieve this";\ + ${ECHO_MSG} "=> file manually into ${_DISTDIR} and try again."; \ + exit 1; \ + fi; \ + fi + +_CHECK_DIST_PATH= \ + if [ "X${DIST_PATH}" != "X" ]; then \ + for d in "" ${DIST_PATH:S/:/ /g}; do \ + case $$d in "" | ${DISTDIR}) continue;; esac; \ + if [ -f $$d/${DIST_SUBDIR}/$$bfile ]; then \ + ${ECHO} "Using $$d/${DIST_SUBDIR}/$$bfile"; \ + ${RM} -f $$bfile; \ + ${LN} -s $$d/${DIST_SUBDIR}/$$bfile $$bfile; \ + break; \ + fi; \ + done; \ + fi + +# +# Set up ORDERED_SITES to work out the exact list of sites for every file, +# using the dynamic sites script, or sorting according to the master site +# list or the patterns in MASTER_SORT or MASTER_SORT_REGEX as appropriate. +# No actual sorting is done until ORDERED_SITES is expanded. +# +.if defined(MASTER_SORT) || defined(MASTER_SORT_REGEX) +MASTER_SORT?= +MASTER_SORT_REGEX?= +MASTER_SORT_REGEX+= ${MASTER_SORT:S/./\\./g:C/.*/:\/\/[^\/]*&\//} + +MASTER_SORT_AWK= BEGIN { RS = " "; ORS = " "; IGNORECASE = 1 ; gl = "${MASTER_SORT_REGEX:S/\\/\\\\/g}"; } +. for srt in ${MASTER_SORT_REGEX} +MASTER_SORT_AWK+= /${srt:C/\//\\\//g}/ { good["${srt:S/\\/\\\\/g}"] = good["${srt:S/\\/\\\\/g}"] " " $$0 ; next; } +. endfor +MASTER_SORT_AWK+= { rest = rest " " $$0; } END { n=split(gl, gla); for(i=1;i<=n;i++) { print good[gla[i]]; } print rest; } + +SORT_SITES_CMD= ${ECHO} $$unsorted_sites | ${AWK} '${MASTER_SORT_AWK}' +ORDERED_SITES= ${_MASTER_SITE_OVERRIDE} `${SORT_SITES_CMD:S/\\/\\\\/g:C/"/\"/g}` +.else +ORDERED_SITES= ${_MASTER_SITE_OVERRIDE} $$unsorted_sites +.endif + +# +# Associate each file to fetch with the correct site(s). +# +.if defined(DYNAMIC_MASTER_SITES) +. for fetchfile in ${_ALLFILES} +SITES_${fetchfile:T:S/=/--/}?= `${SH} ${FILESDIR}/getsite.sh ${fetchfile:T}` +SITES.${fetchfile:T:S/=/--/}?= ${SITES_${fetchfile:T:S/=/--/}} +. endfor +.endif +.if !empty(_DISTFILES) +. for fetchfile in ${_DISTFILES} +SITES_${fetchfile:T:S/=/--/}?= ${MASTER_SITES} +SITES.${fetchfile:T:S/=/--/}?= ${SITES_${fetchfile:T:S/=/--/}} +. endfor +.endif +.if !empty(_PATCHFILES) +. for fetchfile in ${_PATCHFILES} +SITES_${fetchfile:T:S/=/--/}?= ${PATCH_SITES} +SITES.${fetchfile:T:S/=/--/}?= ${SITES_${fetchfile:T:S/=/--/}} +. endfor +.endif + +# This code is only called in a batch case, to check for the presence of +# the distfiles +.PHONY: batch-check-distfiles +batch-check-distfiles: + ${_PKG_SILENT}${_PKG_DEBUG} \ + gotfiles=yes; \ + for file in "" ${_ALLFILES}; do \ + case "$$file" in \ + "") continue ;; \ + *) if [ ! -f ${DISTDIR}/$$file ]; then \ + gotfiles=no; \ + fi ;; \ + esac; \ + done; \ + case "$$gotfiles" in \ + no) ${ECHO} "*** This package requires user intervention to download the distfiles"; \ + ${ECHO} "*** Please fetch the distfiles manually and place them in"; \ + ${ECHO} "*** ${DISTDIR}"; \ + [ ! -z "${MASTER_SITES}" ] && \ + ${ECHO} "*** The distfiles are available from ${MASTER_SITES}"; \ + [ ! -z "${HOMEPAGE}" ] && \ + ${ECHO} "*** See ${HOMEPAGE} for more details"; \ + ${ECHO}; \ + ${TOUCH} ${_INTERACTIVE_COOKIE}; \ + ${FALSE} ;; \ + esac + +.PHONY: do-fetch +.if !target(do-fetch) +do-fetch: +. if !defined(ALLOW_VULNERABLE_PACKAGES) + ${_PKG_SILENT}${_PKG_DEBUG} \ + if [ -f ${PKGVULNDIR}/pkg-vulnerabilities ]; then \ + ${PHASE_MSG} "Checking for vulnerabilities in ${PKGNAME}"; \ + vul=`${MAKE} ${MAKEFLAGS} check-vulnerable`; \ + case "$$vul" in \ + "") ;; \ + *) ${ECHO} "$$vul"; \ + ${ECHO} "or define ALLOW_VULNERABLE_PACKAGES if this package is absolutely essential"; \ + ${FALSE} ;; \ + esac; \ + else \ + ${WARNING_MSG} "No ${PKGVULNDIR}/pkg-vulnerabilities file found,"; \ + ${WARNING_MSG} "skipping vulnerability checks. To fix, install"; \ + ${WARNING_MSG} "the pkgsrc/security/audit-packages package and run"; \ + ${WARNING_MSG} "\`\`${LOCALBASE}/sbin/download-vulnerability-list''."; \ + fi +. endif +. if !empty(_ALLFILES) + ${_PKG_SILENT}${_PKG_DEBUG} \ + ${TEST} -d ${_DISTDIR} || ${MKDIR} ${_DISTDIR} +. if !empty(INTERACTIVE_STAGE:Mfetch) && defined(BATCH) + ${_PKG_SILENT}${_PKG_DEBUG} \ + ${MAKE} ${MAKEFLAGS} batch-check-distfiles +. else +. for fetchfile in ${_ALLFILES} +. if defined(FETCH_MESSAGE) && !empty(FETCH_MESSAGE) + ${_PKG_SILENT}${_PKG_DEBUG} set -e; \ + ${TEST} -f ${DISTDIR:Q}/${fetchfile:Q} || { \ + h="==============="; h="$$h$$h$$h$$h$$h"; \ + ${ECHO} "$$h"; ${ECHO} ""; \ + for l in ${FETCH_MESSAGE}; do ${ECHO} "$$l"; done; \ + ${ECHO} ""; ${ECHO} "$$h"; \ + exit 1; \ + } +. elif defined(_FETCH_MESSAGE) + ${_PKG_SILENT}${_PKG_DEBUG} \ + file="${fetchfile}"; \ + if [ ! -f ${DISTDIR}/$$file ]; then \ + ${_FETCH_MESSAGE}; \ + fi +. else + ${_PKG_SILENT}${_PKG_DEBUG} \ + cd ${_DISTDIR}; \ + file="${fetchfile}"; \ + bfile="${fetchfile:T}"; \ + unsorted_sites="${SITES.${fetchfile:T:S/=/--/}} ${_MASTER_SITE_BACKUP}"; \ + sites="${ORDERED_SITES}"; \ + ${_CHECK_DIST_PATH}; \ + if ${TEST} "${PKG_RESUME_TRANSFERS:M[Yy][Ee][Ss]}" ; then \ + ${_FETCH_FILE}; ${_RESUME_TRANSFER}; \ + else \ + ${_FETCH_FILE}; \ + fi +. endif # defined(_FETCH_MESSAGE) +. endfor +. endif # INTERACTIVE_STAGE == fetch +. endif # !empty(_ALLFILES) +.endif + +.PHONY: show-distfiles +.if !target(show-distfiles) +show-distfiles: +. if defined(PKG_FAIL_REASON) + ${_PKG_SILENT}${_PKG_DEBUG}${DO_NADA} +. else + ${_PKG_SILENT}${_PKG_DEBUG} \ + for file in "" ${_CKSUMFILES}; do \ + if [ "X$$file" = "X" ]; then continue; fi; \ + ${ECHO} $$file; \ + done +. endif +.endif + +.if !target(pre-fetch) +pre-fetch: + @${DO_NADA} +.endif +.if !target(post-fetch) +post-fetch: + @${DO_NADA} +.endif + +# This is for the use of sites which store distfiles which others may +# fetch - only fetch the distfile if it is allowed to be +# re-distributed freely +.PHONY: mirror-distfiles +mirror-distfiles: +.if !defined(NO_SRC_ON_FTP) + @${_PKG_SILENT}${_PKG_DEBUG}${MAKE} ${MAKEFLAGS} fetch NO_SKIP=yes +.endif + +# Prints out a script to fetch all needed files (no checksumming). +.PHONY: fetch-list +.if !target(fetch-list) + +fetch-list: + @${ECHO} '#!/bin/sh' + @${ECHO} '#' + @${ECHO} '# This is an auto-generated script, the result of running' + @${ECHO} '# `${MAKE} fetch-list'"'"' in directory "'"`${PWD_CMD}`"'"' + @${ECHO} '# on host "'"`${UNAME} -n`"'" on "'"`date`"'".' + @${ECHO} '#' + @${MAKE} ${MAKEFLAGS} fetch-list-recursive +.endif # !target(fetch-list) + +.PHONY: fetch-list-recursive +.if !target(fetch-list-recursive) + +fetch-list-recursive: + ${_PKG_SILENT}${_PKG_DEBUG} \ + for dir in `${MAKE} ${MAKEFLAGS} show-all-depends-dirs`; do \ + (cd ../../$$dir && \ + ${MAKE} ${MAKEFLAGS} fetch-list-one-pkg \ + | ${AWK} ' \ + /^[^#]/ { FoundSomething = 1 } \ + /^unsorted/ { gsub(/[[:space:]]+/, " \\\n\t") } \ + /^echo/ { gsub(/;[[:space:]]+/, "\n") } \ + { block[line_c++] = $$0 } \ + END { if (FoundSomething) \ + for (line = 0; line < line_c; line++) \ + print block[line] } \ + ') \ + done +.endif # !target(fetch-list-recursive) + +.PHONY: fetch-list-one-pkg +.if !target(fetch-list-one-pkg) + +fetch-list-one-pkg: +. if !empty(_ALLFILES) + @${ECHO} + @${ECHO} '#' + @location=`${PWD_CMD} | ${AWK} -F / '{ print $$(NF-1) "/" $$NF }'`; \ + ${ECHO} '# Need additional files for ${PKGNAME} ('$$location')...' +. for fetchfile in ${_ALLFILES} +. if defined(_FETCH_MESSAGE) + @(if [ ! -f ${_DISTDIR}/${fetchfile:T} ]; then \ + ${ECHO}; \ + filesize=`${AWK} ' \ + /^Size/ && $$2 == "(${fetchfile})" { print $$4 } \ + ' ${DISTINFO_FILE}` || true; \ + ${ECHO} '# Prompt user to get ${fetchfile} ('$${filesize-???}' bytes) manually:'; \ + ${ECHO} '#'; \ + ${ECHO} ${_FETCH_MESSAGE:Q}; \ + fi) +. elif defined(DYNAMIC_MASTER_SITES) + @(if [ ! -f ${_DISTDIR}/${fetchfile:T} ]; then \ + ${ECHO}; \ + filesize=`${AWK} ' \ + /^Size/ && $$2 == "(${fetchfile})" { print $$4 } \ + ' ${DISTINFO_FILE}` || true; \ + ${ECHO} '# Fetch ${fetchfile} ('$${filesize-???}' bytes):'; \ + ${ECHO} '#'; \ + ${ECHO} '${SH} -s ${fetchfile:T} <<"EOF" |('; \ + ${CAT} ${FILESDIR}/getsite.sh; \ + ${ECHO} EOF; \ + ${ECHO} read unsorted_sites; \ + ${ECHO} 'unsorted_sites="$${unsorted_sites} ${_MASTER_SITE_BACKUP}"'; \ + ${ECHO} sites='"'${ORDERED_SITES:Q}'"'; \ + ${ECHO} "${MKDIR} ${_DISTDIR}"; \ + ${ECHO} 'cd ${_DISTDIR} && [ -f ${fetchfile} -o -f ${fetchfile:T} ] ||'; \ + ${ECHO} 'for site in $$sites; do'; \ + ${ECHO} ' ${FETCH_CMD} ${FETCH_BEFORE_ARGS} "$${site}${fetchfile:T}" ${FETCH_AFTER_ARGS} && break ||'; \ + ${ECHO} ' ${ECHO} ${fetchfile:T} not fetched'; \ + ${ECHO} done; \ + ${ECHO} ')'; \ + fi) +. else + @(if [ ! -f ${_DISTDIR}/${fetchfile:T} ]; then \ + ${ECHO}; \ + filesize=`${AWK} ' \ + /^Size/ && $$2 == "(${fetchfile})" { print $$4 } \ + ' ${DISTINFO_FILE}` || true; \ + ${ECHO} '# Fetch ${fetchfile} ('$${filesize-???}' bytes):'; \ + ${ECHO} '#'; \ + ${ECHO} 'unsorted_sites="${SITES.${fetchfile:T:S/=/--/}} ${_MASTER_SITE_BACKUP}"'; \ + ${ECHO} sites='"'${ORDERED_SITES:Q}'"'; \ + ${ECHO} "${MKDIR} ${_DISTDIR}"; \ + ${ECHO} 'cd ${_DISTDIR} && [ -f ${fetchfile} -o -f ${fetchfile:T} ] ||'; \ + ${ECHO} 'for site in $$sites; do'; \ + ${ECHO} ' ${FETCH_CMD} ${FETCH_BEFORE_ARGS} "$${site}${fetchfile:T}" ${FETCH_AFTER_ARGS} && break ||'; \ + ${ECHO} ' ${ECHO} ${fetchfile:T} not fetched'; \ + ${ECHO} done; \ + fi) +. endif # defined(_FETCH_MESSAGE) || defined(DYNAMIC_MASTER_SITES) +. endfor +. endif # !empty(_ALLFILES) +.endif # !target(fetch-list-one-pkg) diff --git a/mk/bsd.sites.mk b/mk/fetch/sites.mk index 563e051b140..8ee4cb604a3 100644 --- a/mk/bsd.sites.mk +++ b/mk/fetch/sites.mk @@ -1,9 +1,8 @@ -# $NetBSD: bsd.sites.mk,v 1.63 2006/05/19 02:21:24 perry Exp $ +# $NetBSD: sites.mk,v 1.1 2006/06/06 03:05:48 jlam Exp $ +# +# This Makefile fragment defines read-only MASTER_SITE_* variables +# representing some well-known master distribution sites for software. # -# Default MASTER_SITES - -.if !defined(BSD_SITES_MK) -BSD_SITES_MK= 1 MASTER_SITE_XCONTRIB+= \ ftp://ftp.gwdg.de/pub/x11/x.org/contrib/ \ @@ -397,5 +396,3 @@ MASTER_SITE_BACKUP?= \ ftp://ftp.NetBSD.org/pub/NetBSD/packages/distfiles/ \ http://ftp.NetBSD.org/pub/NetBSD/packages/distfiles/ \ ftp://ftp.FreeBSD.org/pub/FreeBSD/distfiles/ - -.endif # BSD_SITES_MK |