path: root/mk/bulk/upload

#!/bin/sh
# $NetBSD: upload,v 1.41 2007/10/09 19:19:13 martti Exp $

#
# Upload non-restricted binary pkgs to ftp server
#

AWK=${AWK:-/usr/bin/awk}
PKG_ADMIN="pkg_admin"
PKG_INFO="pkg_info"

set -eu

#
# Find out where we are
#
scriptdir=`dirname "$0"`
scriptdir=`cd "${scriptdir}" && pwd`

usage()
{
cat << EOF

$prog:  Uploads binary packages.

Usage:  $prog [-c|--config <file>] [-n|--no-upload] [-d|--debug] [-v|--verbose]
	$prog -h|--help
        $prog -V|--version

    -c | --config <file>
			Load the following configuration file instead of
			the default one.

    -d|--debug          Preserves some intermediate files to help
			debug any problems.

    -h|--help           Displays this help message and exits.

    -n|--no-upload      Does all of the work except for the actual upload.
			This option may be used to generate the upload script
			along with the list of packages to be excluded.

    -i|--no-install	Do not install the required packages; assume instead
			that they are already available.

    -v|--verbose        Enables verbose output.

    -V|--version        Displays the version of this script and exits.

Example:

    cd /usr/pkgsrc && $prog

Environment Variables:

    AWK  --  If specified, sets the awk program to be used

Files:

    pkgsrc/mk/bulk/build.conf

EOF
}

# print out error message and exit 1
die () {
	echo "$0: error:" 1>&2
	for i in "$@"; do
		echo "    $i" 1>&2
	done
	exit 1
}

prog=$0
debug=no
do_upload=yes
do_install=yes
verbose=no
while test $# -gt 0; do
	case "$1" in
	-c|--config)
		shift
		BULK_BUILD_CONF=$1; shift
		;;

	-d|--debug)
		debug=yes
		shift
		;;

	-h|--help)
		usage
		exit 0
		;;

	-i|--no-install)
		do_install=no
		shift
		;;

	-n|--no-upload)
		do_upload=no
		shift
		;;

	-v|--verbose)
		verbose=yes
		shift
		;;

	-V|--version)
		${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
		exit 0
		;;

	-*)
		echo "$prog:  Unknown option:  $1" 1>&2
		usage
		exit 1
		;;

	*)
		break
		;;
	esac
done
if test "$#" -gt 0; then
	echo "$prog:  Unknown argument: $1" 1>&2
	usage
	exit 1
fi

install_required()
{

	[ "$do_install" = "yes" ] || return 0
	pkg=$1
	if [ "${verbose}" = "yes" ]; then
		echo "upload> Installing ${pkg}"
	fi
	(cd "$USR_PKGSRC" && cd "$pkg" && ${BMAKE} bulk-install) \
	|| {
		echo "upload> ERROR: Unable to install required package $pkg!" 1>&2
		echo "        Bailing out -- you're on your own." 1>&2
		exit 1
	}
}

MD5="digest md5";
SHA1="digest sha1";

_INFO_VER=`${PKG_INFO} -V`;
if ${PKG_ADMIN} pmatch 'pkg_install<20070714' pkg_install-${_INFO_VER}; then
	REQUIRED_PACKAGES="pkgtools/pkglint net/rsync security/audit-packages"
else
	REQUIRED_PACKAGES="pkgtools/pkglint net/rsync"
fi

opsys=`uname -s`
case "$opsys" in
NetBSD)	BMAKE=make;
	BSDSUM="sum -o 1";
	CKSUM="cksum";
	SYSVSUM="sum -o 2";
	;;
IRIX*)  BMAKE=bmake;
	BSDSUM="sum -r";
	CKSUM="cksum";
	SYSVSUM="sum";
	;;
*)	BMAKE=bmake ;;
esac

export BMAKE

# Bring in variables for bulk-install
BATCH=1
DEPENDS_TARGET=bulk-install
export BATCH DEPENDS_TARGET

#
# Find the configuration file.
#
BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}"
case $BULK_BUILD_CONF in
	/*) ;;
	 *)     BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}"
esac

#
# Get the variables MAKECONF, RSYNC_DST, RSYNC_OPTS from the bulk build
# configuration file.
#
{ test -f "${BULK_BUILD_CONF}"		\
  && . "${BULK_BUILD_CONF}"		\
  && . "${scriptdir}/post-build-conf"	\
  && check_config_vars			\
  && export_config_vars
} || die "Cannot load config file ${BULK_BUILD_CONF}, aborting."

cd $USR_PKGSRC

if [ -z "$RSYNC_DST" ]; then
	echo "upload> ERROR: You must set the variable RSYNC_DST, see build.conf-example." 1>&2
	exit 1
fi

#
# Some temp files
#

umask 022
TMPDIR="${TMPDIR:-/tmp}"
TMP="${TMPDIR}"/pkg_upload.$$
(umask 077 && mkdir "${TMP}") \
|| {
        echo "upload> ERROR: cannot create temporary directory \"${TMP}\"." 1>&2
        exit 1
}

vulnerable_packages="$TMP/vulnerable_packages"
restricted_packages="$TMP/restricted_packages"
old_packages="$TMP/old_packages"
good_packages="$TMP/regular_packages"
all_good_packages="$TMP/all_regular_packages"

upload_general="$TMP"/upload_general
upload_vulnerable="$TMP"/upload_vulnerable

# May be different than $USR_PKGSRC:
echo "upload> Running ${BMAKE} to get the pkgsrc variables"
pkgsrcdir=`cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=_PKGSRCDIR`
packages=`cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=PACKAGES`
distdir=`cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=DISTDIR`
gzip_cmd=`cd pkgtools/pkglint; ${BMAKE} show-var VARNAME=GZIP_CMD USE_TOOLS=gzip`
pkg_info=`cd pkgtools/pkglint && ${BMAKE} show-var VARNAME=PKG_INFO`

# Pull in some pkgs needed
for pkg in ${REQUIRED_PACKAGES}; do
	install_required $pkg
done

echo "upload> Making sure vulnerability-list is up-to-date:"
if [ -z "$UPDATE_VULNERABILITY_LIST" -o "$UPDATE_VULNERABILITY_LIST" = "yes" ]
then
	if ${PKG_ADMIN} pmatch 'pkg_install<20070714' pkg_install-${_INFO_VER}
	then
		env PKGVULNDIR=${distdir} download-vulnerability-list
	else
		_PKGVULNDIR=`audit-packages ${AUDIT_PACKAGES_FLAGS} -Q PKGVULNDIR`
		download-vulnerability-list ${DOWNLOAD_VULNERABILITY_LIST_FLAGS}
		if [ "x${_PKGVULNDIR}" != "x${distdir}" ]; then
			cp ${_PKGVULNDIR}/pkg-vulnerabilities ${distdir}
		fi
	fi
	echo "        done."
else
	echo "        (skipped)"
fi

case $LINTPKGSRC_CACHE in
yes|YES)
	lintpkgsrc_cache="-I `cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=LINTPKGSRC_DB`"
	;;
*)
	lintpkgsrc_cache=''
	;;
esac

echo "upload> Checking for out of date packages:"
# -p  =  report old versions of packages
lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -p > "${old_packages}.tmp"
sed 's@'$packages'/@@' < "${old_packages}.tmp" > "$old_packages"

RSFLAGS="-vap --progress $RSYNC_OPTS"

failed=no
cd $packages

echo "upload> Checking for restricted and vulnerable packages"
(cd All && env PKG_INFO="${pkg_info}" OUTDIR="${TMP}" PKGVULNDIR="${distdir}" sh "${pkgsrcdir}/mk/bulk/sort-packages")

# Add the name of the package file, including all its symlinks to the
# list of files to be uploaded.
while read package; do
	ls -1 */"$package"
done < "$good_packages" > "$all_good_packages"

if [ "${MKSUMS}" = "yes" -o "${MKSUMS}" = "YES" ]; then

	echo "upload> Calculating checksum files..."

	SUMFILES="BSDSUM CKSUM MD5 SHA1 SYSVSUM"

	rm -f ${SUMFILES}

	if [ "${SIGN_AS-}" != "" ]; then
		install_required "security/gnupg"
		for i in ${SUMFILES}; do
			echo > $i
			echo "This file is signed with ${SIGN_AS}'s PGP key." >> $i
			echo >> $i
		done
	fi

	install_required "pkgtools/digest"

	[ -z "${BSDSUM}" ] && BSDSUM="echo"
	[ -z "${CKSUM}" ] && CKSUM="echo"
	[ -z "${SYSVSUM}" ] && SYSVSUM="echo"

	for pkg in `cat "${good_packages}" "${vulnerable_packages}"`; do
		pkg="All/$pkg"
		${BSDSUM}	"$pkg" >> BSDSUM
		${CKSUM}	"$pkg" >> CKSUM
		${MD5}		"$pkg" >> MD5
		${SHA1}		"$pkg" >> SHA1
		${SYSVSUM}	"$pkg" >> SYSVSUM
	done

	[ "${BSDSUM}" = "echo" ] && rm BSDSUM
	[ "${CKSUM}" = "echo" ] && rm CKSUM
	[ "${SYSVSUM}" = "echo" ] && rm SYSVSUM

	if [ "${SIGN_AS-}" != "" ]; then
		for i in ${SUMFILES}; do
			if [ -s $i ]; then
				echo "upload> Signing $i"
				gpg --clearsign $i && rm $i
			fi
		done
	else
		echo "upload> Checksum files not PGP-signed. Please do so manually!"
		echo "        (Run 'gpg --clearsign' on all of them)"
	fi
fi

if [ "${MKSUMMARY-}" = "yes" -o "${MKSUMMARY-}" = "YES" ]; then
	echo "upload> Creating summary file..."
	(cd "${packages}/All" \
		&& ls -t | grep '\.t[gb]z$' | while read n; do pkg_info -X "$n"; done) \
		| ${gzip_cmd} > "${packages}"/All/pkg_summary.gz
fi

cat <<EOF > "$upload_general"
#! /bin/sh
set -e
cd "$packages"
rsync $RSFLAGS --files-from="${all_good_packages}" --exclude-from="${old_packages}" . "$RSYNC_DST/"
EOF
chmod +x "$upload_general"

if [ "$do_upload" = "yes" ]; then
	echo "upload> Uploading non-vulnerable packages"
	sh "$upload_general" \
	|| {
		echo "upload> ERROR: rsync failed.  To retry later, you can run $upload_general" 1>&2
		failed=yes
	}
else
	echo "upload> Skipping upload of non-vulnerable packages."
	echo "        Run \"$upload_general\" to upload them later."
fi

cat <<EOF > "$upload_vulnerable"
#! /bin/sh
set -e
cd "$packages/All"
rsync $RSFLAGS --files-from="${vulnerable_packages}" --exclude-from="${old_packages}" . "$RSYNC_DST/vulnerable/"
EOF
chmod +x "$upload_vulnerable"

if [ "$do_upload" = "yes" ]; then
	echo "upload> Uploading vulnerable packages"
	sh "$upload_vulnerable" \
	|| {
		echo "upload> ERROR: rsync failed.  To retry later, you can run $upload_vulnerable" 1>&2
		failed=yes
	}
else
	echo "upload> Skipping upload of vulnerable packages."
	echo "        Run \"$upload_vulnerable\" to upload them later."
fi

# clean up temp files
if [ "$failed,$debug,$do_upload" = "no,no,yes" ]; then
	rm -fr "$TMP"
else
	echo "upload> Preserving temporary directory ${TMP}"
fi

if [ "$failed" = "yes" ]; then
	exit 1
fi