path: root/mk/bulk/upload

#!/bin/sh
# $NetBSD: upload,v 1.32 2006/08/01 00:53:24 dmcmahill Exp $

#
# Upload non-restricted binary pkgs to ftp server
#

AWK=${AWK:-/usr/bin/awk}

usage()
{
cat << EOF

$prog:  Uploads binary packages.

Usage:  $prog [-n|--dry-run] [-d|--debug] [-v|--verbose]
	$prog -h|--help
        $prog -V|--version

    -d|--debug          Preserves some intermediate files to help
			debug any problems.

    -h|--help           Displays this help message and exits.

    -n|--no-upload      Does all of the work except for the actual upload.
			This option may be used to generate the upload script
			along with the list of packages to be excluded.

    -v|--verbose        Enables verbose output.

    -V|--version        Displays the version of this script and exits.

Example:

    cd /usr/pkgsrc && $prog

Environment Variables:
    
    AWK  --  If specified, sets the awk program to be used

Files:

    pkgsrc/mk/bulk/build.conf

EOF
}

prog=$0
debug=no
do_upload=yes
verbose=no
while
	test -n "$1"
do
	case "$1"
	in

	-d|--debug)
		debug=yes
		shift
		;;

	-h|--help)
		usage
		exit 0
		;;

	-n|--no-upload)
		do_upload=no
		shift
		;;

	-v|--verbose)
		verbose=yes
		shift
		;;

	-V|--version)
		${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
		exit 0
		;;

	-*)
		echo "$prog:  Unknown option:  $1"
		usage
		exit 1
		;;

	*)
		break
		;;
	esac
done
if test -n "$1" ; then
	echo "$prog:  Unknown argument: $1"
	usage
	exit 1
fi

install_required()
{
	pkg=$1
	if [ "${verbose}" = "yes" ]; then
		echo "Installing ${pkg}"
	fi
	( cd $pkg; ${BMAKE} bulk-install )
	if [ $? -gt 0 ]; then
		echo "Unable to install required package $pkg!"
		echo "Bailing out -- you're on your own."
		exit 1
	fi
}

MD5="digest md5";
SHA1="digest sha1";

REQUIRED_PACKAGES="pkgtools/pkglint net/rsync security/audit-packages"

opsys=`uname -s`
case "$opsys" in
NetBSD)	BMAKE=make;
	BSDSUM="sum -o 1";
	CKSUM="cksum";
	SYSVSUM="sum -o 2";
	;;
IRIX*)  BMAKE=bmake;
	BSDSUM="sum -r";
	CKSUM="cksum";
	SYSVSUM="sum";
	;;
*)	BMAKE=bmake ;;
esac

export BMAKE

# Bring in variables for bulk-install
BATCH=1
DEPENDS_TARGET=bulk-install
export BATCH DEPENDS_TARGET

# Pull in RSYNC_DST, RSYNC_OPTS:
if [ -f "$BULK_BUILD_CONF" ]; then
    . $BULK_BUILD_CONF
else
    . `dirname $0`/build.conf
fi

cd $USR_PKGSRC

if [ -z "$RSYNC_DST" ]; then
	echo "You must set the variable RSYNC_DST, see build.conf-example."
	exit 1
fi

#
# Some temp files
#

umask 022
TMPDIR="${TMPDIR:-/tmp}"
TMP="${TMPDIR}"/pkg_upload.$$
(umask 077 && mkdir "${TMP}")
if [ $? -ne 0 ]
then
        echo $0: cannot create temporary directory \""${TMP}"\" >&2
        exit 1
fi

exf="$TMP"/exclude
vf="$TMP"/vulnerable
upload="$TMP"/upload
upload_general="$TMP"/upload_general
upload_vulnerable="$TMP"/upload_vulnerable

# May be different than $USR_PKGSRC:
if [ "${verbose}" = "yes" ]; then
	echo "Extracting variables"
fi
pkgsrcdir=`cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=_PKGSRCDIR`
packages=`cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=PACKAGES`
distdir=`cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=DISTDIR`
gzip_cmd=`cd pkgtools/pkglint; ${BMAKE} show-var VARNAME=GZIP_CMD USE_TOOLS=gzip`

# Pull in some pkgs needed
for pkg in ${REQUIRED_PACKAGES}; do
	install_required $pkg
done

echo "Making sure vulnerability-list is up-to-date:"
if [ -z "$UPDATE_VULNERABILITY_LIST" -o "$UPDATE_VULNERABILITY_LIST" = "yes" ]
then
	env PKGVULNDIR=${distdir} download-vulnerability-list
else
	echo '(skipped)'
fi

case $LINTPKGSRC_CACHE in
yes|YES)
	lintpkgsrc_cache="-I `cd pkgtools/pkglint ; ${BMAKE} show-var VARNAME=LINTPKGSRC_DB`"
	;;
*)
	lintpkgsrc_cache=''
	;;
esac

echo "Checking for restricted and out of date packages:"
# -p  =  report old versions of packages
# -R  =  report restricted packages
lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -pR  | sed 's@'$packages'/@@' > "$exf"

echo "Checking for vulnerable packages:"
lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -V  | sed 's@'$packages'/@@' > "$vf"

RSFLAGS="-vap --progress $RSYNC_OPTS"

failed=no
cd $packages

if [ "${MKSUMS}" = "yes" -o "${MKSUMS}" = "YES" ]; then

	echo "Calculating checksum files..."

	SUMFILES="BSDSUM CKSUM MD5 SHA1 SYSVSUM"

	rm -f ${SUMFILES}

	if [ x"${SIGN_AS}" != x"" ]; then
		( cd ${pkgsrcdir}/security/gnupg; ${BMAKE} bulk-install )
		for i in ${SUMFILES}; do
			echo > $i
			echo "This file is signed with ${SIGN_AS}'s PGP key." >> $i
			echo >> $i
		done
	fi

	( cd ${pkgsrcdir}/pkgtools/digest; ${BMAKE} bulk-install )

	[ -z "${BSDSUM}" ] && BSDSUM="echo"
	[ -z "${CKSUM}" ] && CKSUM="echo"
	[ -z "${SYSVSUM}" ] && SYSVSUM="echo"

	for i in All/*; do
		if grep $i $exf >/dev/null; then
			:
		else
			${BSDSUM} $i >> BSDSUM
			${CKSUM} $i >> CKSUM
			${MD5} $i >> MD5
			${SHA1} $i >> SHA1
			${SYSVSUM} $i >> SYSVSUM
		fi
	done

	[ "${BSDSUM}" = "echo" ] && rm BSDSUM
	[ "${CKSUM}" = "echo" ] && rm CKSUM
	[ "${SYSVSUM}" = "echo" ] && rm SYSVSUM
	
	if [ x"${SIGN_AS}" != x"" ]; then
		for i in ${SUMFILES}; do
			if [ -s $i ]; then
				echo "Signing $i"
				gpg --clearsign $i && rm $i
			fi
		done
	else
		echo "Checksum files not PGP-signed. Please do so manually!"
		echo "(Run 'gpg --clearsign' on all of them)"
	fi
fi

if [ "${MKSUMMARY}" = "yes" -o "${MKSUMMARY}" = "YES" ]; then
	echo "Creating summary file..."
	(cd "${packages}/All" \
		&& ls -t | grep '\.t[gb]z$' | while read n; do pkg_info -X "$n"; done) \
		| ${gzip_cmd} > "${packages}"/All/pkg_summary.gz
fi

cat << EOF > "$upload"
#!/bin/sh

packages=$packages
if cd $packages; then
	:
else
	echo "could not cd to $packages"
	exit 1
fi

EOF

cmd="rsync $RSFLAGS --exclude-from=\"$exf\" --exclude-from=\"$vf\" . \"$RSYNC_DST\""
cp -f "$upload" "$upload_general"
echo "$cmd" >> "$upload_general"
chmod 755 "$upload_general"
echo "$cmd"
if [ "X${do_upload}" = "Xyes" ] ; then
	echo "Uploading non-vulnerable pkgs"
	sh "$upload_general"
	rc=$?
else
	echo "Skipping upload of non-vulnerable pkgs"
	echo "Non-vulnerable upload script = ${upload_general}"
	rc=0
fi
if [ ${rc} != 0 ]; then
	echo "--------------------------------------------------"
	echo " "
	echo "WARNING: rsync failed.  To retry later, you can run"
	echo "    $upload_general"
	echo " "
	echo "--------------------------------------------------"
	failed=yes
fi

sed -n "s@All/@@p" "$exf" > "$exf.new"
sed -n "s@All/@@p" "$vf" > "$vf.new"
cmd="rsync $RSFLAGS --exclude-from=\"$exf.new\" --include-from=\"$vf.new\" --exclude='*' All/ \"$RSYNC_DST/vulnerable/\""
cp -f "$upload" "$upload_vulnerable"
echo "$cmd" >> "$upload_vulnerable"
chmod 755 "$upload_vulnerable"
echo "$cmd"
if [ "X${do_upload}" = "Xyes" ] ; then
	echo "Uploading vulnerable pkgs"
	sh "$upload_vulnerable"
	rc=$?
else
	echo "Skipping upload of vulnerable pkgs"
	echo "Vulnerable upload script = ${upload_vulnerable}"
	rc=0
fi
if [ ${rc} != 0 ]; then
	echo "--------------------------------------------------"
	echo " "
	echo "WARNING: rsync failed.  To retry later, you can run"
	echo "    $upload_vulnerable"
	echo " "
	echo "--------------------------------------------------"
	failed=yes
fi

# clean up temp files
if [ "$failed" = "no" -a "${debug}" = "no" ]; then
	rm -fr "$TMP"
else
	echo "Preserving temp directory ${TMP}"
fi