path: root/mk/scripts/mkreadme

#!/bin/sh
# $NetBSD: mkreadme,v 1.17 2007/08/02 22:42:05 adrianp Exp $
#
# Script for README.html generation
#
# Copyright (c) 2002, 2005 The NetBSD Foundation, Inc.
# All rights reserved.
#
# This code is derived from software contributed to The NetBSD Foundation
# by Dan McMahill.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#        This product includes software developed by the NetBSD
#        Foundation, Inc. and its contributors.
# 4. Neither the name of The NetBSD Foundation nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#


TMPDIR=${TMPDIR:-/tmp/mkreadme}
PKGSRCDIR=${PKGSRCDIR:-/usr/pkgsrc}
AWK=${AWK:-/usr/bin/awk}

opsys=`uname -s`
case "$opsys" in
	*BSD)
		makeprog=make
		;;

	*)
		makeprog=bmake
		;;
esac

BMAKE=${BMAKE:-${makeprog}}

usage(){
    echo "$prog - Generates README.html files for a pkgsrc tree"
    echo "Usage:      $prog [-c|--cdrom] [-C|--prune] [-d|--debug] [-f|--ftp] "
    echo "                  [-p|--pkgsrc directory] "
    echo "                  [-P|--packages directory] [-r|--restart] "
    echo "                  [-s|--summary]"
    echo "                  [-S|--save-database]"
    echo "                  [-V|--pkg-vuln directory]"
    echo " "
    echo "            $prog -h|--help"
    echo " "
    echo "            $prog -v|--version"
    echo " "
    echo "The options supported by $prog are: "
    echo " "
    echo "  -C|--prune          Prune unused README.html files which may exist in"
    echo "                      pkg directories which have been removed from CVS."
    echo " "
    echo "  -c|--cdrom          Generates CD-ROM README.html files"
    echo " "
    echo "  -d|--debug          Enables (extremely verbose) debug output"
    echo " "
    echo "  -f|--ftp            Generates FTP README.html files"
    echo " "
    echo "  -h|--help           Displays this help message"
    echo " "
    echo "  -p|--pkgsrc dir     Specifies the pkgsrc directory.  Defaults to"
    echo "                      the value of the PKGSRCDIR environment variable"
    echo "                      if set or /usr/pkgsrc otherwise."
    echo " "
    echo "  -P|--packages dir   Specifies the packages directory."
    echo " "
    echo "  -r|--restart        Restart.  This option assumes that the database file"
    echo "                      from a previous run still exists and that the script"
    echo "                      should use that instead of recreating the database."
    echo " "
    echo "  -s|--summary        Generate pkg_summary.gz files for the binary packages"
    echo "                      directories while processing them."
    echo " "
    echo "  -S|--save-database  Does not delete the database file after the run."
    echo "                      This is useful for debugging or re-running this script"
    echo "                      with the -r option."
    echo " "
    echo "  -V|--pkg-vuln dir   Specify the pkg-vulnerability directory."
    echo "                      Defaults to the value of the DISTDIR."
    echo " "
    echo "  -v|--version        Displays the version of this script and exits."
    echo " "
    echo "Example:    $prog -p /pub/NetBSD/packages/pkgsrc -P /pub/NetBSD/packages -f"
    echo " "
}

clean_and_exit(){
    if [ "x$DEBUG" = "xno" -a "x$restart" = "xno" -a "x$save" = "xno" ]; then
	rm -fr $TMPDIR
    else
	echo "Debugging output left in $TMP"
    fi
    exit 1
}

prog=$0


######################################################################
#
#  Handle command line options
#
######################################################################

cmdline=$*

ftp_readme=no
restart=no
prune=no
DEBUG=no
save=no
pv=default

summary=""

while
    test -n "$1"
do
    case "$1"
    in

    # We're generating README.html's for a CD-ROM
    -c|--cdrom)
	ftp_readme=no
	shift
	;;

    # Prune old README.html files from pkgs which no longer exist
    -C|--prune)
	prune=yes
	shift
	;;

    # Turn on debugging
    -d|--debug)
	DEBUG=yes
	shift
	;;

    # We're generating README.html's for a CD-ROM
    -f|--ftp)
	ftp_readme=yes
	shift
	;;


    # Help
    -h|--help)
	usage
	exit 0
	;;

    # Specify pkgsrc directory
    -p|--pkgsrc)
	PKGSRCDIR=$2
	shift 2
	;;

    # Specify PACKAGES directory
    -P|--packages)
	PKGDIR=$2
	shift 2
	;;

    # Restart (ie, don't re-generate the database file)
    -r|--restart)
	restart=yes
	shift
	;;

    # Generate the pkg_summary.gz files
    -s|--summary)
	summary="--summary"
	shift
	;;

    # Save the database files
    -S|--save-database)
	save=yes
	shift
	;;

    # Version
    -v|--version)
	${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
	exit 0
        ;;

    # Specify pkg-vulnerability directory
    -V|--pkg-vuln)
	PVDIR=$2
	pv=custom
	shift 2
	;;

    -*) echo "$prog:  ERROR:  $1 is not a valid option"
	usage
	clean_and_exit
	;;

    *)
	break
	;;

    esac
done

if [ "x$DEBUG" = "xyes" ]; then
    set -v
fi

if [ ! -d ${PKGSRCDIR} ]; then
    echo "ERROR:  package source directory ${PKGSRCDIR} does not exist"
    echo ""
    clean_and_exit
fi

if [ ! -d $TMPDIR ]; then
	mkdir -p $TMPDIR
fi

DEPENDSTREEFILE=$TMPDIR/dependstree
export DEPENDSTREEFILE
DEPENDSFILE=$TMPDIR/depends
export DEPENDSFILE
SUPPORTSFILE=$TMPDIR/supports
export SUPPORTSFILE
INDEXFILE=$TMPDIR/index
export SUPPORTSFILE
ORDERFILE=$TMPDIR/order
export ORDERFILE
DATABASEFILE=$TMPDIR/database
export DATABASEFILE
BINPKGFILE=$TMPDIR/binpkglist

echo "Starting README.html generation: `date`"

######################################################################
#
#  Extract key pkgsrc configuration variables
#
######################################################################

echo " "
echo "Extracting tool variables:"
echo " "
if [ -d ${PKGSRCDIR}/pkgtools/prereq-readme ]; then
    cd ${PKGSRCDIR}/pkgtools/prereq-readme
    eval "`${BMAKE} show-tools`"
    for v in AWK CMP ECHO EXPR FGREP FIND GREP GZIP_CMD SED SETENV SORT
    do
	eval "echo '---->'  ${v}=\"\${${v}}\""
    done
else
    echo "Error:   ${PKGSRCDIR}/pkgtools/prereq-readme does not seem to exist"
    exit 1
fi

echo " "
echo "Extracting configuration variables:"
echo " "
if [ -d ${PKGSRCDIR}/pkgtools/prereq-readme ]; then
    cd ${PKGSRCDIR}/pkgtools/prereq-readme
    for v in CDROM_PKG_URL_HOST CDROM_PKG_URL_DIR DISTDIR \
	FTP_PKG_URL_HOST FTP_PKG_URL_DIR PACKAGES PKG_INFO PKG_SUFX PKG_ADMIN \
	AUDIT_PACKAGES_FLAGS PKGVULNDIR PKGTOOLS_VERSION
    do
	val=`${BMAKE} show-var VARNAME=${v}`
	if [ $? != 0 ]; then
	    echo "Error:  make show-var VARNAME=${v} in `pwd` "
	    echo "Failed.  This is a fatal error"
	    clean_and_exit
	fi
	echo "---->  ${v}=\"${val}\""
	eval "${v}=\"${val}\""
    done
else
    echo "Error:   ${PKGSRCDIR}/pkgtools/prereq-readme does not seem to exist"
    exit 1
fi

# this needs to happen here as the variables referenced have not been 
# populated until this stage (see above)
#
if [ "$pv" = "default" ]; then
    if ${PKG_ADMIN} pmatch 'pkg_install<20070714' pkg_install-${PKGTOOLS_VERSION}; then
        PVDIR=$PKGVULNDIR
    else
	PVDIR=`audit-packages ${AUDIT_PACKAGES_FLAGS} -Q PKGVULNDIR`
    fi
fi
	echo "---->  PVDIR=\"${PVDIR}\""


######################################################################
#
#  Decide on FTP vs CDROM README.html files
#
######################################################################

if [ "$ftp_readme" = "yes" ]; then
    PKG_URL=${FTP_PKG_URL_HOST}${FTP_PKG_URL_DIR}
    echo "Will generate FTP readme files with PKG_URL=$PKG_URL"
else
    PKG_URL=${CDROM_PKG_URL_HOST}${CDROM_PKG_URL_DIR}
    echo "Will generate CD-ROM readme files with PKG_URL=$PKG_URL"
fi

######################################################################
#
#  Check for command line switch for packages directory
#
######################################################################

# we've been given the directory as a command line switch
if [ ! -z "$PKGDIR" ]; then
    PACKAGES=$PKGDIR
    echo "PACKAGES specified on command line to be $PKGDIR"
fi


######################################################################
#
#  Extract Database for All Packages (longest step)
#
######################################################################

if [ "x$restart" = "xno" ] ; then
    echo " "
    echo "Extracting data.  This could take a while"
    echo "Started at: `date` "
    echo " "
    npkg=1

    # make sure we don't have an old database lying around
    rm -fr $DATABASEFILE

    cd ${PKGSRCDIR}
    cats=`${BMAKE} show-subdir-var VARNAME=SUBDIR`
    for c in ${cats} ; do
	if [ ! -d ${PKGSRCDIR}/${c} ]; then
		echo "WARNING:  The category directory ${c} does not seem to" > /dev/stderr
		echo "          exist under ${PKGSRCDIR}" > /dev/stderr
	else
		echo " "
		echo "Extracting data for category ${c}"
		cd ${PKGSRCDIR}/${c}
		list=`${BMAKE} show-subdir-var VARNAME=SUBDIR`
    		for pkgdir in $list ; do
			cd ${PKGSRCDIR}/${c}
	    		if [ ! -d $pkgdir ]; then
		    		echo " "
		    		echo "WARNING:  the package directory $pkgdir is listed in" > /dev/stderr
		    		echo "          ${PKGSRCDIR}/${c}/Makefile" > /dev/stderr
		    		echo "          but the directory does not exist.  Please fix this!" > /dev/stderr
	    		else
		    		cd ${PKGSRCDIR}/${c}/${pkgdir}
		    		l=`${BMAKE} print-summary-data`
		    		if [ $? != 0 ]; then
			    		echo "WARNING (printdepends):  the package in ${c}/${pkgdir} had problem with" \
						> /dev/stderr
			    		echo "    ${BMAKE} print-summary-data" > /dev/stderr
			    		echo "    database information for this package" > /dev/stderr
			    		echo "    will be dropped." > /dev/stderr
			    		${BMAKE} print-summary-data  2>&1 > /dev/stderr
		    		else
			    		echo "$l" >> $DATABASEFILE
		    		fi
	    		fi
	    		${ECHO} -n "."
	    		if [ `${EXPR} $npkg % 100 = 0` -eq 1 ]; then
				echo " "
				echo "$npkg"
	    		fi
	  		npkg=`${EXPR} $npkg + 1`
	    		cd ${PKGSRCDIR}
    		done
	fi
    done
    echo " "
    echo "Finished extracting data for ${npkg} packages at: `date` "
else
    echo " "
    echo "Using existing database (are you sure you wanted the -r/--restart flag?)"
    echo " "
    if [ ! -f $DATABASEFILE ]; then
	echo " "
	echo "ERROR:  You have use the -r/--restart flag but the database "
	echo "        file $DATABASEFILE does not exist"
	echo " "
	exit 1
    fi
fi

######################################################################
#
#  Generate the package and category README.html files
#
######################################################################

echo " "
echo "Generating package README.html files"
echo " "
if [ "x$DEBUG" = "xyes" ]; then
    debug=1;
else
    debug=0;
fi

${AWK} -f ${PKGSRCDIR}/mk/scripts/genreadme.awk  \
    builddependsfile=${TMPDIR}/pkgsrc.builddepends.debug \
    debug=$debug \
    dependsfile=${TMPDIR}/pkgsrc.depends.debug \
    summary=${summary} \
    AWK=$AWK \
    CMP=$CMP \
    DISTDIR=$DISTDIR \
    FIND=$FIND \
    GREP=$GREP \
    GZIP_CMD="$GZIP_CMD" \
    PACKAGES=$PACKAGES \
    PKG_INFO="$PKG_INFO" \
    PKG_SUFX=$PKG_SUFX \
    PKG_URL=$PKG_URL \
    PKGSRCDIR=$PKGSRCDIR \
    PVDIR=$PVDIR \
    SED=$SED \
    SETENV=$SETENV \
    SORT=$SORT \
    TMPDIR=$TMPDIR \
	${DATABASEFILE}

if [ $? != 0 ]; then
    echo "Error:  genreadme.awk failed to create README.html files"
    clean_and_exit
fi

######################################################################
#
#  Generate the README-IPv6.html file
#
######################################################################

echo " "
echo "Generating the README-IPv6.html file"
echo " "
cd ${PKGSRCDIR}
ipv6=${TMPDIR}/ipv6pkgs
ipv6_entries=${TMPDIR}/ipv6_entries
${GREP} -l -e '^BUILD_DEFS.*=.*USE_INET6' -e '^PKG_SUPPORTED_OPTIONS.*=.*inet6' */*/Makefile */*/options.mk | ${SED} -e 's;Makefile;;g' -e 's;options.mk;;g' > $ipv6
${FGREP} -f $ipv6 README-all.html | sort -t/ +1 > $ipv6_entries
${SED} \
    -e "/%%TRS%%/r${ipv6_entries}" \
    -e '/%%TRS%%/d' \
    templates/README.ipv6 > ${TMPDIR}/README-IPv6.html
if [ $? != 0 ]; then
    echo "Error:  README-IPv6.html generation failed (on sed script)"
    clean_and_exit
fi

if [ ! -f README-IPv6.html ]; then
	mv -f ${TMPDIR}/README-IPv6.html README-IPv6.html
elif cmp -s ${TMPDIR}/README-IPv6.html README-IPv6.html ; then
	echo "README-IPv6.html is unchanged (no changes were needed)"
else
	mv -f ${TMPDIR}/README-IPv6.html README-IPv6.html
fi

######################################################################
#
#  Prune README.html files which are no longer needed
#
######################################################################
if [ "x$prune" = "xyes" ]; then
    echo " "
    echo "Pruning unused README.html files"
    echo " "
    cd ${PKGSRCDIR}
    for d in `ls -d */*` ; do
	if [ -d $d -a ! -f ${d}/Makefile -a -f ${d}/README.html ]; then
	    echo "Pruning ${d}/README.html which is no longer used"
	    rm -f ${d}/README.html
	fi
    done
fi

######################################################################
#
# All done.  Clean (if needed) and exit
#
######################################################################
echo " "
echo "README.html generation finished:  `date`"
echo " "
if [ "x$DEBUG" = "xno" -a "x$restart" = "xno" -a "x$save" = "xno" ]; then
    rm -fr $TMPDIR
else
    echo "Debugging output left in $TMPDIR"
fi