diff options
| -rw-r--r-- | Makefile | 18 | ||||
| -rw-r--r-- | mk/bsd.pkg.mk | 14 | ||||
| -rwxr-xr-x | mk/scripts/binpkg-cache | 312 | ||||
| -rwxr-xr-x | mk/scripts/genreadme.awk | 283 | ||||
| -rwxr-xr-x | mk/scripts/mkreadme | 121 |
5 files changed, 554 insertions, 194 deletions
@@ -1,4 +1,4 @@ -# $NetBSD: Makefile,v 1.70 2005/05/16 19:59:54 wiz Exp $ +# $NetBSD: Makefile,v 1.71 2005/05/17 21:46:59 dmcmahill Exp $ # # tools used by this Makefile @@ -78,22 +78,10 @@ PKGSRCTOP= yes _README_TYPE_FLAG?= none README.html: .PRECIOUS .if ${_README_TYPE_FLAG} == "--ftp" || ${_README_TYPE_FLAG} == "--cdrom" - @if [ -e ${PACKAGES} ]; then \ - cd ${PACKAGES}; \ - case `pwd` in \ - ${.CURDIR}/packages) \ - MULTIARCH=; \ - ;; \ - *) \ - MULTIARCH=--multi-arch; \ - ;; \ - esac; \ - cd ${.CURDIR} ; \ - fi; \ ${SETENV} TMPDIR=${TMPDIR:U/tmp}/mkreadme \ - BMAKE=${MAKE} AWK=${AWK} EXPR=${EXPR} \ + BMAKE=${MAKE} \ ./mk/scripts/mkreadme --pkgsrc ${.CURDIR} \ - --packages ${PACKAGES} ${_README_TYPE_FLAG} $$MULTIARCH \ + --packages ${PACKAGES} ${_README_TYPE_FLAG} \ --prune .else @${ECHO} "ERROR: please do not use the README.html target directly." diff --git a/mk/bsd.pkg.mk b/mk/bsd.pkg.mk index 50cb04be8ef..a11fdc8fd2e 100644 --- a/mk/bsd.pkg.mk +++ b/mk/bsd.pkg.mk @@ -1,4 +1,4 @@ -# $NetBSD: bsd.pkg.mk,v 1.1667 2005/05/17 06:31:00 jlam Exp $ +# $NetBSD: bsd.pkg.mk,v 1.1668 2005/05/17 21:46:59 dmcmahill Exp $ # # This file is in the public domain. # @@ -3994,16 +3994,18 @@ README.html: .PRECIOUS ${AWK} -f ../../mk/scripts/genreadme.awk \ builddependsfile=/dev/null \ dependsfile=/dev/null \ + AWK=${AWK} \ + CMP=${CMP} \ DISTDIR=${DISTDIR} \ - MACHINE_ARCH=${MACHINE_ARCH} \ - MULTIARCH=$$MULTIARCH \ - OPSYS=${OPSYS} \ - OS_VERSION=${OS_VERSION} \ + GREP=${GREP} \ PACKAGES=${PACKAGES} \ + PKG_INFO="${PKG_INFO}" \ PKG_SUFX=${PKG_SUFX} \ PKG_URL=${PKG_URL} \ - PKGREPOSITORYSUBDIR=${PKGREPOSITORYSUBDIR} \ PKGSRCDIR=${.CURDIR:C|/[^/]*/[^/]*$||} \ + SED=${SED} \ + SETENV=${SETENV} \ + SORT=${SORT} \ TMPDIR=${TMPDIR:U/tmp} \ SINGLEPKG=${PKGPATH} \ $@.tmp1 diff --git a/mk/scripts/binpkg-cache b/mk/scripts/binpkg-cache new file mode 100755 index 00000000000..79ed68eb0db --- /dev/null +++ b/mk/scripts/binpkg-cache @@ -0,0 +1,312 @@ +#!/bin/sh +# +# $NetBSD: binpkg-cache,v 1.1 2005/05/17 21:46:59 dmcmahill Exp $ +# +# Script for generating a cache file with information about +# all binary packages contained in a directory. +# +# Copyright (c) 2005 The NetBSD Foundation, Inc. +# All rights reserved. +# +# This code is derived from software contributed to The NetBSD Foundation +# by Dan McMahill. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. All advertising materials mentioning features or use of this software +# must display the following acknowledgement: +# This product includes software developed by the NetBSD +# Foundation, Inc. and its contributors. +# 4. Neither the name of The NetBSD Foundation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS +# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS +# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# + + +TMPDIR=${TMPDIR:-/tmp} +PACKAGES=${PACKAGES:-/usr/pkgsrc/packages/} +AWK=${AWK:-awk} +CMP=${CMP:-cmp} +GREP=${GREP:-grep} +PKG_INFO=${PKG_INFO:-pkg_info} +PKG_SUFX=${PKG_SUFX:-.tgz} +SED=${SED:-sed} +SORT=${SORT:-sort} +STAT=${STAT:-stat} + +cachefile=.pkgcache +cacheversion=20050428 + +prompt="----> " +tab=" " + +tmpd=${TMPDIR}/pkg-cache.$$ +mkdir -m 0700 ${tmpd} +if test $? -ne 0 ; then + echo "ERROR: Could not create temporary directory ${tmpd}" + echo "Either you do not have write permission to ${tmpd} or" + echo "${tmpd} already exists" + exit 1 +fi + +prog=$0 + +usage(){ + echo "$prog - Generates cache files for each directory containing binary" + echo " packages. This cache file can then be used by the README.html" + echo " generation code to avoid having to call pkg_info(1) over and over" + echo " on the same binary package." + echo " " + echo "Usage: $prog [-d|--debug] [-v|--verbose] [-p|--packages <dir>]" + echo " " + echo " $prog -h|--help" + echo " " + echo " $prog -V|--version" + echo " " + echo "The options supported by $prog are: " + echo " " + echo " -d|--debug Enables debugging output" + echo " " + echo " -f|--database <file> Writes the database into file specified by <file>" + echo " " + echo " -h|--help Displays this help message" + echo " " + echo " -p|--packages <dir> Specifies the top level directory to be searched" + echo " for binary packages." + echo " " + echo " -v|--version Displays the version of this script and exits." + echo " " + echo "Example: $prog -v --packages /usr/pkgsrc/packages" + echo " " +} + +clean_and_exit0(){ + rm -fr ${tmpd} + exit 0 +} + +clean_and_exit1(){ + rm -fr ${tmpd} + exit 1 +} + +all_cache_files="" + +process_binpkg_dir(){ + all_cache_files="${all_cache_files} ${d}/${cachefile}" + + need_update=no + if test -f ${d}/${cachefile} ; then + stale_entries=`find ${d} -type f -name \*${PKG_SUFX} -newer ${d}/${cachefile} -print` + + # FIX_ME + # + # We also should find cache entries for files which no longer exist + # and nuke them. Right now we simply declare the entire cache out + # of date. Once we implement incremental updates to the cache, + # we need to remove the entries but not mark the entire cache as + # bad. + if test "X${DEBUG}" = "Xyes" ; then + echo " Checking for cache entries with no corresponding pkg." + fi + # get the list of what pkgs belong in the cache + rm -f ${tmpd}/pkg_list ${tmpd}/cache_pkg_list + ls ${d}/*${PKG_SUFX} | ${SED} "s;^${PACKAGES}/*;;g" | ${SORT} > ${tmpd}/pkg_list + + # and get the list of what is in the cache + ${AWK} '/pkgcache_begin/ {gsub(/pkgcache_begin[ \t]*/, ""); print}' \ + ${d}/${cachefile} | ${SORT} > ${tmpd}/cache_pkg_list + + if ${CMP} -s ${tmpd}/pkg_list ${tmpd}/cache_pkg_list ; then + if test "X${DEBUG}" = "Xyes" ; then + echo " No extra cache entries in ${d}/${cachefile}" + fi + else + echo " Entries found in ${d}/${cachefile} but no packages found" + need_update=yes + fi + else + stale_entries="" + fi + + if test "X${force}" = "Xyes" -o "X${need_update}" = "Xyes" ; then + need_update=yes + echo "${tab}Forcing rebuild of cache ${d}/${cachefile}." + elif test ! -f ${d}/${cachefile} ; then + need_update=yes + echo "${tab}Missing cache file. ${d}/${cachefile} will be generated." + elif test -n "${stale_entries}" ; then + need_update=yes + echo "${tab}Stale cache file. ${d}/${cachefile} will be regenerated." + else + ${GREP} "pkgcache_version ${cacheversion}" ${d}/${cachefile} >/dev/null 2>&1 + if test $? -ne 0 ; then + need_update=yes + echo "${tab}Old version cache file. ${d}/${cachefile} will be regenerated." + else + echo "${tab}Cache file ${d}/${cachefile} is up to date." + fi + fi + + # FIX_ME + # We should use stale_entries in a way where we only update the + # cache file entries corresponding to these if we're rebuilding + # due to stale entries. That should save a good bit of time. + # + if test "X${need_update}" = "Xyes" ; then + echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile} + for f in ${d}/*${PKG_SUFX} ; do + fn=`echo $f | ${SED} "s;^${PACKAGES}/*;;g"` + if test "X${DEBUG}" = "Xyes" ; then + echo " Adding ${fn} (${f}) to the cache" + fi + echo " " >> ${tmpd}/${cachefile} + # stat(1) needs to be added to the bootstrap kit + # first if we want to use it here + #eval $(${STAT} -s ${f} 2>/dev/null) + echo "pkgcache_begin ${fn}" >> ${tmpd}/${cachefile} + #echo "pkgcache_mtime=${st_mtime}" >> ${tmpd}/${cachefile} + if test "X${DEBUG}" = "Xyes" ; then + echo "${PKG_INFO} -q -B ${f}" + fi + ${PKG_INFO} -q -B ${f} >> ${tmpd}/${cachefile} + echo "pkgcache_end ${fn}" >> ${tmpd}/${cachefile} + done + mv ${tmpd}/${cachefile} ${d}/${cachefile} + fi + + +} + +process_cache_files(){ + echo "${prompt}Checking master cache file ${PACKAGES}/${cachefile}" + echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile} + if test -n "${all_cache_files}" ; then + for c in ${all_cache_files} ; do + echo "pkgcache_cachefile ${c}" >> ${tmpd}/${cachefile} + done + fi + if test ! -f ${PACKAGES}/${cachefile} ; then + echo "${tab}Creating master cache file ${PACKAGES}/${cachefile}" + cp ${tmpd}/${cachefile} ${PACKAGES}/${cachefile} + fi + if ${CMP} -s ${tmpd}/${cachefile} ${PACKAGES}/${cachefile} ; then + echo "${tab}Master cache file ${PACKAGES}/${cachefile} is up to date" + else + echo "${tab}Updating master cache file ${PACKAGES}/${cachefile}" + mv ${tmpd}/${cachefile} ${PACKAGES}/${cachefile} + fi +} + +###################################################################### +# +# Handle command line options +# +###################################################################### + +DEBUG=no +verbose=no +force=no + +while + test -n "$1" +do + case "$1" in + + # Turn on debugging + -d|--debug) + DEBUG=yes + verbose=yes + shift + ;; + + # Force a rebuilde of the cache + -F|--force) + force=yes + shift + ;; + # Help + -h|--help) + usage + exit 0 + ;; + + # Use the specified packages directory + -p|--packages) + PACKAGES=$2 + shift 2 + ;; + + # Version + -V|--version) + ${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s: Version %s, %s\n",$3,$4,$5); exit 0;}' $prog + exit 0 + ;; + + # Turn on verbose output, but not as noisy as DEBUG + -v|--verbose) + verbose=yes + shift + ;; + + -*) echo "$prog: ERROR: $1 is not a valid option" + usage + clean_and_exit1 + ;; + + *) + break + ;; + + esac +done + +# put a trailing / after ${PACKAGES} in case ${PACKAGES} is +# a link. +for d in `find ${PACKAGES}/ -type d -print` ; do + if test "X${DEBUG}" = "Xyes" ; then + echo "${prompt}Processing directory ${d}" + fi + is_pkg_dir=no + for f in ${d}/*${PKG_SUFX} ; do + if test -f "${f}" -a ! -h "${f}" ; then + ${PKG_INFO} ${f} >/dev/null 2>&1 + if test $? -eq 0 ; then + is_pkg_dir=yes + break + fi + fi + done + if test "X${is_pkg_dir}" = "Xyes" ; then + echo "${prompt}Checking cache in ${d}" + process_binpkg_dir + else + if test "X${DEBUG}" = "Xyes" ; then + echo "${prompt}no binary packages in ${d}" + fi + fi + +done + +process_cache_files + +clean_and_exit0 + diff --git a/mk/scripts/genreadme.awk b/mk/scripts/genreadme.awk index a653f592511..fdc9c507c6c 100755 --- a/mk/scripts/genreadme.awk +++ b/mk/scripts/genreadme.awk @@ -1,7 +1,7 @@ #!/usr/bin/awk -f -# $NetBSD: genreadme.awk,v 1.12 2004/07/06 22:49:18 wiz Exp $ +# $NetBSD: genreadme.awk,v 1.13 2005/05/17 21:46:59 dmcmahill Exp $ # -# Copyright (c) 2002, 2003 The NetBSD Foundation, Inc. +# Copyright (c) 2002, 2003, 2005 The NetBSD Foundation, Inc. # All rights reserved. # # This code is derived from software contributed to The NetBSD Foundation @@ -53,8 +53,13 @@ BEGIN { do_pkg_readme=1; # set to 1 to use "README-new.html" as the name use_readme_new=0; + if (use_readme_new) { + readme_name = "README-new.html"; + } + else { + readme_name = "README.html"; + } printf("Reading database file\n"); - fflush("/dev/stdout"); } #conflicts /usr/pkgsrc/math/scilab @@ -151,6 +156,10 @@ BEGIN { for (i = 4; i <= NF; i++){ htmlname = htmlname " " $i; } + # If we are using a name other than README.html, change it + # here. This avoids having to process a huge line later which + # makes lesser awks puke. + gsub(/README.html/, readme_name, htmlname); dir2htmlname[dir] = htmlname; if (debug) printf("added dir2htmlname[%s]=%s\n", dir, htmlname); next; @@ -186,19 +195,27 @@ BEGIN { # END { - if (use_readme_new) { - readme_name = "README-new.html"; - } - else { - readme_name = "README.html"; - } readme = TMPDIR "/" readme_name; if ( dependsfile == "" ) dependsfile = "/dev/stdout"; if ( builddependsfile == "" ) builddependsfile = "/dev/stdout"; + + printf("Making sure binary package cache file is up to date...\n"); + cmd = sprintf("%s AWK=%s CMP=%s GREP=%s PKG_INFO=\"%s\" PKG_SUFX=%s SED=%s SORT=%s %s/mk/scripts/binpkg-cache --packages %s", + SETENV, AWK, CMP, GREP, PKG_INFO, PKG_SUFX, SED, SORT, PKGSRCDIR, PACKAGES); + if (debug) printf("\nExecute: %s\n",cmd); + rc = system(cmd); + if (rc != 0) { + printf("\n**** WARNING ****\n") > "/dev/stderr"; + printf("Command: %s\nfailed.", cmd) > "/dev/stderr"; + printf("**** ------- ****\n") > "/dev/stderr"; + exit(1); + } + printf("Loading binary package cache file...\n"); + load_cache_file( PACKAGES "/.pkgcache" ); + printf("Flattening dependencies\n"); - fflush("/dev/stdout"); printf("") > dependsfile; for (toppkg in topdepends){ if (debug) printf("calling find_all_depends(%s, run)\n", toppkg); @@ -214,9 +231,10 @@ END { # clear out the flattened depends list and repeat for the build depends - delete alldepends; + for( key in alldepends ) { + delete alldepends[key]; + } printf("Flattening build dependencies\n"); - fflush("/dev/stdout"); printf("") > builddependsfile; for (toppkg in topbuilddepends){ find_all_depends(toppkg, "build"); @@ -234,7 +252,6 @@ END { # read the vulnerabilities file printf("Reading vulnerability file \"%s\"\n which was updated %s\n", vfile, vuldate); - fflush("/dev/stdout"); i = 1; while((getline < vfile) > 0) { if ($0 !~ /^\#/) { @@ -253,16 +270,16 @@ END { have_vfile = 0; } close(cmd); - fflush("/dev/stdout"); if (SINGLEPKG != "" ) { printf("Only creating README for %s\n",SINGLEPKG); - delete topdepends; + for( key in topdepends ) { + delete topdepends[key]; + } topdepends[SINGLEPKG] = "yes"; } printf("Generating README.html files\n"); - fflush("/dev/stdout"); pkgcnt = 0; if (do_pkg_readme) { templatefile = PKGSRCDIR "/templates/README.pkg"; @@ -278,12 +295,15 @@ END { if ((pkgcnt % 100) == 0) { printf("\n%d\n", pkgcnt); } - fflush("/dev/stdout"); printf("") > readme; htmldeps = ""; - delete dpkgs; + for( key in dpkgs ) { + delete dpkgs[key]; + } split(alldepends[toppkg], dpkgs); i = 1; + htmldeps_file = TMPDIR "/htmldep"; + printf("") > htmldeps_file; while(i in dpkgs){ if (debug) { printf("\tdpkg=%s, pat2dir[%s] = %s\n", @@ -293,23 +313,30 @@ END { } nm=dpkgs[i]; -# we need a zillion escapes here because we need to end up -# with \\< in 'htmldeps' so that when -# we gsub htmldeps into the output file, we end up with < - - gsub(/&/, "\\\\\\&", nm); - gsub(/</, "\\\\\\<", nm); - gsub(/>/, "\\\\\\>", nm); - htmldeps=sprintf("%s<a href=\"../../%s/%s\">%s</a>\n", + gsub(/&/, "\\&", nm); + gsub(/</, "\\<", nm); + gsub(/>/, "\\>", nm); +# htmldeps=sprintf("%s<a href=\"../../%s/%s\">%s</a>\n", +# htmldeps, +# pat2dir[dpkgs[i]], +# readme_name, nm); +# We use a temp file to hold the html dependencies because for +# packages like gnome, this list can get very very large and +# become larger than what some awk implementations can deal +# with. The nawk shipped with solaris 9 is an example of +# such a limited awk. + printf("%s<a href=\"../../%s/%s\">%s</a>\n", htmldeps, pat2dir[dpkgs[i]], - readme_name, nm); + readme_name, nm) >> htmldeps_file; i = i + 1; } - if ( htmldeps == "" ) { - htmldeps = "<EM>none</EM>"; + if ( i == 1 ) { + printf("<EM>none</EM>") >> htmldeps_file; } - if (debug) printf("htmldeps = \"%s\"\n", htmldeps); + close(htmldeps_file); + if (debug) printf("wrote = %d entries to \"%s\"\n", + i-1, htmldeps_file); vul = ""; if (have_vfile) { @@ -344,72 +371,28 @@ END { } } - if (MULTIARCH == "no"){ - cmd="ls -1 "PACKAGES "/" PKGREPOSITORYSUBDIR "/" wildcard[toppkg] PKG_SUFX " 2>/dev/null"; - if (debug) { - printf("Checking for binary package with %s\n", - cmd); - } - binpkgs = ""; - while((cmd | getline) > 0) { - pkgfile = $0; - gsub(/.*\//, "", pkgfile); - pkgnm = pkgfile; - gsub(/\.tgz$/, "", pkgnm); - binpkgs = sprintf("%s\n<TR><TD>%s:<TD><a href=\"%s/%s\">%s</a><TD>(%s %s)", - binpkgs, - MACHINE_ARCH, - PKG_URL, - pkgfile, - pkgnm, - OPSYS, - OS_VERSION); - } - close(cmd); - } - else { - cmd = "ls -1 -d "PACKAGES"/[0-9].*/*/" PKGREPOSITORYSUBDIR "/" wildcard[toppkg] PKG_SUFX " 2>/dev/null"; - oldfs = FS; - FS = "[/]"; - binpkgs = ""; - while((cmd | getline) > 0) { - release = $(NF-3); - arch = $(NF-2); - pkg = $NF; - pkgnm = pkg; - gsub(PKG_SUFX "$", "", pkgnm) - if (debug) { - printf("%s:%s:%s (%s)\n", - release, - arch, - pkg, - pkgnm); - } - binpkgs = sprintf("%s\n<TR><TD>%s:<TD><a href=\"%s/%s/%s/%s/%s\">%s</a><TD>(%s-%s)", - binpkgs, arch, - PKG_URL, release, - arch, - PKGREPOSITORYSUBDIR, - pkg, pkgnm, - OPSYS, release); - } - FS = oldfs; - close(cmd); + + if (debug) { + printf("Checking for binary package with lookup_cache( %s)\n", + wc); } +# lookup_cache( wildcard ) will produce HTML for the packages which are found + binpkgs = lookup_cache( wildcard[toppkg] PKG_SUFX ); + # sort the binary pkgs (XXX would be nice to implement in memory in awk) sfile = TMPDIR "/sorted"; - spipe = "sort > " sfile; + spipe = SORT " > " sfile; printf("%s",binpkgs) | spipe; close(spipe); binpkgs = ""; while((getline < sfile) > 0) { - binpkgs = sprintf("%s\n%s", binpkgs, $0); + binpkgs = sprintf("%s\n%s", binpkgs, $0); } close(sfile); - - if (debug) printf("binary packages: \n%s\n\n", - binpkgs); + + if (debug) printf("binary packages: \n%s\n\n", + binpkgs); if ( flatdepends[toppkg] ~ /^[ \t]*$/ ) { rundeps = "<EM>none</EM>"; @@ -435,11 +418,21 @@ END { } gsub(/%%VULNERABILITIES%%/, ""vul""); gsub(/%%VULDATE%%/, ""vuldate""); - gsub(/%%BUILD_DEPENDS%%/, ""htmldeps""); + gsub(/%%RUN_DEPENDS%%/, ""rundeps""); gsub(/%%BIN_PKGS%%/, ""binpkgs""); - gsub(/README.html/, readme_name); - print >> readme; + + line = $0 + + if( line ~/%%BUILD_DEPENDS%%/ ) { + gsub(/%%BUILD_DEPENDS%%/, "", line); + while((getline < htmldeps_file) > 0) { + print >> readme; + } + close( htmldeps_file ); + } + + print line >> readme; } close(readme); close(templatefile); @@ -484,6 +477,8 @@ END { printf("Category = %s\n", category); cat_make = catdir"/Makefile"; pkgs = ""; + pkgs_file = TMPDIR "/pkgs_file"; + printf("") > pkgs_file; numpkg = 0; print "" > readme; while((getline < cat_make) > 0){ @@ -500,10 +495,19 @@ END { pkgdir2name[dir], comment[dir]); } - pkgs = sprintf("%s<TR><TD VALIGN=TOP><a href=\"%s/%s\">%s</a>: %s<TD>\n", - pkgs, pkg, readme_name, +# pkgs = sprintf("%s<TR><TD VALIGN=TOP><a href=\"%s/%s\">%s</a>: %s<TD>\n", +# pkgs, pkg, readme_name, +# pkgdir2name[dir], +# comment[dir]); +# We use a temp file to hold the list of all packages because +# this list can get very very large and +# become larger than what some awk implementations can deal +# with. The nawk shipped with solaris 9 is an example of +# such a limited awk. + printf("<TR><TD VALIGN=TOP><a href=\"%s/%s\">%s</a>: %s<TD>\n", + pkg, readme_name, pkgdir2name[dir], - comment[dir]); + comment[dir]) >> pkgs_file; allpkg[tot_numpkg] = sprintf("<!-- %s (for sorting) --><TR VALIGN=TOP><TD><a href=\"%s/%s/%s\">%s</a>: <TD>(<a href=\"%s/%s\">%s</a>) <td>%s\n", pkgdir2name[dir], category, pkg, @@ -526,9 +530,19 @@ END { gsub(/%%CATEGORY%%/, category); gsub(/%%NUMITEMS%%/, numpkg); gsub(/%%DESCR%%/, descr); - gsub(/%%SUBDIR%%/, pkgs); - gsub(/README.html/, readme_name); - print >> readme; + + line = $0 + + if( $0 ~/%%SUBDIR%%/ ) { + gsub(/%%SUBDIR%%/, ""); + while((getline < pkgs_file) > 0) { + gsub(/README.html/, readme_name); + print >> readme; + } + close( pkgs_file ); + } + + print line >> readme; } close(readme); close(templatefile); @@ -682,6 +696,18 @@ function reg2str(reg){ } # +# take a string which has a shell glob pattern and turn it into +# an awk regular expression. +# +function glob2reg(reg){ + gsub(/\./, "\\\.", reg); + gsub(/\+/, "\\\+", reg); + gsub(/\*/, ".*", reg); + gsub(/\?/, ".?", reg); + return(reg); +} + +# # accepts a full path to a package directory, like "/usr/pkgsrc/math/scilab" # and returns just the last 2 directories, like "math/scilab" # @@ -712,7 +738,7 @@ function uniq(list, deps, i, ulist){ } function fatal_check_file(file, cmd){ - cmd="test -f "file ; + cmd="test -f " file ; if (debug) printf("Execute: %s\n",cmd); if (system(cmd) != 0) { printf("**** FATAL ****\nRequired file %s does not exist\n", @@ -747,7 +773,8 @@ function copy_readme(old, new, cmd, rc) { # Compare the existing README.html file to the one we created. If they are # not the same, then copy over the one we created - cmd = " if ! cmp -s "new" "old" ; then mv -f " new " " old " ; fi"; + cmd = sprintf("%s -s %s %s ; if test $? -ne 0 ; then mv -f %s %s ; fi", + CMP, new, old, new, old); if (debug) printf("copy_readme() execute: %s\n",cmd); rc = system(cmd); if (rc != 0) { @@ -768,3 +795,65 @@ function copy_readme(old, new, cmd, rc) { } + +function load_cache_file( file ) { + + if( debug ) printf("load_cache_file(%s)\n", file); + + # read the cache file + while( getline < file ) { + + # if this line points to another cache file, then recursively + # load it + if( $0 ~ /^pkgcache_cachefile/ ) { + if( debug ) printf("\tFound pkgcache_cachefile line.\n"); + load_cache_file( $2 ); + } else if( $0 ~/^pkgcache_begin /) { + pkgfile = $2; + if( debug ) printf("\tStarting %s\n", pkgfile); + opsys = "unknown"; + osver = "unknown"; + march = "unknown"; + } else if( $0 ~/^OPSYS=/ ) { + opsys = $1; + gsub(/OPSYS=/, "", opsys); + } else if( $0 ~/^OS_VERSION=/ ) { + osver = $1; + gsub(/OS_VERSION=/, "", osver); + } else if( $0 ~/^MACHINE_ARCH=/ ) { + march = $1; + gsub(/MACHINE_ARCH=/, "", march); + } else if( $0 ~/^pkgcache_end /) { + if( debug ) printf("\t%s, OPSYS=%s, OS_VERSION=%s, MACHINE_ARCH=%s\n", + pkgfile, opsys, osver, march); + opsys_list[pkgfile] = opsys; + osver_list[pkgfile] = osver; + march_list[pkgfile] = march; + pkg_list[pkgfile] = pkgfile; + pkgnm_list[pkgfile] = pkgfile; + gsub(/.*\//, "", pkgnm_list[pkgfile]); + } else { + # skip this line + } + } + + # close the cache file + close( file ); +} + +function lookup_cache( wc, r, binpkgs) { + if( debug ) printf("lookup_cache( %s )\n", wc); + r = ".*/" glob2reg( wc ); + if( debug ) printf("lookup_cache(): Searching for \"%s\"\n", r ); + + binpkgs = ""; + for( key in pkg_list ) { + if( key ~ r ) { + binpkgs = sprintf("%s\n<TR><TD>%s:<TD><a href=\"%s/%s\">%s</a><TD>(%s %s)\n", + binpkgs, march_list[key], PKG_URL, key, pkgnm_list[key], + opsys_list[key], osver_list[key]); + + } + } + return( binpkgs ); +} diff --git a/mk/scripts/mkreadme b/mk/scripts/mkreadme index 231cdb36ee1..8eea9444dc3 100755 --- a/mk/scripts/mkreadme +++ b/mk/scripts/mkreadme @@ -1,9 +1,9 @@ #!/bin/sh -# $NetBSD: mkreadme,v 1.4 2003/09/23 01:51:44 yyamano Exp $ +# $NetBSD: mkreadme,v 1.5 2005/05/17 21:46:59 dmcmahill Exp $ # # Script for README.html generation # -# Copyright (c) 2002 The NetBSD Foundation, Inc. +# Copyright (c) 2002, 2005 The NetBSD Foundation, Inc. # All rights reserved. # # This code is derived from software contributed to The NetBSD Foundation @@ -41,16 +41,27 @@ TMPDIR=${TMPDIR:-/tmp/mkreadme} PKGSRCDIR=${PKGSRCDIR:-/usr/pkgsrc} -BMAKE=${BMAKE:-make} AWK=${AWK:-/usr/bin/awk} -EXPR=${EXPR:-expr} + +opsys=`uname -s` +case "$opsys" in + *BSD) + makeprog=make + ;; + + *) + makeprog=bmake + ;; +esac + +BMAKE=${BMAKE:-${makeprog}} usage(){ echo "$prog - Generates README.html files for a pkgsrc tree" echo "Usage: $prog [-c|--cdrom] [-C|--prune] [-d|--debug] [-f|--ftp] " - echo " [-m|--multi-arch] [-p|--pkgsrc directory] " + echo " [-p|--pkgsrc directory] " echo " [-P|--packages directory] [-r|--restart] " - echo " [-s|--single-arch] [-S|--save-database]" + echo " [-S|--save-database]" echo " " echo " $prog -h|--help" echo " " @@ -69,10 +80,6 @@ usage(){ echo " " echo " -h|--help Displays this help message" echo " " - echo " -m|--multi-arch Assumes a packages directory layout like:" - echo " OS_VERSION/MACHINE_ARCH/ for example:" - echo " 1.{5,6}/{alpha,i386,sparc,vax}" - echo " " echo " -p|--pkgsrc dir Specifies the pkgsrc directory. Defaults to" echo " The value of the PKGSRCDIR environment variable" echo " if set or /usr/pkgsrc otherwise." @@ -83,9 +90,6 @@ usage(){ echo " from a previous run still exists and that the script" echo " should use that instead of recreating the database." echo " " - echo " -s|--single-arch Assumes a single OS release and MACHINE_ARCH binary" - echo " packages directory structure." - echo " " echo " -S|--save-database Does not delete the database file after the run." echo " This is useful for debugging or re-running this script" echo " with the -r option." @@ -114,6 +118,8 @@ prog=$0 # ###################################################################### +cmdline=$* + ftp_readme=no restart=no prune=no @@ -157,12 +163,6 @@ do exit 0 ;; - # assume a OS_RELEASE/MACHINE_ARCH/ layout for the binary pkgs - -m|--multi-arch) - multiarch=yes - shift - ;; - # Specify pkgsrc directory -p|--pkgsrc) PKGSRCDIR=$2 @@ -181,12 +181,6 @@ do shift ;; - # assume a single directory structure layout for the binary pkgs - -s|--single-arch) - multiarch=no - shift - ;; - # Save the database files -S|--save-database) save=yes @@ -252,8 +246,9 @@ echo "Extracting configuration variables:" echo " " if [ -d ${PKGSRCDIR}/pkgtools/pkglint ]; then cd ${PKGSRCDIR}/pkgtools/pkglint - - for v in DISTDIR PACKAGES PKGREPOSITORYSUBDIR PKG_SUFX MACHINE_ARCH PKG_URL OPSYS OS_VERSION FTP_PKG_URL_HOST FTP_PKG_URL_DIR CDROM_PKG_URL_HOST CDROM_PKG_URL_DIR + for v in AWK CDROM_PKG_URL_HOST CDROM_PKG_URL_DIR CMP DISTDIR ECHO EXPR \ + FGREP FTP_PKG_URL_HOST FTP_PKG_URL_DIR GREP PACKAGES PKG_INFO PKG_SUFX \ + SED SETENV SORT do val=`${BMAKE} show-var VARNAME=${v}` if [ $? != 0 ]; then @@ -261,15 +256,14 @@ if [ -d ${PKGSRCDIR}/pkgtools/pkglint ]; then echo "Failed. This is a fatal error" clean_and_exit fi - echo "$v=$val" - eval ${v}=${val} + echo "----> ${v}=\"${val}\"" + eval "${v}=\"${val}\"" done else echo "Error: ${PKGSRCDIR}/pkgtools/pkglint does not seem to exist" exit 1 fi - ###################################################################### # # Decide on FTP vs CDROM README.html files @@ -296,39 +290,6 @@ if [ ! -z "$PKGDIR" ]; then echo "PACKAGES specified on command line to be $PKGDIR" fi -###################################################################### -# -# Decide in binary package directory layout -# -###################################################################### - - -# If PACKAGES is set to the default (../../pkgsrc/packages), the current -# ${MACHINE_ARCH} and "release" (uname -r) will be used. Otherwise a directory -# structure of ...pkgsrc/packages/`uname -r`/${MACHINE_ARCH} is assumed. -# -# This is the logic from bsd.pkg.mk, but I think I'd like to change it to -# come from a command line switch - -if [ -z "$multiarch" -a -e ${PACKAGES} ]; then - cd ${PACKAGES} - case `pwd` in - */pkgsrc/packages) - multiarch=no - ;; - - *) - multiarch=yes - ;; - esac -fi - -if [ "x$multiarch" = "xyes" ]; then - echo "Will generate multi-release, multi-arch readme files" -else - echo "Will generate single-release, single-arch readme files" -fi - ###################################################################### # @@ -339,6 +300,7 @@ fi if [ "x$restart" = "xno" ] ; then echo " " echo "Extracting data. This could take a while" + echo "Started at: `date` " echo " " npkg=1 @@ -346,13 +308,13 @@ if [ "x$restart" = "xno" ] ; then rm -fr $DATABASEFILE cd ${PKGSRCDIR} - list=`grep '^[[:space:]]*'SUBDIR */Makefile | sed 's,/Makefile.*=[[:space:]]*,/,'` + list=`${GREP} '^[ \t]*'SUBDIR */Makefile | ${GREP} -v regress/ | ${SED} 's,/Makefile.*=[[:space:]]*,/,'` for pkgdir in $list do if [ ! -d $pkgdir ]; then echo " " echo "WARNING: the package directory $pkgdir is listed in" > /dev/stderr - echo $pkgdir | sed 's;/.*;/Makefile;g' > /dev/stderr + echo $pkgdir | ${SED} 's;/.*;/Makefile;g' > /dev/stderr echo "but the directory does not exist. Please fix this!" > /dev/stderr else cd $pkgdir @@ -368,7 +330,7 @@ if [ "x$restart" = "xno" ] ; then echo "$l" >> $DATABASEFILE fi fi - echo -n "." + ${ECHO} -n "." if [ `${EXPR} $npkg % 100 = 0` -eq 1 ]; then echo " " echo "$npkg" @@ -376,6 +338,8 @@ if [ "x$restart" = "xno" ] ; then npkg=`${EXPR} $npkg + 1` cd $PKGSRCDIR done + echo " " + echo "Finished extracting data for ${npkg} packages at: `date` " else echo " " echo "Using existing database (are you sure you wanted the -r/--restart flag?)" @@ -408,16 +372,18 @@ ${AWK} -f ${PKGSRCDIR}/mk/scripts/genreadme.awk \ builddependsfile=${TMPDIR}/pkgsrc.builddepends.debug \ debug=$debug \ dependsfile=${TMPDIR}/pkgsrc.depends.debug \ + AWK=$AWK \ + CMP=$CMP \ DISTDIR=$DISTDIR \ - MACHINE_ARCH=$MACHINE_ARCH \ - MULTIARCH=$multiarch \ - OPSYS=$OPSYS \ - OS_VERSION=$OS_VERSION \ + GREP=$GREP \ PACKAGES=$PACKAGES \ + PKG_INFO="$PKG_INFO" \ PKG_SUFX=$PKG_SUFX \ PKG_URL=$PKG_URL \ - PKGREPOSITORYSUBDIR=$PKGREPOSITORYSUBDIR \ PKGSRCDIR=$PKGSRCDIR \ + SED=$SED \ + SETENV=$SETENV \ + SORT=$SORT \ TMPDIR=$TMPDIR \ ${DATABASEFILE} @@ -438,9 +404,9 @@ echo " " cd ${PKGSRCDIR} ipv6=${TMPDIR}/ipv6pkgs ipv6_entries=${TMPDIR}/ipv6_entries -grep -l '^BUILD_DEFS.*=.*USE_INET6' */*/Makefile | sed 's;Makefile;;g' > $ipv6 -fgrep -f $ipv6 README-all.html | sort -t/ +1 > $ipv6_entries -sed \ +${GREP} -l '^BUILD_DEFS.*=.*USE_INET6' */*/Makefile | ${SED} 's;Makefile;;g' > $ipv6 +${FGREP} -f $ipv6 README-all.html | sort -t/ +1 > $ipv6_entries +${SED} \ -e "/%%TRS%%/r${ipv6_entries}" \ -e '/%%TRS%%/d' \ templates/README.ipv6 > ${TMPDIR}/README-IPv6.html @@ -448,10 +414,13 @@ if [ $? != 0 ]; then echo "Error: README-IPv6.html generation failed (on sed script)" clean_and_exit fi -if [ ! -f README-IPv6.html ] || ! cmp -s ${TMPDIR}/README-IPv6.html README-IPv6.html ; then + +if [ ! -f README-IPv6.html ]; then mv -f ${TMPDIR}/README-IPv6.html README-IPv6.html -else +elif cmp -s ${TMPDIR}/README-IPv6.html README-IPv6.html ; then echo "README-IPv6.html is unchanged (no changes were needed)" +else + mv -f ${TMPDIR}/README-IPv6.html README-IPv6.html fi ###################################################################### |