Add pkg_check (renamed to lintpkgsrc, and add orginal lintpkgsrc functionality

(fixed up to work a little better).
lintpkgsrc can:
    -d : Check each Makefile's 'DEPENDS' matches current pkgsrc versions.
    -i : Check versions of installed packages against pkgsrc.
    -l : Run pkglint on every package in pkgsrc.
    -m : Report md5 mismatches for files in 'distfiles'.
    -o : Report old/obsolete 'distfiles' (not referenced by any md5).
    -r : Remove any 'bad' distfiles (Without -m or -o, implies both).

6 files changed, 485 insertions, 50 deletions

diff --git a/pkgtools/pkglint/Makefile b/pkgtools/pkglint/Makefile
index d5e38c4a823..ee3f1485edd 100644
--- a/pkgtools/pkglint/Makefile
+++ b/pkgtools/pkglint/Makefile
@@ -1,7 +1,7 @@
-# $NetBSD: Makefile,v 1.8 1999/03/10 08:24:16 agc Exp $
+# $NetBSD: Makefile,v 1.9 1999/04/19 07:32:09 abs Exp $
 #
 
-DISTNAME=		pkglint-1.69
+DISTNAME=		pkglint-1.70
 CATEGORIES=		pkgtools devel
 MASTER_SITES=		# empty
 DISTFILES=		# empty
@@ -23,29 +23,27 @@ MAKE_ENV=	PKGSRCDIR=${PKGSRCDIR}
 .include "../../mk/bsd.prefs.mk"
 
 do-build:
+.for file in pkglint lintpkgsrc plist-clash
 	${SED} -e 's|@PREFIX@|${PREFIX}|g'		\
 		-e 's|@PORTSDIR@|${PKGSRCDIR}|g'	\
-		< ${FILESDIR}/pkglint.pl 		\
-		> ${WRKSRC}/pkglint
+		< ${FILESDIR}/${file}.pl 		\
+		> ${WRKSRC}/${file}
+.endfor
+.for file in pkglint lintpkgsrc
 	${SED} -e 's|@PREFIX@|${PREFIX}|g'		\
 		-e 's|@PORTSDIR@|${PKGSRCDIR}|g'	\
-		< ${FILESDIR}/lintpkgsrc.sh		\
-		> ${WRKSRC}/lintpkgsrc
-	${SED} -e 's|@PREFIX@|${PREFIX}|g'		\
-		-e 's|@PORTSDIR@|${PKGSRCDIR}|g'	\
-		< ${FILESDIR}/plist-clash.pl		\
-		> ${WRKSRC}/plist-clash
-	${SED} -e 's|@PREFIX@|${PREFIX}|g'		\
-		-e 's|@PORTSDIR@|${PKGSRCDIR}|g'	\
-		< ${FILESDIR}/pkglint.1			\
-		> ${WRKSRC}/pkglint.1
-	nroff -mandoc ${WRKSRC}/pkglint.1 >${WRKSRC}/pkglint.0
+		< ${FILESDIR}/${file}.1			\
+		> ${WRKSRC}/${file}.1
+	nroff -mandoc ${WRKSRC}/${file}.1 >${WRKSRC}/${file}.0
+.endfor
 
 do-install:
 	${INSTALL_SCRIPT} ${WRKSRC}/pkglint ${PREFIX}/bin/pkglint
 	${INSTALL_SCRIPT} ${WRKSRC}/lintpkgsrc ${PREFIX}/bin/lintpkgsrc
 	${INSTALL_SCRIPT} ${WRKSRC}/plist-clash ${PREFIX}/bin/plist-clash
-	${INSTALL_MAN} ${WRKSRC}/pkglint.1 ${PREFIX}/man/man1
+	${INSTALL_MAN} ${WRKSRC}/lintpkgsrc.0 ${PREFIX}/man/cat1
+	${INSTALL_MAN} ${WRKSRC}/lintpkgsrc.1 ${PREFIX}/man/man1
 	${INSTALL_MAN} ${WRKSRC}/pkglint.0 ${PREFIX}/man/cat1
+	${INSTALL_MAN} ${WRKSRC}/pkglint.1 ${PREFIX}/man/man1
 
 .include "../../mk/bsd.pkg.mk"
diff --git a/pkgtools/pkglint/files/lintpkgsrc.1 b/pkgtools/pkglint/files/lintpkgsrc.1
new file mode 100644
index 00000000000..32446c99bc7
--- /dev/null
+++ b/pkgtools/pkglint/files/lintpkgsrc.1
@@ -0,0 +1,49 @@
+.\"	$NetBSD: lintpkgsrc.1,v 1.1 1999/04/19 07:32:10 abs Exp $
+.\"
+.\" Copyright (c) 1999 by David Brownlee (abs@netbsd.org)
+.\" Absolutely no warranty.
+.\"
+.Dd April 19, 1999
+.Dt LINTPKGSRC 1
+.Sh NAME
+.Nm lintpkgsrc
+.Nd a verifier for the pkgsrc tree
+.Sh SYNOPSIS
+.Nm
+.Op Fl dhilmor
+.Sh DESCRIPTION
+.Nm
+tries to verify the entire contents of the pkgsrc tree.
+.Nm
+uses simple regular-expression matching for verifying
+files that make up a pkgsrc directory.
+Note that it does NOT implement complete Makefile parser.
+.Pp
+.Sy Options
+.Bl -tag -width Fl
+.It Fl h
+Basic help and usage.
+.It Fl d
+Extract the 'DEPENDS' listing from each pkgsrc package Makefile, then
+verify the version specified in each DEPEND correctly matches a current
+pkgsrc package.
+.It Fl i
+Check the version of each installed package against the current version in
+pkgsrc.
+.It Fl l
+Run pkglint on every package in pkgsrc.
+.It Fl m
+For each current distfile, verify its checksum against the appropriate
+package's md5 file. Also report if any package's md5 file references
+the same distfile with a different md5 checksum.
+.It Fl o
+Report any old distfile (not referenced by any package's md5 file).
+.It Fl r
+Remove distfiles which are not referenced in any package's md5 file, or has
+an incorrect checksum (usually due to an interrupted download). To remove
+one or the other type, additionally specify the -u or -m flag respectively.
+.El
+.Sh AUTHOR
+David Brownlee <abs@netbsd.org>
+.Sh BUGS
+Potentially, but non serious to date.
diff --git a/pkgtools/pkglint/files/lintpkgsrc.pl b/pkgtools/pkglint/files/lintpkgsrc.pl
new file mode 100755
index 00000000000..2e842f741b8
--- /dev/null
+++ b/pkgtools/pkglint/files/lintpkgsrc.pl
@@ -0,0 +1,416 @@
+#!/usr/bin/env perl
+
+# $NetBSD: lintpkgsrc.pl,v 1.1 1999/04/19 07:32:10 abs Exp $
+
+# (Somewhat quickly) Written by David Brownlee <abs@anim.dreamworks.com>.
+# Caveats:
+#	The 'Makefile parsing' algorithym used to obtain package versions
+#	and DEPENDS information is geared towards speed rather than perfection.
+#
+#	The 'invalid distfiles' code picks up a couple of false positives in
+#	fastcap (which does strange things anyway).
+
+require 'getopts.pl';
+if (! &Getopts('dhilmor') || $opt_h || ! (defined($opt_o) || defined($opt_d) ||
+					defined($opt_l) || defined($opt_i) ||
+					defined($opt_m) || defined($opt_r) ))
+    { &usage_and_exit; }
+$|=1;
+
+$^W=1;
+use strict;
+use vars ('$opt_o','$opt_d','$opt_h','$opt_i','$opt_l','$opt_m','$opt_r');
+
+# main
+    {
+    my($pkgsrcdir,%pkg2dir,$pkglint_flags);
+
+    $pkglint_flags='-a -b -c -v';
+    $pkgsrcdir=&set_pkgsrcdir;  # Parse /etc/mk.conf (if present) for PKGSRCDIR
+
+    if ($opt_r && !$opt_o && !$opt_m)
+	{ $opt_o=$opt_m=1; }
+    if ($opt_o || $opt_m)
+	{
+	my(@baddist);
+
+	@baddist=&scan_pkgsrc_distfiles_vs_md5($pkgsrcdir,$opt_o,$opt_m);
+	if ($opt_r)
+	    {
+	    &safe_chdir("$pkgsrcdir/distfiles");
+	    &verbose("Unlinking 'bad' distfiles\n");
+	    foreach (@baddist)
+		{ unlink($_); }
+	    }
+	}
+    if ($opt_d)
+	{ %pkg2dir=&scan_pkgsrc_makefiles($pkgsrcdir,1); }
+    if ($opt_i)
+	{
+	my(@pkgs,$pkg);
+
+	@pkgs=&list_installed_packages;
+	if (!%pkg2dir)
+	    { %pkg2dir=&scan_pkgsrc_makefiles($pkgsrcdir); }
+	foreach $pkg ( @pkgs )
+	    { &list_possible_versions($pkg,%pkg2dir); }
+	}
+    if ($opt_l)
+	{ &pkglint_all_pkgsrc($pkgsrcdir,$pkglint_flags); }
+    }
+exit;
+
+sub fail
+    { print STDERR @_,"\n"; exit(3); }
+
+# Given a package, strip version and report possible matches
+#
+sub list_possible_versions
+    {
+    my($pkg,%pkg2dir)=@_;
+    my($pkgname,@maybe,$fail);
+
+    if( !defined($pkg2dir{$pkg}) )
+	{
+	$pkgname=$pkg;
+	$pkgname =~ s/-[^-]+$//;
+	foreach ( sort keys %pkg2dir )
+	    {
+	    if( substr($_,0,length($pkgname)) eq $pkgname )
+		{ push(@maybe,$_); }
+	    }
+	$_="Unknown package: '$pkg'";
+	if( @maybe )
+	    { $_.=" (Maybe @maybe)"; }
+	else
+	    { $_.=" (DBG $pkgname)"; }
+	print "$_\n";
+	$fail=1;
+	}
+    $fail;
+    }
+
+# List (recursive) non diretory contents of specified directory
+#
+sub listdir
+    {
+    my($base,$dir)=@_;
+    my($thisdir,$entry);
+    my(@list,@thislist);
+
+    $thisdir=$base;
+    if (defined($dir))
+	{
+	$thisdir.="/$dir";
+	$dir.='/';
+	}
+    else
+	{ $dir=''; }
+    opendir(DIR,$thisdir) || &fail("Unable to opendir($thisdir): $!");
+    @thislist=grep(substr($_,0,1) ne '.' && $_ ne 'CVS',readdir(DIR));
+    closedir(DIR);
+    foreach $entry (@thislist)
+	{
+	if (-d "$thisdir/$entry")
+	    { push(@list,&listdir($base,"$dir$entry")); }
+	else
+	    { push(@list,"$dir$entry"); }
+	}
+    @list;
+    }
+
+# Use pkg_info to list installed packages
+#
+sub list_installed_packages
+    {
+    my(@pkgs);
+
+    open(PKG_INFO,'pkg_info -a|') || &fail("Unable to run pkg_info: $!");
+    while ( <PKG_INFO> )
+	{ push(@pkgs,(split)[0]); }
+    close(PKG_INFO);
+    @pkgs;
+    }
+
+# List top level pkgsrc categories
+#
+sub list_pkgsrc_categories
+    {
+    my($pkgsrcdir)=@_;
+    my(@categories);
+
+    opendir(BASE,$pkgsrcdir) || die("Unable to opendir($pkgsrcdir): $!");
+    @categories=grep(substr($_,0,1) ne '.' && -f "$pkgsrcdir/$_/Makefile",
+							readdir(BASE));
+    closedir(BASE);
+    @categories;
+    }
+
+# Extract variable assignments from Makefile, include pkgname.
+# Much unpalatable magic to avoid having to use make (all for speed)
+#
+sub parse_makefile
+    {
+    my($file)=@_;
+    my($pkgname,%vars);
+    my($key,$plus,$value);
+
+    if (open(FILE,$file))
+	{
+	while( <FILE> )
+	    {
+	    s/#.*//;
+	    if (/^\s*(\w+)(\+?)=\s+(\S*)/)
+		{
+		$key=$1;
+		$plus=$2;
+		$value=$3;
+		if ($plus)
+		    { $vars{$key}.="\n$value"; }
+		else
+		    { $vars{$key}=$value; }
+		} 
+	    }
+	foreach $key ( keys %vars, keys %vars)
+	    {
+	    foreach $value ( keys %vars )
+		{
+		if ($vars{$key} =~ m#\${(\w+):S/(\w+)/(\w+)/}#)
+		    {
+		    my($var,$from,$to)=($1,$2,$3);
+		    $_=$vars{$var};
+		    s/$from/$to/;
+		    $vars{$key} =~ s#\${$var:S/$from/$to/}#$_#;
+		    }
+		$vars{$key} =~ s/\$\{$value\}/$vars{$value}/g;
+		}
+	    }
+	close(FILE);
+	}
+    if (defined($vars{'PKGNAME'}))
+	{ $pkgname=$vars{'PKGNAME'}; }
+    elsif (defined($vars{'DISTNAME'}))
+	{ $pkgname=$vars{'DISTNAME'}; }
+    if (defined($pkgname))
+	{ return($pkgname,%vars); }
+    else
+	{ return(undef); }
+    }
+
+# Run pkglint on every pkgsrc entry
+#
+sub pkglint_all_pkgsrc
+    {
+    my($pkgsrcdir,$pkglint_flags)=@_;
+    my($cat,$pkg,@categories,@output);
+
+    @categories=&list_pkgsrc_categories($pkgsrcdir);
+    foreach $cat ( sort @categories )
+	{
+	&safe_chdir("$pkgsrcdir/$cat");
+	if (! opendir(CAT,'.'))
+	    { die("Unable to opendir($pkgsrcdir/$cat): $!"); }
+	foreach $pkg ( grep(substr($_,0,1) ne '.',readdir(CAT) ) )
+	    {
+	    if (-f "$pkg/Makefile")
+		{
+		if (!open(PKGLINT,"pkglint $pkglint_flags $pkg|"))
+		    { &fail("Unable to run pkglint: $!"); }
+		@output=grep(!/^OK:/ &&
+			     !/^WARN: be sure to cleanup .* work before/ &&
+			     !/^WARN: is it a new port/ &&
+			     !/^\d+ fatal errors and \d+ warnings found/
+			     ,<PKGLINT> );
+		close(PKGLINT);
+		if (@output)
+		    {
+		    print "===> $cat/$pkg\n",
+			  "pkglint $pkglint_flags $pkg\n",
+			  @output,"\n";
+		    }
+		}
+	    }
+	close(CAT);
+	}
+    }
+
+# chdir() || &fail()
+#
+sub safe_chdir
+    {
+    my($dir)=@_;
+
+    if (! chdir($dir))
+	{ &fail("Unable to chdir($dir): $!"); }
+    }
+
+# Generate pkgname->category/pkg mapping, optionally check DEPENDS
+#
+sub scan_pkgsrc_makefiles
+    {
+    my($pkgsrcdir,$check_depends)=@_;
+    my($cat,@categories,$pkg,$pkgname);
+    my(%pkg2dir,%depends);
+
+    @categories=&list_pkgsrc_categories($pkgsrcdir);
+    &verbose("Scanning pkgsrc Makefiles: ".'_'x@categories."\b"x@categories);
+    foreach $cat ( sort @categories )
+	{
+	if (! opendir(CAT,"$pkgsrcdir/$cat"))
+	    { die("Unable to opendir($pkgsrcdir/$cat): $!"); }
+	foreach $pkg ( grep(substr($_,0,1) ne '.',readdir(CAT) ) )
+	    {
+	    my(%vars);
+	    ($pkgname,%vars)=&parse_makefile("$pkgsrcdir/$cat/$pkg/Makefile");
+	    if ($pkgname)
+		{
+		$pkg2dir{$pkgname}="$cat/$pkg";
+		if (defined($vars{'DEPENDS'}))
+		    { $depends{$pkgname}=$vars{'DEPENDS'}; }
+		}
+	    }
+	close(CAT);
+	&verbose('.');
+	}
+    &verbose(' (',scalar(keys %pkg2dir)," packages)\n");
+
+    if ($check_depends)
+	{
+	foreach $pkg ( sort keys %depends )
+	    {
+	    my($err);
+	    foreach (split("\n",$depends{$pkg}))
+		{
+		s/:.*// || next;
+		if (!defined($pkg2dir{$_}))
+		    {
+		    if (!defined($err))
+			{ print "DEPENDS errors for $pkg2dir{$pkg}:\n"; }
+		    $err=1;
+		    print "\t";
+		    &list_possible_versions($_);
+		    }
+		}
+	    }
+	}
+    %pkg2dir;
+    }
+
+# Extract all md5 entries, then verify contents of distfiles
+#
+sub scan_pkgsrc_distfiles_vs_md5
+    {
+    my($pkgsrcdir,$check_unref,$check_md5)=@_;
+    my($cat,@categories,$pkg);
+    my(%distfiles,%md5,@distwarn,$file,$numpkg);
+    my(@distfiles,@bad_distfiles);
+
+    @categories=&list_pkgsrc_categories($pkgsrcdir);
+
+    &verbose("Scanning pkgsrc md5s: ".'_'x@categories."\b"x@categories);
+    $numpkg=0;
+    foreach $cat ( sort @categories )
+	{
+	if (! opendir(CAT,"$pkgsrcdir/$cat"))
+	    { die("Unable to opendir($pkgsrcdir/$cat): $!"); }
+	foreach $pkg ( grep(substr($_,0,1) ne '.',readdir(CAT) ) )
+	    {
+	    if (open(MD5,"$pkgsrcdir/$cat/$pkg/files/md5"))
+		{
+		++$numpkg;
+		while( <MD5> )
+		    {
+		    if (m/^MD5 \(([^\)]+)\) = (\S+)/)
+			{
+			if (!defined($distfiles{$1}))
+			    {
+			    $distfiles{$1}="$cat/$pkg";
+			    $md5{$1}=$2;
+			    }
+			elsif( $md5{$1} ne $2 )
+			    {
+			    push(@distwarn,"md5 mismatch between '$1' in ".
+			    "$cat/$pkg and $distfiles{$1}\n");
+			    }
+			}
+		    }
+		close(MD5);
+		}
+	    }
+	close(CAT);
+	&verbose('.');
+	}
+    &verbose(" ($numpkg packages)\n");
+
+    foreach $file (&listdir("$pkgsrcdir/distfiles"))
+	{
+	if (!defined($distfiles{$file}))
+	    { push(@bad_distfiles,$file); }
+	else
+	    { push(@distfiles,$file); }
+	}
+    if ($check_unref && @bad_distfiles)
+	{
+	&verbose(scalar(@bad_distfiles),
+			" unreferenced file(s) in '$pkgsrcdir/distfiles':\n");
+	print join("\n",sort @bad_distfiles),"\n";
+	}
+    if ($check_md5)
+	{
+	if (@distwarn)
+	    { &verbose(@distwarn); }
+	&verbose("md5 mismatches\n");
+	@distfiles=sort @distfiles;
+	&safe_chdir("$pkgsrcdir/distfiles");
+	open(MD5,"md5 @distfiles|") || &fail("Unable to run md5: $!");
+	while (<MD5>)
+	    {
+	    if (m/^MD5 \(([^\)]+)\) = (\S+)/)
+		{
+		if ($md5{$1} ne 'IGNORE' && $md5{$1} ne $2)
+		    {
+		    print $1,"\n";
+		    push(@bad_distfiles,$1);
+		    }
+		}
+	    }
+	close(MD5);
+	}
+    @bad_distfiles;
+    }
+
+sub set_pkgsrcdir # Parse /etc/mk.conf (if present) for PKGSRCDIR
+    {
+    my($pkgsrcdir);
+
+    $pkgsrcdir='/usr/pkgsrc';
+    if (open(MK_CONF,'/etc/mk.conf'))
+	{
+	while (<MK_CONF>)
+	    {
+	    if( /PKGSRCDIR\s*=\s*(\S+)/ )
+		{ $pkgsrcdir=$1; last; }
+	    }
+	close(MK_CONF);
+	}
+    $pkgsrcdir;
+    }
+
+sub usage_and_exit
+    {
+    print "Usage: pkg_check [opts]
+opts:	-d : Check each Makefile's 'DEPENDS' matches current pkgsrc versions.
+	-h : This help.
+	-i : Check versions of installed packages against pkgsrc.
+	-l : Run pkglint on every package in pkgsrc.
+	-m : Report md5 mismatches for files in 'distfiles'.
+	-o : Report old/obsolete 'distfiles' (not referenced by any md5).
+	-r : Remove any 'bad' distfiles (Without -m or -o, implies both).
+
+If pkgsrc is not in /usr/pkgsrc, set PKGSRCDIR in /etc/mk.conf
+";
+    exit;
+    }
+
+sub verbose
+    { print STDERR @_; }
diff --git a/pkgtools/pkglint/files/lintpkgsrc.sh b/pkgtools/pkglint/files/lintpkgsrc.sh
deleted file mode 100644
index d57de1e9699..00000000000
--- a/pkgtools/pkglint/files/lintpkgsrc.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/sh
-# $NetBSD: lintpkgsrc.sh,v 1.2 1998/08/08 15:19:11 tsarna Exp $
-
-PORTLINT=@PREFIX@/bin/pkglint
-PORTLINTFLAGS="-a -b -c -v"
-
-cd @PORTSDIR@
-for i in *
-do
-	if [ -d $i/. -a $i != distfiles -a $i != packages ]; then
-		cd $i
-		for j in *
-		do
-			if [ -d $j/. -a $j != "CVS" -a $j != "pkg" ]; then
-				${PORTLINT} ${PORTLINTFLAGS} $j \
-					| grep -v '^OK' \
-					| grep -v '^WARN: be sure to cleanup .*/work before committing the port' \
-					| grep -v '^WARN: is it a new port' \
-					>tmp$$
-				if [ `cat tmp$$ | wc -l` -gt 1 ]; then
-					echo ""
-					echo "===> $i/$j"
-					echo ${PORTLINT} ${PORTLINTFLAGS} $j
-					cat tmp$$
-				fi
-				rm -f tmp$$
-			fi
-		done
-		cd ..
-	fi
-done
diff --git a/pkgtools/pkglint/files/pkglint.1 b/pkgtools/pkglint/files/pkglint.1
index 990746b1b13..c68d6ea5d3c 100644
--- a/pkgtools/pkglint/files/pkglint.1
+++ b/pkgtools/pkglint/files/pkglint.1
@@ -1,4 +1,4 @@
-.\"	$NetBSD: pkglint.1,v 1.1 1998/08/07 22:13:43 tsarna Exp $
+.\"	$NetBSD: pkglint.1,v 1.2 1999/04/19 07:32:10 abs Exp $
 .\"	From FreeBSD: portlint.1,v 1.8 1997/11/25 14:53:14 itojun Exp
 .\"
 .\" Copyright (c) 1997 by Jun-ichiro Itoh <itojun@itojun.org>.
@@ -117,6 +117,7 @@ This type of messages are used in verbose mode
 .It Pa /usr/pkgsrc/mk/bsd.pkg.mk
 master Makefile for pkgsrc
 .It Pa /usr/pkgsrc/*
+.El
 pkgsrc collection
 .Sh AUTHORS
 Jun-ichiro Itoh <itojun@itojun.org>
diff --git a/pkgtools/pkglint/pkg/PLIST b/pkgtools/pkglint/pkg/PLIST
index d8b9609774d..5897bf11535 100644
--- a/pkgtools/pkglint/pkg/PLIST
+++ b/pkgtools/pkglint/pkg/PLIST
@@ -1,6 +1,8 @@
-@comment $NetBSD: PLIST,v 1.1 1998/08/07 22:13:44 tsarna Exp $
+@comment $NetBSD: PLIST,v 1.2 1999/04/19 07:32:10 abs Exp $
 bin/lintpkgsrc
 bin/pkglint
 bin/plist-clash
-man/man1/pkglint.1
+man/cat1/lintpkgsrc.0
 man/cat1/pkglint.0
+man/man1/lintpkgsrc.1
+man/man1/pkglint.1