update to cdpack-1.4

- by default cdpack now excludes NO_BIN_ON_CDROM packages.  This is the most
  important change.  Previously all pkgs were included.
- added flag to allow NO_BIN_ON_CDROM packages
- added flag to exclude NO_BIN_ON_FTP packages
- increased the verbosity resulting from the verbose flag
- added a debug flag to preserve tmp files

4 files changed, 253 insertions, 62 deletions

diff --git a/pkgtools/cdpack/Makefile b/pkgtools/cdpack/Makefile
index b5e1208cca9..a0d59cfd6f9 100644
--- a/pkgtools/cdpack/Makefile
+++ b/pkgtools/cdpack/Makefile
@@ -1,7 +1,7 @@
-# $NetBSD: Makefile,v 1.6 2001/09/27 23:18:35 jlam Exp $
+# $NetBSD: Makefile,v 1.7 2002/05/19 07:59:46 dmcmahill Exp $
 #
 
-DISTNAME=	cdpack-1.3
+DISTNAME=	cdpack-1.4
 CATEGORIES=	pkgtools
 MASTER_SITES=	# empty
 DISTFILES=	# empty
diff --git a/pkgtools/cdpack/files/cdgen.awk b/pkgtools/cdpack/files/cdgen.awk
index a3009bd85e0..97c528d7dcb 100755
--- a/pkgtools/cdpack/files/cdgen.awk
+++ b/pkgtools/cdpack/files/cdgen.awk
@@ -1,5 +1,5 @@
 #!/usr/bin/awk -f
-# $NetBSD: cdgen.awk,v 1.2 2001/06/02 02:03:52 dmcmahill Exp $
+# $NetBSD: cdgen.awk,v 1.3 2002/05/19 07:59:46 dmcmahill Exp $
 #
 # Copyright (c) 2001 Dan McMahill, All rights reserved.
 #
@@ -37,13 +37,14 @@ BEGIN {
 # ARGV[1] = packages directory (/usr/pkgsrc/packages/All)
 # ARGV[2] = output directory
 # ARGV[3] = Depends tree file.  Has packages in tsort(1) input format.
-# ARGV[4] = Depends order file.  Has packages in tsort(1) output format.
-# ARGV[5] = CD list.  This script leaves a list of the CD directories in this file
-# ARGV[6] = dup flag.  "dup=yes" for package duplication, "dup=no" for no duplication. 
-# ARGV[7] = verbose flag.  "verbose=yes" for verbose output
-# ARGV[8] = xtra_size.  How many kB are needed per CD for common files
-
-    if (ARGC != 10){
+# ARGV[4] = Exclude file.  Packages listed here are excluded.
+# ARGV[5] = Depends order file.  Has packages in tsort(1) output format.
+# ARGV[6] = CD list.  This script leaves a list of the CD directories in this file
+# ARGV[7] = dup flag.  "dup=yes" for package duplication, "dup=no" for no duplication. 
+# ARGV[8] = verbose flag.  "verbose=yes" for verbose output
+# ARGV[9] = xtra_size.  How many kB are needed per CD for common files
+
+    if (ARGC != 11){
 	printf("%s:  wrong number of arguments\n",ARGV[0]);
 	usage();
 	exit(1);
@@ -53,18 +54,19 @@ BEGIN {
     packages = ARGV[1];
     cddir    = ARGV[2];
     deptree  = ARGV[3];
-    order    = ARGV[4];
-    cdlist   = ARGV[5];
-    xtra_size= ARGV[8];
-    other_size= ARGV[9];
+    exclude  = ARGV[4];
+    order    = ARGV[5];
+    cdlist   = ARGV[6];
+    xtra_size= ARGV[9];
+    other_size= ARGV[10];
 
-    if (ARGV[6] ~ "dup=yes"){
+    if (ARGV[7] ~ "dup=yes"){
 	dup=1;
     }
     else{
 	dup=0;
     }
-    if (ARGV[7] ~ "verbose=yes"){
+    if (ARGV[8] ~ "verbose=yes"){
 	verbose=1;
     }
     else{
@@ -94,18 +96,25 @@ BEGIN {
 	usage();
 	exit(1);
     }
-
+    
+    cmd="test -f " exclude ;
+    if(system(cmd) != 0){
+	printf("%s:  exclude file \"%s\" does not exist\n",prog,exclude);
+	usage();
+	exit(1);
+    }
+    
     cmd="test -f " order ;
     if(system(cmd) != 0){
 	printf("%s:  build order file \"%s\" does not exist\n",prog,order);
 	usage();
 	exit(1);
     }
-
-
+    
+    
     now = strftime("%a %b %d %H:%M:%S %Z %Y");
     printf("%s starting %28s\n",prog,now);
-
+    
 #
 # Read in the build order.  This gives the list of all possible 
 # packages (note that some may actually not be available as binary
@@ -123,6 +132,14 @@ BEGIN {
     npkgs = n-1;
     printf("%d packages to go on CD-ROM!\n",npkgs);
 
+#
+# Read in the list of excluded packages
+#
+    printf("Reading list of packages to be excluded\n");
+    while(getline < exclude > 0){
+	excludes[$1] = 1 ;
+    }
+    close(exclude);
 
 #
 # Read in the depends tree and flatten it.
@@ -183,30 +200,36 @@ BEGIN {
 	pkgn=0;
 	cdtot[cdn]=xtra_size;
 	cdpkgs[cdn]=0;
-
+	
 	for (n=1; n<=npkgs ; n=n+1){
+	    if (verbose) printf("Processing: %s\n",pkgorder[n]);
+	    if (pkgorder[n] in excludes) {
+		if (verbose) printf("Skipping excluded package: %s\n",pkgorder[n]);
+	    }
+	    else {
 # only process the package if it exists.
-	    if (pkgsize[pkgorder[n]] > 0){
-		if (cdtot[cdn] < (maxcd-pkgsize[pkgorder[n]]) ){
-		    cdtot[cdn] = cdtot[cdn] +pkgsize[pkgorder[n]];
-		    cdcontents[cdn":"pkgorder[n]] = 1;
-		    pkgn = pkgn + 1;
-		}
-		else{
+		if (pkgsize[pkgorder[n]] > 0){
+		    if (cdtot[cdn] < (maxcd-pkgsize[pkgorder[n]]) ){
+			cdtot[cdn] = cdtot[cdn] +pkgsize[pkgorder[n]];
+			cdcontents[cdn":"pkgorder[n]] = 1;
+			pkgn = pkgn + 1;
+		    }
+		    else{
 # the CD is full
-		    printf("cd number %d is full (%g Mb)\n",cdn,
-			   cdtot[cdn]/1024);
-		    cdpkgs[cdn] = pkgn;
+			printf("cd number %d is full (%g Mb)\n",cdn,
+			       cdtot[cdn]/1024);
+			cdpkgs[cdn] = pkgn;
 # increment the CD counter
-		    cdn = cdn + 1;
-		    pkgn = 1;
-		    cdtot[cdn] = xtra_size + pkgsize[pkgorder[n]];
-		    cdcontents[cdn":"pkgorder[n]] = 1;
+			cdn = cdn + 1;
+			pkgn = 1;
+			cdtot[cdn] = xtra_size + pkgsize[pkgorder[n]];
+			cdcontents[cdn":"pkgorder[n]] = 1;
+		    }
 		}
 	    }
 	}
 	cdpkgs[cdn] = pkgn;
-
+	
 # see if the extra files will fit on the last CD
 	if ( (cdtot[cdn] + other_size) < maxcd ){
 	    printf("cd number %d is partially full (%g Mb)\n",cdn,
@@ -221,39 +244,69 @@ BEGIN {
 	    printf("cd number %d is partially full (%g Mb)\n",cdn,
 		   cdtot[cdn]/1024);
 	}
-	    
+	
     }
-
+    
 #
 # We will duplicate some packages to eliminate inter-CD dependencies.
 #
     else{
 	cdn=1;
 	pkgn=0;
+# initialize the size count for the current CD with the extras that we
+# are putting on all CD's
 	cdtot[cdn]=xtra_size;
 	cdpkgs[cdn]=0;
 	n=npkgs;
 	while (n > 0){
+	    if (verbose) printf("Begin processing %s\n",pkgorder[n]);
 	    if ( !pkg_done[pkgorder[n]]){
 		size_needed = pkgsize[pkgorder[n]];
 		deps_needed = 0;
 		split(alldepends[pkgorder[n]],pkgdeps);
 		for (dep in pkgdeps){
-		    if(!cdcontents[cdn":"pkgdeps[dep]]){
-			size_needed = size_needed + pkgsize[pkgdeps[dep]];
-			deps_needed++;
+		    if (verbose) printf("   Examining dependency: %s\n",pkgdeps[dep]);
+		    if (pkgdeps[dep] in excludes) {
+			if (verbose) printf("   Skipping excluded dependency in count: %s\n",pkgdeps[dep]);
+		    }
+		    else {
+			if(!cdcontents[cdn":"pkgdeps[dep]]){
+			    size_needed = size_needed + pkgsize[pkgdeps[dep]];
+			    deps_needed++;
+			}
 		    }
 		}
 		if (cdtot[cdn] + size_needed < maxcd){
-		    cdcontents[cdn":"pkgorder[n]] = 1;
+		    if (verbose) printf("   Processing %s\n",pkgorder[n]);
+		    if (pkgorder[n] in excludes) {
+			if (verbose) printf("   Skipping excluded package in packing: %s\n",pkgorder[n]);
+		    }
+		    else {
+			cdcontents[cdn":"pkgorder[n]] = 1;
+		    }
 		    pkg_done[pkgorder[n]] = 1;
+		    if (verbose) printf("   Marked %s as processed\n",pkgorder[n]);
 		    for (dep in pkgdeps){
-			cdcontents[cdn":"pkgdeps[dep]] = 1;
-			pkg_done[pkgdeps[dep]] = 1;
+			if (pkgdeps[dep] in excludes) {
+			    if (verbose) printf("   Skipping excluded dependency in packing: %s\n",pkgdeps[dep]);
+			}
+			else {
+			    cdcontents[cdn":"pkgdeps[dep]] = 1;
+			    pkg_done[pkgdeps[dep]] = 1;
+			    if (verbose) printf("   Marked dependency pkg %s as processed\n",pkgdeps[dep]);
+			}
 		    }
 		    cdtot[cdn] = cdtot[cdn] + size_needed;
+		    if (pkgorder[n] in excludes) {
+# don't include this one in the count if its excluded
+			if (verbose) printf("   Added  %d dependencies to the image\n",deps_needed);
+			pkgn = pkgn + deps_needed;
+		    }
+		    else {
+			if (verbose) printf("   Added %s plus %d dependencies to the image\n",pkgorder[n],deps_needed);
+			pkgn = pkgn + 1 + deps_needed;
+		    }
 		    n--;
-		    pkgn = pkgn + 1 + deps_needed;
 		}
 		else{
 # the CD is full
@@ -267,9 +320,10 @@ BEGIN {
 	    }
 	    else{
 # we've already done this pkg
+		if (verbose) printf("   %s has already been processed\n",pkgorder[n]);
 		n--;
 	    }
-
+	    
 	}
 	cdpkgs[cdn] = pkgn;
 
@@ -293,15 +347,20 @@ BEGIN {
     ncd=cdn;
 
 # print some stats
+    tot_ex=0;
+    for (ex in excludes){
+	tot_ex++;
+    }
     if (dup){
 	tot_cdpkgs=0;
 	for (cdn=1; cdn<=ncd; cdn=cdn+1){
 	    tot_cdpkgs = tot_cdpkgs + cdpkgs[cdn];
 	}
 	printf("CD images with package duplication resulted in %d packages total\n",tot_cdpkgs);
-	printf("This is an increase of %d over the base %d packages\n",tot_cdpkgs-npkgs,npkgs);
+	printf("This is an increase of %d over the base %d packages\n",tot_cdpkgs-(npkgs-tot_ex),npkgs-tot_ex);
     }
 	
+    printf("%d packages out of %d have been excluded due to redistribution restrictions\n",tot_ex,npkgs);
 #
 # Next, create a subdirectory for each CD and populate the directory
 # with links to the actual binary pkgs
diff --git a/pkgtools/cdpack/files/cdpack.1 b/pkgtools/cdpack/files/cdpack.1
index e5c9717b9df..f502bbfecef 100644
--- a/pkgtools/cdpack/files/cdpack.1
+++ b/pkgtools/cdpack/files/cdpack.1
@@ -1,4 +1,4 @@
-.\" $NetBSD: cdpack.1,v 1.6 2001/12/03 19:03:22 wiz Exp $
+.\" $NetBSD: cdpack.1,v 1.7 2002/05/19 07:59:47 dmcmahill Exp $
 .\"
 .\" Copyright (c) 2001 Dan McMahill, All rights reserved.
 .\"
@@ -37,8 +37,9 @@
 .Nd produce multi-CD binary package images
 .Sh SYNOPSIS
 .Nm ""
+.Op -ef
 .Op -l logfile
-.Op -nRvV
+.Op -adnRvV
 .Op -x common_directory
 .Op -X extras_directory
 .Ar packages_directory
@@ -64,6 +65,18 @@ that directory.  A file,
 which lists the contents of each CD will be left in the root
 directory of each CD image.
 .Bl -tag -width "-x directory "
+.It Fl a
+Include binary packages marked as NO_BIN_ON_CDROM on the CD set.
+The default is to omit any
+packages which were marked as NO_BIN_ON_CDROM.  This option should
+only be used when making a CD set for personal use and not for
+distribution unless you are sure you have not violated any licenses.
+.It Fl d
+Enable debugging by not deleting temporary files at the end of the
+run.  These files are left in TMPDIR.
+.It Fl ef
+Exclude packages marked as NO_BIN_ON_FTP from the CD set.  The default
+is to allow packages which are NO_BIN_ON_FTP.
 .It Fl l Ar logfile
 Log the output of mkisofs -v -v to the file
 .Ar logfile .
diff --git a/pkgtools/cdpack/files/cdpack.sh b/pkgtools/cdpack/files/cdpack.sh
index f0490efa74c..6179760b92c 100755
--- a/pkgtools/cdpack/files/cdpack.sh
+++ b/pkgtools/cdpack/files/cdpack.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
-# $NetBSD: cdpack.sh,v 1.4 2001/06/24 21:46:34 dmcmahill Exp $
+# $NetBSD: cdpack.sh,v 1.5 2002/05/19 07:59:47 dmcmahill Exp $
 #
-# Copyright (c) 2001 Dan McMahill, All rights reserved.
+# Copyright (c) 2001, 2002 Dan McMahill, All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -44,14 +44,17 @@ TMP=${TMPDIR}/${prog}.$$
 depf=$TMP/depf
 depf2=$TMP/depf2
 deptree=$TMP/deptree
+exclude=$TMP/exclude
 order=$TMP/order
 cdlist=$TMP/cdlist
 indexf=$TMP/indexf
 readme=$TMP/readme
 warnings=$TMP/warnings
+restricted=$TMP/restricted
 
 rm -fr $TMP
 mkdir $TMP 
+touch $exclude
 
 usage(){
 	echo "$prog - generates ISO9660 images for a multi-cd binary package collection"
@@ -61,7 +64,11 @@ usage(){
 }
 
 clean_and_exit(){
-	rm -fr $TMP
+	if [ "x$DEBUG" = "xno" ]; then
+	    rm -fr $TMP
+        else
+	    echo "Debugging output left in $TMP"
+	fi
 	exit 1
 }
 
@@ -80,6 +87,7 @@ fullpath(){
 }
 
 ADD_README=no
+DEBUG=no
 DUP=yes
 VERBOSE=no
 VERSION=no
@@ -87,11 +95,29 @@ USE_XTRA=no
 USE_OTHERS=no
 mkisofslog=/dev/null
 
+ALLOW_NO_BIN_ON_CDROM=no
+ALLOW_NO_BIN_ON_FTP=yes
+
 while
     test -n "$1"
 do
     case "$1"
     in
+	# allow NO_BIN_ON_CDROM packages
+	-a) ALLOW_NO_BIN_ON_CDROM=yes
+	    shift
+	    ;;
+
+	# enable debugging
+	-d) DEBUG=yes
+	    shift
+	    ;;
+
+	# exclude NO_BIN_ON_FTP packages
+	-ef) ALLOW_NO_BIN_ON_FTP=no
+	    shift
+	    ;;
+
 	# log file for the output of mkisofs -v -v
 	-l) mkisofslog=$2
 	    shift 2
@@ -159,6 +185,16 @@ cddir=$2
 
 if [ $VERBOSE = "yes" ]; then
     echo "Verbose output is on"
+    if [ "$ALLOW_NO_BIN_ON_CDROM" = "yes" ]; then
+	echo "NO_BIN_ON_CDROM Packages will be included in the images"
+    else
+	echo "NO_BIN_ON_CDROM Packages will be excluded in the images"
+    fi
+    if [ "$ALLOW_NO_BIN_ON_FTP" = "yes" ]; then
+	echo "NO_BIN_ON_FTP Packages will be included in the images"
+    else
+	echo "NO_BIN_ON_FTP Packages will be excluded in the images"
+    fi
 fi
 
 #
@@ -234,7 +270,7 @@ do
 		pkgname=`basename $pkg .tgz`
 
 		# extract the packing list
-		cat $pkg | (cd $TMP; tar --fast-read -xzf - +CONTENTS)
+		cat $pkg | (cd $TMP; tar --fast-read -xzf - +BUILD_INFO +CONTENTS)
 
 		# extract the depends
 		deps=`awk '/^@pkgdep/ {printf("%s ",$2)}' $TMP/+CONTENTS`
@@ -242,7 +278,30 @@ do
 		# extract any conflicts
 		cfls=`awk '/^@pkgcfl/ {printf("%s ",$2)}' $TMP/+CONTENTS`
 
-		rm $TMP/+CONTENTS
+		# check to see if we're allowed to add this package to the CD set
+		NO_BIN_ON_CDROM=`awk -F "=" '/NO_BIN_ON_CDROM/ {print $2}' $TMP/+BUILD_INFO`
+		if [ ! -z "$NO_BIN_ON_CDROM" ]; then
+		    echo "$pkg:  NO_BIN_ON_CDROM=$NO_BIN_ON_CDROM" >> $restricted
+		    if [ "$ALLOW_NO_BIN_ON_CDROM" = "no" ]; then
+			if [ "$VERBOSE" = "yes" ]; then
+			    echo "Excluding $pkg because NO_BIN_ON_CDROM=$NO_BIN_ON_CDROM"
+			    echo "$pkgname" >> $exclude
+			fi
+		    fi
+		fi
+		NO_BIN_ON_FTP=`awk -F "=" '/NO_BIN_ON_FTP/ {print $2}' $TMP/+BUILD_INFO`
+		if [ ! -z "$NO_BIN_ON_FTP" ]; then
+		    echo "$pkg:  NO_BIN_ON_FTP=$NO_BIN_ON_FTP" >> $restricted
+		    if [ "$ALLOW_NO_BIN_ON_FTP" = "no" ]; then
+			if [ "$VERBOSE" = "yes" ]; then
+			    echo "Excluding $pkg because NO_BIN_ON_FTP=$NO_BIN_ON_FTP"
+			    echo "$pkgname" >> $exclude
+			fi
+		    fi
+		fi
+
+		# cleanup
+		rm $TMP/+CONTENTS $TMP/+BUILD_INFO
 		
 		# store the results
 		echo "$pkgname | $deps | $cfls" >> $depf
@@ -300,13 +359,14 @@ tsort $deptree > $order
 #    packages = ARGV[1];
 #    cddir    = ARGV[2];
 #    deptree  = ARGV[3];
-#    order    = ARGV[4];
-#    cdlist   = ARGV[5];
+#    exclude  = ARGV[4];
+#    order    = ARGV[5];
+#    cdlist   = ARGV[6];
 #
 if [ "$VERBOSE" = "yes" ]; then
-    echo "awk -f @prefix@/libexec/cdgen.awk $packages $cddir $deptree $order $cdlist dup=$DUP verbose=$VERBOSE $XTRA_SIZE $OTHER_SIZE"
+    echo "awk -f @prefix@/libexec/cdgen.awk $packages $cddir $deptree $exclude $order $cdlist dup=$DUP verbose=$VERBOSE $XTRA_SIZE $OTHER_SIZE"
 fi
-awk -f @prefix@/libexec/cdgen.awk $packages $cddir $deptree $order $cdlist dup=$DUP verbose=$VERBOSE $XTRA_SIZE $OTHER_SIZE
+awk -f @prefix@/libexec/cdgen.awk $packages $cddir $deptree $exclude $order $cdlist dup=$DUP verbose=$VERBOSE $XTRA_SIZE $OTHER_SIZE
 
 if [ $? -ne 0 ]; then
     echo "$prog:  ERROR:  cdgen.awk has failed"
@@ -346,6 +406,46 @@ a collection of packages.
 EOF
 fi
 
+#
+# NO_BIN_ON_CDROM Comment
+#
+
+if [ "$ALLOW_NO_BIN_ON_CDROM" = "no" ]; then
+cat <<EOF >> $readme
+All packages with NO_BIN_ON_CDROM set have been excluded from
+this cd collection.
+EOF
+else
+cat <<EOF >> $readme
+This CD collection includes packaged with NO_BIN_ON_CDROM set.
+Please do not violate license agreements by selling this CD
+without verifying that you are allowed to.  A list of these
+packages may be found in the ".restricted" file on this CD.
+EOF
+fi
+
+#
+# NO_BIN_ON_FTP Comment
+#
+
+if [ "$ALLOW_NO_BIN_ON_FTP" = "no" ]; then
+cat <<EOF >> $readme
+All packages with NO_BIN_ON_FTP set have been excluded from
+this cd collection.
+EOF
+else
+cat <<EOF >> $readme
+This CD collection includes packages with NO_BIN_ON_FTP set.
+Please do not violate license agreements by placing this 
+image on a public FTP site without verifying that you are
+allowed to.  A list of these packages may be found in the
+ ".restricted" file on this CD.
+EOF
+fi
+
+#
+# cdpack Comment
+#
 cat <<EOF >> $readme
 
 This README, along with the CD layout was created using the
@@ -355,7 +455,7 @@ ftp://ftp.netbsd.org/pub/NetBSD/packages/pkgsrc/pkgtools/cdpack
 .
 
 EOF
-
+    
 # 
 # Generate an index file which lists the contents of each CD.
 #
@@ -364,6 +464,21 @@ echo "Creating CD Index File"
 
 for cdname in `cat $cdlist`
 do
+    #
+    # cdgen shouldn't have included any restricted pkgs, but
+    # make 100% sure now!
+    #
+    if [ $VERBOSE = "yes" ]; then
+	echo "Removing any leftover restricted packages from $cdname"
+    fi
+    for pkg in `cat $exclude`
+    do
+	rm -f ${cddir}/${cdname}/packages/*/$pkg
+    done
+
+    if [ $VERBOSE = "yes" ]; then
+	echo "Creating index for $cdname"
+    fi
     for pkg in ${cddir}/${cdname}/packages/All/*
     do
 	echo "`basename $pkg`  $cdname" >> $indexf
@@ -382,13 +497,13 @@ sort ${indexf}.tmp > $indexf
 #
 
 if [ $VERBOSE = "yes" ]; then
-    echo "Copying .index file to the image directories."
+    echo "Copying .index and .restricted files to the image directories."
 fi
 
 ncds=0
 for cdname in `cat $cdlist`
 do
-    (cd ${cddir}/${cdname} && cp $indexf .index)
+    (cd ${cddir}/${cdname} && cp $indexf .index ; cp $restricted .restricted)
 
     if [ $ADD_README = "yes" ]; then
        (cd ${cddir}/${cdname} && cp $readme README.txt)
@@ -450,7 +565,11 @@ echo " "
 echo "$prog finished: `date`" 
 echo " "
 
-rm -fr $TMP
+if [ "x$DEBUG" = "xno" ]; then
+    rm -fr $TMP
+else
+    echo "Debugging output left in $TMP"
+fi
 
 exit 0