diff options
author | rillig <rillig@pkgsrc.org> | 2019-10-06 10:33:34 +0000 |
---|---|---|
committer | rillig <rillig@pkgsrc.org> | 2019-10-06 10:33:34 +0000 |
commit | 93e5a6886222a823d6149d3e3c122af50c0b011e (patch) | |
tree | 37afd3239635fe4fdd0ed9ccf0930851852aa3bc | |
parent | 77f6d5121c502802ecea0d6d2510549d74fe36f2 (diff) | |
download | pkgsrc-93e5a6886222a823d6149d3e3c122af50c0b011e.tar.gz |
pkgtools/pkglint4: update to 4.193.0, remove check for absolute paths
The check for absolute paths is not necessary since it doesn't provide
any benefit. It had been removed from pkgtools/pkglint already.
19 files changed, 22 insertions, 1561 deletions
diff --git a/pkgtools/pkglint4/Makefile b/pkgtools/pkglint4/Makefile index 9a0122e4984..a8a50ea5942 100644 --- a/pkgtools/pkglint4/Makefile +++ b/pkgtools/pkglint4/Makefile @@ -1,7 +1,6 @@ -# $NetBSD: Makefile,v 1.12 2019/08/11 13:22:36 wiz Exp $ +# $NetBSD: Makefile,v 1.13 2019/10/06 10:33:34 rillig Exp $ -PKGNAME= pkglint4-4.164 -PKGREVISION= 2 +PKGNAME= pkglint4-4.193.0 CATEGORIES= pkgtools OWNER= rillig@NetBSD.org @@ -36,7 +35,7 @@ SUBST_SED.pkglint+= -e s\|@PKGSRCDIR@\|/usr/pkgsrc\|g .else SUBST_VARS.pkglint= PKGSRCDIR .endif -SUBST_VARS.pkglint= PREFIX +SUBST_VARS.pkglint+= PREFIX SUBST_SED.pkglint+= -e s\|@DISTVER@\|${PKGNAME:S/pkglint-//}\|g SUBST_VARS.pkglint+= MAKE SUBST_SED.pkglint+= -e s\|@PERL@\|${PERL5:Q}\|g @@ -57,7 +56,7 @@ do-build: && mv pkglint.pl.inlined pkglint.pl do-test: - cd ${WRKSRC} && prove pkglint.t + cd ${WRKSRC} && prove -v -I. pkglint.t do-install: ${INSTALL_SCRIPT} ${WRKSRC}/pkglint.pl ${DESTDIR}${PREFIX}/bin/pkglint diff --git a/pkgtools/pkglint4/README b/pkgtools/pkglint4/README deleted file mode 100644 index 6c5eb9d7479..00000000000 --- a/pkgtools/pkglint4/README +++ /dev/null @@ -1,27 +0,0 @@ -$NetBSD: README,v 1.1 2015/11/25 16:42:21 rillig Exp $ - -== Current problems == - -There are finally some automated tests that document a few intended -and actual behaviors. There is still plenty of poorly expressed -code not yet under test and therefore not yet safe to refactor. - -The current pkglint architecture will not scale much further. What is -needed next are parsers for nested, non-context-free languages (make(1), -sh(1), sed(1)). The parsers should be able to recognize partial -structures, as well as structures containing foreign parts. This is -because most of pkgsrc is heavily based on preprocessors: - -- The .if and .for directives in Makefiles are preprocessed by make(1) - before building dependencies and shell commands out of the remaining - text. - -- make(1) assembles shell commands from literal text and variables like - ${PKGNAME}. - -- Shell commands often use dynamic evaluation of variables. - -All this makes enhancing pkglint non-trivial. If you know of any -academic papers that might be of help in this case, please tell me. - -The pkglint source code is much too big for a single file. diff --git a/pkgtools/pkglint4/TODO b/pkgtools/pkglint4/TODO deleted file mode 100644 index 52b0d5d2ffa..00000000000 --- a/pkgtools/pkglint4/TODO +++ /dev/null @@ -1,37 +0,0 @@ -$NetBSD: TODO,v 1.1 2015/11/25 16:42:21 rillig Exp $ - -Please add your own entries at the bottom of this file. If possible, -include the name of an example package where a warning should occur. - -* When you understand some behavior of the code, document it by - adding automated tests to pkglint.t! -* warn about the use of ${WRKDIR:=...}, as this construct should only - be used with lists. -* Add checks for binary packages. See Debian/lintian for ideas. -* Of the user-defined variables, some may be used at load-time and some - don't. Find out how pkglint can distinguish them. -* Make sure that no variable is modified at load-time after it has been - used once. This should at least flag BUILD_DEFS in bsd.pkg.mk. -* Invent an annotation scheme for files that intentionally define - variables for use in other files. -* ${MACHINE_ARCH}-${LOWER_OPSYS}elf in PLISTs etc. is a NetBSD config.guess - problem ==> use of ${APPEND_ELF} -* Packages including lang/python/extension.mk must follow the Python version - scheme. Enforcing PYPKGPREFIX for those is most likely a good idea. -* Check for parallel files/dirs whose names differ only in case. -* Check for license files that are completely unused. -* If a dependency depends on an option (in options.mk), it should also - depend on the same option in the buildlink3.mk file. -* Complain about ${PKGSRC_COMPILER} == "sunpro", which should be - !empty(PKGSRC_COMPILER:Msunpro). -* If USE_TOOLS has autoconf213, and the package does stuff like - cd ${WRKSRC} && autoconf, then an incorrect warning is issued. -* LOCALBASE should not be used in normal Makefiles -* don't complain about "procedure calls", like for pkg-build-options in - the various buildlink3.mk files. -* if package A conflicts with B, then B should also conflict with A. -* When pkglint runs on a case-insensitive filesystem, it should still - point out problems that only occur on case-sensitive filesystems. For - example, devel/p5-Net-LDAP and devel/p5-Net-ldap should be considered - different paths. -* Warn about using REPLACE_PYTHON without including application.mk. diff --git a/pkgtools/pkglint4/files/PkgLint/Patches.pm b/pkgtools/pkglint4/files/PkgLint/Patches.pm index b96e7daa5f6..8bc8ff0c275 100644 --- a/pkgtools/pkglint4/files/PkgLint/Patches.pm +++ b/pkgtools/pkglint4/files/PkgLint/Patches.pm @@ -1,4 +1,4 @@ -# $NetBSD: Patches.pm,v 1.1 2015/11/25 16:42:21 rillig Exp $ +# $NetBSD: Patches.pm,v 1.2 2019/10/06 10:33:34 rillig Exp $ # # Everything concerning checks for patch files. # @@ -6,55 +6,6 @@ use strict; use warnings; -# Guess the type of file based on the filename. This is used to select -# the proper subroutine for detecting absolute pathnames. -# -# Returns one of "source", "shell", "make", "text", "configure", -# "ignore", "unknown". -# -sub get_filetype($$) { - my ($line, $fname) = @_; - my $basename = basename($fname); - - # The trailig .in part is not needed, since it does not - # influence the type of contents. - $basename =~ s,\.in$,,; - - # Let's assume that everything else that looks like a Makefile - # is indeed a Makefile. - if ($basename =~ m"^I?[Mm]akefile(?:\..*|)?|.*\.ma?k$") { - return "make"; - } - - # Too many false positives for shell scripts, so configure - # scripts get their own category. - if ($basename =~ m"^configure(?:|\.ac)$") { - $opt_debug_unchecked and $line->log_debug("Skipped check for absolute pathnames."); - return "configure"; - } - - if ($basename =~ m"\.(?:sh|m4)$"i) { - return "shell"; - } - - if ($basename =~ m"\.(?:cc?|cpp|cxx|el|hh?|hpp|l|pl|pm|py|s|t|y)$"i) { - return "source"; - } - - if ($basename =~ m"^.+\.(?:\d+|conf|html|info|man|po|tex|texi|texinfo|txt|xml)$"i) { - return "text"; - } - - # Filenames without extension are hard to guess right. :( - if ($basename !~ m"\.") { - return "unknown"; - } - - $opt_debug_misc and $line->log_debug("Don't know the file type of ${fname}."); - - return "unknown"; -} - sub checkline_cpp_macro_names($$) { my ($line, $text) = @_; my ($rest); @@ -120,85 +71,11 @@ sub checkline_cpp_macro_names($$) { } } -# Checks whether the line contains text that looks like absolute -# pathnames, assuming that the file uses the common syntax with -# single or double quotes to represent strings. -# -sub checkline_source_absolute_pathname($$) { - my ($line, $text) = @_; - my ($abspath); - - $opt_debug_trace and $line->log_debug("checkline_source_absolute_pathname(${text})"); - - if ($text =~ m"(.*)([\"'])(/[^\"']*)\2") { - my ($before, $delim, $string) = ($1, $2, $3); - - $opt_debug_misc and $line->log_debug("checkline_source_absolute_pathname(before=${before}, string=${string})"); - if ($before =~ m"[A-Z_]+\s*$") { - # allowed: PREFIX "/bin/foo" - - } elsif ($string =~ m"^/[*/]") { - # This is more likely to be a C or C++ comment. - - } elsif ($string !~ m"^/\w") { - # Assume that pathnames start with a letter or digit. - - } elsif ($before =~ m"\+\s*$") { - # Something like foodir + '/lib' - - } else { - $abspath = $string; - } - } - - if (defined($abspath)) { - checkword_absolute_pathname($line, $abspath); - } -} - -# Last resort if the file does not look like a Makefile or typical -# source code. All strings that look like pathnames and start with -# one of the typical Unix prefixes are found. -# -sub checkline_other_absolute_pathname($$) { - my ($line, $text) = @_; - - $opt_debug_trace and $line->log_debug("checkline_other_absolute_pathname(\"${text}\")"); - - if ($text =~ m"^#[^!]") { - # Don't warn for absolute pathnames in comments, - # except for shell interpreters. - - } elsif ($text =~ m"^(.*?)((?:/[\w.]+)*/(?:bin|dev|etc|home|lib|mnt|opt|proc|sbin|tmp|usr|var)\b[\w./\-]*)(.*)$") { - my ($before, $path, $after) = ($1, $2, $3); - - if ($before =~ m"\@$") { - # Something like @PREFIX@/bin - - } elsif ($before =~ m"[)}]$") { - # Something like ${prefix}/bin or $(PREFIX)/bin - - } elsif ($before =~ m"\+\s*[\"']$") { - # Something like foodir + '/lib' - - } elsif ($before =~ m"\w$") { - # Something like $dir/lib - - } elsif ($before =~ m"\.$") { - # ../foo is not an absolute pathname. - - } else { - $opt_debug_misc and $line->log_debug("before=${before}"); - checkword_absolute_pathname($line, $path); - } - } -} - sub checkfile_patch($) { my ($fname) = @_; my ($lines); my ($state, $redostate, $nextstate, $dellines, $addlines, $hunks); - my ($seen_comment, $current_fname, $current_ftype, $patched_files); + my ($seen_comment, $current_fname, $patched_files); my ($leading_context_lines, $trailing_context_lines, $context_scanning_leading); # Abbreviations used: @@ -271,42 +148,6 @@ sub checkfile_patch($) { return unless $m->has(1); $text = $m->text(1); checkline_cpp_macro_names($line, $text); - - # XXX: This check is not as accurate as the similar one in - # checkline_mk_shelltext(). - if (defined($current_fname)) { - if ($current_ftype eq "shell" || $current_ftype eq "make") { - my ($mm, $rest) = match_all($text, $regex_shellword); - - foreach my $m (@{$mm}) { - my $shellword = $m->text(1); - - if ($shellword =~ m"^#") { - last; - } - checkline_mk_absolute_pathname($line, $shellword); - } - - } elsif ($current_ftype eq "source") { - checkline_source_absolute_pathname($line, $text); - - } elsif ($current_ftype eq "configure") { - if ($text =~ m": Avoid regenerating within pkgsrc$") { - $line->log_error("This code must not be included in patches."); - $line->explain_error( -"It is generated automatically by pkgsrc after the patch phase.", -"", -"For more details, look for \"configure-scripts-override\" in", -"mk/configure/gnu-configure.mk."); - } - - } elsif ($current_ftype eq "ignore") { - # Ignore it. - - } else { - checkline_other_absolute_pathname($line, $text); - } - } }; my $check_hunk_end = sub($$$) { @@ -431,8 +272,6 @@ sub checkfile_patch($) { PST_CFA() => [ [re_patch_cfa, PST_CH, sub() { $current_fname = $m->text(1); - $current_ftype = get_filetype($line, $current_fname); - $opt_debug_patches and $line->log_debug("fname=$current_fname ftype=$current_ftype"); $patched_files++; $hunks = 0; }]], @@ -500,8 +339,6 @@ sub checkfile_patch($) { PST_UFA() => [ [re_patch_ufa, PST_UH, sub() { $current_fname = $m->text(1); - $current_ftype = get_filetype($line, $current_fname); - $opt_debug_patches and $line->log_debug("fname=$current_fname ftype=$current_ftype"); $patched_files++; $hunks = 0; }]], @@ -558,7 +395,6 @@ sub checkfile_patch($) { $patched_files = 0; $seen_comment = false; $current_fname = undef; - $current_ftype = undef; $hunks = undef; for (my $lineno = 0; $lineno <= $#{$lines}; ) { diff --git a/pkgtools/pkglint4/files/PkgLint/Shell.pm b/pkgtools/pkglint4/files/PkgLint/Shell.pm index c9251b9ba04..661808c6906 100644 --- a/pkgtools/pkglint4/files/PkgLint/Shell.pm +++ b/pkgtools/pkglint4/files/PkgLint/Shell.pm @@ -1,4 +1,4 @@ -# $NetBSD: Shell.pm,v 1.1 2015/11/25 16:42:21 rillig Exp $ +# $NetBSD: Shell.pm,v 1.2 2019/10/06 10:33:34 rillig Exp $ # # Parsing and checking shell commands embedded in Makefiles # @@ -541,10 +541,6 @@ sub checkline_mk_shelltext($$) { "\"||\" operator."); } - if (($state != SCST_PAX_S && $state != SCST_SED_E && $state != SCST_CASE_LABEL)) { - checkline_mk_absolute_pathname($line, $shellword); - } - if (($state == SCST_INSTALL_D || $state == SCST_MKDIR) && $shellword =~ m"^(?:\$\{DESTDIR\})?\$\{PREFIX(?:|:Q)\}/") { $line->log_warning("Please use AUTO_MKDIRS instead of " . (($state == SCST_MKDIR) ? "\${MKDIR}" : "\${INSTALL} -d") diff --git a/pkgtools/pkglint4/files/doc/Makefile b/pkgtools/pkglint4/files/doc/Makefile deleted file mode 100644 index 049a20531c0..00000000000 --- a/pkgtools/pkglint4/files/doc/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -# $NetBSD: Makefile,v 1.1 2015/11/25 16:42:21 rillig Exp $ -# - -XMLDOCS+= pkglint.xml -XMLDOCS+= chap.intro.xml -XMLDOCS+= chap.defs.xml -XMLDOCS+= chap.types.xml -XMLDOCS+= chap.code.xml -XMLDOCS+= chap.statemachines.xml -XMLDOCS+= chap.future.xml - -IMAGES+= statemachine.patch.png -IMAGES+= statemachine.shellcmd.png - -.PHONY: all -all: pkglint.html - -pkglint.html: ${XMLDOCS} ${IMAGES} stylesheet.xsl - xmlto -m stylesheet.xsl html-nochunks pkglint.xml - -.PHONY: clean -clean: - rm -f *.html *.png - -.SUFFIXES: .dia .png -.dia.png: - dia -e ${.TARGET:Q} -t png ${.IMPSRC:Q} diff --git a/pkgtools/pkglint4/files/doc/chap.code.xml b/pkgtools/pkglint4/files/doc/chap.code.xml deleted file mode 100644 index 44b02f9f128..00000000000 --- a/pkgtools/pkglint4/files/doc/chap.code.xml +++ /dev/null @@ -1,307 +0,0 @@ -<!-- $NetBSD: chap.code.xml,v 1.1 2015/11/25 16:42:21 rillig Exp $ --> - -<chapter id="code"> -<title>Code structure</title> - - <para>In this chapter, I give an overview of how the &pkglint; - code is organized, starting with the <function>main</function> - function, passing the functions that check a single line and - finally arriving at the infrastructure that makes writing the - other functions easier.</para> - -<sect1 id="code.overview"> -<title>Overview</title> - - <para>The &pkglint; code is structured in modular, easy to - understand procedures. These procedures can be further - classified with respect to what they do. There are procedures - that check a file, others check the lines of a file, again - others check a single line. These classes of procedures are - described in the following sections in a top-down - fashion.</para> - - <para>If nothing special is said about which procedures call - which others, you may assume that procedures of a certain rank - only call procedures that are of a strictly lower rank. For - example, no <function>checkline_*</function> will ever call - <function>checkfile_*</function>. Sometimes, functions of the - same rank are called, but these cases are documented - explicitly.</para> - -</sect1> - -<sect1 id="code.select"> -<title>Selecting the proper checking function</title> - - <para>The <function>main</function> procedure of &pkglint; is a - simple loop around a TODO list containing pathnames of items (I - couldn't think of a better name here). The decision of which - checks to apply to a given item is done in - <function>checkitem</function>, which checks whether the item is - a file or a directory and dispatches the actual checking to - specialized procedures.</para> - -</sect1> - -<sect1 id="code.dir"> -<title>Checking a directory</title> - - <para>The procedures that check a directory are - <function>checkdir_root</function> for the pkgsrc root - directory, <function>checkdir_category</function> for a category - of packages and <function>checkdir_package</function> for a - single package.</para> - -</sect1> - -<sect1 id="code.file"> -<title>Checking a file</title> - - <para>Since the dispatching for files requires much code, it has - been put into a separate procedure called - <function>checkfile</function>, which further dispatches the - call to the other procedures.</para> - - <para>The procedures that check a specific file are - <function>checkfile_ALTERNATIVES</function>, - <function>checkfile_DESCR</function>, - <function>checkfile_distinfo</function>, - <function>checkfile_extra</function>, - <function>checkfile_INSTALL</function>, - <function>checkfile_MESSAGE</function>, - <function>checkfile_mk</function>, - <function>checkfile_patch</function> and - <function>checkfile_PLIST</function>. For most of the - procedures, it should be obvious to which files they are - applied. A distinction is made between buildlink3 files and - other <filename>Makefiles</filename>, as some additional checks - apply to buildlink3 files. Of course, these procedures use - pretty much the same code for checking, and this is where the - <function>checklines_*</function> functions step in.</para> - - <para>The <function>checkfile_package_Makefile</function> - function is somewhat special in that it expects four parameters - instead of only one. This is because loading the package data - has been separated from the actual checking.</para> - -</sect1> - -<sect1 id="code.lines"> -<title>Checking the lines in a file</title> - - <para>This class of procedures consists of - <function>checklines_trailing_empty_lines</function>, - <function>checklines_package_Makefile_varorder</function> and - <function>checklines_mk</function>. The middle one is too - complex to be included in - <function>checkfile_package_Makefile</function>, and the other - ones are of so generic use that they deserved to be procedures - of their own.</para> - - <para>The <function>checklines_mk</function> makes heavy use of - the various <function>checkline_*</function> functions that are - explained in the next chapter.</para> - -</sect1> - -<sect1 id="code.line"> -<title>Checking a single line in a file</title> - - <para>This class of procedures checks a single line of a file. - The number of parameters differs for most of these procedures, - as some need more context information and others don't.</para> - - <para>The procedures that are applicable to any file type are - <function>checkline_length</function>, - <function>checkline_valid_characters</function>, - <function>checkline_valid_characters_in_variable</function>, - <function>checkline_trailing_whitespace</function>, - <function>checkline_rcsid_regex</function>, - <function>checkline_rcsid</function>, - <function>checkline_relative_path</function>, - <function>checkline_relative_pkgdir</function>, - <function>checkline_spellcheck</function> and - <function>checkline_cpp_macro_names</function>.</para> - - <para>The rest of the procedures is specific to - <filename>Makefile</filename>s: - <function>checkline_mk_text</function>, - <function>checkline_mk_shellword</function>, - <function>checkline_mk_shelltext</function>, - <function>checkline_mk_shellcmd</function>, - <function>checkline_mk_vartype_basic</function>, - <function>checkline_mk_vartype_basic</function>, - <function>checkline_mk_vartype</function> and - <function>checkline_mk_varassign</function>.</para> - - <para>This class of procedures contains the most code in - &pkglint;. The procedures that check shell commands and shell - words both have around 200 lines, and the largest procedure is - the check for predefined variable types, which has almost 500 - lines. But the code is not complex at all, since this procedure - contains a large switch for all the predefined types. The checks - for a single type usually fit on a single screen.</para> - -</sect1> - -<sect1 id="code.infrastructure"> -<title>The &pkglint; infrastructure</title> - - <para>To keep the code in the checking procedures small and - legible, an additional layer of procedures is needed that - provides basic operations and abstractions for handling files as - a collection of lines and to print all diagnostics in a common - format that is suitable to further processing by software - tools.</para> - - <para>Since October 2004, this part of &pkglint; makes use of - some of the object oriented features of the Perl programming - language. It has worked quite well upto now, but it has not been - fun to write object-oriented code in Perl. The most basic - feature I am missing is that the compiler checks whether an - object has a specific method or not, as I have often written - <code>$line->warning()</code> instead of - <code>$line->log_warning()</code>. This makes refacturing quite - difficult if you don't have a 100 % coverage test, and I - don't have that.</para> - - <para>The classes are all defined in the - <varname>PkgLint</varname> namespace.</para> - - <para>The traditional class is <classname>Line</classname>, - which represents a logical line of a file. In case of - <filename>Makefile</filename>s, line continuations are parsed - properly and combined into a single line. For all other files, - each logical line corresponds to a physical line. The - <classname>Line</classname> class has accessor methods to its - fields <methodname>fname</methodname>, - <methodname>lines</methodname> and - <methodname>text</methodname>. It also has the methods - <methodname>log_fatal</methodname>, - <methodname>log_error</methodname>, - <methodname>log_warning</methodname>, - <methodname>log_info</methodname> and - <methodname>log_debug</methodname> that all have one parameter, - the diagnostics message. The other methods are used less - often.</para> - - <para>In January 2006, the logging has been improved in - functionality. Before that, a logical line could well consist of - 300 physical lines, so a diagnostic would say <quote>you have a - bug somewhere between line 100 and 400</quote>. This is not - helpful. Therefore, a new class has been invented that allows to - map each character of a logical line to its corresponding - physical location in the file. The new representation of a - logical line is called a <classname>String</classname>. This - feature is still experimental, since the only method for logging - a string is <methodname>log_warning</methodname>. The others are - still missing. It is also completely unclear how lines that have - been fixed by &pkglint; are represented since this moves - characters around in the physical lines.</para> - - <para>To make pattern matching with the new - <classname>String</classname> easy to use, the additional class - <classname>StringMatch</classname> has been created. It saves - the result of a <classname>String</classname> that is matched - against a regular expression. The canonical way to get such a - <classname>StringMatch</classname> is to call the - <methodname>String::match</methodname> method.</para> - - <para>Since the <classname>StringMatch</classname> was - convenient to use, the <classname>SimpleMatch</classname> class - represents the result of matching a Perl string against a - regular expression. The class <classname>Location</classname> is - currently unused.</para> - -</sect1> -<sect1 id="code.style"> -<title>Perl programming style</title> - - <para>The &pkglint; source code has evolved from FreeBSD's portlint, - which has been written in Perl, and up to now, &pkglint; is written - in Perl. Since one of the main ingredients to &pkglint; are regular - expressions, this choice seems natural, and indeed the Perl regular - expressions are a great help to keep the code short. But &pkglint; - is more than just throwing regular expressions at the package - files.</para> - - <para>In 2004, when the &pkglint; source code comprised about - 40 kilobytes, this was quite appropriate. Since then, the code - has become much more structured and various abstraction layers have - been inserted. It became more and more clear that the Perl - programming language has not been designed with nice-looking source - code in mind.</para> - - <para>The first example are subroutines and their parameters. In - most other languages, the names of the parameters are mentioned in - the subroutine definition. Not so in Perl. The parameters to each - subroutine are passed in the <literal>@_</literal> array. The usual - way to get named parameters is to write assign the parameter array - to a list of local variables. This extra statement is a nuisance, - but it is merely syntactical.</para> - - <para>More serious is the way the arguments are passed to a - subroutine. Perl allows the programmer to define subroutines with a - weak form of prototypes, which helps to catch calls to subroutines - that provide a wrong number of arguments. This feature catches many - bugs that are easily overlooked. The downside is that anything - besides using scalars as parameter types is difficult to understand - and quickly leads to unexpected behavior. Therefore the subroutines - in &pkglint; only use this style for parameter passing. Oh, and by - the way, the subroutine prototypes are only checked for in certain - situations like direct calls. In method calls, nothing is checked at - all. Since almost all diagnostics are produced by calling - <code>$line->log_warning()</code> or - <code>$line->log_error()</code>, most of the subroutine calls in - &pkglint; go unchecked.</para> - - <para>Instead of using magic numbers, well written code defines - named constants for these numbers and then refers to them using - their names, giving the reader extra information that plain numbers - could not give. Although the constant definitions look quite good in - &pkglint; there is one big caveat. The Perl programming language - does not know constants. So these definitions are rather shortcuts - for defining functions that return the value of the constant. And as - functions in Perl have package-wide scope, so have these constants. - This is why the namespace prefixes like <varname>SWST_</varname> are - necessary to avoid name clashes.</para> - - <para>Most of the constants would be written as an enumeration data - type if Perl had one. The same limitation applies for many of the - classes (implemented as packages in Perl) that are simply structs. - The typical Perl implementation of structs are classes, er, packages - which then use methods for accessing the fields. Again, the names of - these methods are only checked at runtime, so there is no language - support for detecting spelling mistakes in field names.</para> - - <para>Another area where Perl fails to detect many errors is the - loose type system. You can apply almost every operator to almost - every data type, and the Perl language will give you more or less - what you want. Especially it does not prevent you from matching - a regular expression against a reference. It will simply compute - a string representation of the reference and match the regular - expression against that.</para> - - <para>The current Perl interpreter is very inefficient when - copying strings. This happens really often in pkglint, for - example when passing arguments to functions or saving the result - of a regular expression match in <quote>real</quote> variables. - For a great speed-up, an implementation that handles string - objects by reference-counting them would be better. (Lua comes - to mind.)</para> - -</sect1> -<sect1 id="code.lang"> -<title>Switching to another language</title> - - <para>Switching to C++ is not an option, since the typing - overhead would be more than twice the current amount. As a - consequence the code would become much less readable.</para> - - <para>Switching to OCaml looks nice (because of the type - inference), but the regular expressions that are provided by the - system are by no means sufficient. On the other hand, since - today there is a PCRE package for OCaml in pkgsrc.</para> - -</sect1> -</chapter> diff --git a/pkgtools/pkglint4/files/doc/chap.defs.xml b/pkgtools/pkglint4/files/doc/chap.defs.xml deleted file mode 100644 index 1bbee6f911a..00000000000 --- a/pkgtools/pkglint4/files/doc/chap.defs.xml +++ /dev/null @@ -1,26 +0,0 @@ -<!-- $NetBSD: chap.defs.xml,v 1.1 2015/11/25 16:42:21 rillig Exp $ --> -<!-- don't include useless definitions. -<chapter id="defs"> -<title>Definitions</title> - - <para>In every non-toy program, the need arises to define new - words or redefine and clarify existing words. This is the list - of words that are used in pkglint.</para> - - <variablelist> - - <varlistentry><term>function</term><listitem><para>A subroutine - that is called to obtain a return value, rather than for its - side effects. Functions should restrict the user-visible side - effects to the necessary minimum.</para> - </listitem></varlistentry> - - <varlistentry><term>procedure</term><listitem><para>A subroutine - that is not called to obtain a return value, but rather called - because of its side effects, like input/output.</para> - </listitem></varlistentry> - - </variablelist> - -</chapter> ---> diff --git a/pkgtools/pkglint4/files/doc/chap.design.xml b/pkgtools/pkglint4/files/doc/chap.design.xml deleted file mode 100644 index 15ef2b844d9..00000000000 --- a/pkgtools/pkglint4/files/doc/chap.design.xml +++ /dev/null @@ -1,117 +0,0 @@ -<!-- $NetBSD: chap.design.xml,v 1.2 2017/10/02 14:41:21 wiz Exp $ --> - -<chapter id="design"> -<title>Design goals</title> - - <para>&pkglint; should be simple to use. It should be consistent - and predictable in what it does. The diagnostics should be - understandable. The number of false positive and false negative - diagnostics should be minimal.</para> - -<sect1 id="design.simple-to-use"> -<title>Simple to use</title> - - <para><emphasis>Requirement:</emphasis> Using &pkglint; should - not require any knowledge about obscure command line options or - hidden features.</para> - - <para>Calling &pkglint; without options gives a useful amount of - warnings. No further knowledge is needed. Users that are - accustomed to GNU software will quickly find the - <literal>--help</literal> command line option, which gives a - quite verbose description of the available options. Users that - know the GNU compilers will easily remember the - <literal>-W</literal> class of options, especially - <literal>-Wall</literal>. Other than with the GNU compilers, the - latter option enables really <emphasis>all</emphasis> warnings - that are intended to be user-visible.</para> - - <para>The command line options come in two flavors: short and - long options. The long options are meant to be used when - explaining them to others, while the short options are meant to - be used when invoking &pkglint; in an interactive shell.</para> - -</sect1> -<sect1 id="design.consistent-and-predictable"> -<title>Consistent and predictable</title> - - <para><emphasis>Requirement:</emphasis> &pkglint; should behave - such that the user quickly gets an impression about what - &pkglint; does. This impression should be persistent, that is, - the output format for diagnostics should be stable over time, - and diagnostic messages should not be changed without - reason.</para> - - <para>There are only two cases of what the output of - &pkglint; is. One is a single line containing the text - <quote>Looks fine.</quote>, the other is a list of diagnostics, - followed by a summary on the number of diagnostics.</para> - - <para>If no warnings are printed, the single line <quote>looks - fine.</quote> gives a little motivation to the user. This - message is one of the few things that have been kept in - &pkglint; since it has been adopted from FreeBSD. It just makes - pkglint a more friendly tool. :)</para> - - <para>All error and warning messages are formatted by a single - procedure, <function>PkgLint::Logging::log_message</function>. This - way, all messages are formatted the same way, which allows easy - recognition by human users as well as other tools. There are two - different formats available, the traditional one and the gcc-like - one. In both formats, each diagnostic occupies exactly one line. Up - to the year 2005, some of the longer messages used to take more than - one line, but this behavior has been removed.</para> - - <para>The default format is the traditional one. It consists of the - severity, in upper-case letters, followed by the filename, the line - number and finally the message text. It allows easy recognition of - the severity of the messages. Even if errors and warnings are - intermixed in the output, the filenames start almost in the same - column.</para> - - <para>The gcc-like output format consists of the filename, the line - numbers, the severity and finally the message text. It has been - added to make it easier to integrate the &pkglint; diagnostics into - various text editors, for example Emacs. Since in this format the - filename is the first word, it can be easily seen which warning - originates in which file.</para> - - <para>There are some other procedures that affect the output, but - they have to be enabled explicitly.</para> - -</sect1> -<sect1 id="design.understandable"> -<title>Understandable diagnostics</title> - - <para><emphasis>Requirement:</emphasis> The diagnostics are - intended to help the user in writing better package definitions. - They should use an unambiguous, clear language and point - directly to the problem. If possible, they should include a hint - on how the problem can be fixed properly.</para> - -</sect1> -<sect1 id="design.false-diagnostics"> -<title>Few false diagnostics</title> - - <para><emphasis>Requirement:</emphasis> The number of - <firstterm>false positives</firstterm>, that is diagnostics - where no problem actually exists, should be minimal. On the - other hand, &pkglint; should detect as many problems as - possible. If it fails to detect a problem, this is called a - <firstterm>false negative</firstterm>.</para> - - <para>Currently, there are very few false positives. The way - &pkglint; parses the files is already close to the way - <command>make</command> and <command>sh</command> parse them, so - the structure of the files is modelled quite well.</para> - - <para>Since &pkglint; is also very strict in what it accepts, - many problems can already be detected. But since the pkgsrc - developers are quite creative when it comes to solving problems, - &pkglint; cannot detect everything. After all, the language used - to define packages is turing-complete, so it cannot be decided - in every case whether a package is valid or not. Luckily, most - packages are quite simple.</para> - -</sect1> -</chapter> diff --git a/pkgtools/pkglint4/files/doc/chap.future.xml b/pkgtools/pkglint4/files/doc/chap.future.xml deleted file mode 100644 index e76cd43c4f6..00000000000 --- a/pkgtools/pkglint4/files/doc/chap.future.xml +++ /dev/null @@ -1,91 +0,0 @@ -<!-- $NetBSD: chap.future.xml,v 1.1 2015/11/25 16:42:21 rillig Exp $ --> - -<chapter id="future"> -<title>Future directions</title> - -<sect1 id="future.tokenize"> -<title>Tokenizing the input</title> - - <para>For providing more exact diagnostics, it would be nice if - &pkglint; could point the user to the exact character position - of the smallest problematic text in a file. To do this, the - file's contents has to be splitted into tokens.</para> - - <para>Doing this is nontrivial, since the tokenizing scheme - depends on the context in which the tokens are used. For - example, the <varname>COMMENT</varname> variable may contain - arbitrary characters (including <literal>'</literal> and - <literal>"</literal>), whereas in many other contexts these are - parts of quoted shell words.</para> - -</sect1> -<sect1 id="future.ast"> -<title>Working on abstract syntax trees (AST)</title> - - <para>When the tokenizing above is done, the tokens could be - parsed by a grammar to form abstract syntax trees. These would - consist mainly of function application so that pkglint can infer - types and valid values over these trees. The following functions - are likely to appear.</para> - - <table id="future.ast.func"> - <title>Functions in the abstract syntax trees</title> - <tgroup cols="2"> - <thead><row><entry>Function</entry><entry>Purpose</entry></row></thead> - <tbody> - <row><entry><function>quote</function>(Val)</entry><entry>The <literal>:Q</literal> modifier</entry></row> - <row><entry><function>append</function>(Val, Val)</entry><entry>The <literal>+=</literal> operator</entry></row> - <row><entry><function>concat</function>(Val, Val)</entry><entry>The direct concatenation of two values</entry></row> - <row><entry><function>subst</function>(Val, Subst)</entry><entry>The <literal>:S</literal> and <literal>:C</literal> modifiers</entry></row> - <row><entry><function>shell</function>(Val)</entry><entry>The <literal>!=</literal> operator and the <literal>:sh</literal> modifier</entry></row> - <row><entry><function>literal</function>(Val)</entry><entry>Introduces literal values</entry></row> - </tbody> - </tgroup> - </table> - - <para>Examples:</para> - -<programlisting> - WRKSRC= ${WRKDIR} - SUBST_SED.pkglint+= -e s\|@DATADIR@\|${PREFIX:Q}/share/pkglint\|g -</programlisting> - - <para>The first line would be parsed as - <literal>assign(var("WRKSRC"), varuse("WRKDIR"))</literal>. The - second line would be parsed as - <literal>assign(var("SUBST_SED.pkglint"), - append(varuse("SUBST_SED.pkglint"), concat(concat(str("-e - s\\|@DATADIR@\\|"), quote(varuse("PREFIX"))), - str("/share/pkglint\\|g"))))</literal>.</para> - - <para>At this point, unification together with a pattern matcher - on tree structures would come in handy, to allow the parser for - the shell commands to still operate on this parse tree. This - might eventually enable cross-language type inference.</para> - -</sect1> -<sect1 id="future.vars"> -<title>Even more restricted variables</title> - - <para>Currently there are mainly two restrictions for variables: - What values they may contain (data types) and where they may be - defined and used, on a per-file basis.</para> - - <para>The <filename>makevars.map</filename> file already - contains annotations to distinguish user-defined from - system-defined variables, but they are currently only used as - abbreviations and not further exploited. Based on these - definitions, sequence points may be defined in the pkgsrc - infrastructure where the values of these variables must have - certain properties, like being defined or being fixed (which - means that the variable will not change further).</para> - - <para>For example, user-defined variables may then be specified - as follows. They are given default values in - <filename>mk/defaults/mk.conf</filename>, may be overridden by - any file that is included inside <varname>MAKECONF</varname>, - and after that, their value is fixed. They may then be used at - both load and run time.</para> - -</sect1> -</chapter> diff --git a/pkgtools/pkglint4/files/doc/chap.intro.xml b/pkgtools/pkglint4/files/doc/chap.intro.xml deleted file mode 100644 index f9c118eff8c..00000000000 --- a/pkgtools/pkglint4/files/doc/chap.intro.xml +++ /dev/null @@ -1,15 +0,0 @@ -<!-- $NetBSD: chap.intro.xml,v 1.1 2015/11/25 16:42:21 rillig Exp $ --> - -<chapter id="intro"> -<title>Introduction</title> - - <para>&pkglint; is a static analysis tool for pkgsrc packages. - It finds many errors and problematic issues in those packages. - Starting in June 2004, &pkglint; has evolved into a powerful - tool that gives precise warnings wherever possible. With that - power comes much additional complexity, which cannot be - understood from reading the source code alone. This document - provides the necessary background information to understand what - the actual code does and why it is done this way.</para> - -</chapter> diff --git a/pkgtools/pkglint4/files/doc/chap.statemachines.xml b/pkgtools/pkglint4/files/doc/chap.statemachines.xml deleted file mode 100644 index 42f5b1792fe..00000000000 --- a/pkgtools/pkglint4/files/doc/chap.statemachines.xml +++ /dev/null @@ -1,77 +0,0 @@ -<!-- $NetBSD: chap.statemachines.xml,v 1.1 2015/11/25 16:42:21 rillig Exp $ --> - -<chapter id="statemachines"> -<title>State machines</title> - - <para>This chapter explains the various state machines that are - used in &pkglint;. It also provides graphical representations of - them that are much easier to read than the source code.</para> - - <para>The opaque arrows in the figures represent transitions - that have a regular expression as condition. The hollow arrows - are the default transitions if nothing else matches. When - multiple regular expressions match in a state, the one that - appears first in the source code is chosen.</para> - -<sect1 id="statemachines.shellword"> -<title>The state machine for shell words</title> - - <para>The state machine for single shell words is pretty simple, - and I think it can be understood from the source code alone. So - no graphical representation is provided.</para> - -</sect1> - -<sect1 id="statemachines.shellcommand"> -<title>The state machine for shell commands</title> - - <figure id="statemachine.shellcommand"> - <title>The state transitions for shell commands</title> - <mediaobject> - <imageobject> - <imagedata fileref="statemachine.shellcmd.png" format="PNG"/> - </imageobject> - <textobject><para>(Here should be a drawing of the state transitions.)</para></textobject> - </mediaobject> - </figure> - - <para>The punch card symbols provide a means to go to a certain - state whenever the input matches the text on the punch - card.</para> - -</sect1> - -<sect1 id="statemachines.patch"> -<title>The state machine for patch files</title> - - <para>The state machine for patch files is the newest of the - state machines. Here, the state transitions are separated from - the code, which makes the code itself pretty small. I don't know - yet if this programming style is elegant or not. Time will - show.</para> - - <figure id="statemachine.patch"> - <title>The state transitions for patch files</title> - <mediaobject> - <imageobject> - <imagedata fileref="statemachine.patch.png" format="PNG"/> - </imageobject> - <textobject><para>(Here should be a drawing of the state transitions.)</para></textobject> - </mediaobject> - </figure> - - <para>The states on the left side are for parsing context diffs, - the ones on the right side are for unified diffs. Some of the - state names are highly abbreviated as follows. The first letter - gives the format of the patch, which is <quote>c</quote> for - context diffs and <quote>u</quote> for unified diffs. The second - letter gives the current syntactical level, which is - <quote>f</quote> for a file header, <quote>h</quote> for a hunk - header, or <quote>l</quote> for the hunk lines. The third letter - describes the action that belongs to the line, which is - <quote>a</quote> for an addition, and <quote>d</quote> for a - deletion.</para> - -</sect1> - -</chapter> diff --git a/pkgtools/pkglint4/files/doc/chap.types.xml b/pkgtools/pkglint4/files/doc/chap.types.xml deleted file mode 100644 index 77f21c6548f..00000000000 --- a/pkgtools/pkglint4/files/doc/chap.types.xml +++ /dev/null @@ -1,545 +0,0 @@ -<!-- $NetBSD: chap.types.xml,v 1.1 2015/11/25 16:42:21 rillig Exp $ --> - -<chapter id="types"> -<title>The &pkglint; type system</title> - - <para>One of the most notable additions to &pkglint; is the - introduction of typed variables. Traditionally, in - <filename>Makefile</filename>s, all variables have the type - <type>String</type>. This prevents many useful checks from being - done before executing the code.</para> - - <para>Up to 2004, &pkglint; already did some checks based on - the value of the variables, but these checks had no common - structure that could be described easily.</para> - -<sect1 id="types.history"> -<title>History</title> - - <para>In February 2005, initial support for the &pkglint; type - system has been added. Some of the common variables have been - assigned types such as <literal><type>Boolean</type></literal> - or <literal><type>Yes_Or_Undefined</type></literal>, which are - the two common ways to represent boolean variables in pkgsrc. - The list of typed variables has been moved from the &pkglint; - code to an external file, <filename>makevars.map</filename>. - Many more basic types have been added later.</para> - - <para>In October 2005, the type system has been extended to - allow <literal><type>List of - <replaceable>simple-type</replaceable></type></literal>, which - allowed to handle variables like <varname>DEPENDS</varname> and - <varname>CFLAGS</varname>. One month later, enumeration types - have been added, allowing the type of - <varname>PTHREAD_OPTS</varname> to be expressed as <literal>List - of { require native }</literal>.</para> - - <para>In May 2006, the definition and use of variables has been - further restricted by introducing ACLs, which define the - permitted operations (write, append, default, read, preprocess-read) - depending on the current file.</para> - -</sect1> - -<sect1 id="types.syntax"> -<title>Syntax for defining types</title> - -<programlisting> - type ::= (list-type)? simple-type (acls)? - - list-type ::= ("List" | "InternalList") "of" - - simple-type ::= predefined-type - | enumeration - predefined-type ::= [A-Za-z][0-9A-Z_a-z]* - enumeration ::= "{" (enumeration-item)* "}" - enumeration-item ::= [-0-9A-Z_a-z]+ - - acls ::= "[" (acl-entry ("," acl-entry)*)? "]" - acl-entry ::= acl-subject ":" acl-perms - acl-subject ::= [.0-9A-Za-z]+ | "_" - acl-perms ::= [adprs]* -</programlisting> - -</sect1> -<sect1 id="types.semantics"> -<title>Semantics of the types</title> - - <para>The <firstterm>simple types</firstterm> in &pkglint; are - either predefined types or enumeration types. A - <firstterm>predefined type</firstterm> is used by its name. See - <xref linkend="types.predefined"/> for the list of predefined - types.</para> - - <para>An expression of an enumeration type may have either of - the enumeration-items as a value. It may not reference other - variables.</para> - - <para>A list type can be constructed from a predefined type or - an enumeration. It is not possible to construct lists of lists, - since I have never needed that. There are two types of lists, - called <literal>List</literal> and - <literal>InternalList</literal>, which are described in the - <ulink url="&pkgsrc-guide;/makefile.html">pkgsrc guide, the - chapter about <filename>Makefile</filename>s</ulink>.</para> - -</sect1> -<sect1 id="types.acls"> -<title>Access Control Lists</title> - - <para>Additionally to the data type, which specifies - <emphasis>what</emphasis> a variable can contain, the ACLs - define <emphasis>where</emphasis> the variable can be defined or - used (this is called the <firstterm>ACL subject</firstterm>) and - which operations are allowed (these are the <firstterm>ACL - permissions</firstterm>).</para> - - <para>The ACL subjects are specified by the filename. For - example, <filename>Makefile</filename> and - <filename>buildlink3.mk</filename> are valid ACL subjects. Since - some names occur over an over in pkgsrc, these can be - abbreviated as shown in <xref linkend="types.acl.subjects.abbr" - />. The character <literal>*</literal> is a placeholder for zero - or more arbitrary characters, like in the shell. The possible - actions on a variable are shown in <xref - linkend="types.acl.perms" />.</para> - - <table id="types.acl.subjects.abbr"> - <title>ACL Subjects</title> - <tgroup cols="2"> - <thead><row><entry>Subject</entry><entry>Abbreviation</entry></row></thead> - <tbody> - <row><entry><filename>Makefile</filename></entry><entry>m</entry></row> - <row><entry><filename>Makefile.common</filename></entry><entry>c</entry></row> - <row><entry><filename>buildlink3.mk</filename></entry><entry>b</entry></row> - <row><entry><filename>hacks.mk</filename></entry><entry>h</entry></row> - <row><entry><filename>options.mk</filename></entry><entry>o</entry></row> - <row><entry>any file</entry><entry>*</entry></row> - </tbody> - </tgroup> - </table> - - <table id="types.acl.perms"> - <title>ACL Permissions</title> - <tgroup cols="2"> - <thead><row><entry>Permission</entry><entry>Description</entry></row></thead> - <tbody> - <row><entry><filename>a</filename></entry><entry>Append to the - variable using the <literal>+=</literal> operator.</entry></row> - <row><entry><filename>d</filename></entry><entry>Provide a - default value for the variable using the <literal>?=</literal> - operator.</entry></row> - <row><entry><filename>s</filename></entry><entry>Set the - variable unconditionally using the <literal>=</literal>, - <literal>:=</literal> or <literal>!=</literal> - operator.</entry></row> - <row><entry><filename>u</filename></entry><entry>Use the value - of the variable.</entry></row> - <row><entry><filename>p</filename></entry><entry>Use the value - of the variable during preprocessing.</entry></row> - </tbody> - </tgroup> - </table> - - <para>If a variable has no ACL definition at all, all operations - are allowed on it. Otherwise exactly those operations of the - first ACL entry whose subject matches the current filename are - allowed. If no entry matches, nothing is allowed.</para> - - <para>For determining if a variable is used in the correct - place, the filename is only one part of the whole decision. The - other one is the context in which the variable appears. There - are many factors that influence whether the variable is used - correctly.</para> - - <itemizedlist> - - <listitem><para>The variable may be either used at preprocessing time - or at runtime. Some variables are defined in - <filename>bsd.pkg.mk</filename> and thus are not available until - that file has been included. As this should always be the very - last thing a package includes, it practically means that these - variables are only available at runtime.</para></listitem> - - <listitem><para>The variable may appear as the whole right hand side - of an assignment, as a single word, or even as part of a word. - First, the types on the right and left side should be - compatible. Second, some variables need to be quoted correctly, - depending on whether they are part of a word or not.</para></listitem> - - <listitem><para>In shell commands, the variable may also appear as a - whole word or as part of a word. This is similar to the case - above.</para></listitem> - - <listitem><para>The variable may appear inside some sort of quotes. - For some variables this is acceptable, as they are assumed to - never contain special characters. For others it - isn't.</para></listitem> - - <listitem><para>The various operators in conditional statements like - <literal>.if</literal> may further restrict the valid values. - For example, the <varname>OPSYS</varname> variable should only - be compared to well-known operating system names. The - <varname>exists()</varname> function should only be called on - pathnames. The <varname>defined()</varname> operator only checks - if the variable is defined and does not access its - value.</para></listitem> - - </itemizedlist> - -<sect2 id="types.acls.future"> -<title>Future Directions</title> - - <para>Currently the ACLs only cover the <quote>user - space</quote> of pkgsrc. They will be extended later to also - check for valid variable definition and use in the pkgsrc - infrastructure, as well as the user configuration file. For - completeness, those variables that are intended to be specified - on the command line will be added to the - <filename>makevars.map</filename> file.</para> - -</sect2> -</sect1> -<sect1 id="types.predefined"> -<title>Predefined types</title> - - <para>There are many predefined types in &pkglint;, which are - described below.</para> - - <!-- reference: pkglint.pl, revision 1.532 --> - <variablelist> - - <varlistentry><term><literal><type>AwkCommand</type></literal></term> - <listitem><para>An awk command. Currently nothing is checked - here.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>BuildlinkDepmethod</type></literal></term> - <listitem><para>Must be either <literal>build</literal> or - <literal>full</literal>.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>BuildlinkDepth</type></literal></term> - <listitem><para>This type is only intended for one variable, - namely <varname>BUILDLINK_DEPTH</varname>, which is only - modified in <filename>buildlink3.mk</filename> - files.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>BuildlinkPackages</type></literal></term> - <listitem><para>The type of the variable - <varname>BUILDLINK_PACKAGES</varname>. Like - <literal><type>BuildlinkDepth</type></literal> above, this is - only used in <filename>buildlink3.mk</filename> files. This - variable has two different patterns to be modified. The first is - to remove the current package from itself, and the second is to - append the current package. This prevents a package from showing - up twice in the list.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>Category</type></literal></term> - <listitem><para>One of the categories that a package may be - placed in. The list of categories has been assembled manually - when the type was introduced. There is no further agreement on - which valid categories are valid, besides the top level - directory names in pkgsrc.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>CFlag</type></literal></term> - <listitem><para>One word in a <varname>CFLAGS</varname> or - <varname>CPPFLAGS</varname> variable. &pkglint; knows the flags - starting with <literal>-D</literal>, <literal>-U</literal>, - <literal>-I</literal>. Flags starting with - <literal>-O</literal>, <literal>-W</literal>, - <literal>-f</literal>, <literal>-g</literal> or - <literal>-m</literal> are silently accepted since they are - commonly used for the GNU compilers. As the pkgsrc framework - does not know how to handle most of these flags, care should be - taken.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>Comment</type></literal></term> - <listitem><para>The comment of a package.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>Dependency</type></literal></term> - <listitem><para>A simple dependency like - <literal>foopkg>=1.0</literal>, <literal>foopkg-[0-9]*</literal> - or <literal>foopkg-1.0</literal>.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>DependencyWithPath</type></literal></term> - <listitem><para>A dependency (see above), followed by a colon - and a relative directory. For some packages, special variables - like <varname>USE_TOOLS</varname> should be used instead of an - explicit dependency.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>DistSuffix</type></literal></term> - <listitem><para>The value of the variable - <varname>EXTRACT_SUFX</varname>. The difference in the name is - intentional here, since <varname>EXTRACT_SUFX</varname> is a - misnomer. <varname>DIST_SUFX</varname> or - <varname>DIST_SUFFIX</varname> would be more appropriate.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>EmulPlatform</type></literal></term> - <listitem><para>An emulated platform consists of the operating - system (in lowercase, as opposed to <type>PlatformTriple</type>) - and the hardware architecture.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>Filename</type></literal></term> - <listitem><para>A filename, as defined in <ulink - url="http://www.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap03.html#tag_03_169">POSIX</ulink>. - This type further restricts the set of allowed characters. - See also <literal><type>Pathname</type></literal>.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>Filemask</type></literal></term> - <listitem><para>A shell globbing pattern that does not contain a - slash. See also <literal><type>Pathmask</type></literal>.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>Identifier</type></literal></term> - <listitem><para>In various places in pkgsrc, identifiers are - used. This type collects the most common naming conventions. - When you need a more specific check, you have to write your own - check.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>LdFlag</type></literal></term> - <listitem><para>A flag that is passed to the linker. Flags - starting with <literal>-L</literal> or <literal>-l</literal> are - accepted, as well as some others that are assumed to be handled - by the wrapper framework.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>Mail_Address</type></literal></term> - <listitem><para>Checks for a very restricted subset of <ulink - url="http://www.ietf.org/rfc/rfc2822.txt">RFC - 2822</ulink>.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>Message</type></literal></term> - <listitem><para>Messages are printed to the user as status - indicators. <ulink - url="http://www.freebsd.org/cgi/cvsweb.cgi/ports/devel/portlint/src/portlint.pl#rev1.77">As - opposed to FreeBSD</ulink>, they should not be quoted since they - may be used in contexts where quoting should be done - differently.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>Option</type></literal></term> - <listitem><para>An option from the - <literal>PKG_OPTIONS</literal> framework. Options should not - contain underscores. They should be documented in - <filename>pkgsrc/mk/defaults/options.description</filename>.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>Pathlist</type></literal></term> - <listitem><para>A list of directories that are separated by - colons, like the popular environment variable - <varname>PATH</varname>. This type differs from the type - <literal><type>List of Pathname</type></literal> in the - character that is used as a separator.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>Pathmask</type></literal></term> - <listitem><para>A shell globbing expression that may include - slashes.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>Pathname</type></literal></term> - <listitem><para>A pathname, as defined in <ulink - url="http://www.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap03.html#tag_03_266">POSIX</ulink>. - See also <literal><type>Filename</type></literal>.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>Perl5Packlist</type></literal></term> - <listitem><para>A common error has been to refer to - <varname>INSTALLARCHLIB</varname> in the location of the packing - list. Therefore no references to other variables are - allowed.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>PkgName</type></literal></term> - <listitem><para>A package name should conform to some - restrictions, since the filename of the binary package is - created from it, which is then interpreted by pkg_add and the - like.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>PkgOptionsVar</type></literal></term> - <listitem><para>I had once made the mistake of referencing - <varname>PKGBASE</varname> in this variable, not knowing that - <varname>PKG_OPTIONS_VAR</varname> is used during preprocessing, - when <varname>PKGBASE</varname> is not yet defined. This type - prevent that mistake from being done again.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>PkgPath</type></literal></term> - <listitem><para>A directory name that is relative to the - top-level pkgsrc directory. This is only used in specifying - specific packages in bulk builds. Despite its name, this type is - more similar to <type>RelativePkgDir</type> than to - <type>RelativePkgPath</type>.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>PkgRevision</type></literal></term> - <listitem><para>The package revision must be a small integer. - The only place where this definition may occur is the package - <filename>Makefile</filename> itself, as this variable says - something about the individual package. There is no mechanism in - pkgsrc for something similar to <varname>PKGREVISION</varname> - that can be used in <filename>Makefile.common</filename> - files.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>PlatformTriple</type></literal></term> - <listitem><para>pkgsrc has been ported to many platforms, all of - which are identified using a triple of operating system, - operating system version and hardware - architecture.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>Readonly</type></literal></term> - <listitem><para>This type is used to mark a variable as being - read-only to a package author. As this is not really a data type - but an access restriction, it will disappear in the next version - of the type system.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>RelativePkgDir</type></literal></term> - <listitem><para>A directory name that is relative to the package - directory. Mostly used for dependencies. See also - <literal><type>RelativePkgPath</type></literal>.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>RelativePkgPath</type></literal></term> - <listitem><para>A pathname that is relative to the package - directory. It may point to either a regular file or a directory. - See also <literal><type>RelativePkgDir</type></literal>.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>ShellCommand</type></literal></term> - <listitem><para>A shell command is similar to a - <literal><type>List of ShellWord</type></literal>, except that - additional checks are performed on the direct use of tool names - or certain other deprecated shell commands.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>ShellWord</type></literal></term> - <listitem><para>A shell word is what the shell would regard as a - single word.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>Stage</type></literal></term> - <listitem><para>In pkgsrc, there are phases, stages and steps. - Especially for the <varname>SUBST_STAGE</varname> variable, this - should always be one of the few predefined names, otherwise the - whole substitution group will be ignored.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>Tool</type></literal></term> - <listitem><para>The pkgsrc tools framework contains very few - plausibility checks. To prevent spelling mistakes, the list of - valid tool names is loaded from the pkgsrc infrastructure files - and compared with the names that are used in the - <varname>USE_TOOLS</varname> variable.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>URL</type></literal></term> - <listitem><para>URLs appear in <varname>MASTER_SITES</varname> - and the <varname>HOMEPAGE</varname>. If a - <varname>MASTER_SITES</varname> group exists for a given URL, it - should be used instead of listing the URL directly.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>UserGroupName</type></literal></term> - <listitem><para>User and group names should consist only of - alphanumeric characters and the underscore. This restriction - ensures maximum portability of pkgsrc.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>Userdefined</type></literal></term> - <listitem><para>Another instance of misuse of the type system. - But it helps to catch some errors in packages. This type will - disappear in the next version of the type system. See also - <literal><type>Readonly</type></literal>.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>Varname</type></literal></term> - <listitem><para>Variable names are restricted to only uppercase - letters and the underscore in the basename, and arbitrary - characters in the parameterized part, following the dot.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>WrkdirSubdirectory</type></literal></term> - <listitem><para>The variable <varname>WRKSRC</varname> is - usually defined with reference to <varname>WRKDIR</varname>. - This check currently does nothing, and I don't know if it's - worth to check anything here.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>WrksrcSubdirectory</type></literal></term> - <listitem><para>Subdirectories of <varname>WRKSRC</varname> can - be used in <varname>CONFIGURE_DIRS</varname> and some other - variables. For convenience, they are interpreted relative to - <varname>WRKSRC</varname>, so package authors don't have to type - <literal>${WRKSRC}</literal> all the time.</para> - </listitem></varlistentry> - - <varlistentry><term><literal><type>Yes</type></literal></term> - <listitem><para>This type is used for variables that are checked - using <literal>defined(VARNAME)</literal>. Their value is - interpreted as <quote>true</quote> if they are defined, no - matter if they are set to <literal>yes</literal> or - <literal>no</literal>.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>YesNo</type></literal></term> - <listitem><para>This type is used for variables that are checked - using <literal>defined(VARNAME) && - !empty(VARNAME:M[Yy][Ee][Ss])</literal>. A value of - <varname>no</varname> means <quote>no</quote> for - them.</para></listitem></varlistentry> - - <varlistentry><term><literal><type>YesNoFromCommand</type></literal></term> - <listitem><para>Like <literal><type>YesNo</type></literal>, but - the value may be produced by a shell command using the - <literal>!=</literal> operator.</para></listitem></varlistentry> - - </variablelist> - -</sect1> -<sect1 id="types.future"> -<title>Future directions</title> - -<sect2 id="types.kinds"> -<title>Different interpretation of the same data types</title> - - <para>As explained above, there are internal lists and external - lists in pkgsrc. But that is not the only attribute that a list - can have. They also differ in the way they are defined, which - files may access them, and what it means to append to append a - value to it.</para> - - <para>For example, <varname>NOT_FOR_PLATFORM</varname> is a list - that every file may append to without leading to unexpected - behavior. Compare this with - <varname>ONLY_FOR_PLATFORM</varname>, which should only be set - in a single place throughout pkgsrc. Let's say in the package - <filename>Makefile</filename> it is set to - <literal>NetBSD-*-*</literal>, because this file's author knows - for sure that the package is only usable on NetBSD. Now when - some <filename>*.mk</filename> file from a dependency package - adds <literal>DragonFly-*-*</literal> to it, the intent of the - package <filename>Makefile</filename> is undermined by the - dependency package, because now it is possible to build the - package on DragonFly, too.</para> - - <para>The same problem arises with the various variables that - can be either <literal>yes</literal> or undefined. They should - always be chosen so that two definitions in different files - don't undermine each other. A good example is - <varname>USE_LIBTOOL</varname>, a bad example is - <varname>NO_BUILD</varname>.</para> - - <para>TODO: What are the general properties of - <quote>good</quote> and <quote>bad</quote> variables? How can it - be decided of which kind a certain variable is?</para> - - <para>For most lists, the only valid operation is to append - something at the end. Therefore it is good practice to warn if a - list is assigned using another operator that - <literal>+=</literal>. For <varname>SUBST_CLASSES</varname> this - fits perfectly. But for <varname>SUBST_FILES.*</varname> it - doesn't. Usually all occurences of a - <varname>SUBST_FILES.*</varname> variable occur in the same - file, and there should be no other file modifying these - variables. Therefore it is better to use the - <literal>=</literal> operator for the first of the - assignments.</para> - -</sect2> -</sect1> -</chapter> diff --git a/pkgtools/pkglint4/files/doc/pkglint.xml b/pkgtools/pkglint4/files/doc/pkglint.xml deleted file mode 100644 index d49282e747d..00000000000 --- a/pkgtools/pkglint4/files/doc/pkglint.xml +++ /dev/null @@ -1,38 +0,0 @@ -<?xml version="1.0" encoding="iso-8859-1"?> -<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.4//EN" - "http://www.oasis-open.org/docbook/xml/4.4/docbookx.dtd" -[ - <!ENTITY pkglint "<literal>pkglint</literal>"> - <!ENTITY pkgsrc-guide "http://www.NetBSD.org/docs/pkgsrc"> - - <!ENTITY chap.intro SYSTEM "chap.intro.xml"> - <!ENTITY chap.defs SYSTEM "chap.defs.xml"> - <!ENTITY chap.design SYSTEM "chap.design.xml"> - <!ENTITY chap.types SYSTEM "chap.types.xml"> - <!ENTITY chap.code SYSTEM "chap.code.xml"> - <!ENTITY chap.statemachines SYSTEM "chap.statemachines.xml"> - <!ENTITY chap.future SYSTEM "chap.future.xml"> -]> - -<!-- $NetBSD: pkglint.xml,v 1.1 2015/11/25 16:42:21 rillig Exp $ --> - -<book> -<title>Design and implementation of &pkglint;</title> - -<bookinfo> -<author> - <firstname>Roland</firstname> - <surname>Illig</surname> - <email>rillig@NetBSD.org</email> -</author> -</bookinfo> - -&chap.intro; -&chap.defs; -&chap.design; -&chap.types; -&chap.code; -&chap.statemachines; -&chap.future; - -</book> diff --git a/pkgtools/pkglint4/files/doc/statemachine.patch.dia b/pkgtools/pkglint4/files/doc/statemachine.patch.dia Binary files differdeleted file mode 100644 index bf1aac9919d..00000000000 --- a/pkgtools/pkglint4/files/doc/statemachine.patch.dia +++ /dev/null diff --git a/pkgtools/pkglint4/files/doc/statemachine.shellcmd.dia b/pkgtools/pkglint4/files/doc/statemachine.shellcmd.dia Binary files differdeleted file mode 100644 index 76b7442f210..00000000000 --- a/pkgtools/pkglint4/files/doc/statemachine.shellcmd.dia +++ /dev/null diff --git a/pkgtools/pkglint4/files/doc/stylesheet.xsl b/pkgtools/pkglint4/files/doc/stylesheet.xsl deleted file mode 100644 index 078cb37470d..00000000000 --- a/pkgtools/pkglint4/files/doc/stylesheet.xsl +++ /dev/null @@ -1,6 +0,0 @@ -<!-- $NetBSD: stylesheet.xsl,v 1.1 2015/11/25 16:42:21 rillig Exp $ --> - -<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" - version="1.0"> - <xsl:param name="html.longdesc" select="0"/> -</xsl:stylesheet> diff --git a/pkgtools/pkglint4/files/pkglint.pl b/pkgtools/pkglint4/files/pkglint.pl index 9720d265efd..8f40d923079 100644 --- a/pkgtools/pkglint4/files/pkglint.pl +++ b/pkgtools/pkglint4/files/pkglint.pl @@ -1,5 +1,5 @@ #! @PERL@ -# $NetBSD: pkglint.pl,v 1.6 2017/10/08 23:25:06 rillig Exp $ +# $NetBSD: pkglint.pl,v 1.7 2019/10/06 10:33:34 rillig Exp $ # # pkglint - static analyzer and checker for pkgsrc packages @@ -2247,36 +2247,6 @@ sub warn_about_PLIST_imake_mannewsuffix($) { # Subroutines to check part of a single line. # -sub checkword_absolute_pathname($$) { - my ($line, $word) = @_; - - $opt_debug_trace and $line->log_debug("checkword_absolute_pathname(\"${word}\")"); - - if ($word =~ m"^/dev/(?:null|tty|zero)$") { - # These are defined by POSIX. - - } elsif ($word eq "/bin/sh") { - # This is usually correct, although on Solaris, it's pretty - # feature-crippled. - - } elsif ($word !~ m"/(?:[a-z]|\$[({])") { - # Assume that all pathnames start with a lowercase letter. - - } else { - $line->log_warning("Found absolute pathname: ${word}"); - $line->explain_warning( -"Absolute pathnames are often an indicator for unportable code. As", -"pkgsrc aims to be a portable system, absolute pathnames should be", -"avoided whenever possible.", -"", -"A special variable in this context is \${DESTDIR}, which is used in GNU", -"projects to specify a different directory for installation than what", -"the programs see later when they are executed. Usually it is empty, so", -"if anything after that variable starts with a slash, it is considered", -"an absolute pathname."); - } -} - sub check_unused_licenses() { for my $licensefile (glob("${cwd_pkgsrcdir}/licenses/*")) { @@ -2397,31 +2367,6 @@ sub checkline_rcsid($$) { checkline_rcsid_regex($line, quotemeta($prefix), $prefix); } -sub checkline_mk_absolute_pathname($$) { - my ($line, $text) = @_; - my $abspath; - - $opt_debug_trace and $line->log_debug("checkline_mk_absolute_pathname(${text})"); - - # In the GNU coding standards, DESTDIR is defined as a (usually - # empty) prefix that can be used to install files to a different - # location from what they have been built for. Therefore - # everything following it is considered an absolute pathname. - # Another commonly used context is in assignments like - # "bindir=/bin". - if ($text =~ m"(?:^|\$\{DESTDIR\}|\$\(DESTDIR\)|[\w_]+\s*=\s*)(/(?:[^\"'\`\s]|\"[^\"*]\"|'[^']*'|\`[^\`]*\`)*)") { - my $path = $1; - - if ($path =~ m"^/\w") { - $abspath = $path; - } - } - - if (defined($abspath)) { - checkword_absolute_pathname($line, $abspath); - } -} - sub checkline_relative_path($$$) { my ($line, $path, $must_exist) = @_; my ($res_path); @@ -3242,14 +3187,12 @@ sub checkline_mk_vartype_basic($$$$$$$$) { if ($value_novar !~ m"^[#\-0-9A-Za-z._~+%*?/\[\]]*$") { $line->log_warning("\"${value}\" is not a valid pathname mask."); } - checkline_mk_absolute_pathname($line, $value); }, Pathname => sub { if ($value_novar !~ m"^[#\-0-9A-Za-z._~+%/]*$") { $line->log_warning("\"${value}\" is not a valid pathname."); } - checkline_mk_absolute_pathname($line, $value); }, Perl5Packlist => sub { diff --git a/pkgtools/pkglint4/files/pkglint.t b/pkgtools/pkglint4/files/pkglint.t index 34f07a4d6b4..41ee25772a1 100644 --- a/pkgtools/pkglint4/files/pkglint.t +++ b/pkgtools/pkglint4/files/pkglint.t @@ -1,5 +1,5 @@ #! @PERL@ -# $NetBSD: pkglint.t,v 1.2 2017/10/02 14:41:21 wiz Exp $ +# $NetBSD: pkglint.t,v 1.3 2019/10/06 10:33:34 rillig Exp $ # require 'pkglint.pl'; # so we can test its internals @@ -58,8 +58,8 @@ sub test_program { if (defined $exitcode) { is($ret, $exitcode, qq{exits $exitcode}); } - like($stdout, qr/$stdout_re/sm, qq{stdout matches $stdout_re}); - like($stderr, qr/$stderr_re/sm, qq{stderr matches $stderr_re}); + like($stdout, qr/$stdout_re/sm, qq{stdout $stdout matches $stdout_re}); + like($stderr, qr/$stderr_re/sm, qq{stderr $stderr matches $stderr_re}); # return @results; } @@ -130,40 +130,40 @@ sub test_pkglint_main { @ARGV = ('-h'); test_unit($unit, undef, 0, '^usage: pkglint ', '^$'); - @ARGV = ('..'); + @ARGV = ('@PKGSRCDIR@/pkgtools/pkglint4'); test_unit($unit, undef, 0, '^Looks fine', '^$'); @ARGV = ('.'); - test_unit($unit, undef, 1, '^ERROR:.+how to check', '^$'); + test_unit($unit, undef, 1, '^ERROR:.+outside a pkgsrc tree', '^$'); @ARGV = (); - test_unit($unit, undef, 1, '^ERROR:.+how to check', '^$'); + test_unit($unit, undef, 1, '^ERROR:.+outside a pkgsrc tree', '^$'); @ARGV = ('/does/not/exist'); test_unit($unit, undef, 1, '^ERROR:.+not exist', '^$'); @ARGV = ($ENV{HOME}); - test_unit($unit, undef, 1, '^ERROR:.+outside a pkgsrc', '^$'); + test_unit($unit, undef, 1, '^ERROR:.+outside a pkgsrc tree', '^$'); } sub test_lint_some_reference_packages { my %reference_packages = ( 'devel/syncdir' => { stdout_re => <<EOT, -^ERROR: .*Makefile: Each package must define its LICENSE\. -ERROR: .*patches/patch-aa:[0-9]+: Comment expected\. -2 errors and 0 warnings found\..*\$ +Looks fine\..*\$ EOT stderr_re => undef, - exitcode => 1, + exitcode => 0, }, 'mail/qmail' => { stdout_re => <<EOT, -^WARN: .*Makefile:[0-9]+: USERGROUP_PHASE is defined but not used\. Spelling mistake\\? -0 errors and 1 warnings found\..*\$ +^WARN: .*Makefile:[0-9]+: QMAILPATCHES is defined but not used\..* +.* +ERROR: .*/options.mk:\\d+: Unknown dependency pattern.* +.*\$ EOT stderr_re => undef, - exitcode => 0, + exitcode => 1, }, 'mail/getmail' => { stdout_re => <<EOT, |