diff options
| author | rillig <rillig@pkgsrc.org> | 2018-10-03 22:27:53 +0000 |
|---|---|---|
| committer | rillig <rillig@pkgsrc.org> | 2018-10-03 22:27:53 +0000 |
| commit | fdec192e50af89d39f564b123cd46ae83a9e836e (patch) | |
| tree | 4252f392afbad9b5007a239f913c22ecc5bd6b4f /pkgtools | |
| parent | 4c1a9481bb0841d4e85ba5eb7ad9a8b3d165fc4a (diff) | |
| download | pkgsrc-fdec192e50af89d39f564b123cd46ae83a9e836e.tar.gz | |
pkgtools/pkglint: Update to 5.6.3
Changes since 5.6.2:
* Add check for version patterns 1.5*, which should rather be 1.5.*
* Re-enable check for "set -e" and commands that may silently fail
because of missing error checking
* Lots of internal clean-up and tests
Diffstat (limited to 'pkgtools')
68 files changed, 3933 insertions, 1594 deletions
diff --git a/pkgtools/pkglint/Makefile b/pkgtools/pkglint/Makefile index d82fe57d7ab..f2703f86ec0 100644 --- a/pkgtools/pkglint/Makefile +++ b/pkgtools/pkglint/Makefile @@ -1,6 +1,6 @@ -# $NetBSD: Makefile,v 1.548 2018/09/05 17:56:22 rillig Exp $ +# $NetBSD: Makefile,v 1.549 2018/10/03 22:27:53 rillig Exp $ -PKGNAME= pkglint-5.6.2 +PKGNAME= pkglint-5.6.3 DISTFILES= # none CATEGORIES= pkgtools diff --git a/pkgtools/pkglint/files/alternatives.go b/pkgtools/pkglint/files/alternatives.go index 3f725278061..bc493000dcf 100644 --- a/pkgtools/pkglint/files/alternatives.go +++ b/pkgtools/pkglint/files/alternatives.go @@ -1,9 +1,6 @@ package main -import ( - "netbsd.org/pkglint/regex" - "strings" -) +import "strings" func CheckfileAlternatives(filename string, plistFiles map[string]bool) { lines := Load(filename, NotEmpty|LogErrors) @@ -19,7 +16,7 @@ func CheckfileAlternatives(filename string, plistFiles map[string]bool) { } relImplementation := strings.Replace(implementation, "@PREFIX@/", "", 1) - plistName := regex.Compile(`@(\w+)@`).ReplaceAllString(relImplementation, "${$1}") + plistName := replaceAll(relImplementation, `@(\w+)@`, "${$1}") if !plistFiles[plistName] && !G.Pkg.vars.Defined("ALTERNATIVES_SRC") { if plistName != implementation { line.Errorf("Alternative implementation %q must appear in the PLIST as %q.", implementation, plistName) diff --git a/pkgtools/pkglint/files/alternatives_test.go b/pkgtools/pkglint/files/alternatives_test.go index b5415c1556b..a7e47ddb594 100644 --- a/pkgtools/pkglint/files/alternatives_test.go +++ b/pkgtools/pkglint/files/alternatives_test.go @@ -2,25 +2,30 @@ package main import "gopkg.in/check.v1" -func (s *Suite) Test_Alternatives_PLIST(c *check.C) { +func (s *Suite) Test_CheckfileAlternatives__PLIST(c *check.C) { t := s.Init(c) + t.SetupPackage("category/package") t.Chdir("category/package") - t.SetupFileLines("ALTERNATIVES", + t.CreateFileLines("ALTERNATIVES", "sbin/sendmail @PREFIX@/sbin/sendmail.postfix@POSTFIXVER@", "sbin/sendmail @PREFIX@/sbin/sendmail.exim@EXIMVER@", "bin/echo bin/gnu-echo", "bin/editor bin/vim -e", "invalid") + t.CreateFileLines("PLIST", + PlistRcsID, + "bin/echo", + "bin/vim", + "sbin/sendmail.exim${EXIMVER}") - G.Pkg = NewPackage(".") - G.Pkg.PlistFiles["bin/echo"] = true - G.Pkg.PlistFiles["bin/vim"] = true - G.Pkg.PlistFiles["sbin/sendmail.exim${EXIMVER}"] = true - - CheckfileAlternatives("ALTERNATIVES", G.Pkg.PlistFiles) + G.CheckDirent(".") + // TODO: Remove redundant diagnostics. t.CheckOutputLines( + "NOTE: ALTERNATIVES:1: @PREFIX@/ can be omitted from the file name.", + "NOTE: ALTERNATIVES:2: @PREFIX@/ can be omitted from the file name.", + "ERROR: ALTERNATIVES:5: Invalid ALTERNATIVES line \"invalid\".", "ERROR: ALTERNATIVES:1: Alternative implementation \"@PREFIX@/sbin/sendmail.postfix@POSTFIXVER@\" must appear in the PLIST as \"sbin/sendmail.postfix${POSTFIXVER}\".", "NOTE: ALTERNATIVES:1: @PREFIX@/ can be omitted from the file name.", "NOTE: ALTERNATIVES:2: @PREFIX@/ can be omitted from the file name.", @@ -33,7 +38,7 @@ func (s *Suite) Test_CheckfileAlternatives__empty(c *check.C) { t := s.Init(c) t.Chdir("category/package") - t.SetupFileLines("ALTERNATIVES") + t.CreateFileLines("ALTERNATIVES") G.Pkg = NewPackage(".") diff --git a/pkgtools/pkglint/files/autofix.go b/pkgtools/pkglint/files/autofix.go index 88dce2db593..beb636c1b56 100644 --- a/pkgtools/pkglint/files/autofix.go +++ b/pkgtools/pkglint/files/autofix.go @@ -46,8 +46,9 @@ func NewAutofix(line Line) *Autofix { // If printAutofix or autofix is true, the fix should be done in // memory as far as possible (e.g. changes to the text of the line). // -// If autofix is true, the fix should be done permanently -// (e.g. direct changes to the file system). +// If autofix is true, the fix should be done persistently +// (e.g. direct changes to the file system). Except if the fix only +// affects the current line, then SaveAutofixChanges will do that. func (fix *Autofix) Custom(fixer func(printAutofix, autofix bool)) { if fix.skip() { return @@ -104,7 +105,7 @@ func (fix *Autofix) ReplaceRegex(from regex.Pattern, toText string, howOften int return toText } - if replaced := regex.Compile(from).ReplaceAllStringFunc(rawLine.textnl, replace); replaced != rawLine.textnl { + if replaced := replaceAllFunc(rawLine.textnl, from, replace); replaced != rawLine.textnl { if G.opts.PrintAutofix || G.opts.Autofix { rawLine.textnl = replaced } @@ -134,12 +135,12 @@ func (fix *Autofix) Realign(mkline MkLine, newWidth int) { } for _, rawLine := range fix.lines[1:] { - _, comment, space := regex.Match2(rawLine.textnl, `^(#?)([ \t]*)`) + _, comment, space := match2(rawLine.textnl, `^(#?)([ \t]*)`) width := tabWidth(comment + space) if (oldWidth == 0 || width < oldWidth) && width >= 8 && rawLine.textnl != "\n" { oldWidth = width } - if !regex.Matches(space, `^\t* {0,7}$`) { + if !matches(space, `^\t* {0,7}$`) { normalized = false } } @@ -156,7 +157,7 @@ func (fix *Autofix) Realign(mkline MkLine, newWidth int) { } for _, rawLine := range fix.lines[1:] { - _, comment, oldSpace := regex.Match2(rawLine.textnl, `^(#?)([ \t]*)`) + _, comment, oldSpace := match2(rawLine.textnl, `^(#?)([ \t]*)`) newWidth := tabWidth(oldSpace) - oldWidth + newWidth newSpace := strings.Repeat("\t", newWidth/8) + strings.Repeat(" ", newWidth%8) replaced := strings.Replace(rawLine.textnl, comment+oldSpace, comment+newSpace, 1) @@ -311,6 +312,7 @@ func SaveAutofixChanges(lines []Line) (autofixed bool) { for _, line := range lines { if line.autofix != nil && line.autofix.modified { G.autofixAvailable = true + G.fileCache.Evict(line.Filename) } } return @@ -338,6 +340,7 @@ func SaveAutofixChanges(lines []Line) (autofixed bool) { } for fname := range changed { + G.fileCache.Evict(fname) changedLines := changes[fname] tmpname := fname + ".pkglint.tmp" text := "" diff --git a/pkgtools/pkglint/files/autofix_test.go b/pkgtools/pkglint/files/autofix_test.go index 6e28540fa9f..01987bb3c22 100644 --- a/pkgtools/pkglint/files/autofix_test.go +++ b/pkgtools/pkglint/files/autofix_test.go @@ -7,7 +7,7 @@ import ( "strings" ) -func (s *Suite) Test_Autofix_ReplaceRegex(c *check.C) { +func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix(c *check.C) { t := s.Init(c) t.SetupCommandLine("--show-autofix") @@ -36,7 +36,7 @@ func (s *Suite) Test_Autofix_ReplaceRegex(c *check.C) { "AUTOFIX: ~/Makefile:2: Replacing \"2\" with \"X\".") } -func (s *Suite) Test_Autofix_ReplaceRegex_with_autofix(c *check.C) { +func (s *Suite) Test_Autofix_ReplaceRegex__autofix(c *check.C) { t := s.Init(c) t.SetupCommandLine("--autofix", "--source") @@ -75,7 +75,7 @@ func (s *Suite) Test_Autofix_ReplaceRegex_with_autofix(c *check.C) { "line3") } -func (s *Suite) Test_Autofix_ReplaceRegex_with_show_autofix(c *check.C) { +func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix_and_source(c *check.C) { t := s.Init(c) t.SetupCommandLine("--show-autofix", "--source") @@ -111,11 +111,11 @@ func (s *Suite) Test_Autofix_ReplaceRegex_with_show_autofix(c *check.C) { "+\tYXXXX") } -func (s *Suite) Test_autofix_MkLines(c *check.C) { +func (s *Suite) Test_SaveAutofixChanges(c *check.C) { t := s.Init(c) t.SetupCommandLine("--autofix") - t.SetupFileLines("category/basename/Makefile", + t.CreateFileLines("category/basename/Makefile", "line1 := value1", "line2 := value2", "line3 := value3") @@ -149,6 +149,27 @@ func (s *Suite) Test_autofix_MkLines(c *check.C) { "XXXe3 := value3") } +func (s *Suite) Test_SaveAutofixChanges__no_changes_necessary(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("--autofix") + lines := t.SetupFileLines("DESCR", + "Line 1", + "Line 2") + + fix := lines[0].Autofix() + fix.Warnf("Dummy warning.") + fix.Replace("X", "Y") + fix.Apply() + + // Since nothing has been effectively changed, + // nothing needs to be saved. + SaveAutofixChanges(lines) + + // And therefore, no AUTOFIX action must appear in the log. + t.CheckOutputEmpty() +} + func (s *Suite) Test_Autofix__multiple_modifications(c *check.C) { t := s.Init(c) @@ -255,7 +276,7 @@ func (s *Suite) Test_Autofix__multiple_modifications(c *check.C) { "AUTOFIX: fname:1: Deleting this line.") } -func (s *Suite) Test_Autofix_show_source_code(c *check.C) { +func (s *Suite) Test_Autofix__show_autofix_and_source(c *check.C) { t := s.Init(c) t.SetupCommandLine("--show-autofix", "--source") @@ -319,7 +340,7 @@ func (s *Suite) Test_Autofix_Delete(c *check.C) { // Demonstrates that the --show-autofix option only shows those diagnostics // that would be fixed. -func (s *Suite) Test_Autofix_suppress_unfixable_warnings(c *check.C) { +func (s *Suite) Test_Autofix__suppress_unfixable_warnings(c *check.C) { t := s.Init(c) t.SetupCommandLine("--show-autofix", "--source") @@ -328,14 +349,14 @@ func (s *Suite) Test_Autofix_suppress_unfixable_warnings(c *check.C) { "line2", "line3") - lines[0].Warnf("This warning is not shown since it is not automatically fixed.") + lines[0].Warnf("This warning is not shown since it is not part of a fix.") fix := lines[1].Autofix() fix.Warnf("Something's wrong here.") fix.ReplaceRegex(`.`, "X", -1) fix.Apply() - fix.Warnf("The XXX marks are usually not fixed, use TODO instead.") + fix.Warnf("Since XXX marks are usually not fixed, use TODO instead to draw attention.") fix.Replace("XXX", "TODO") fix.Apply() @@ -351,14 +372,14 @@ func (s *Suite) Test_Autofix_suppress_unfixable_warnings(c *check.C) { "-\tline2", "+\tXXXXX", "", - "WARN: Makefile:2: The XXX marks are usually not fixed, use TODO instead.", + "WARN: Makefile:2: Since XXX marks are usually not fixed, use TODO instead to draw attention.", "AUTOFIX: Makefile:2: Replacing \"XXX\" with \"TODO\".", "-\tline2", "+\tTODOXX") } // If an Autofix doesn't do anything it must not log any diagnostics. -func (s *Suite) Test_Autofix_failed_replace(c *check.C) { +func (s *Suite) Test_Autofix__noop_replace(c *check.C) { t := s.Init(c) line := t.NewLine("Makefile", 14, "Original text") @@ -372,28 +393,9 @@ func (s *Suite) Test_Autofix_failed_replace(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_SaveAutofixChanges(c *check.C) { - t := s.Init(c) - - t.SetupCommandLine("--autofix") - lines := t.SetupFileLines("DESCR", - "Line 1", - "Line 2") - - fix := lines[0].Autofix() - fix.Warnf("Dummy warning.") - fix.Replace("X", "Y") - fix.Apply() - - // Since nothing has been effectively changed, - // nothing needs to be saved. - SaveAutofixChanges(lines) - - // And therefore, no AUTOFIX action must appear in the log. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Autofix_CustomFix(c *check.C) { +// When using Autofix.CustomFix, it is tricky to get all the details right. +// For best results, see the existing examples and the documentation. +func (s *Suite) Test_Autofix_Custom(c *check.C) { t := s.Init(c) lines := t.NewLines("Makefile", diff --git a/pkgtools/pkglint/files/buildlink3_test.go b/pkgtools/pkglint/files/buildlink3_test.go index 78c9c07b407..3b4fe3df1bb 100644 --- a/pkgtools/pkglint/files/buildlink3_test.go +++ b/pkgtools/pkglint/files/buildlink3_test.go @@ -37,14 +37,14 @@ func (s *Suite) Test_ChecklinesBuildlink3Mk(c *check.C) { // Before version 5.3, pkglint wrongly warned here. // The mk/haskell.mk file takes care of constructing the correct PKGNAME, // but pkglint had not looked at that file. -func (s *Suite) Test_ChecklinesBuildlink3Mk_name_mismatch(c *check.C) { +func (s *Suite) Test_ChecklinesBuildlink3Mk__name_mismatch(c *check.C) { t := s.Init(c) - t.SetupVartypes() - G.Pkg = NewPackage(t.File("x11/hs-X11")) - G.Pkg.EffectivePkgbase = "X11" - G.Pkg.EffectivePkgnameLine = t.NewMkLine("Makefile", 3, "DISTNAME=\tX11-1.0") - mklines := t.NewMkLines("buildlink3.mk", + t.SetupCommandLine("-Wall") + t.SetupPackage("x11/hs-X11", + "DISTNAME=\tX11-1.0") + t.Chdir("x11/hs-X11") + t.CreateFileLines("buildlink3.mk", MkRcsID, "", "BUILDLINK_TREE+=\ths-X11", @@ -59,13 +59,14 @@ func (s *Suite) Test_ChecklinesBuildlink3Mk_name_mismatch(c *check.C) { "", "BUILDLINK_TREE+=\t-hs-X11") - ChecklinesBuildlink3Mk(mklines) + G.CheckDirent(".") + // This warning only occurs because pkglint cannot see mk/haskell.mk in this test. t.CheckOutputLines( "ERROR: buildlink3.mk:3: Package name mismatch between \"hs-X11\" in this file and \"X11\" from Makefile:3.") } -func (s *Suite) Test_ChecklinesBuildlink3Mk_name_mismatch_multiple_inclusion(c *check.C) { +func (s *Suite) Test_ChecklinesBuildlink3Mk__name_mismatch_multiple_inclusion(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -88,7 +89,7 @@ func (s *Suite) Test_ChecklinesBuildlink3Mk_name_mismatch_multiple_inclusion(c * "WARN: buildlink3.mk:9: Definition of BUILDLINK_API_DEPENDS is missing.") } -func (s *Suite) Test_ChecklinesBuildlink3Mk_name_mismatch_abi_api(c *check.C) { +func (s *Suite) Test_ChecklinesBuildlink3Mk__name_mismatch_abi_api(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -115,7 +116,7 @@ func (s *Suite) Test_ChecklinesBuildlink3Mk_name_mismatch_abi_api(c *check.C) { "WARN: buildlink3.mk:10: Only buildlink variables for \"hs-X11\", not \"hs-X12\" may be set in this file.") } -func (s *Suite) Test_ChecklinesBuildlink3Mk_abi_api_versions(c *check.C) { +func (s *Suite) Test_ChecklinesBuildlink3Mk__abi_api_versions(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -140,7 +141,7 @@ func (s *Suite) Test_ChecklinesBuildlink3Mk_abi_api_versions(c *check.C) { "WARN: buildlink3.mk:9: ABI version \"1.6.0\" should be at least API version \"1.6.1\" (see line 8).") } -func (s *Suite) Test_ChecklinesBuildlink3Mk_no_BUILDLINK_TREE_at_beginning(c *check.C) { +func (s *Suite) Test_ChecklinesBuildlink3Mk__no_BUILDLINK_TREE_at_beginning(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -164,7 +165,7 @@ func (s *Suite) Test_ChecklinesBuildlink3Mk_no_BUILDLINK_TREE_at_beginning(c *ch "WARN: buildlink3.mk:3: Expected a BUILDLINK_TREE line.") } -func (s *Suite) Test_ChecklinesBuildlink3Mk_no_BUILDLINK_TREE_at_end(c *check.C) { +func (s *Suite) Test_ChecklinesBuildlink3Mk__no_BUILDLINK_TREE_at_end(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -193,7 +194,7 @@ func (s *Suite) Test_ChecklinesBuildlink3Mk_no_BUILDLINK_TREE_at_end(c *check.C) "WARN: buildlink3.mk:15: This line should contain the following text: BUILDLINK_TREE+=\t-hs-X11") } -func (s *Suite) Test_ChecklinesBuildlink3Mk_multiple_inclusion_wrong(c *check.C) { +func (s *Suite) Test_ChecklinesBuildlink3Mk__multiple_inclusion_wrong(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -212,7 +213,7 @@ func (s *Suite) Test_ChecklinesBuildlink3Mk_multiple_inclusion_wrong(c *check.C) "WARN: buildlink3.mk:6: This line should contain the following text: HS_X11_BUILDLINK3_MK:=") } -func (s *Suite) Test_ChecklinesBuildlink3Mk_missing_endif(c *check.C) { +func (s *Suite) Test_ChecklinesBuildlink3Mk__missing_endif(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -230,7 +231,7 @@ func (s *Suite) Test_ChecklinesBuildlink3Mk_missing_endif(c *check.C) { "WARN: buildlink3.mk:EOF: Expected \".endif\".") } -func (s *Suite) Test_ChecklinesBuildlink3Mk_unknown_dependency_patterns(c *check.C) { +func (s *Suite) Test_ChecklinesBuildlink3Mk__unknown_dependency_patterns(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -257,7 +258,7 @@ func (s *Suite) Test_ChecklinesBuildlink3Mk_unknown_dependency_patterns(c *check "WARN: buildlink3.mk:10: Unknown dependency pattern \"hs-X11!=1.6.1.2nb2\".") } -func (s *Suite) Test_ChecklinesBuildlink3Mk_PKGBASE_with_variable(c *check.C) { +func (s *Suite) Test_ChecklinesBuildlink3Mk__PKGBASE_with_variable(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -282,7 +283,7 @@ func (s *Suite) Test_ChecklinesBuildlink3Mk_PKGBASE_with_variable(c *check.C) { "WARN: buildlink3.mk:3: Please use \"py\" instead of \"${PYPKGPREFIX}\" (also in other variables in this file).") } -func (s *Suite) Test_ChecklinesBuildlink3Mk_PKGBASE_with_unknown_variable(c *check.C) { +func (s *Suite) Test_ChecklinesBuildlink3Mk__PKGBASE_with_unknown_variable(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -312,7 +313,7 @@ func (s *Suite) Test_ChecklinesBuildlink3Mk_PKGBASE_with_unknown_variable(c *che // This special exception might have been for backwards-compatibility, // but ideally should be handled like everywhere else. // See MkLineChecker.checkInclude. -func (s *Suite) Test_ChecklinesBuildlink3Mk_indentation(c *check.C) { +func (s *Suite) Test_ChecklinesBuildlink3Mk__indentation(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") diff --git a/pkgtools/pkglint/files/category_test.go b/pkgtools/pkglint/files/category_test.go index 16cb2c656a2..6cbd750ba07 100644 --- a/pkgtools/pkglint/files/category_test.go +++ b/pkgtools/pkglint/files/category_test.go @@ -6,7 +6,7 @@ func (s *Suite) Test_CheckdirCategory__totally_broken(c *check.C) { t := s.Init(c) t.SetupVartypes() - t.SetupFileLines("archivers/Makefile", + t.CreateFileLines("archivers/Makefile", "# $", "SUBDIR+=pkg1", "SUBDIR+=\u0020aaaaa", @@ -36,16 +36,16 @@ func (s *Suite) Test_CheckdirCategory__invalid_comment(c *check.C) { t := s.Init(c) t.SetupVartypes() - t.SetupFileLines("archivers/Makefile", + t.CreateFileLines("archivers/Makefile", MkRcsID, "COMMENT=\t\\Make $$$$ fast\"", "", "SUBDIR+=\tpackage", "", ".include \"../mk/misc/category.mk\"") - t.SetupFileLines("archivers/package/Makefile", + t.CreateFileLines("archivers/package/Makefile", "# dummy") - t.SetupFileLines("mk/misc/category.mk", + t.CreateFileLines("mk/misc/category.mk", "# dummy") CheckdirCategory(t.File("archivers")) @@ -59,10 +59,10 @@ func (s *Suite) Test_CheckdirCategory__wip(c *check.C) { t.SetupPkgsrc() t.SetupVartypes() - t.SetupFileLines("mk/misc/category.mk") - t.SetupFileLines("wip/package/Makefile") - t.SetupFileLines("wip/fs-only/Makefile") - t.SetupFileLines("wip/Makefile", + t.CreateFileLines("mk/misc/category.mk") + t.CreateFileLines("wip/package/Makefile") + t.CreateFileLines("wip/fs-only/Makefile") + t.CreateFileLines("wip/Makefile", MkRcsID, "COMMENT=\tCategory comment", "", diff --git a/pkgtools/pkglint/files/check_test.go b/pkgtools/pkglint/files/check_test.go index fb00b814ab4..d8e8eef7366 100644 --- a/pkgtools/pkglint/files/check_test.go +++ b/pkgtools/pkglint/files/check_test.go @@ -57,7 +57,8 @@ func (s *Suite) SetUpTest(c *check.C) { t := &Tester{checkC: c} s.Tester = t - G = Pkglint{Testing: true} + G = NewPkglint() + G.Testing = true textproc.Testing = true G.logOut = NewSeparatorWriter(&t.stdout) G.logErr = NewSeparatorWriter(&t.stderr) @@ -89,7 +90,7 @@ func (s *Suite) TearDownTest(c *check.C) { G = Pkglint{} // unusable because of missing logOut and logErr textproc.Testing = false if out := t.Output(); out != "" { - fmt.Fprintf(os.Stderr, "Unchecked output in %q; check with: t.CheckOutputLines(%v)", + fmt.Fprintf(os.Stderr, "Unchecked output in %q; check with: t.CheckOutputLines(%#v)", c.TestName(), strings.Split(out, "\n")) } t.tmpdir = "" @@ -238,20 +239,112 @@ func (t *Tester) SetupPkgsrc() { MkRcsID) } -func (t *Tester) CreateFileLines(relativeFilename string, lines ...string) (filename string) { +// SetupCategory makes the given category valid by creating a dummy Makefile. +func (t *Tester) SetupCategory(name string) { + if _, err := os.Stat(name + "/Makefile"); os.IsNotExist(err) { + t.CreateFileLines(name+"/Makefile", + MkRcsID) + } +} + +// SetupPackage sets up all files for a package so that it does not produce +// any warnings. +// +// The given makefileLines start in line 20. Except if they are variable +// definitions for already existing variables, then they replace that line. +// +// Returns the path to the package, ready to be used with Pkglint.CheckDirent. +func (t *Tester) SetupPackage(pkgpath string, makefileLines ...string) string { + category := path.Dir(pkgpath) + + t.SetupPkgsrc() + t.SetupVartypes() + t.SetupCategory(category) + + t.CreateFileLines(pkgpath+"/DESCR", + "Package description") + t.CreateFileLines(pkgpath+"/PLIST", + PlistRcsID, + "bin/program") + t.CreateFileLines(pkgpath+"/distinfo", + RcsID, + "", + "SHA1 (distfile-1.0.tar.gz) = 12341234...", + "RMD160 (distfile-1.0.tar.gz) = 12341234...", + "SHA512 (distfile-1.0.tar.gz) = 12341234...", + "Size (distfile-1.0.tar.gz) = 12341234") + + var mlines []string + mlines = append(mlines, + MkRcsID, + "", + "DISTNAME=\tdistname-1.0", + "CATEGORIES=\t"+category, + "MASTER_SITES=\t# none", + "", + "MAINTAINER=\tpkgsrc-users@NetBSD.org", + "HOMEPAGE=\t# none", + "COMMENT=\tDummy package", + "LICENSE=\t2-clause-bsd", + "") + for len(mlines) < 19 { + mlines = append(mlines, "# empty") + } + +line: + for _, line := range makefileLines { + if m, prefix := match1(line, `^(\w+=)`); m { + for i, existingLine := range mlines { + if hasPrefix(existingLine, prefix) { + mlines[i] = line + continue line + } + } + } + mlines = append(mlines, line) + } + + mlines = append(mlines, + "", + ".include \"../../mk/bsd.pkg.mk\"") + + t.CreateFileLines(pkgpath+"/Makefile", + mlines...) + + return t.File(pkgpath) +} + +func (t *Tester) CreateFileLines(relativeFilename string, lines ...string) (fileName string) { content := "" for _, line := range lines { content += line + "\n" } - filename = t.File(relativeFilename) - err := os.MkdirAll(path.Dir(filename), 0777) + fileName = t.File(relativeFilename) + err := os.MkdirAll(path.Dir(fileName), 0777) t.c().Assert(err, check.IsNil) - err = ioutil.WriteFile(filename, []byte(content), 0666) + err = ioutil.WriteFile(fileName, []byte(content), 0666) t.c().Check(err, check.IsNil) - return filename + G.fileCache.Evict(fileName) + + return fileName +} + +// CreateFileDummyPatch creates a patch file with the given name in the +// temporary directory. +func (t *Tester) CreateFileDummyPatch(relativeFileName string) { + t.CreateFileLines(relativeFileName, + RcsID, + "", + "Documentation", + "", + "--- oldfile", + "+++ newfile", + "@@ -1 +1 @@", + "-old", + "+new") } // File returns the absolute path to the given file in the @@ -295,6 +388,14 @@ func (t *Tester) Chdir(relativeFilename string) { t.relcwd = relativeFilename } +// Remove removes the file from the temporary directory. The file must exist. +func (t *Tester) Remove(relativeFilename string) { + fileName := t.File(relativeFilename) + err := os.Remove(fileName) + t.c().Check(err, check.IsNil) + G.fileCache.Evict(fileName) +} + // ExpectFatal runs the given action and expects that this action calls // Line.Fatalf or uses some other way to panic with a pkglintFatal. // diff --git a/pkgtools/pkglint/files/distinfo_test.go b/pkgtools/pkglint/files/distinfo_test.go index 573a850d885..b5287d47097 100644 --- a/pkgtools/pkglint/files/distinfo_test.go +++ b/pkgtools/pkglint/files/distinfo_test.go @@ -6,10 +6,10 @@ func (s *Suite) Test_ChecklinesDistinfo(c *check.C) { t := s.Init(c) t.Chdir("category/package") - t.SetupFileLines("patches/patch-aa", + t.CreateFileLines("patches/patch-aa", RcsID+" line is ignored for computing the SHA1 hash", "patch contents") - t.SetupFileLines("patches/patch-ab", + t.CreateFileLines("patches/patch-ab", "patch contents") lines := t.SetupFileLines("distinfo", "should be the RCS ID", @@ -34,7 +34,7 @@ func (s *Suite) Test_ChecklinesDistinfo(c *check.C) { "WARN: distinfo:9: Patch file \"patch-nonexistent\" does not exist in directory \"patches\".") } -func (s *Suite) Test_ChecklinesDistinfo_global_hash_mismatch(c *check.C) { +func (s *Suite) Test_ChecklinesDistinfo__global_hash_mismatch(c *check.C) { t := s.Init(c) otherLine := t.NewLine("other/distinfo", 7, "dummy") @@ -53,11 +53,11 @@ func (s *Suite) Test_ChecklinesDistinfo_global_hash_mismatch(c *check.C) { "ERROR: distinfo:EOF: Expected SHA1, RMD160, SHA512, Size checksums for \"pkgname-1.1.tar.gz\", got SHA512.") } -func (s *Suite) Test_ChecklinesDistinfo_uncommitted_patch(c *check.C) { +func (s *Suite) Test_ChecklinesDistinfo__uncommitted_patch(c *check.C) { t := s.Init(c) t.Chdir("category/package") - t.SetupFileLines("patches/patch-aa", + t.CreateFileLines("patches/patch-aa", RcsID, "", "--- oldfile", @@ -65,7 +65,7 @@ func (s *Suite) Test_ChecklinesDistinfo_uncommitted_patch(c *check.C) { "@@ -1,1 +1,1 @@", "-old", "+new") - t.SetupFileLines("CVS/Entries", + t.CreateFileLines("CVS/Entries", "/distinfo/...") lines := t.SetupFileLines("distinfo", RcsID, @@ -79,13 +79,13 @@ func (s *Suite) Test_ChecklinesDistinfo_uncommitted_patch(c *check.C) { "WARN: distinfo:3: patches/patch-aa is registered in distinfo but not added to CVS.") } -func (s *Suite) Test_ChecklinesDistinfo_unrecorded_patches(c *check.C) { +func (s *Suite) Test_ChecklinesDistinfo__unrecorded_patches(c *check.C) { t := s.Init(c) t.Chdir("category/package") - t.SetupFileLines("patches/CVS/Entries") - t.SetupFileLines("patches/patch-aa") - t.SetupFileLines("patches/patch-src-Makefile") + t.CreateFileLines("patches/CVS/Entries") + t.CreateFileLines("patches/patch-aa") + t.CreateFileLines("patches/patch-src-Makefile") lines := t.SetupFileLines("distinfo", RcsID, "", @@ -102,7 +102,7 @@ func (s *Suite) Test_ChecklinesDistinfo_unrecorded_patches(c *check.C) { "ERROR: distinfo: patch \"patches/patch-src-Makefile\" is not recorded. Run \""+confMake+" makepatchsum\".") } -func (s *Suite) Test_ChecklinesDistinfo_manual_patches(c *check.C) { +func (s *Suite) Test_ChecklinesDistinfo__manual_patches(c *check.C) { t := s.Init(c) t.Chdir("category/package") @@ -142,7 +142,7 @@ func (s *Suite) Test_ChecklinesDistinfo__missing_php_patches(c *check.C) { MkRcsID, "", "PHPEXT_MK= # defined", - "PHPPKGSRCDIR= lang/php72", + "PHPPKGSRCDIR= ../../lang/php72", "LICENSE?= unknown-license", "COMMENT?= Some PHP package", "GENERATE_PLIST+=# none", @@ -163,7 +163,7 @@ func (s *Suite) Test_ChecklinesDistinfo__missing_php_patches(c *check.C) { "@@ -1,1 +1,1 @@", "-old", "+new") - t.SetupFileLines("lang/php72/distinfo", + t.CreateFileLines("lang/php72/distinfo", RcsID, "", "SHA1 (patch-php72) = c109b2089f5ddbc5372b2ab28115ff558ee4187d") diff --git a/pkgtools/pkglint/files/expecter.go b/pkgtools/pkglint/files/expecter.go index a94a20fc81b..58bdc7ede41 100644 --- a/pkgtools/pkglint/files/expecter.go +++ b/pkgtools/pkglint/files/expecter.go @@ -57,7 +57,7 @@ func (exp *Expecter) AdvanceIfMatches(re regex.Pattern) bool { } if !exp.EOF() { - if m := regex.Match(exp.lines[exp.index].Text, re); m != nil { + if m := G.res.Match(exp.lines[exp.index].Text, re); m != nil { exp.index++ exp.m = m return true diff --git a/pkgtools/pkglint/files/files.go b/pkgtools/pkglint/files/files.go index a57c6c0e256..7c8dc228e0f 100644 --- a/pkgtools/pkglint/files/files.go +++ b/pkgtools/pkglint/files/files.go @@ -16,6 +16,11 @@ const ( ) func Load(fileName string, options LoadOptions) []Line { + fromCache := G.fileCache.Get(fileName, options) + if fromCache != nil { + return fromCache + } + rawBytes, err := ioutil.ReadFile(fileName) if err != nil { switch { @@ -42,7 +47,9 @@ func Load(fileName string, options LoadOptions) []Line { G.loaded.Add(path.Clean(fileName), 1) } - return convertToLogicalLines(fileName, rawText, options&Makefile != 0) + result := convertToLogicalLines(fileName, rawText, options&Makefile != 0) + G.fileCache.Put(fileName, options, result) + return result } func LoadMk(fileName string, options LoadOptions) *MkLines { diff --git a/pkgtools/pkglint/files/files_test.go b/pkgtools/pkglint/files/files_test.go index b6aa1d564c7..f9aa93e93f8 100644 --- a/pkgtools/pkglint/files/files_test.go +++ b/pkgtools/pkglint/files/files_test.go @@ -4,7 +4,7 @@ import ( "gopkg.in/check.v1" ) -func (s *Suite) Test_convertToLogicalLines_no_continuation(c *check.C) { +func (s *Suite) Test_convertToLogicalLines__no_continuation(c *check.C) { rawText := "" + "first line\n" + "second line\n" @@ -16,7 +16,7 @@ func (s *Suite) Test_convertToLogicalLines_no_continuation(c *check.C) { c.Check(lines[1].String(), equals, "fname_nocont:2: second line") } -func (s *Suite) Test_convertToLogicalLines_continuation(c *check.C) { +func (s *Suite) Test_convertToLogicalLines__continuation(c *check.C) { rawText := "" + "first line \\\n" + "second line\n" + @@ -121,7 +121,7 @@ func (s *Suite) Test_convertToLogicalLines__comments(c *check.C) { "ERROR: ~/comment.mk:23: Unknown Makefile line format: \"This is no comment\".") } -func (s *Suite) Test_convertToLogicalLines_continuationInLastLine(c *check.C) { +func (s *Suite) Test_convertToLogicalLines__continuation_in_last_line(c *check.C) { t := s.Init(c) rawText := "" + diff --git a/pkgtools/pkglint/files/fuzzer_test.go b/pkgtools/pkglint/files/fuzzer_test.go new file mode 100644 index 00000000000..ee5621cab96 --- /dev/null +++ b/pkgtools/pkglint/files/fuzzer_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "gopkg.in/check.v1" + "math/rand" +) + +type Fuzzer struct { + seed int64 + rnd *rand.Rand + stock []struct { + r rune + weight int + } + total int + last string + ok bool +} + +func NewFuzzer(seed ...int64) *Fuzzer { + var actualSeed int64 + if len(seed) != 0 { + actualSeed = seed[0] + } else { + actualSeed = rand.Int63() + } + return &Fuzzer{seed: actualSeed, rnd: rand.New(rand.NewSource(actualSeed))} +} + +// Char randomly generates a character from the given set. +// Each character has the given weight. +func (f *Fuzzer) Char(set string, weight int) { + for _, r := range set { + f.addChar(r, weight) + } +} + +// Range randomly generates a character from the given range. +// Each character has the given weight. +func (f *Fuzzer) Range(minIncl, maxIncl rune, weight int) { + for r := minIncl; r <= maxIncl; r++ { + f.addChar(r, weight) + } +} + +func (f *Fuzzer) addChar(r rune, weight int) { + f.stock = append(f.stock, struct { + r rune + weight int + }{r, weight}) + f.total += weight +} + +func (f *Fuzzer) Generate(length int) string { + s := "" + for i := 0; i < length; i++ { + s += string(f.randomChar()) + } + f.last = s + return s +} + +func (f *Fuzzer) randomChar() rune { + i := int(f.rnd.Int31n(int32(f.total))) + for _, entry := range f.stock { + i -= entry.weight + if i < 0 { + return entry.r + } + } + panic("Out of stock") +} + +func (f *Fuzzer) CheckOk() { + if !f.ok { + dummyLine.Errorf("Fuzzing failed with seed %d, last generated value: %s", f.seed, f.last) + } +} + +func (f *Fuzzer) Ok() { f.ok = true } + +func (s *Suite) Test_Fuzzer__out_of_stock(c *check.C) { + fuzzer := NewFuzzer(0) + fuzzer.total = 1 // Trick the fuzzer to achieve full code coverage. + + c.Check( + func() { fuzzer.Generate(1) }, + check.Panics, + "Out of stock") +} diff --git a/pkgtools/pkglint/files/getopt/getopt_test.go b/pkgtools/pkglint/files/getopt/getopt_test.go index 9a0ed788d61..091a41368a5 100644 --- a/pkgtools/pkglint/files/getopt/getopt_test.go +++ b/pkgtools/pkglint/files/getopt/getopt_test.go @@ -137,3 +137,43 @@ func (s *Suite) Test_Options_Parse_string_list(c *check.C) { c.Check(err.Error(), check.Equals, "progname: option requires an argument: --include") } + +func (s *Suite) Test_Options_Parse__long_flags(c *check.C) { + var aflag, bflag, cflag, dflag, eflag, fflag, gflag, hflag, iflag, jflag bool + + opts := NewOptions() + opts.AddFlagVar('a', "aflag", &aflag, false, "") + opts.AddFlagVar('b', "bflag", &bflag, false, "") + opts.AddFlagVar('c', "cflag", &cflag, false, "") + opts.AddFlagVar('d', "dflag", &dflag, false, "") + opts.AddFlagVar('e', "eflag", &eflag, true, "") + opts.AddFlagVar('f', "fflag", &fflag, true, "") + opts.AddFlagVar('g', "gflag", &gflag, true, "") + opts.AddFlagVar('h', "hflag", &hflag, true, "") + opts.AddFlagVar('i', "iflag", &iflag, false, "") + opts.AddFlagVar('j', "jflag", &jflag, false, "") + + args, err := opts.Parse([]string{"progname", + "--aflag=true", + "--bflag=on", + "--cflag=enabled", + "--dflag=1", + "--eflag=false", + "--fflag=off", + "--gflag=disabled", + "--hflag=0", + "--iflag", + "--jflag=unknown"}) + + c.Check(args, check.HasLen, 0) + c.Check(aflag, check.Equals, true) + c.Check(bflag, check.Equals, true) + c.Check(cflag, check.Equals, true) + c.Check(dflag, check.Equals, true) + c.Check(eflag, check.Equals, false) + c.Check(fflag, check.Equals, false) + c.Check(gflag, check.Equals, false) + c.Check(hflag, check.Equals, false) + c.Check(iflag, check.Equals, true) + c.Check(err, check.ErrorMatches, `^progname: invalid argument for option --jflag$`) +} diff --git a/pkgtools/pkglint/files/licenses.go b/pkgtools/pkglint/files/licenses.go index e1d1d760541..bd57b4840b4 100644 --- a/pkgtools/pkglint/files/licenses.go +++ b/pkgtools/pkglint/files/licenses.go @@ -30,7 +30,7 @@ type LicenseChecker struct { func (lc *LicenseChecker) Check(value string, op MkOperator) { expanded := resolveVariableRefs(value) // For ${PERL5_LICENSE} - licenses := licenses.Parse(ifelseStr(op == opAssignAppend, "append-placeholder ", "") + expanded) + licenses := licenses.Parse(ifelseStr(op == opAssignAppend, "append-placeholder ", "")+expanded, &G.res) if licenses == nil { if op == opAssign { diff --git a/pkgtools/pkglint/files/licenses/licenses.go b/pkgtools/pkglint/files/licenses/licenses.go index 4055ab4d96c..b81d2b156f7 100644 --- a/pkgtools/pkglint/files/licenses/licenses.go +++ b/pkgtools/pkglint/files/licenses/licenses.go @@ -1,6 +1,9 @@ package licenses -import "netbsd.org/pkglint/textproc" +import ( + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/textproc" +) // Condition describes a complex license condition. // It has either `Name` or `Paren` or `Children` set. @@ -14,8 +17,8 @@ type Condition struct { Children []*Condition `json:",omitempty"` } -func Parse(licenses string) *Condition { - lexer := &licenseLexer{repl: textproc.NewPrefixReplacer(licenses)} +func Parse(licenses string, res *regex.Registry) *Condition { + lexer := &licenseLexer{repl: textproc.NewPrefixReplacer(licenses, res)} result := liyyNewParser().Parse(lexer) if result == 0 { return lexer.result diff --git a/pkgtools/pkglint/files/licenses/licenses_test.go b/pkgtools/pkglint/files/licenses/licenses_test.go index 871a44aba04..8e9b08455af 100644 --- a/pkgtools/pkglint/files/licenses/licenses_test.go +++ b/pkgtools/pkglint/files/licenses/licenses_test.go @@ -3,6 +3,7 @@ package licenses import ( "encoding/json" "gopkg.in/check.v1" + "netbsd.org/pkglint/regex" "strings" "testing" ) @@ -10,11 +11,15 @@ import ( type Suite struct{} func (s *Suite) Test_Parse(c *check.C) { + res := regex.NewRegistry() checkParse := func(cond string, expected string) { - c.Check(toJSON(Parse(cond)), check.Equals, expected) + c.Check(toJSON(Parse(cond, &res)), check.Equals, expected) + } + checkParseDeep := func(cond string, expected *Condition) { + c.Check(Parse(cond, &res), check.DeepEquals, expected) } - c.Check(Parse("gnu-gpl-v2"), check.DeepEquals, NewSingleton(NewName("gnu-gpl-v2"))) + checkParseDeep("gnu-gpl-v2", NewSingleton(NewName("gnu-gpl-v2"))) checkParse("gnu-gpl-v2", "{Children:[{Name:gnu-gpl-v2}]}") checkParse("a AND b", "{And:true,Children:[{Name:a},{Name:b}]}") @@ -24,21 +29,18 @@ func (s *Suite) Test_Parse(c *check.C) { checkParse("(a OR b) AND c", "{And:true,Children:[{Paren:{Or:true,Children:[{Name:a},{Name:b}]}},{Name:c}]}") checkParse("a AND b AND c AND d", "{And:true,Children:[{Name:a},{Name:b},{Name:c},{Name:d}]}") - c.Check( - Parse("a AND b AND c AND d"), - check.DeepEquals, + checkParseDeep( + "a AND b AND c AND d", NewAnd(NewName("a"), NewName("b"), NewName("c"), NewName("d"))) checkParse("a OR b OR c OR d", "{Or:true,Children:[{Name:a},{Name:b},{Name:c},{Name:d}]}") - c.Check( - Parse("a OR b OR c OR d"), - check.DeepEquals, + checkParseDeep( + "a OR b OR c OR d", NewOr(NewName("a"), NewName("b"), NewName("c"), NewName("d"))) checkParse("(a OR b) AND (c AND d)", "{And:true,Children:[{Paren:{Or:true,Children:[{Name:a},{Name:b}]}},{Paren:{And:true,Children:[{Name:c},{Name:d}]}}]}") - c.Check( - (Parse("(a OR b) AND (c AND d)")), - check.DeepEquals, + checkParseDeep( + "(a OR b) AND (c AND d)", NewAnd( NewParen(NewOr(NewName("a"), NewName("b"))), NewParen(NewAnd(NewName("c"), NewName("d"))))) @@ -46,9 +48,9 @@ func (s *Suite) Test_Parse(c *check.C) { checkParse("a AND b OR c AND d", "{And:true,Or:true,Children:[{Name:a},{Name:b},{Name:c},{Name:d}]}") checkParse("((a AND (b AND c)))", "{Children:[{Paren:{Children:[{Paren:{And:true,Children:[{Name:a},{Paren:{And:true,Children:[{Name:b},{Name:c}]}}]}}]}}]}") - c.Check(Parse("a AND b OR c AND d").String(), check.Equals, "a MIXED b MIXED c MIXED d") + c.Check(Parse("a AND b OR c AND d", &res).String(), check.Equals, "a MIXED b MIXED c MIXED d") - c.Check(Parse("AND artistic"), check.IsNil) + c.Check(Parse("AND artistic", &res), check.IsNil) } func (s *Suite) Test_Condition_String(c *check.C) { diff --git a/pkgtools/pkglint/files/licenses_test.go b/pkgtools/pkglint/files/licenses_test.go index f0ef45dbf6e..2538692ea90 100644 --- a/pkgtools/pkglint/files/licenses_test.go +++ b/pkgtools/pkglint/files/licenses_test.go @@ -4,10 +4,10 @@ import ( "gopkg.in/check.v1" ) -func (s *Suite) Test_checklineLicense(c *check.C) { +func (s *Suite) Test_LicenseChecker_Check(c *check.C) { t := s.Init(c) - t.SetupFileLines("licenses/gnu-gpl-v2", + t.CreateFileLines("licenses/gnu-gpl-v2", "Most software \u2026") mkline := t.NewMkLine("Makefile", 7, "LICENSE=dummy") @@ -48,15 +48,15 @@ func (s *Suite) Test_checkToplevelUnusedLicenses(c *check.C) { t := s.Init(c) t.SetupPkgsrc() - t.SetupFileLines("mk/misc/category.mk") - t.SetupFileLines("licenses/2-clause-bsd") - t.SetupFileLines("licenses/gnu-gpl-v3") + t.CreateFileLines("mk/misc/category.mk") + t.CreateFileLines("licenses/2-clause-bsd") + t.CreateFileLines("licenses/gnu-gpl-v3") - t.SetupFileLines("Makefile", + t.CreateFileLines("Makefile", MkRcsID, "SUBDIR+=\tcategory") - t.SetupFileLines("category/Makefile", + t.CreateFileLines("category/Makefile", MkRcsID, "COMMENT=\tExample category", "", @@ -64,14 +64,14 @@ func (s *Suite) Test_checkToplevelUnusedLicenses(c *check.C) { "", ".include \"../mk/misc/category.mk\"") - t.SetupFileLines("category/package/Makefile", + t.CreateFileLines("category/package/Makefile", MkRcsID, "CATEGORIES=\tcategory", "", "COMMENT=Example package", "LICENSE=\t2-clause-bsd", "NO_CHECKSUM=\tyes") - t.SetupFileLines("category/package/PLIST", + t.CreateFileLines("category/package/PLIST", PlistRcsID, "bin/program") @@ -88,9 +88,9 @@ func (s *Suite) Test_LicenseChecker_checkLicenseName__LICENSE_FILE(c *check.C) { t.SetupPkgsrc() t.SetupCommandLine("-Wno-space") - t.SetupFileLines("category/package/DESCR", + t.CreateFileLines("category/package/DESCR", "Package description") - t.SetupFileMkLines("category/package/Makefile", + t.CreateFileLines("category/package/Makefile", MkRcsID, "", "CATEGORIES= chinese", @@ -102,10 +102,10 @@ func (s *Suite) Test_LicenseChecker_checkLicenseName__LICENSE_FILE(c *check.C) { "NO_CHECKSUM= yes", "", ".include \"../../mk/bsd.pkg.mk\"") - t.SetupFileLines("category/package/PLIST", + t.CreateFileLines("category/package/PLIST", PlistRcsID, "bin/program") - t.SetupFileLines("category/package/my-license", + t.CreateFileLines("category/package/my-license", "An individual license file.") G.Main("pkglint", t.File("category/package")) @@ -113,5 +113,6 @@ func (s *Suite) Test_LicenseChecker_checkLicenseName__LICENSE_FILE(c *check.C) { // FIXME: It should be allowed to place a license file directly into // the package directory. t.CheckOutputLines( - "WARN: ~/category/package/my-license: Unexpected file found.", "0 errors and 1 warning found.") + "WARN: ~/category/package/my-license: Unexpected file found.", + "0 errors and 1 warning found.") } diff --git a/pkgtools/pkglint/files/line.go b/pkgtools/pkglint/files/line.go index d18c180db71..de6995482b9 100644 --- a/pkgtools/pkglint/files/line.go +++ b/pkgtools/pkglint/files/line.go @@ -39,6 +39,7 @@ type LineImpl struct { Text string raw []*RawLine autofix *Autofix + Once } func NewLine(fname string, lineno int, text string, rawLines []*RawLine) Line { @@ -47,7 +48,7 @@ func NewLine(fname string, lineno int, text string, rawLines []*RawLine) Line { // NewLineMulti is for logical Makefile lines that end with backslash. func NewLineMulti(fname string, firstLine, lastLine int, text string, rawLines []*RawLine) Line { - return &LineImpl{fname, path.Base(fname), int32(firstLine), int32(lastLine), text, rawLines, nil} + return &LineImpl{fname, path.Base(fname), int32(firstLine), int32(lastLine), text, rawLines, nil, Once{}} } // NewLineEOF creates a dummy line for logging, with the "line number" EOF. diff --git a/pkgtools/pkglint/files/linechecker.go b/pkgtools/pkglint/files/linechecker.go index d0c8f7b7bd9..e8282015898 100644 --- a/pkgtools/pkglint/files/linechecker.go +++ b/pkgtools/pkglint/files/linechecker.go @@ -19,8 +19,8 @@ func CheckLineAbsolutePathname(line Line, text string) { // // Another context where absolute pathnames usually appear is in // assignments like "bindir=/bin". - if m, path := regex.Match1(text, `(?:^|\$[{(]DESTDIR[)}]|[\w_]+\s*=\s*)(/(?:[^"'\s]|"[^"*]"|'[^']*')*)`); m { - if regex.Matches(path, `^/\w`) { + if m, path := match1(text, `(?:^|\s|\$[{(]DESTDIR[)}]|[\w_]+\s*=\s*)(/(?:[^"'\s\\]|"[^"*]"|'[^']*')*)`); m { + if matches(path, `^/\w`) { CheckwordAbsolutePathname(line, path) } } @@ -36,13 +36,14 @@ func CheckLineLength(line Line, maxlength int) { } } -func CheckLineValidCharacters(line Line, reChar regex.Pattern) { - rest := regex.Compile(reChar).ReplaceAllString(line.Text, "") - if rest != "" { - uni := "" - for _, c := range rest { - uni += fmt.Sprintf(" %U", c) +func CheckLineValidCharacters(line Line) { + uni := "" + for _, r := range line.Text { + if r != '\t' && !(' ' <= r && r <= '~') { + uni += fmt.Sprintf(" %U", r) } + } + if uni != "" { line.Warnf("Line contains invalid characters (%s).", uni[1:]) } } @@ -64,7 +65,7 @@ func CheckLineRcsid(line Line, prefixRe regex.Pattern, suggestedPrefix string) b defer trace.Call(prefixRe, suggestedPrefix)() } - if regex.Matches(line.Text, `^`+prefixRe+`\$`+`NetBSD(?::[^\$]+)?\$$`) { + if matches(line.Text, `^`+prefixRe+`\$`+`NetBSD(?::[^\$]+)?\$$`) { return true } @@ -86,13 +87,16 @@ func CheckwordAbsolutePathname(line Line, word string) { } switch { - case regex.Matches(word, `^/dev/(?:null|tty|zero)$`): - // These are defined by POSIX. + case matches(word, `^/dev/(?:null|tty|zero)$`): + // These are defined by POSIX. + case word == "/bin/sh": - // This is usually correct, although on Solaris, it's pretty feature-crippled. - case regex.Matches(word, `^/s\W`): - // Probably a sed(1) command - case regex.Matches(word, `^/(?:[a-z]|\$[({])`): + // This is usually correct, although on Solaris, it's pretty feature-crippled. + + case matches(word, `/s\W`): + // Probably a sed(1) command, e.g. /find/s,replace,with, + + case matches(word, `^/(?:[a-z]|\$[({])`): // Absolute paths probably start with a lowercase letter. line.Warnf("Found absolute pathname: %s", word) if contains(line.Text, "DESTDIR") { diff --git a/pkgtools/pkglint/files/linechecker_test.go b/pkgtools/pkglint/files/linechecker_test.go index 34deab41c17..29718e368a5 100644 --- a/pkgtools/pkglint/files/linechecker_test.go +++ b/pkgtools/pkglint/files/linechecker_test.go @@ -4,22 +4,30 @@ import ( "gopkg.in/check.v1" ) -func (s *Suite) Test_LineChecker_CheckAbsolutePathname(c *check.C) { +func (s *Suite) Test_CheckLineAbsolutePathname(c *check.C) { t := s.Init(c) line := t.NewLine("Makefile", 1, "# dummy") CheckLineAbsolutePathname(line, "bindir=/bin") CheckLineAbsolutePathname(line, "bindir=/../lib") - CheckLineAbsolutePathname(line, "cat /dev/null") // FIXME: Not classified as absolute path. - CheckLineAbsolutePathname(line, "cat /dev//tty") // FIXME: Not classified as absolute patFIXMEh. - CheckLineAbsolutePathname(line, "cat /dev/zero") // FIXME: Not classified as absolute path. - CheckLineAbsolutePathname(line, "cat /dev/stdin") // FIXME: Not classified as absolute path. - CheckLineAbsolutePathname(line, "cat /dev/stdout") // FIXME: Not classified as absolute path. - CheckLineAbsolutePathname(line, "cat /dev/stderr") // FIXME: Not classified as absolute path. + CheckLineAbsolutePathname(line, "cat /dev/null") + CheckLineAbsolutePathname(line, "cat /dev/tty") + CheckLineAbsolutePathname(line, "cat /dev/zero") + CheckLineAbsolutePathname(line, "cat /dev/stdin") + CheckLineAbsolutePathname(line, "cat /dev/stdout") + CheckLineAbsolutePathname(line, "cat /dev/stderr") + CheckLineAbsolutePathname(line, "printf '#! /bin/sh\\nexit 0'") + + // This is not a file name at all, but certainly looks like one. + // Nevertheless, pkglint doesn't fall into the trap. + CheckLineAbsolutePathname(line, "sed -e /usr/s/usr/var/g") t.CheckOutputLines( - "WARN: Makefile:1: Found absolute pathname: /bin") + "WARN: Makefile:1: Found absolute pathname: /bin", + "WARN: Makefile:1: Found absolute pathname: /dev/stdin", + "WARN: Makefile:1: Found absolute pathname: /dev/stdout", + "WARN: Makefile:1: Found absolute pathname: /dev/stderr") } func (s *Suite) Test_CheckLineTrailingWhitespace(c *check.C) { diff --git a/pkgtools/pkglint/files/logging.go b/pkgtools/pkglint/files/logging.go index 3582c8f6e99..375f63b0a78 100644 --- a/pkgtools/pkglint/files/logging.go +++ b/pkgtools/pkglint/files/logging.go @@ -113,13 +113,16 @@ func Explain(explanation ...string) { for _, s := range explanation { if l := tabWidth(s); l > 68 && contains(s, " ") { lastSpace := strings.LastIndexByte(s[:68], ' ') - G.logErr.Write(fmt.Sprintf("Long explanation line: %s\nBreak after: %s\n", s, s[:lastSpace])) + G.logErr.Printf("Long explanation line: %s\nBreak after: %s\n", s, s[:lastSpace]) } if m, before := match1(s, `(.+)\. [^ ]`); m { if !matches(before, `\d$|e\.g`) { - G.logErr.Write(fmt.Sprintf("Short space after period: %s\n", s)) + G.logErr.Printf("Short space after period: %s\n", s) } } + if hasSuffix(s, " ") || hasSuffix(s, "\t") { + G.logErr.Printf("Trailing whitespace: %q\n", s) + } } } @@ -174,8 +177,10 @@ func (wr *SeparatorWriter) Write(text string) { io.WriteString(wr.out, "\n") wr.needSeparator = false } - io.WriteString(wr.out, text) - wr.wroteSomething = true + n, err := io.WriteString(wr.out, text) + if err == nil && n > 0 { + wr.wroteSomething = true + } } func (wr *SeparatorWriter) Printf(format string, args ...interface{}) { diff --git a/pkgtools/pkglint/files/logging_test.go b/pkgtools/pkglint/files/logging_test.go index 4dcc73209f7..64e6235d9e5 100644 --- a/pkgtools/pkglint/files/logging_test.go +++ b/pkgtools/pkglint/files/logging_test.go @@ -15,7 +15,7 @@ import "gopkg.in/check.v1" // To keep the output layout consistent between all these // modes, the source code is written below the diagnostic // also in the default (check-only) mode. -func (s *Suite) Test_show_source_separator(c *check.C) { +func (s *Suite) Test__show_source_separator(c *check.C) { t := s.Init(c) t.SetupCommandLine("--source") @@ -47,7 +47,7 @@ func (s *Suite) Test_show_source_separator(c *check.C) { ">\tThe third line") } -func (s *Suite) Test_show_source_separator_show_autofix(c *check.C) { +func (s *Suite) Test__show_source_separator_show_autofix(c *check.C) { t := s.Init(c) t.SetupCommandLine("--source", "--show-autofix") @@ -80,7 +80,7 @@ func (s *Suite) Test_show_source_separator_show_autofix(c *check.C) { "+\tThe bronze medal line") } -func (s *Suite) Test_show_source_separator_autofix(c *check.C) { +func (s *Suite) Test__show_source_separator_autofix(c *check.C) { t := s.Init(c) t.SetupCommandLine("--source", "--autofix") @@ -114,7 +114,7 @@ func (s *Suite) Test_show_source_separator_autofix(c *check.C) { // Demonstrates how to filter log messages. // This is useful in combination with the --autofix option, // to restrict the fixes to exactly one group or topic. -func (s *Suite) Test_Line_log_only(c *check.C) { +func (s *Suite) Test_Line_log__only(c *check.C) { t := s.Init(c) t.SetupCommandLine("--autofix", "--source", "--only", "interesting") @@ -136,7 +136,7 @@ func (s *Suite) Test_Line_log_only(c *check.C) { "+\tThe new2 song") } -func (s *Suite) Test_collect_explanations_with_only(c *check.C) { +func (s *Suite) Test_Pkglint_PrintSummary__explanations_with_only(c *check.C) { t := s.Init(c) t.SetupCommandLine("--only", "interesting") @@ -161,7 +161,7 @@ func (s *Suite) Test_collect_explanations_with_only(c *check.C) { "(Run \"pkglint -e\" to show explanations.)") } -func (s *Suite) Test_explain_with_only(c *check.C) { +func (s *Suite) Test_Explain__only(c *check.C) { t := s.Init(c) t.SetupCommandLine("--only", "interesting", "--explain") @@ -232,10 +232,20 @@ func (s *Suite) Test_Explain__long_lines(c *check.C) { t := s.Init(c) Explain( - "123456789 12345678. abcdefghi. 123456789 123456789 123456789 123456789 123456789 ") + "123456789 12345678. abcdefghi. 123456789 123456789 123456789 123456789 123456789") t.CheckOutputLines( - "Long explanation line: 123456789 12345678. abcdefghi. 123456789 123456789 123456789 123456789 123456789 ", + "Long explanation line: 123456789 12345678. abcdefghi. 123456789 123456789 123456789 123456789 123456789", "Break after: 123456789 12345678. abcdefghi. 123456789 123456789 123456789", - "Short space after period: 123456789 12345678. abcdefghi. 123456789 123456789 123456789 123456789 123456789 ") + "Short space after period: 123456789 12345678. abcdefghi. 123456789 123456789 123456789 123456789 123456789") +} + +func (s *Suite) Test_Explain__trailing_whitespace(c *check.C) { + t := s.Init(c) + + Explain( + "This is a space: ") + + t.CheckOutputLines( + "Trailing whitespace: \"This is a space: \"") } diff --git a/pkgtools/pkglint/files/mkline.go b/pkgtools/pkglint/files/mkline.go index e56ac1c312b..37355504306 100644 --- a/pkgtools/pkglint/files/mkline.go +++ b/pkgtools/pkglint/files/mkline.go @@ -4,7 +4,6 @@ package main import ( "fmt" - "netbsd.org/pkglint/regex" "netbsd.org/pkglint/trace" "path" "strings" @@ -327,12 +326,12 @@ func (mkline *MkLineImpl) ValueTokens() []*MkToken { func (mkline *MkLineImpl) WithoutMakeVariables(value string) string { valueNovar := value for { - var m []string // TODO: properly parse nested variables - m, valueNovar = regex.ReplaceFirst(valueNovar, `\$\{[^{}]*\}`, "") - if m == nil { + replaced := replaceFirst(valueNovar, `\$\{[^{}]*\}`, "") + if replaced == valueNovar { return valueNovar } + valueNovar = replaced } } @@ -357,7 +356,7 @@ func (mkline *MkLineImpl) ResolveVarsInRelativePath(relativePath string, adjustD tmp = strings.Replace(tmp, "${PHPPKGSRCDIR}", G.Pkgsrc.Latest("lang", `^php[0-9]+$`, "../../lang/$0"), -1) } if contains(tmp, "${SUSE_DIR_PREFIX}") { - suseDirPrefix := G.Pkgsrc.Latest("emulators", `^(suse[0-9]+)_base`, "$1") + suseDirPrefix := G.Pkgsrc.Latest("emulators", `^(suse[0-9]+)_base$`, "$1") tmp = strings.Replace(tmp, "${SUSE_DIR_PREFIX}", suseDirPrefix, -1) } if contains(tmp, "${PYPKGSRCDIR}") { @@ -372,14 +371,14 @@ func (mkline *MkLineImpl) ResolveVarsInRelativePath(relativePath string, adjustD } if adjustDepth { - if m, pkgpath := match1(tmp, `^\.\./\.\./([^.].*)$`); m { - tmp = pkgsrcdir + "/" + pkgpath + if hasPrefix(tmp, "../../") && !hasPrefix(tmp[6:], ".") { + tmp = pkgsrcdir + "/" + tmp[6:] } } tmp = cleanpath(tmp) - if trace.Tracing { + if trace.Tracing && relativePath != tmp { trace.Step2("resolveVarsInRelativePath: %q => %q", relativePath, tmp) } return tmp @@ -584,7 +583,7 @@ func (mkline *MkLineImpl) VariableNeedsQuoting(varname string, vartype *Vartype, // TODO: merge with determineUsedVariables func (mkline *MkLineImpl) ExtractUsedVariables(text string) []string { - re := regex.Compile(`^(?:[^\$]+|\$[\$*<>?@]|\$\{([.0-9A-Z_a-z]+)(?::(?:[^\${}]|\$[^{])+)?\})`) + re := G.res.Compile(`^(?:[^\$]+|\$[\$*<>?@]|\$\{([.0-9A-Z_a-z]+)(?::(?:[^\${}]|\$[^{])+)?\})`) rest := text var result []string for { @@ -635,7 +634,7 @@ func (mkline *MkLineImpl) DetermineUsedVariables() (varnames []string) { } rest = rest[min:] - m := regex.Compile(`(?:\$\{|\$\(|defined\(|empty\()([*+\-.0-9A-Z_a-z]+)[:})]`).FindStringSubmatchIndex(rest) + m := G.res.Compile(`(?:\$\{|\$\(|defined\(|empty\()([*+\-.0-9A-Z_a-z]+)[:})]`).FindStringSubmatchIndex(rest) if m == nil { return } @@ -918,9 +917,17 @@ func (ind *Indentation) TrackAfter(mkline MkLine) { switch directive { case "if": // For multiple-inclusion guards, the indentation stays at the same level. - if m, varname := match1(args, `^!defined\(([\w]+_MK)\)$`); m { - ind.AddVar(varname) - } else { + guard := false + if hasPrefix(args, "!defined") && hasSuffix(args, "_MK)") { + if hasPrefix(args, "!defined(") && hasSuffix(args, ")") { + varname := args[9 : len(args)-1] + if varname != "" && isalnum(varname) { + ind.AddVar(varname) + guard = true + } + } + } + if !guard { ind.top().depth += 2 } @@ -1055,5 +1062,27 @@ func MatchVarassign(text string) (m, commented bool, varname, spaceAfterVarname, } func MatchMkInclude(text string) (m bool, indentation, directive, filename string) { - return match3(text, `^\.(\s*)(s?include)\s+\"([^\"]+)\"\s*(?:#.*)?$`) + repl := G.NewPrefixReplacer(text) + if repl.AdvanceStr(".") { + if repl.AdvanceHspace() { + indentation = repl.Str() + } + if repl.AdvanceStr("include") || repl.AdvanceStr("sinclude") { + directive = repl.Str() + repl.AdvanceHspace() + if repl.AdvanceByte('"') { + if repl.AdvanceBytesFunc(func(c byte) bool { return c != '"' }) { + filename = repl.Str() + if repl.AdvanceByte('"') { + repl.AdvanceHspace() + if repl.EOF() || repl.PeekByte() == '#' { + m = true + return + } + } + } + } + } + } + return false, "", "", "" } diff --git a/pkgtools/pkglint/files/mkline_test.go b/pkgtools/pkglint/files/mkline_test.go index e65fc55565f..d38bfb0f99f 100644 --- a/pkgtools/pkglint/files/mkline_test.go +++ b/pkgtools/pkglint/files/mkline_test.go @@ -2,7 +2,7 @@ package main import "gopkg.in/check.v1" -func (s *Suite) Test_VaralignBlock_Check_autofix(c *check.C) { +func (s *Suite) Test_VaralignBlock_Check__autofix(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wspace", "--show-autofix") @@ -68,7 +68,7 @@ func (s *Suite) Test_VaralignBlock_Check__reduce_indentation(c *check.C) { "NOTE: file.mk:3: This variable value should be aligned to column 9.") } -func (s *Suite) Test_VaralignBlock_Check_longest_line_no_space(c *check.C) { +func (s *Suite) Test_VaralignBlock_Check__longest_line_no_space(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wspace") @@ -91,7 +91,7 @@ func (s *Suite) Test_VaralignBlock_Check_longest_line_no_space(c *check.C) { "NOTE: file.mk:4: This variable value should be aligned to column 33.") } -func (s *Suite) Test_VaralignBlock_Check_only_spaces(c *check.C) { +func (s *Suite) Test_VaralignBlock_Check__only_spaces(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wspace") @@ -219,7 +219,7 @@ func (s *Suite) Test_NewMkLine__autofix_space_after_varname(c *check.C) { } // Guessing the variable type works for both plain and parameterized variable names. -func (s *Suite) Test_MkLine_VariableType_varparam(c *check.C) { +func (s *Suite) Test_Pkgsrc_VariableType__varparam(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -242,13 +242,13 @@ func (s *Suite) Test_VarUseContext_String(c *check.C) { vartype := G.Pkgsrc.VariableType("PKGNAME") vuc := &VarUseContext{vartype, vucTimeUnknown, vucQuotBackt, false} - c.Check(vuc.String(), equals, "(PkgName time:unknown quoting:backt wordpart:false)") + c.Check(vuc.String(), equals, "(Pkgname time:unknown quoting:backt wordpart:false)") } // In variable assignments, a plain '#' introduces a line comment, unless // it is escaped by a backslash. In shell commands, on the other hand, it // is interpreted literally. -func (s *Suite) Test_NewMkLine_numbersign(c *check.C) { +func (s *Suite) Test_NewMkLine__number_sign(c *check.C) { t := s.Init(c) mklineVarassignEscaped := t.NewMkLine("fname", 1, "SED_CMD=\t's,\\#,hash,g'") @@ -273,7 +273,7 @@ func (s *Suite) Test_NewMkLine_numbersign(c *check.C) { "WARN: fname:1: The # character starts a comment.") } -func (s *Suite) Test_NewMkLine_leading_space(c *check.C) { +func (s *Suite) Test_NewMkLine__leading_space(c *check.C) { t := s.Init(c) _ = t.NewMkLine("rubyversion.mk", 427, " _RUBYVER=\t2.15") @@ -311,7 +311,7 @@ func (s *Suite) Test_MkLines_Check__extra(c *check.C) { "NOTE: options.mk:11: You can use \"../build\" instead of \"${WRKSRC}/../build\".") } -func (s *Suite) Test_MkLine_variableNeedsQuoting__unknown_rhs(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__unknown_rhs(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("fname", 1, "PKGNAME := ${UNKNOWN}") @@ -323,7 +323,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__unknown_rhs(c *check.C) { c.Check(nq, equals, nqDontKnow) } -func (s *Suite) Test_MkLine_variableNeedsQuoting__append_URL_to_list_of_URLs(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__append_URL_to_list_of_URLs(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -341,7 +341,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__append_URL_to_list_of_URLs(c * t.CheckOutputEmpty() // Up to pkglint 5.3.6, it warned about a missing :Q here, which was wrong. } -func (s *Suite) Test_MkLine_variableNeedsQuoting__append_list_to_list(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__append_list_to_list(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -355,7 +355,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__append_list_to_list(c *check.C t.CheckOutputEmpty() } -func (s *Suite) Test_MkLine_variableNeedsQuoting__eval_shell(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__eval_shell(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -369,7 +369,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__eval_shell(c *check.C) { "NOTE: builtin.mk:3: The :Q operator isn't necessary for ${BUILTIN_PKG.Xfixes} here.") } -func (s *Suite) Test_MkLine_variableNeedsQuoting__command_in_single_quotes(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_single_quotes(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -382,7 +382,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__command_in_single_quotes(c *ch "WARN: Makefile:3: Please use ${INSTALL:Q} instead of ${INSTALL} and make sure the variable appears outside of any quoting characters.") } -func (s *Suite) Test_MkLine_variableNeedsQuoting__command_in_command(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_command(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -401,7 +401,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__command_in_command(c *check.C) "WARN: Makefile:2: The exitcode of \"${FIND}\" at the left of the | operator is ignored.") } -func (s *Suite) Test_MkLine_variableNeedsQuoting__word_as_part_of_word(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__word_as_part_of_word(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -421,7 +421,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__word_as_part_of_word(c *check. // therefore no warning is issued in both these cases. // // Based on graphics/circos/Makefile. -func (s *Suite) Test_MkLine_variableNeedsQuoting__command_as_command_argument(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_as_command_argument(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -441,7 +441,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__command_as_command_argument(c } // Based on mail/mailfront/Makefile. -func (s *Suite) Test_MkLine_variableNeedsQuoting__URL_as_part_of_word_in_list(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__URL_as_part_of_word_in_list(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -460,7 +460,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__URL_as_part_of_word_in_list(c // modifier. // // Based on www/firefox31/xpi.mk. -func (s *Suite) Test_MkLine_variableNeedsQuoting__command_in_subshell(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_subshell(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -483,7 +483,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__command_in_subshell(c *check.C // LDFLAGS (and even more so CPPFLAGS and CFLAGS) may contain special // shell characters like quotes or backslashes. Therefore, quoting them // correctly is more tricky than with other variables. -func (s *Suite) Test_MkLine_variableNeedsQuoting__LDFLAGS_in_single_quotes(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__LDFLAGS_in_single_quotes(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -506,7 +506,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__LDFLAGS_in_single_quotes(c *ch // requires the variable to be declared as "lkSpace". // In this case it doesn't matter though since each option is an identifier, // and these do not pose any quoting or escaping problems. -func (s *Suite) Test_MkLine_variableNeedsQuoting__package_options(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__package_options(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -542,7 +542,7 @@ func (s *Suite) Test_MkLines_Check__MASTER_SITE_in_HOMEPAGE(c *check.C) { "WARN: devel/catch/Makefile:5: HOMEPAGE should not be defined in terms of MASTER_SITEs.") } -func (s *Suite) Test_MkLine_variableNeedsQuoting__tool_in_quotes_in_subshell_in_shellwords(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__tool_in_quotes_in_subshell_in_shellwords(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -559,7 +559,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__tool_in_quotes_in_subshell_in_ t.CheckOutputEmpty() } -func (s *Suite) Test_MkLine_variableNeedsQuoting__LDADD_in_BUILDLINK_TRANSFORM(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__LDADD_in_BUILDLINK_TRANSFORM(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -574,7 +574,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__LDADD_in_BUILDLINK_TRANSFORM(c "WARN: x11/qt5-qtbase/Makefile.common:1: Please use ${BUILDLINK_LDADD.dl:Q} instead of ${BUILDLINK_LDADD.dl:M*}.") } -func (s *Suite) Test_MkLine_variableNeedsQuoting__command_in_message(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_message(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -588,7 +588,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__command_in_message(c *check.C) t.CheckOutputEmpty() } -func (s *Suite) Test_MkLine_variableNeedsQuoting__guessed_list_variable_in_quotes(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__guessed_list_variable_in_quotes(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -604,7 +604,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__guessed_list_variable_in_quote "WARN: audio/jack-rack/Makefile:3: The variable LADSPA_PLUGIN_PATH should be quoted as part of a shell word.") } -func (s *Suite) Test_MkLine_variableNeedsQuoting__list_in_list(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__list_in_list(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -618,7 +618,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__list_in_list(c *check.C) { t.CheckOutputEmpty() // Don't warn about missing :Q modifiers. } -func (s *Suite) Test_MkLine_variableNeedsQuoting__PKGNAME_and_URL_list_in_URL_list(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__PKGNAME_and_URL_list_in_URL_list(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -633,7 +633,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__PKGNAME_and_URL_list_in_URL_li t.CheckOutputEmpty() // Don't warn about missing :Q modifiers. } -func (s *Suite) Test_MkLine_variableNeedsQuoting__tool_in_CONFIGURE_ENV(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__tool_in_CONFIGURE_ENV(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -654,7 +654,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__tool_in_CONFIGURE_ENV(c *check "NOTE: Makefile:3: The :Q operator isn't necessary for ${TOOLS_TAR} here.") } -func (s *Suite) Test_MkLine_variableNeedsQuoting__backticks(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__backticks(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -682,7 +682,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__backticks(c *check.C) { // the :Q modifier can be safely removed since pkgsrc will never support // having special characters in these directory names. // For guessed variable types be cautious and don't autofix them. -func (s *Suite) Test_MkLine_variableNeedsQuoting__only_remove_known(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__only_remove_known(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall", "--autofix") @@ -709,7 +709,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__only_remove_known(c *check.C) // TODO: COMPILER_RPATH_FLAG and LINKER_RPATH_FLAG have different types // defined in vardefs.go; examine why. -func (s *Suite) Test_MkLine_variableNeedsQuoting__shellword_part(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__shellword_part(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall,no-space") @@ -729,7 +729,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__shellword_part(c *check.C) { } // Tools, when used in a shell command, must not be quoted. -func (s *Suite) Test_MkLine_variableNeedsQuoting__tool_in_shell_command(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__tool_in_shell_command(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall,no-space") @@ -747,7 +747,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__tool_in_shell_command(c *check } // These examples from real pkgsrc end up in the final nqDontKnow case. -func (s *Suite) Test_MkLine_variableNeedsQuoting__uncovered_cases(c *check.C) { +func (s *Suite) Test_MkLine_VariableNeedsQuoting__uncovered_cases(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall,no-space") @@ -770,7 +770,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__uncovered_cases(c *check.C) { "WARN: ~/Makefile:4: LINKER_RPATH_FLAG should not be evaluated at load time.") } -func (s *Suite) Test_MkLine_Pkgmandir(c *check.C) { +func (s *Suite) Test_ShellLine_CheckWord__PKGMANDIR(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -820,7 +820,7 @@ func (s *Suite) Test_MkLines_Check__shell_command_as_wordpart_in_ENV_list(c *che "WARN: x11/lablgtk1/Makefile:2: Please use ${CC:Q} instead of ${CC}.") } -func (s *Suite) Test_MkLine_shell_varuse_in_backt_dquot(c *check.C) { +func (s *Suite) Test_MkLine__shell_varuse_in_backt_dquot(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -838,7 +838,7 @@ func (s *Suite) Test_MkLine_shell_varuse_in_backt_dquot(c *check.C) { } // See PR 46570, Ctrl+F "3. In lang/perl5". -func (s *Suite) Test_MkLine_VariableType(c *check.C) { +func (s *Suite) Test_Pkgsrc_VariableType(c *check.C) { t := s.Init(c) t.SetupVartypes() diff --git a/pkgtools/pkglint/files/mklinechecker.go b/pkgtools/pkglint/files/mklinechecker.go index 4ef128383c1..b9c7981baf7 100644 --- a/pkgtools/pkglint/files/mklinechecker.go +++ b/pkgtools/pkglint/files/mklinechecker.go @@ -18,7 +18,7 @@ func (ck MkLineChecker) Check() { mkline := ck.MkLine CheckLineTrailingWhitespace(mkline.Line) - CheckLineValidCharacters(mkline.Line, `[\t -~]`) + CheckLineValidCharacters(mkline.Line) switch { case mkline.IsVarassign(): @@ -379,15 +379,11 @@ func (ck MkLineChecker) CheckVaruse(varuse *MkVarUse, vuc *VarUseContext) { ck.checkVarusePermissions(varname, vartype, vuc) if varname == "LOCALBASE" && !G.Infrastructure { - ck.WarnVaruseLocalbase() + ck.MkLine.Warnf("Please use PREFIX instead of LOCALBASE.") } needsQuoting := mkline.VariableNeedsQuoting(varname, vartype, vuc) - if vuc.quoting == vucQuotFor { - ck.checkVaruseFor(varname, vartype, needsQuoting) - } - if G.opts.WarnQuoting && vuc.quoting != vucQuotUnknown && needsQuoting != nqDontKnow { ck.CheckVaruseShellword(varname, vartype, vuc, varuse.Mod(), needsQuoting) } @@ -538,57 +534,6 @@ func (ck MkLineChecker) checkVaruseLoadTime(varname string, isIndirect bool) { } } -func (ck MkLineChecker) WarnVaruseLocalbase() { - ck.MkLine.Warnf("Please use PREFIX instead of LOCALBASE.") - Explain( - // from jlam via private mail. - "Currently, LOCALBASE is typically used in these cases:", - "", - "(1) To locate a file or directory from another package.", - "(2) To refer to own files after installation.", - "", - "Example for (1):", - "", - " STRLIST= ${LOCALBASE}/bin/strlist", - "", - " do-build:", - " cd ${WRKSRC} && ${STRLIST} *.str", - "", - "This should better be:", - "", - " EVAL_PREFIX= STRLIST_PREFIX=strlist", - " STRLIST= ${STRLIST_PREFIX}/bin/strlist", - "", - " do-build:", - " cd ${WRKSRC} && ${STRLIST} *.str", - "", - "Example for (2):", - "", - " CONFIGURE_ENV+= --with-datafiles=${LOCALBASE}/share/pkgbase", - "", - "This should better be:", - "", - " CONFIGURE_ENV+= --with-datafiles=${PREFIX}/share/pkgbase") -} - -func (ck MkLineChecker) checkVaruseFor(varname string, vartype *Vartype, needsQuoting NeedsQuoting) { - if trace.Tracing { - defer trace.Call(varname, vartype, needsQuoting)() - } - - if false && // Too many false positives - vartype != nil && - vartype.kindOfList != lkSpace && - needsQuoting != nqDoesntMatter { - ck.MkLine.Warnf("The variable %s should not be used in .for loops.", varname) - Explain( - "The .for loop splits its argument at sequences of white-space, as", - "opposed to all other places in make(1), which act like the shell.", - "Therefore only variables that are split at whitespace or don't", - "contain any special characters should be used here.") - } -} - // CheckVaruseShellword checks whether a variable use of the form ${VAR} // or ${VAR:Modifier} is allowed in a certain context. func (ck MkLineChecker) CheckVaruseShellword(varname string, vartype *Vartype, vuc *VarUseContext, mod string, needsQuoting NeedsQuoting) { @@ -601,7 +546,7 @@ func (ck MkLineChecker) CheckVaruseShellword(varname string, vartype *Vartype, v // configure scripts. // // When doing checks outside a package, the :M* operator is needed for safety. - needMstar := matches(varname, `^(?:.*_)?(?:CFLAGS||CPPFLAGS|CXXFLAGS|FFLAGS|LDFLAGS|LIBS)$`) && + needMstar := matches(varname, `^(?:.*_)?(?:CFLAGS|CPPFLAGS|CXXFLAGS|FFLAGS|LDFLAGS|LIBS)$`) && (G.Pkg == nil || G.Pkg.vars.Defined("GNU_CONFIGURE")) strippedMod := mod @@ -759,23 +704,12 @@ func (ck MkLineChecker) checkVarassign() { ck.checkVarassignSpecific() - if varname == "EVAL_PREFIX" { - if m, evalVarname := match1(value, `^([\w_]+)=`); m { - - // The variables mentioned in EVAL_PREFIX will later be - // defined by find-prefix.mk. Therefore, they are marked - // as known in the current file. - G.Mk.vars.Define(evalVarname, mkline) - } - } - if fix := G.Pkgsrc.Deprecated[varname]; fix != "" { mkline.Warnf("Definition of %s is deprecated. %s", varname, fix) - } else if fix := G.Pkgsrc.Deprecated[varcanon]; fix != "" { + } else if fix = G.Pkgsrc.Deprecated[varcanon]; fix != "" { mkline.Warnf("Definition of %s is deprecated. %s", varname, fix) } - ck.checkVarassignPlistComment(varname, value) ck.checkVarassignVaruse() } @@ -839,10 +773,10 @@ func (ck MkLineChecker) checkVarassignVaruseShell(vartype *Vartype, time vucTime mkline := ck.MkLine atoms := NewShTokenizer(mkline.Line, mkline.Value(), false).ShAtoms() for i, atom := range atoms { - if atom.Type == shtVaruse { + if varuse := atom.VarUse(); varuse != nil { isWordPart := isWordPart(atoms, i) vuc := &VarUseContext{vartype, time, atom.Quoting.ToVarUseContext(), isWordPart} - ck.CheckVaruse(atom.Data.(*MkVarUse), vuc) + ck.CheckVaruse(varuse, vuc) } } } @@ -922,34 +856,6 @@ func (ck MkLineChecker) checkVarassignBsdPrefs() { "bsd.prefs.mk file, which will take care of everything.") } -func (ck MkLineChecker) checkVarassignPlistComment(varname, value string) { - if false && // This is currently neither correct nor helpful - contains(value, "@comment") && !matches(value, `="@comment "`) { - ck.MkLine.Warnf("Please don't use @comment in %s.", varname) - Explain( - "If you are defining a PLIST condition here, use one of the", - "following patterns instead:", - "", - "1. The direct way, without intermediate variable", - "", - "\tPLIST_SUBST+=\tMY_VAR=\"@comment \"", - "", - "2. The indirect way, with a separate variable", - "", - "\tPLIST_VARS+=\tMY_VAR", - "\t.if ...", - "\tMY_VAR?=\tyes", - "\t.endif") - } - - // Mark the variable as PLIST condition. This is later used in checkfile_PLIST. - if G.Pkg != nil { - if m, plistVarname := match1(value, `(.+)=.*@comment.*`); m { - G.Pkg.plistSubstCond[plistVarname] = true - } - } -} - func (ck MkLineChecker) CheckVartype(varname string, op MkOperator, value, comment string) { if trace.Tracing { defer trace.Call(varname, op, value, comment)() @@ -1040,7 +946,7 @@ func (ck MkLineChecker) checkText(text string) { rest := text for { - m, r := regex.ReplaceFirst(rest, `(?:^|[^$])\$\{([-A-Z0-9a-z_]+)(\.[\-0-9A-Z_a-z]+)?(?::[^\}]+)?\}`, "") + m, r := G.res.ReplaceFirst(rest, `(?:^|[^$])\$\{([-A-Z0-9a-z_]+)(\.[\-0-9A-Z_a-z]+)?(?::[^\}]+)?\}`, "") if m == nil { break } @@ -1145,7 +1051,7 @@ func (ck MkLineChecker) checkCompareVarStr(varname, op, value string) { func (ck MkLineChecker) CheckValidCharactersInValue(reValid regex.Pattern) { mkline := ck.MkLine - rest := regex.Compile(reValid).ReplaceAllString(mkline.Value(), "") + rest := replaceAll(mkline.Value(), reValid, "") if rest != "" { uni := "" for _, c := range rest { diff --git a/pkgtools/pkglint/files/mklinechecker_test.go b/pkgtools/pkglint/files/mklinechecker_test.go index 0e0a684b164..21976d57c83 100644 --- a/pkgtools/pkglint/files/mklinechecker_test.go +++ b/pkgtools/pkglint/files/mklinechecker_test.go @@ -2,6 +2,131 @@ package main import "gopkg.in/check.v1" +func (s *Suite) Test_MkLineChecker_Check__url2pkg(c *check.C) { + t := s.Init(c) + + t.SetupVartypes() + + mkline := t.NewMkLine("fname.mk", 1, "# url2pkg-marker") + + MkLineChecker{mkline}.Check() + + t.CheckOutputLines( + "ERROR: fname.mk:1: This comment indicates unfinished work (url2pkg).") +} + +func (s *Suite) Test_MkLineChecker_Check__buildlink3_include_prefs(c *check.C) { + t := s.Init(c) + + t.SetupVartypes() + + t.CreateFileLines("mk/bsd.prefs.mk") + mklines := t.SetupFileMkLines("category/package/buildlink3.mk", + ".include \"../../mk/bsd.prefs.mk\"") + // If the buildlink3.mk file is not actually created, resolving the + // relative path fails since that depends on the actual file system, + // not on syntactical paths; see os.Stat in CheckRelativePath. + + MkLineChecker{mklines.mklines[0]}.Check() + + t.CheckOutputLines( + "NOTE: ~/category/package/buildlink3.mk:1: For efficiency reasons, please include bsd.fast.prefs.mk instead of bsd.prefs.mk.") +} + +func (s *Suite) Test_MkLineChecker_checkInclude(c *check.C) { + t := s.Init(c) + + t.SetupVartypes() + + t.CreateFileLines("pkgtools/x11-links/buildlink3.mk") + t.CreateFileLines("graphics/jpeg/buildlink3.mk") + t.CreateFileLines("devel/intltool/buildlink3.mk") + t.CreateFileLines("devel/intltool/builtin.mk") + mklines := t.SetupFileMkLines("category/package/fname.mk", + MkRcsID, + "", + ".include \"../../pkgtools/x11-links/buildlink3.mk\"", + ".include \"../../graphics/jpeg/buildlink3.mk\"", + ".include \"../../devel/intltool/buildlink3.mk\"", + ".include \"../../devel/intltool/builtin.mk\"") + + mklines.Check() + + t.CheckOutputLines( + "ERROR: ~/category/package/fname.mk:3: ../../pkgtools/x11-links/buildlink3.mk must not be included directly. "+ + "Include \"../../mk/x11.buildlink3.mk\" instead.", + "ERROR: ~/category/package/fname.mk:4: ../../graphics/jpeg/buildlink3.mk must not be included directly. "+ + "Include \"../../mk/jpeg.buildlink3.mk\" instead.", + "WARN: ~/category/package/fname.mk:5: Please write \"USE_TOOLS+= intltool\" instead of this line.", + "ERROR: ~/category/package/fname.mk:6: ../../devel/intltool/builtin.mk must not be included directly. "+ + "Include \"../../devel/intltool/buildlink3.mk\" instead.") +} + +func (s *Suite) Test_MkLineChecker_checkDirective(c *check.C) { + t := s.Init(c) + + t.SetupVartypes() + + mklines := t.NewMkLines("category/package/fname.mk", + MkRcsID, + "", + ".for", + ".endfor", + "", + ".if", + ".else don't", + ".endif invalid-arg", + "", + ".ifdef FNAME_MK", + ".endif", + ".ifndef FNAME_MK", + ".endif", + "", + ".for var in a b c", + ".endfor", + ".undef var", + "", + ".for VAR in a b c", + ".endfor", + "", + ".for $ in a b c", + ".endfor") + + mklines.Check() + + t.CheckOutputLines( + "ERROR: category/package/fname.mk:3: \".for\" requires arguments.", + "ERROR: category/package/fname.mk:6: \".if\" requires arguments.", + "ERROR: category/package/fname.mk:7: \".else\" does not take arguments. If you meant \"else if\", use \".elif\".", + "ERROR: category/package/fname.mk:8: \".endif\" does not take arguments.", + "WARN: category/package/fname.mk:10: The \".ifdef\" directive is deprecated. Please use \".if defined(FNAME_MK)\" instead.", + "WARN: category/package/fname.mk:12: The \".ifndef\" directive is deprecated. Please use \".if !defined(FNAME_MK)\" instead.", + "NOTE: category/package/fname.mk:17: Using \".undef\" after a \".for\" loop is unnecessary.", + "WARN: category/package/fname.mk:19: .for variable names should not contain uppercase letters.", + "ERROR: category/package/fname.mk:22: Invalid variable name \"$\".") +} + +func (s *Suite) Test_MkLineChecker_checkDependencyRule(c *check.C) { + t := s.Init(c) + + t.SetupVartypes() + + mklines := t.NewMkLines("category/package/fname.mk", + MkRcsID, + "", + ".PHONY: target-1", + "target-2: .PHONY", + ".ORDER: target-1 target-2", + "target-1:", + "target-2:", + "target-3:") + + mklines.Check() + + t.CheckOutputLines( + "WARN: category/package/fname.mk:8: Unusual target \"target-3\".") +} + func (s *Suite) Test_MkLineChecker_CheckVartype__simple_type(c *check.C) { t := s.Init(c) @@ -36,6 +161,30 @@ func (s *Suite) Test_MkLineChecker_CheckVartype(c *check.C) { t.CheckOutputEmpty() } +func (s *Suite) Test_MkLineChecker_CheckVartype__skip(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wno-types") + t.SetupVartypes() + mkline := t.NewMkLine("fname", 1, "DISTNAME=invalid:::distname") + + MkLineChecker{mkline}.Check() + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLineChecker_CheckVartype__append_to_non_list(c *check.C) { + t := s.Init(c) + + t.SetupVartypes() + mkline := t.NewMkLine("fname", 1, "DISTNAME+=suffix") + + MkLineChecker{mkline}.Check() + + t.CheckOutputLines( + "WARN: fname:1: The \"+=\" operator should only be used with lists.") +} + // Pkglint once interpreted all lists as consisting of shell tokens, // splitting this URL at the ampersand. func (s *Suite) Test_MkLineChecker_checkVarassign__URL_with_shell_special_characters(c *check.C) { @@ -138,34 +287,39 @@ func (s *Suite) Test_MkLineChecker_checkVarassign(c *check.C) { func (s *Suite) Test_MkLineChecker_checkVarassignPermissions(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall") + t.SetupCommandLine("-Wall,no-space") t.SetupVartypes() - mkline := t.NewMkLine("options.mk", 2, "PKG_DEVELOPER?=\tyes") + mklines := t.NewMkLines("options.mk", + MkRcsID, + "PKG_DEVELOPER?= yes", + "BUILD_DEFS?= VARBASE") - MkLineChecker{mkline}.checkVarassignPermissions() + mklines.Check() t.CheckOutputLines( - "WARN: options.mk:2: The variable PKG_DEVELOPER may not be given a default value by any package.") + "WARN: options.mk:2: The variable PKG_DEVELOPER may not be given a default value by any package.", + "WARN: options.mk:2: Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".", + "WARN: options.mk:3: The variable BUILD_DEFS may not be given a default value (only appended to) in this file.") } // Don't check the permissions for infrastructure files since they have their own rules. -func (s *Suite) Test_MkLineChecker_checkVarassignDefPermissions__infrastructure(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVarassignPermissions__infrastructure(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") t.SetupVartypes() - t.SetupFileMkLines("mk/infra.mk", + t.CreateFileLines("mk/infra.mk", MkRcsID, "", "PKG_DEVELOPER?=\tyes") - t.SetupFileMkLines("mk/bsd.pkg.mk") + t.CreateFileLines("mk/bsd.pkg.mk") G.CheckDirent(t.File("mk/infra.mk")) t.CheckOutputEmpty() } -func (s *Suite) Test_MkLineChecker_CheckVarusePermissions(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVarusePermissions(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -186,7 +340,7 @@ func (s *Suite) Test_MkLineChecker_CheckVarusePermissions(c *check.C) { "NOTE: options.mk:4: This variable value should be aligned to column 17.") } -func (s *Suite) Test_MkLineChecker_CheckVarusePermissions__load_time(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -202,12 +356,13 @@ func (s *Suite) Test_MkLineChecker_CheckVarusePermissions__load_time(c *check.C) "NOTE: options.mk:2: This variable value should be aligned to column 17.") } -func (s *Suite) Test_MkLineChecker_WarnVaruseLocalbase(c *check.C) { +func (s *Suite) Test_MkLineChecker__warn_varuse_LOCALBASE(c *check.C) { t := s.Init(c) + t.SetupVartypes() mkline := t.NewMkLine("options.mk", 56, "PKGNAME=${LOCALBASE}") - MkLineChecker{mkline}.WarnVaruseLocalbase() + MkLineChecker{mkline}.Check() t.CheckOutputLines( "WARN: options.mk:56: Please use PREFIX instead of LOCALBASE.") @@ -256,7 +411,7 @@ func (s *Suite) Test_MkLineChecker__Varuse_Modifier_L(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_MkLineChecker_CheckCond__comparison_with_shell_command(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparison_with_shell_command(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -273,7 +428,7 @@ func (s *Suite) Test_MkLineChecker_CheckCond__comparison_with_shell_command(c *c "WARN: security/openssl/Makefile:2: Use ${PKGSRC_COMPILER:Mgcc} instead of the == operator.") } -func (s *Suite) Test_MkLine_CheckCond_comparing_PKGSRC_COMPILER_with_eqeq(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparing_PKGSRC_COMPILER_with_eqeq(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -313,7 +468,7 @@ func (s *Suite) Test_MkLineChecker_CheckVartype__CFLAGS_with_backticks(c *check. // See PR 46570, Ctrl+F "4. Shell quoting". // Pkglint is correct, since the shell sees this definition for // CPPFLAGS as three words, not one word. -func (s *Suite) Test_MkLineChecker_CheckVartype_CFLAGS(c *check.C) { +func (s *Suite) Test_MkLineChecker_CheckVartype__CFLAGS(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -330,7 +485,7 @@ func (s *Suite) Test_MkLineChecker_CheckVartype_CFLAGS(c *check.C) { // Up to 2018-01-28, pkglint applied the autofix also to the continuation // lines, which is incorrect. It replaced the dot in "4.*" with spaces. -func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation_autofix(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall", "--autofix") @@ -382,6 +537,64 @@ func (s *Suite) Test_MkLineChecker_CheckVaruseShellword(c *check.C) { "WARN: ~/options.mk:4: The variable PATH should be quoted as part of a shell word.") } +func (s *Suite) Test_MkLineChecker_CheckVaruseShellword__mstar(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-space") + t.SetupVartypes() + mklines := t.SetupFileMkLines("options.mk", + MkRcsID, + "CONFIGURE_ARGS+= ${CFLAGS:Q}", + "CONFIGURE_ARGS+= ${CFLAGS:M*:Q}", + "CONFIGURE_ARGS+= ${ADA_FLAGS:Q}", + "CONFIGURE_ARGS+= ${ADA_FLAGS:M*:Q}", + "CONFIGURE_ENV+= ${CFLAGS:Q}", + "CONFIGURE_ENV+= ${CFLAGS:M*:Q}", + "CONFIGURE_ENV+= ${ADA_FLAGS:Q}", + "CONFIGURE_ENV+= ${ADA_FLAGS:M*:Q}") + + mklines.Check() + + // FIXME: There should be some notes and warnings; prevented by the PERL5 case in VariableNeedsQuoting. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLineChecker_CheckVaruseShellword__mstar_not_needed(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-space") + pkg := t.SetupPackage("category/package", + "MAKE_FLAGS+=\tCFLAGS=${CFLAGS:M*:Q}", + "MAKE_FLAGS+=\tLFLAGS=${LDFLAGS:M*:Q}") + G.Pkgsrc.LoadInfrastructure() + // FIXME: It is too easy to forget this important call. + + // This package is guaranteed to not use GNU_CONFIGURE. + // Since the :M* hack is only needed for GNU_CONFIGURE, it is not necessary here. + G.CheckDirent(pkg) + + // FIXME: Duplicate diagnostics. + t.CheckOutputLines( + "NOTE: ~/category/package/Makefile:20: The :M* modifier is not needed here.", + "NOTE: ~/category/package/Makefile:20: The :M* modifier is not needed here.", + "NOTE: ~/category/package/Makefile:21: The :M* modifier is not needed here.", + "NOTE: ~/category/package/Makefile:21: The :M* modifier is not needed here.") +} + +func (s *Suite) Test_MkLineChecker_CheckVaruseShellword__q_not_needed(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + pkg := t.SetupPackage("category/package", + "MASTER_SITES=\t${HOMEPAGE:Q}") + G.Pkgsrc.LoadInfrastructure() + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:5: The :Q operator should not be used for ${HOMEPAGE} here.") +} + // The ${VARNAME:=suffix} expression should only be used with lists. // It typically appears in MASTER_SITE definitions. func (s *Suite) Test_MkLineChecker_CheckVaruse__eq_nonlist(c *check.C) { @@ -455,7 +668,10 @@ func (s *Suite) Test_MkLineChecker_checkVarassignSpecific(c *check.C) { "_TOOLS_VARNAME.sed= SED", "DIST_SUBDIR= ${PKGNAME}", "WRKSRC= ${PKGNAME}", - "SITES_distfile.tar.gz= ${MASTER_SITES_GITHUB:=user/}") + "SITES_distfile.tar.gz= ${MASTER_SITES_GITHUB:=user/}", + // TODO: The first of the below assignments should be flagged as redundant by RedundantScope. + "PYTHON_VERSIONS_ACCEPTED= -13", + "PYTHON_VERSIONS_ACCEPTED= 27 36") mklines.Check() @@ -466,7 +682,14 @@ func (s *Suite) Test_MkLineChecker_checkVarassignSpecific(c *check.C) { "WARN: ~/module.mk:4: PKGNAME should not be used in DIST_SUBDIR, as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", "WARN: ~/module.mk:5: PKGNAME should not be used in WRKSRC, as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", "WARN: ~/module.mk:6: SITES_distfile.tar.gz is defined but not used.", - "WARN: ~/module.mk:6: SITES_* is deprecated. Please use SITES.* instead.") + "WARN: ~/module.mk:6: SITES_* is deprecated. Please use SITES.* instead.", + "WARN: ~/module.mk:7: The variable PYTHON_VERSIONS_ACCEPTED may not be set "+ + "(only given a default value, appended to) in this file; it would be ok in Makefile, Makefile.common, options.mk.", + "WARN: ~/module.mk:7: Invalid version number \"-13\".", + "ERROR: ~/module.mk:7: All values for PYTHON_VERSIONS_ACCEPTED must be positive integers.", + "WARN: ~/module.mk:8: The variable PYTHON_VERSIONS_ACCEPTED may not be set "+ + "(only given a default value, appended to) in this file; it would be ok in Makefile, Makefile.common, options.mk.", + "WARN: ~/module.mk:8: The values for PYTHON_VERSIONS_ACCEPTED should be in decreasing order.") } func (s *Suite) Test_MkLineChecker_checkText(c *check.C) { diff --git a/pkgtools/pkglint/files/mklines.go b/pkgtools/pkglint/files/mklines.go index 8433fd50eee..ff328db4849 100644 --- a/pkgtools/pkglint/files/mklines.go +++ b/pkgtools/pkglint/files/mklines.go @@ -112,8 +112,8 @@ func (mklines *MkLines) Check() { switch mkline.Varcanon() { case "PLIST_VARS": - value := mkline.ValueSplit(resolveVariableRefs(mkline.Value()), "") - for _, id := range value { + ids := mkline.ValueSplit(resolveVariableRefs(mkline.Value()), "") + for _, id := range ids { if !mklines.plistVarSkip && mklines.plistVarSet[id] == nil { mkline.Warnf("%q is added to PLIST_VARS, but PLIST.%s is not defined in this file.", id, id) } @@ -220,8 +220,8 @@ func (mklines *MkLines) DetermineDefinedVariables() { } case "PLIST_VARS": - value := mkline.ValueSplit(resolveVariableRefs(mkline.Value()), "") - for _, id := range value { + ids := mkline.ValueSplit(resolveVariableRefs(mkline.Value()), "") + for _, id := range ids { if trace.Tracing { trace.Step1("PLIST.%s is added to PLIST_VARS.", id) } @@ -255,8 +255,8 @@ func (mklines *MkLines) collectPlistVars() { if mkline.IsVarassign() { switch mkline.Varcanon() { case "PLIST_VARS": - value := mkline.ValueSplit(resolveVariableRefs(mkline.Value()), "") - for _, id := range value { + ids := mkline.ValueSplit(resolveVariableRefs(mkline.Value()), "") + for _, id := range ids { if containsVarRef(id) { mklines.plistVarSkip = true } else { @@ -310,9 +310,13 @@ func (mklines *MkLines) determineDocumentedVariables() { for _, mkline := range mklines.mklines { text := mkline.Text - words := splitOnSpace(text) + switch { + case hasPrefix(text, "#"): + words := splitOnSpace(text) + if len(words) <= 1 { + break + } - if 1 < len(words) && words[0] == "#" { commentLines++ parser := NewMkParser(mkline.Line, words[1], false) @@ -330,9 +334,8 @@ func (mklines *MkLines) determineDocumentedVariables() { if 1 < len(words) && words[1] == "Copyright" { relevant = false } - } - if text == "" { + case mkline.IsEmpty(): finish() } } diff --git a/pkgtools/pkglint/files/mklines_test.go b/pkgtools/pkglint/files/mklines_test.go index 97ce2e3a4a4..dd880ae8728 100644 --- a/pkgtools/pkglint/files/mklines_test.go +++ b/pkgtools/pkglint/files/mklines_test.go @@ -64,7 +64,7 @@ func (s *Suite) Test_MkLineChecker_checkInclude__Makefile(c *check.C) { "ERROR: ~/Makefile:2: Other Makefiles must not be included directly.") } -func (s *Suite) Test_MkLines_quoting_LDFLAGS_for_GNU_configure(c *check.C) { +func (s *Suite) Test_MkLines__quoting_LDFLAGS_for_GNU_configure(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -386,7 +386,7 @@ func (s *Suite) Test_MkLines_DetermineUsedVariables__nested(c *check.C) { c.Check(mklines.vars.FirstUse("outer.*"), equals, mkline) } -func (s *Suite) Test_MkLines_PrivateTool_Undefined(c *check.C) { +func (s *Suite) Test_MkLines__private_tool_undefined(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -402,7 +402,7 @@ func (s *Suite) Test_MkLines_PrivateTool_Undefined(c *check.C) { "WARN: fname:3: Unknown shell command \"md5sum\".") } -func (s *Suite) Test_MkLines_PrivateTool_Defined(c *check.C) { +func (s *Suite) Test_MkLines__private_tool_defined(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -420,7 +420,7 @@ func (s *Suite) Test_MkLines_PrivateTool_Defined(c *check.C) { "WARN: fname:4: The \"md5sum\" tool is used but not added to USE_TOOLS.") } -func (s *Suite) Test_MkLines_Check_indentation(c *check.C) { +func (s *Suite) Test_MkLines_Check__indentation(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -519,7 +519,7 @@ func (s *Suite) Test_MkLines_Check__unbalanced_directives(c *check.C) { // Demonstrates how to define your own make(1) targets for creating // files in the current directory. The pkgsrc-wip category Makefile // does this, while all other categories don't need any custom code. -func (s *Suite) Test_MkLines_wip_category_Makefile(c *check.C) { +func (s *Suite) Test_MkLines__wip_category_Makefile(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall", "--explain") diff --git a/pkgtools/pkglint/files/mkparser.go b/pkgtools/pkglint/files/mkparser.go index 7d20a5ad0b2..cac6856c630 100644 --- a/pkgtools/pkglint/files/mkparser.go +++ b/pkgtools/pkglint/files/mkparser.go @@ -6,6 +6,8 @@ import ( "strings" ) +// MkParser wraps a Parser and provides methods for parsing +// things related to Makefiles. type MkParser struct { *Parser } @@ -193,6 +195,7 @@ loop: } repl.Reset(modifierMark) + // FIXME: Why AdvanceRegexp? This accepts :S,a,b,c,d,e,f but shouldn't. for p.VarUse() != nil || repl.AdvanceRegexp(regex.Pattern(`^([^:$`+closing+`]|\$\$)+`)) { } if suffixSubst := repl.Since(modifierMark); contains(suffixSubst, "=") { diff --git a/pkgtools/pkglint/files/mkparser_test.go b/pkgtools/pkglint/files/mkparser_test.go index 06a17476c82..a917a82dfce 100644 --- a/pkgtools/pkglint/files/mkparser_test.go +++ b/pkgtools/pkglint/files/mkparser_test.go @@ -104,7 +104,9 @@ func (s *Suite) Test_MkParser_MkTokens(c *check.C) { check("${${${PKG_INFO} -E ${d} || echo:L:sh}:L:C/[^[0-9]]*/ /g:[1..3]:ts.}", varuse("${${PKG_INFO} -E ${d} || echo:L:sh}", "L", "C/[^[0-9]]*/ /g", "[1..3]", "ts.")) - check("${VAR:S/-//S/.//}", varuseText("${VAR:S/-//S/.//}", "VAR", "S/-//", "S/.//")) // For :S and :C, the colon can be left out. + // For :S and :C, the colon can be left out. + check("${VAR:S/-//S/.//}", + varuseText("${VAR:S/-//S/.//}", "VAR", "S/-//", "S/.//")) check("${VAR:ts}", varuse("VAR", "ts")) // The separator character can be left out. check("${VAR:ts\\000012}", varuse("VAR", "ts\\000012")) // The separator character can be a long octal number. @@ -130,12 +132,21 @@ func (s *Suite) Test_MkParser_MkTokens(c *check.C) { checkRest("hello, ${W:L:tl}orld", []*MkToken{ literal("hello, "), varuse("W", "L", "tl"), - literal("orld")}, "") + literal("orld")}, + "") checkRest("ftp://${PKGNAME}/ ${MASTER_SITES:=subdir/}", []*MkToken{ literal("ftp://"), varuse("PKGNAME"), literal("/ "), - varuse("MASTER_SITES", "=subdir/")}, "") + varuse("MASTER_SITES", "=subdir/")}, + "") + + // FIXME: Text must match modifiers. + checkRest("${VAR:S,a,b,c,d,e,f}", + []*MkToken{{ + Text: "${VAR:S,a,b,c,d,e,f}", + Varuse: &MkVarUse{varname: "VAR", modifiers: []string{"S,a,b,"}}}}, + "") } func (s *Suite) Test_MkParser_MkCond(c *check.C) { diff --git a/pkgtools/pkglint/files/mkshparser_test.go b/pkgtools/pkglint/files/mkshparser_test.go index 6204d8c2254..40857bcd8fe 100644 --- a/pkgtools/pkglint/files/mkshparser_test.go +++ b/pkgtools/pkglint/files/mkshparser_test.go @@ -27,7 +27,15 @@ type ShSuite struct { var _ = check.Suite(&ShSuite{}) -func (s *ShSuite) Test_ShellParser_program(c *check.C) { +func (s *ShSuite) SetUpTest(c *check.C) { + G = NewPkglint() +} + +func (s *ShSuite) TearDownTest(c *check.C) { + G = Pkglint{} // Make it unusable +} + +func (s *ShSuite) Test_ShellParser__program(c *check.C) { b := s.init(c) s.test("", @@ -103,7 +111,7 @@ func (s *ShSuite) Test_ShellParser_program(c *check.C) { b.List().AddCommand(b.SimpleCommand("action2")).AddSemicolon()))) } -func (s *ShSuite) Test_ShellParser_list(c *check.C) { +func (s *ShSuite) Test_ShellParser__list(c *check.C) { b := s.init(c) s.test("echo1 && echo2", @@ -125,7 +133,7 @@ func (s *ShSuite) Test_ShellParser_list(c *check.C) { AddBackground()) } -func (s *ShSuite) Test_ShellParser_and_or(c *check.C) { +func (s *ShSuite) Test_ShellParser__and_or(c *check.C) { b := s.init(c) s.test("echo1 | echo2", @@ -154,7 +162,7 @@ func (s *ShSuite) Test_ShellParser_and_or(c *check.C) { b.SimpleCommand("echo4"))))) } -func (s *ShSuite) Test_ShellParser_pipeline(c *check.C) { +func (s *ShSuite) Test_ShellParser__pipeline(c *check.C) { b := s.init(c) s.test("command1 | command2", @@ -168,7 +176,7 @@ func (s *ShSuite) Test_ShellParser_pipeline(c *check.C) { b.SimpleCommand("command2"))))) } -func (s *ShSuite) Test_ShellParser_pipe_sequence(c *check.C) { +func (s *ShSuite) Test_ShellParser__pipe_sequence(c *check.C) { b := s.init(c) s.test("command1 | if true ; then : ; fi", @@ -179,7 +187,7 @@ func (s *ShSuite) Test_ShellParser_pipe_sequence(c *check.C) { b.List().AddCommand(b.SimpleCommand(":")).AddSemicolon()))))) } -func (s *ShSuite) Test_ShellParser_command(c *check.C) { +func (s *ShSuite) Test_ShellParser__command(c *check.C) { b := s.init(c) s.test("simple_command", @@ -205,7 +213,7 @@ func (s *ShSuite) Test_ShellParser_command(c *check.C) { b.Redirection(2, ">&", "1")))) } -func (s *ShSuite) Test_ShellParser_compound_command(c *check.C) { +func (s *ShSuite) Test_ShellParser__compound_command(c *check.C) { b := s.init(c) s.test("{ brace ; }", @@ -228,7 +236,7 @@ func (s *ShSuite) Test_ShellParser_compound_command(c *check.C) { } -func (s *ShSuite) Test_ShellParser_subshell(c *check.C) { +func (s *ShSuite) Test_ShellParser__subshell(c *check.C) { b := s.init(c) sub3 := b.Subshell(b.List().AddCommand(b.SimpleCommand("sub3"))) @@ -237,7 +245,7 @@ func (s *ShSuite) Test_ShellParser_subshell(c *check.C) { s.test("( ( ( sub3 ) ; sub2 ) ; sub1 )", b.List().AddCommand(sub1)) } -func (s *ShSuite) Test_ShellParser_compound_list(c *check.C) { +func (s *ShSuite) Test_ShellParser__compound_list(c *check.C) { b := s.init(c) s.test("( \n echo )", @@ -245,13 +253,13 @@ func (s *ShSuite) Test_ShellParser_compound_list(c *check.C) { b.List().AddCommand(b.SimpleCommand("echo"))))) } -func (s *ShSuite) Test_ShellParser_term(c *check.C) { +func (s *ShSuite) Test_ShellParser__term(c *check.C) { b := s.init(c) _ = b } -func (s *ShSuite) Test_ShellParser_for_clause(c *check.C) { +func (s *ShSuite) Test_ShellParser__for_clause(c *check.C) { b := s.init(c) s.test("for var do echo $var ; done", @@ -296,7 +304,7 @@ func (s *ShSuite) Test_ShellParser_for_clause(c *check.C) { b.List().AddCommand(b.SimpleCommand("echo", "$$i$$j")).AddSemicolon()))))) } -func (s *ShSuite) Test_ShellParser_case_clause(c *check.C) { +func (s *ShSuite) Test_ShellParser__case_clause(c *check.C) { b := s.init(c) s.test("case $var in esac", @@ -335,7 +343,7 @@ func (s *ShSuite) Test_ShellParser_case_clause(c *check.C) { } -func (s *ShSuite) Test_ShellParser_if_clause(c *check.C) { +func (s *ShSuite) Test_ShellParser__if_clause(c *check.C) { b := s.init(c) s.test( @@ -354,7 +362,7 @@ func (s *ShSuite) Test_ShellParser_if_clause(c *check.C) { b.List().AddCommand(b.SimpleCommand("action")).AddSemicolon()))))) } -func (s *ShSuite) Test_ShellParser_while_clause(c *check.C) { +func (s *ShSuite) Test_ShellParser__while_clause(c *check.C) { b := s.init(c) s.test("while condition ; do action ; done", @@ -363,7 +371,7 @@ func (s *ShSuite) Test_ShellParser_while_clause(c *check.C) { b.List().AddCommand(b.SimpleCommand("action")).AddSemicolon()))) } -func (s *ShSuite) Test_ShellParser_until_clause(c *check.C) { +func (s *ShSuite) Test_ShellParser__until_clause(c *check.C) { b := s.init(c) s.test("until condition ; do action ; done", @@ -372,13 +380,13 @@ func (s *ShSuite) Test_ShellParser_until_clause(c *check.C) { b.List().AddCommand(b.SimpleCommand("action")).AddSemicolon()))) } -func (s *ShSuite) Test_ShellParser_function_definition(c *check.C) { +func (s *ShSuite) Test_ShellParser__function_definition(c *check.C) { b := s.init(c) _ = b } -func (s *ShSuite) Test_ShellParser_brace_group(c *check.C) { +func (s *ShSuite) Test_ShellParser__brace_group(c *check.C) { b := s.init(c) // No semicolon necessary after the closing brace. @@ -389,7 +397,7 @@ func (s *ShSuite) Test_ShellParser_brace_group(c *check.C) { b.List().AddCommand(b.SimpleCommand("echo", "yes")).AddSemicolon()))))) } -func (s *ShSuite) Test_ShellParser_simple_command(c *check.C) { +func (s *ShSuite) Test_ShellParser__simple_command(c *check.C) { b := s.init(c) s.test( @@ -427,7 +435,7 @@ func (s *ShSuite) Test_ShellParser_simple_command(c *check.C) { b.List().AddCommand(b.SimpleCommand("{OpenGrok", "args"))) } -func (s *ShSuite) Test_ShellParser_io_redirect(c *check.C) { +func (s *ShSuite) Test_ShellParser__io_redirect(c *check.C) { b := s.init(c) s.test("echo >> ${PLIST_SRC}", @@ -472,7 +480,7 @@ func (s *ShSuite) Test_ShellParser_io_redirect(c *check.C) { {-1, ">", b.Token("/dev/stderr")}}}})) } -func (s *ShSuite) Test_ShellParser_io_here(c *check.C) { +func (s *ShSuite) Test_ShellParser__io_here(c *check.C) { b := s.init(c) _ = b diff --git a/pkgtools/pkglint/files/mkshtypes.go b/pkgtools/pkglint/files/mkshtypes.go index d5e66e0a386..c7ab0a732e0 100644 --- a/pkgtools/pkglint/files/mkshtypes.go +++ b/pkgtools/pkglint/files/mkshtypes.go @@ -1,14 +1,16 @@ package main import ( - "fmt" "netbsd.org/pkglint/regex" + "strings" ) +// MkShList is a list of shell commands, separated by newlines or semicolons. +// // Example: cd $dir && echo "In $dir"; cd ..; ls -l type MkShList struct { AndOrs []*MkShAndOr - Separators []MkShSeparator + Separators []MkShSeparator // One less entry than in AndOrs. } func NewMkShList() *MkShList { @@ -25,6 +27,9 @@ func (list *MkShList) AddSeparator(separator MkShSeparator) *MkShList { return list } +// MkShAndOr is a group of commands that are connected with && or || +// conditions. +// // Example: cd $dir && echo "In $dir" || echo "Cannot cd into $dir" type MkShAndOr struct { Pipes []*MkShPipeline @@ -41,6 +46,8 @@ func (andor *MkShAndOr) Add(op string, pipeline *MkShPipeline) *MkShAndOr { return andor } +// MkShPipeline is a group of commands, connected by pipelines. +// // Example: grep word file | sed s,^,---, type MkShPipeline struct { Negated bool @@ -56,6 +63,8 @@ func (pipe *MkShPipeline) Add(cmd *MkShCommand) *MkShPipeline { return pipe } +// MkShCommand is a simple or compound shell command. +// // Example: LC_ALL=C sort */*.c > sorted // Example: dir() { ls -l "$@"; } // Example: { echo "first"; echo "second"; } @@ -66,6 +75,8 @@ type MkShCommand struct { Redirects []*MkShRedirection // For Compound and FuncDef } +// MkShCompoundCommand is a group of commands. +// // Example: { echo "first"; echo "second"; } // Example: for f in *.c; do compile "$f"; done // Example: if [ -f "$file" ]; then echo "It exists"; fi @@ -79,6 +90,8 @@ type MkShCompoundCommand struct { Loop *MkShLoopClause } +// MkShForClause is a "for" loop. +// // Example: for f in *.c; do compile "$f"; done type MkShForClause struct { Varname string @@ -86,12 +99,16 @@ type MkShForClause struct { Body *MkShList } +// MkShCaseClause is a "case" statement, including all its branches. +// // Example: case $filename in *.c) echo "C source" ;; esac type MkShCaseClause struct { Word *ShToken Cases []*MkShCaseItem } +// MkShCaseItem is one branch of a "case" statement. +// // Example: *.c) echo "C source" ;; type MkShCaseItem struct { Patterns []*ShToken @@ -99,6 +116,9 @@ type MkShCaseItem struct { Separator MkShSeparator } +// MkShIfClause is a conditional statement, possibly having +// many branches. +// // Example: if [ -f "$file" ]; then echo "It exists"; fi type MkShIfClause struct { Conds []*MkShList @@ -111,6 +131,8 @@ func (cl *MkShIfClause) Prepend(cond *MkShList, action *MkShList) { cl.Actions = append([]*MkShList{action}, cl.Actions...) } +// MkShLoopClause is a "while" or "until" loop. +// // Example: while sleep 1; do printf .; done type MkShLoopClause struct { Cond *MkShList @@ -118,12 +140,17 @@ type MkShLoopClause struct { Until bool } +// MkShFunctionDefinition is the definition of a shell function. +// // Example: dir() { ls -l "$@"; } type MkShFunctionDefinition struct { Name string Body *MkShCompoundCommand } +// MkShSimpleCommand is a shell command that does not involve any +// pipeline or conditionals. +// // Example: LC_ALL=C sort */*.c > sorted type MkShSimpleCommand struct { Assignments []*ShToken @@ -149,9 +176,9 @@ func NewStrCommand(cmd *MkShSimpleCommand) *StrCommand { return strcmd } -// Similar to MkShSimpleCommand, but all components are converted -// to strings to allow for simpler checks, especially for analyzing -// command line options. +// StrCommand is structurally similar to MkShSimpleCommand, but all +// components are converted to strings to allow for simpler checks, +// especially for analyzing command line options. // // Example: LC_ALL=C sort */*.c > sorted type StrCommand struct { @@ -160,6 +187,7 @@ type StrCommand struct { Args []string } +// HasOption checks whether one of the arguments is exactly the given opt. func (c *StrCommand) HasOption(opt string) bool { for _, arg := range c.Args { if arg == opt { @@ -179,9 +207,21 @@ func (c *StrCommand) AnyArgMatches(pattern regex.Pattern) bool { } func (c *StrCommand) String() string { - return fmt.Sprintf("%v %v %v", c.Assignments, c.Name, c.Args) + var strs []string + for _, assignment := range c.Assignments { + strs = append(strs, assignment) + } + if c.Name != "" { + strs = append(strs, c.Name) + } + for _, arg := range c.Args { + strs = append(strs, arg) + } + return strings.Join(strs, " ") } +// MkShRedirection is a single file descriptor redirection. +// // Example: > sorted // Example: 2>&1 type MkShRedirection struct { diff --git a/pkgtools/pkglint/files/mkshwalker.go b/pkgtools/pkglint/files/mkshwalker.go index d32a5406ff3..0d100229915 100644 --- a/pkgtools/pkglint/files/mkshwalker.go +++ b/pkgtools/pkglint/files/mkshwalker.go @@ -1,211 +1,323 @@ package main +import ( + "fmt" + "reflect" + "strings" +) + type MkShWalker struct { + Callback struct { + List func(list *MkShList) + AndOr func(andor *MkShAndOr) + Pipeline func(pipeline *MkShPipeline) + Command func(command *MkShCommand) + SimpleCommand func(command *MkShSimpleCommand) + CompoundCommand func(command *MkShCompoundCommand) + Case func(caseClause *MkShCaseClause) + CaseItem func(caseItem *MkShCaseItem) + FunctionDefinition func(funcdef *MkShFunctionDefinition) + If func(ifClause *MkShIfClause) + Loop func(loop *MkShLoopClause) + Words func(words []*ShToken) + Word func(word *ShToken) + Redirects func(redirects []*MkShRedirection) + Redirect func(redirect *MkShRedirection) + For func(forClause *MkShForClause) + Varname func(varname string) + } + Context []MkShWalkerPathElement +} + +type MkShWalkerPathElement struct { + Index int + Element interface{} } func NewMkShWalker() *MkShWalker { return &MkShWalker{} } +// Path returns a representation of the path in the AST that is +// currently visited. +func (w *MkShWalker) Path() string { + var path []string + for _, level := range w.Context { + typeName := reflect.TypeOf(level.Element).Elem().Name() + abbreviated := strings.Replace(typeName, "MkSh", "", 1) + if level.Index == -1 { + path = append(path, abbreviated) + } else { + path = append(path, fmt.Sprintf("%s[%d]", abbreviated, level.Index)) + } + } + return strings.Join(path, ".") +} + // Walk calls the given callback for each node of the parsed shell program, // in visiting order from large to small. -func (w *MkShWalker) Walk(list *MkShList, callback *MkShWalkCallback) { - w.walkList(list, callback) +func (w *MkShWalker) Walk(list *MkShList) { + w.walkList(-1, list) + + G.Assertf(len(w.Context) == 0, "MkShWalker.Walk %v", w.Context) } -func (w *MkShWalker) walkList(list *MkShList, callback *MkShWalkCallback) { - if callback.List != nil { - callback.List(list) +func (w *MkShWalker) walkList(index int, list *MkShList) { + w.push(index, list) + + if callback := w.Callback.List; callback != nil { + callback(list) } - for _, andor := range list.AndOrs { - w.walkAndOr(andor, callback) + for i, andor := range list.AndOrs { + w.walkAndOr(i, andor) } + + w.pop() } -func (w *MkShWalker) walkAndOr(andor *MkShAndOr, callback *MkShWalkCallback) { - if callback.AndOr != nil { - callback.AndOr(andor) +func (w *MkShWalker) walkAndOr(index int, andor *MkShAndOr) { + w.push(index, andor) + + if callback := w.Callback.AndOr; callback != nil { + callback(andor) } - for _, pipeline := range andor.Pipes { - w.walkPipeline(pipeline, callback) + for i, pipeline := range andor.Pipes { + w.walkPipeline(i, pipeline) } + + w.pop() } -func (w *MkShWalker) walkPipeline(pipeline *MkShPipeline, callback *MkShWalkCallback) { - if callback.Pipeline != nil { - callback.Pipeline(pipeline) +func (w *MkShWalker) walkPipeline(index int, pipeline *MkShPipeline) { + w.push(index, pipeline) + + if callback := w.Callback.Pipeline; callback != nil { + callback(pipeline) } - for _, command := range pipeline.Cmds { - w.walkCommand(command, callback) + for i, command := range pipeline.Cmds { + w.walkCommand(i, command) } + + w.pop() } -func (w *MkShWalker) walkCommand(command *MkShCommand, callback *MkShWalkCallback) { - if callback.Command != nil { - callback.Command(command) +func (w *MkShWalker) walkCommand(index int, command *MkShCommand) { + w.push(index, command) + + if callback := w.Callback.Command; callback != nil { + callback(command) } switch { case command.Simple != nil: - w.walkSimpleCommand(command.Simple, callback) + w.walkSimpleCommand(-1, command.Simple) case command.Compound != nil: - w.walkCompoundCommand(command.Compound, callback) - w.walkRedirects(command.Redirects, callback) + w.walkCompoundCommand(-1, command.Compound) + w.walkRedirects(-1, command.Redirects) case command.FuncDef != nil: - w.walkFunctionDefinition(command.FuncDef, callback) - w.walkRedirects(command.Redirects, callback) + w.walkFunctionDefinition(-1, command.FuncDef) + w.walkRedirects(-1, command.Redirects) } + + w.pop() } -func (w *MkShWalker) walkSimpleCommand(command *MkShSimpleCommand, callback *MkShWalkCallback) { - if callback.SimpleCommand != nil { - callback.SimpleCommand(command) +func (w *MkShWalker) walkSimpleCommand(index int, command *MkShSimpleCommand) { + w.push(index, command) + + if callback := w.Callback.SimpleCommand; callback != nil { + callback(command) } - w.walkWords(command.Assignments, callback) + w.walkWords(0, command.Assignments) if command.Name != nil { - w.walkWord(command.Name, callback) + w.walkWord(-1, command.Name) } - w.walkWords(command.Args, callback) - w.walkRedirects(command.Redirections, callback) + w.walkWords(2, command.Args) + w.walkRedirects(-1, command.Redirections) + + w.pop() } -func (w *MkShWalker) walkCompoundCommand(command *MkShCompoundCommand, callback *MkShWalkCallback) { - if callback.CompoundCommand != nil { - callback.CompoundCommand(command) +func (w *MkShWalker) walkCompoundCommand(index int, command *MkShCompoundCommand) { + w.push(index, command) + + if callback := w.Callback.CompoundCommand; callback != nil { + callback(command) } switch { case command.Brace != nil: - w.walkList(command.Brace, callback) + w.walkList(-1, command.Brace) case command.Case != nil: - w.walkCase(command.Case, callback) + w.walkCase(command.Case) case command.For != nil: - w.walkFor(command.For, callback) + w.walkFor(command.For) case command.If != nil: - w.walkIf(command.If, callback) + w.walkIf(command.If) case command.Loop != nil: - w.walkLoop(command.Loop, callback) + w.walkLoop(command.Loop) case command.Subshell != nil: - w.walkList(command.Subshell, callback) + w.walkList(-1, command.Subshell) } + + w.pop() } -func (w *MkShWalker) walkCase(caseClause *MkShCaseClause, callback *MkShWalkCallback) { - if callback.Case != nil { - callback.Case(caseClause) +func (w *MkShWalker) walkCase(caseClause *MkShCaseClause) { + w.push(-1, caseClause) + + if callback := w.Callback.Case; callback != nil { + callback(caseClause) } - w.walkWord(caseClause.Word, callback) - for _, caseItem := range caseClause.Cases { - if callback.CaseItem != nil { - callback.CaseItem(caseItem) + w.walkWord(0, caseClause.Word) + for i, caseItem := range caseClause.Cases { + w.push(i, caseItem) + if callback := w.Callback.CaseItem; callback != nil { + callback(caseItem) } - w.walkWords(caseItem.Patterns, callback) - w.walkList(caseItem.Action, callback) + w.walkWords(0, caseItem.Patterns) + w.walkList(1, caseItem.Action) + w.pop() } + + w.pop() } -func (w *MkShWalker) walkFunctionDefinition(funcdef *MkShFunctionDefinition, callback *MkShWalkCallback) { - if callback.FunctionDefinition != nil { - callback.FunctionDefinition(funcdef) +func (w *MkShWalker) walkFunctionDefinition(index int, funcdef *MkShFunctionDefinition) { + w.push(index, funcdef) + + if callback := w.Callback.FunctionDefinition; callback != nil { + callback(funcdef) } - w.walkCompoundCommand(funcdef.Body, callback) + w.walkCompoundCommand(-1, funcdef.Body) + + w.pop() } -func (w *MkShWalker) walkIf(ifClause *MkShIfClause, callback *MkShWalkCallback) { - if callback.If != nil { - callback.If(ifClause) +func (w *MkShWalker) walkIf(ifClause *MkShIfClause) { + w.push(-1, ifClause) + + if callback := w.Callback.If; callback != nil { + callback(ifClause) } for i, cond := range ifClause.Conds { - w.walkList(cond, callback) - w.walkList(ifClause.Actions[i], callback) + w.walkList(2*i, cond) + w.walkList(2*i+1, ifClause.Actions[i]) } if ifClause.Else != nil { - w.walkList(ifClause.Else, callback) + w.walkList(2*len(ifClause.Conds), ifClause.Else) } + + w.pop() } -func (w *MkShWalker) walkLoop(loop *MkShLoopClause, callback *MkShWalkCallback) { - if callback.Loop != nil { - callback.Loop(loop) +func (w *MkShWalker) walkLoop(loop *MkShLoopClause) { + w.push(-1, loop) + + if callback := w.Callback.Loop; callback != nil { + callback(loop) } - w.walkList(loop.Cond, callback) - w.walkList(loop.Action, callback) + w.walkList(0, loop.Cond) + w.walkList(1, loop.Action) + + w.pop() } -func (w *MkShWalker) walkWords(words []*ShToken, callback *MkShWalkCallback) { - if len(words) != 0 { - if callback.Words != nil { - callback.Words(words) - } +func (w *MkShWalker) walkWords(index int, words []*ShToken) { + if len(words) == 0 { + return + } - for _, word := range words { - w.walkWord(word, callback) - } + w.push(index, words) + + if callback := w.Callback.Words; callback != nil { + callback(words) } + + for i, word := range words { + w.walkWord(i, word) + } + + w.pop() } -func (w *MkShWalker) walkWord(word *ShToken, callback *MkShWalkCallback) { - if callback.Word != nil { - callback.Word(word) +func (w *MkShWalker) walkWord(index int, word *ShToken) { + w.push(index, word) + + if callback := w.Callback.Word; callback != nil { + callback(word) } + + w.pop() } -func (w *MkShWalker) walkRedirects(redirects []*MkShRedirection, callback *MkShWalkCallback) { - if len(redirects) != 0 { - if callback.Redirects != nil { - callback.Redirects(redirects) - } +func (w *MkShWalker) walkRedirects(index int, redirects []*MkShRedirection) { + if len(redirects) == 0 { + return + } + + w.push(index, redirects) - for _, redirect := range redirects { - if callback.Redirect != nil { - callback.Redirect(redirect) - } + if callback := w.Callback.Redirects; callback != nil { + callback(redirects) + } - w.walkWord(redirect.Target, callback) + for i, redirect := range redirects { + if callback := w.Callback.Redirect; callback != nil { + callback(redirect) } + + w.walkWord(i, redirect.Target) } + + w.pop() } -func (w *MkShWalker) walkFor(forClause *MkShForClause, callback *MkShWalkCallback) { - if callback.For != nil { - callback.For(forClause) +func (w *MkShWalker) walkFor(forClause *MkShForClause) { + w.push(-1, forClause) + + if callback := w.Callback.For; callback != nil { + callback(forClause) } - if callback.Varname != nil { - callback.Varname(forClause.Varname) + if callback := w.Callback.Varname; callback != nil { + callback(forClause.Varname) } - w.walkWords(forClause.Values, callback) - w.walkList(forClause.Body, callback) + w.walkWords(-1, forClause.Values) + w.walkList(-1, forClause.Body) + + w.pop() +} + +// Current provides access to the element that the walker is currently +// processing, especially its index as seen from its parent element. +func (w *MkShWalker) Current() MkShWalkerPathElement { + return w.Context[len(w.Context)-1] +} + +// Parent returns an ancestor element from the currently visited path. +// Parent(0) is the element that is currently visited, +// Parent(1) is its direct parent, and so on. +func (w *MkShWalker) Parent(steps int) interface{} { + index := len(w.Context) - 1 - steps + if index >= 0 { + return w.Context[index].Element + } + return nil } -type MkShWalkCallback struct { - List func(list *MkShList) - AndOr func(andor *MkShAndOr) - Pipeline func(pipeline *MkShPipeline) - Command func(command *MkShCommand) - SimpleCommand func(command *MkShSimpleCommand) - CompoundCommand func(command *MkShCompoundCommand) - Case func(caseClause *MkShCaseClause) - CaseItem func(caseItem *MkShCaseItem) - FunctionDefinition func(funcdef *MkShFunctionDefinition) - If func(ifClause *MkShIfClause) - Loop func(loop *MkShLoopClause) - Words func(words []*ShToken) - Word func(word *ShToken) - Redirects func(redirects []*MkShRedirection) - Redirect func(redirect *MkShRedirection) - For func(forClause *MkShForClause) - Varname func(varname string) +func (w *MkShWalker) push(index int, element interface{}) { + w.Context = append(w.Context, MkShWalkerPathElement{index, element}) } -func NewMkShWalkCallback() *MkShWalkCallback { - return &MkShWalkCallback{} +func (w *MkShWalker) pop() { + w.Context = w.Context[:len(w.Context)-1] } diff --git a/pkgtools/pkglint/files/mkshwalker_test.go b/pkgtools/pkglint/files/mkshwalker_test.go index ca486d59ba4..7c424c7a47e 100644 --- a/pkgtools/pkglint/files/mkshwalker_test.go +++ b/pkgtools/pkglint/files/mkshwalker_test.go @@ -26,12 +26,16 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { commands = append(commands, fmt.Sprintf("%16s %s", kind, detail)) } - callback := NewMkShWalkCallback() + walker := NewMkShWalker() + callback := &walker.Callback callback.List = func(list *MkShList) { add("List", "with %d andOrs", len(list.AndOrs)) } callback.AndOr = func(andor *MkShAndOr) { add("AndOr", "with %d pipelines", len(andor.Pipes)) } callback.Pipeline = func(pipeline *MkShPipeline) { add("Pipeline", "with %d commands", len(pipeline.Cmds)) } callback.Command = func(command *MkShCommand) { add("Command", "") } - callback.SimpleCommand = func(command *MkShSimpleCommand) { add("SimpleCommand", "%s", NewStrCommand(command).String()) } + callback.SimpleCommand = func(command *MkShSimpleCommand) { + add("SimpleCommand", "%s", NewStrCommand(command).String()) + add("Path", "%s", walker.Path()) + } callback.CompoundCommand = func(command *MkShCompoundCommand) { add("CompoundCommand", "") } callback.Case = func(caseClause *MkShCaseClause) { add("Case", "with %d items", len(caseClause.Cases)) } callback.CaseItem = func(caseItem *MkShCaseItem) { add("CaseItem", "with %d patterns", len(caseItem.Patterns)) } @@ -45,7 +49,7 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { callback.For = func(forClause *MkShForClause) { add("For", "variable %s", forClause.Varname) } callback.Varname = func(varname string) { add("Varname", "%s", varname) } - NewMkShWalker().Walk(list, callback) + walker.Walk(list) c.Check(commands, deepEquals, []string{ " List with 5 andOrs", @@ -58,13 +62,15 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " AndOr with 1 pipelines", " Pipeline with 1 commands", " Command ", - " SimpleCommand [] condition []", + " SimpleCommand condition", + " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.IfClause.List[0].AndOr[0].Pipeline[0].Command[0].SimpleCommand", " Word condition", " List with 1 andOrs", " AndOr with 1 pipelines", " Pipeline with 1 commands", " Command ", - " SimpleCommand [] action []", + " SimpleCommand action", + " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.IfClause.List[1].AndOr[0].Pipeline[0].Command[0].SimpleCommand", " Word action", " List with 1 andOrs", " AndOr with 1 pipelines", @@ -80,19 +86,22 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " AndOr with 1 pipelines", " Pipeline with 1 commands", " Command ", - " SimpleCommand [] case-item-action []", + " SimpleCommand case-item-action", + " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.IfClause.List[2].AndOr[0].Pipeline[0].Command[0].CompoundCommand.CaseClause.CaseItem[0].List[1].AndOr[0].Pipeline[0].Command[0].SimpleCommand", " Word case-item-action", " AndOr with 1 pipelines", " Pipeline with 1 commands", " Command ", - " SimpleCommand [] set [-e]", + " SimpleCommand set -e", + " Path List.AndOr[1].Pipeline[0].Command[0].SimpleCommand", " Word set", " Words with 1 words", " Word -e", " AndOr with 1 pipelines", " Pipeline with 1 commands", " Command ", - " SimpleCommand [] cd [${WRKSRC}/locale]", + " SimpleCommand cd ${WRKSRC}/locale", + " Path List.AndOr[2].Pipeline[0].Command[0].SimpleCommand", " Word cd", " Words with 1 words", " Word ${WRKSRC}/locale", @@ -108,7 +117,8 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " AndOr with 2 pipelines", " Pipeline with 1 commands", " Command ", - " SimpleCommand [] [ [\"$${lang}\" = \"wxstd.po\" ]]", + " SimpleCommand [ \"$${lang}\" = \"wxstd.po\" ]", + " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.ForClause.List.AndOr[0].Pipeline[0].Command[0].SimpleCommand", " Word [", " Words with 4 words", " Word \"$${lang}\"", @@ -117,12 +127,14 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " Word ]", " Pipeline with 1 commands", " Command ", - " SimpleCommand [] continue []", + " SimpleCommand continue", + " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.ForClause.List.AndOr[0].Pipeline[1].Command[0].SimpleCommand", " Word continue", " AndOr with 1 pipelines", " Pipeline with 1 commands", " Command ", - " SimpleCommand [] ${TOOLS_PATH.msgfmt} [-c -o \"$${lang%.po}.mo\" \"$${lang}\"]", + " SimpleCommand ${TOOLS_PATH.msgfmt} -c -o \"$${lang%.po}.mo\" \"$${lang}\"", + " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.ForClause.List.AndOr[1].Pipeline[0].Command[0].SimpleCommand", " Word ${TOOLS_PATH.msgfmt}", " Words with 4 words", " Word -c", @@ -138,7 +150,8 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " AndOr with 1 pipelines", " Pipeline with 1 commands", " Command ", - " SimpleCommand [] : []", + " SimpleCommand :", + " Path List.AndOr[4].Pipeline[0].Command[0].CompoundCommand.LoopClause.List[0].AndOr[0].Pipeline[0].Command[0].SimpleCommand", " Word :", " List with 1 andOrs", " AndOr with 1 pipelines", @@ -150,10 +163,15 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " AndOr with 1 pipelines", " Pipeline with 1 commands", " Command ", - " SimpleCommand [] : []", + " SimpleCommand :", + " Path List.AndOr[4].Pipeline[0].Command[0].CompoundCommand.LoopClause.List[1].AndOr[0].Pipeline[0].Command[0].FunctionDefinition.CompoundCommand.List.AndOr[0].Pipeline[0].Command[0].SimpleCommand", " Word :", " Redirects with 1 redirects", " Redirect >&", " Word 2"}) + + // After parsing, there is not a single level of indentation, + // therefore even Parent(0) returns nil. + c.Check(walker.Parent(0), equals, nil) } } diff --git a/pkgtools/pkglint/files/options.go b/pkgtools/pkglint/files/options.go index e6039c42ebe..779f3d303b6 100755 --- a/pkgtools/pkglint/files/options.go +++ b/pkgtools/pkglint/files/options.go @@ -16,7 +16,7 @@ func ChecklinesOptionsMk(mklines *MkLines) { exp.CurrentLine().Warnf("Expected definition of PKG_OPTIONS_VAR.") Explain( "The input variables in an options.mk file should always be", - "mentioned in the same order: PKG_OPTIONS_VAR, ", + "mentioned in the same order: PKG_OPTIONS_VAR,", "PKG_SUPPORTED_OPTIONS, PKG_SUGGESTED_OPTIONS. This way, the", "options.mk files have the same structure and are easy to understand.") return @@ -49,14 +49,9 @@ loop: // The conditionals are typically for OPSYS and MACHINE_ARCH. case mkline.IsInclude(): - includedFile := mkline.IncludeFile() - switch { - case matches(includedFile, `/[^/]+\.buildlink3\.mk$`): - case matches(includedFile, `/[^/]+\.builtin\.mk$`): - case includedFile == "../../mk/bsd.options.mk": + if mkline.IncludeFile() == "../../mk/bsd.options.mk" { exp.Advance() break loop - case IsPrefs(includedFile): } default: diff --git a/pkgtools/pkglint/files/options_test.go b/pkgtools/pkglint/files/options_test.go index fefcd9206ee..9828f652708 100755 --- a/pkgtools/pkglint/files/options_test.go +++ b/pkgtools/pkglint/files/options_test.go @@ -15,7 +15,7 @@ func (s *Suite) Test_ChecklinesOptionsMk(c *check.C) { t.SetupOption("sqlite", "") t.SetupOption("x11", "") - t.SetupFileMkLines("mk/bsd.options.mk", + t.CreateFileLines("mk/bsd.options.mk", MkRcsID) mklines := t.SetupFileMkLines("category/package/options.mk", @@ -70,7 +70,7 @@ func (s *Suite) Test_ChecklinesOptionsMk__unexpected_line(c *check.C) { t.SetupOption("slang", "") t.SetupOption("x11", "") - t.SetupFileMkLines("mk/bsd.options.mk", + t.CreateFileLines("mk/bsd.options.mk", MkRcsID) mklines := t.SetupFileMkLines("category/package/options.mk", @@ -99,7 +99,7 @@ func (s *Suite) Test_ChecklinesOptionsMk__malformed_condition(c *check.C) { t.SetupOption("slang", "") t.SetupOption("x11", "") - t.SetupFileMkLines("mk/bsd.options.mk", + t.CreateFileLines("mk/bsd.options.mk", MkRcsID) mklines := t.SetupFileMkLines("category/package/options.mk", @@ -109,6 +109,10 @@ func (s *Suite) Test_ChecklinesOptionsMk__malformed_condition(c *check.C) { "PKG_SUPPORTED_OPTIONS= # none", "PKG_SUGGESTED_OPTIONS= # none", "", + "# Comment", + ".if ${OPSYS} == NetBSD", + ".endif", + "", ".include \"../../mk/bsd.options.mk\"", "", ".if ${OPSYS} == 'Darwin'", @@ -117,5 +121,5 @@ func (s *Suite) Test_ChecklinesOptionsMk__malformed_condition(c *check.C) { ChecklinesOptionsMk(mklines) t.CheckOutputLines( - "WARN: ~/category/package/options.mk:9: Invalid condition, unrecognized part: \"${OPSYS} == 'Darwin'\".") + "WARN: ~/category/package/options.mk:13: Invalid condition, unrecognized part: \"${OPSYS} == 'Darwin'\".") } diff --git a/pkgtools/pkglint/files/package.go b/pkgtools/pkglint/files/package.go index 6cbcab44356..297fe3a60b7 100644 --- a/pkgtools/pkglint/files/package.go +++ b/pkgtools/pkglint/files/package.go @@ -31,7 +31,6 @@ type Package struct { vars Scope bl3 map[string]Line // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. - plistSubstCond map[string]bool // varname => true; all variables that are used as conditions (@comment or nothing) in PLISTs. included map[string]Line // fname => line seenMakefileCommon bool // Does the package have any .includes? conditionalIncludes map[string]MkLine @@ -52,24 +51,33 @@ func NewPackage(dir string) *Package { Pkgdir: ".", Filesdir: "files", Patchdir: "patches", - DistinfoFile: "distinfo", + DistinfoFile: "${PKGDIR}/distinfo", PlistDirs: make(map[string]bool), PlistFiles: make(map[string]bool), vars: NewScope(), bl3: make(map[string]Line), - plistSubstCond: make(map[string]bool), included: make(map[string]Line), conditionalIncludes: make(map[string]MkLine), unconditionalIncludes: make(map[string]MkLine), } pkg.vars.DefineAll(G.Pkgsrc.UserDefinedVars) + + pkg.vars.Fallback("PKGDIR", ".") + pkg.vars.Fallback("DISTINFO_FILE", "${PKGDIR}/distinfo") + pkg.vars.Fallback("FILESDIR", "files") + pkg.vars.Fallback("PATCHDIR", "patches") + pkg.vars.Fallback("KRB5_TYPE", "heimdal") + pkg.vars.Fallback("PGSQL_VERSION", "95") + pkg.vars.Fallback(".CURDIR", ".") // FIXME: In reality, this is an absolute pathname. + return pkg } // File returns the (possibly absolute) path to relativeFilename, // as resolved from the package's directory. +// Variables that are known in the package are resolved, e.g. ${PKGDIR}. func (pkg *Package) File(relativeFilename string) string { - return cleanpath(pkg.dir + "/" + relativeFilename) + return cleanpath(resolveVariableRefs(pkg.dir + "/" + relativeFilename)) } func (pkg *Package) checkPossibleDowngrade() { @@ -93,7 +101,7 @@ func (pkg *Package) checkPossibleDowngrade() { } if change.Action == "Updated" { - changeVersion := regex.Compile(`nb\d+$`).ReplaceAllString(change.Version, "") + changeVersion := replaceAll(change.Version, `nb\d+$`, "") if pkgver.Compare(pkgversion, changeVersion) < 0 { mkline.Warnf("The package is being downgraded from %s (see %s) to %s.", change.Version, change.Line.ReferenceFrom(mkline.Line), pkgversion) Explain( @@ -158,7 +166,7 @@ func (pkglint *Pkglint) checkdirPackage(dir string) { files = append(files, dirglob(pkg.File(pkg.Filesdir))...) } files = append(files, dirglob(pkg.File(pkg.Patchdir))...) - if pkg.DistinfoFile != "distinfo" && pkg.DistinfoFile != "./distinfo" { + if pkg.DistinfoFile != pkg.vars.fallback["DISTINFO_FILE"] { files = append(files, pkg.File(pkg.DistinfoFile)) } haveDistinfo := false @@ -181,11 +189,14 @@ func (pkglint *Pkglint) checkdirPackage(dir string) { for _, fname := range files { if containsVarRef(fname) { + if trace.Tracing { + trace.Stepf("Skipping file %q because the name contains an unresolved variable.", fname) + } continue } if fname == pkg.File("Makefile") { if st, err := os.Lstat(fname); err == nil { - pkglint.checkExecutable(st, fname) + pkglint.checkExecutable(st) } if G.opts.CheckMakefile { pkg.checkfilePackageMakefile(fname, lines) @@ -201,7 +212,7 @@ func (pkglint *Pkglint) checkdirPackage(dir string) { pkg.checkLocallyModified(fname) } - if G.opts.CheckDistinfo && G.opts.CheckPatches { + if pkg.Pkgdir == "." && G.opts.CheckDistinfo && G.opts.CheckPatches { if havePatches && !haveDistinfo { NewLineWhole(pkg.File(pkg.DistinfoFile)).Warnf("File not found. Please run \"%s makepatchsum\".", confMake) } @@ -214,10 +225,6 @@ func (pkglint *Pkglint) checkdirPackage(dir string) { } } } - - if !isEmptyDir(pkg.File("scripts")) { - NewLineWhole(pkg.File("scripts")).Warnf("This directory and its contents are deprecated! Please call the script(s) explicitly from the corresponding target(s) in the pkg's Makefile.") - } } func (pkg *Package) loadPackageMakefile() *MkLines { @@ -227,7 +234,8 @@ func (pkg *Package) loadPackageMakefile() *MkLines { } mainLines, allLines := NewMkLines(nil), NewMkLines(nil) - if !pkg.readMakefile(fname, mainLines, allLines, "") { + if _, result := pkg.readMakefile(fname, mainLines, allLines, ""); !result { + LoadMk(fname, NotEmpty|LogErrors) // Just for the LogErrors. return nil } @@ -241,10 +249,10 @@ func (pkg *Package) loadPackageMakefile() *MkLines { allLines.DetermineUsedVariables() allLines.CheckRedundantVariables() - pkg.Pkgdir = pkg.expandVariableWithDefault("PKGDIR", ".") - pkg.DistinfoFile = pkg.expandVariableWithDefault("DISTINFO_FILE", "distinfo") - pkg.Filesdir = pkg.expandVariableWithDefault("FILESDIR", "files") - pkg.Patchdir = pkg.expandVariableWithDefault("PATCHDIR", "patches") + pkg.Pkgdir, _ = pkg.vars.Value("PKGDIR") + pkg.DistinfoFile, _ = pkg.vars.Value("DISTINFO_FILE") + pkg.Filesdir, _ = pkg.vars.Value("FILESDIR") + pkg.Patchdir, _ = pkg.vars.Value("PATCHDIR") // See lang/php/ext.mk if varIsDefined("PHPEXT_MK") { @@ -271,19 +279,20 @@ func (pkg *Package) loadPackageMakefile() *MkLines { return mainLines } -func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkLines, includingFnameForUsedCheck string) bool { +func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkLines, includingFnameForUsedCheck string) (exists bool, result bool) { if trace.Tracing { defer trace.Call1(fname)() } - fileMklines := LoadMk(fname, NotEmpty|LogErrors) + fileMklines := LoadMk(fname, NotEmpty) if fileMklines == nil { - return false + return false, false } + exists = true isMainMakefile := len(mainLines.mklines) == 0 - result := true + result = true lineAction := func(mkline MkLine) bool { if isMainMakefile { mainLines.mklines = append(mainLines.mklines, mkline) @@ -308,7 +317,7 @@ func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkL if includeFile != "" { if path.Base(fname) != "buildlink3.mk" { if m, bl3File := match1(includeFile, `^\.\./\.\./(.*)/buildlink3\.mk$`); m { - G.Pkg.bl3[bl3File] = mkline.Line + pkg.bl3[bl3File] = mkline.Line if trace.Tracing { trace.Step1("Buildlink3 file in package: %q", bl3File) } @@ -316,8 +325,8 @@ func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkL } } - if includeFile != "" && G.Pkg.included[includeFile] == nil { - G.Pkg.included[includeFile] = mkline.Line + if includeFile != "" && pkg.included[includeFile] == nil { + pkg.included[includeFile] = mkline.Line if matches(includeFile, `^\.\./[^./][^/]*/[^/]+`) { mkline.Warnf("References to other packages should look like \"../../category/package\", not \"../package\".") @@ -328,7 +337,7 @@ func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkL if trace.Tracing { trace.Step1("Including %q sets seenMakefileCommon.", includeFile) } - G.Pkg.seenMakefileCommon = true + pkg.seenMakefileCommon = true } skip := contains(fname, "/mk/") || hasSuffix(includeFile, "/bsd.pkg.mk") || IsPrefs(includeFile) @@ -336,30 +345,37 @@ func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkL dirname, _ := path.Split(fname) dirname = cleanpath(dirname) - // Only look in the directory relative to the - // current file and in the current working directory. - // Pkglint doesn't have an include dir list, like make(1) does. - if !fileExists(dirname + "/" + includeFile) { + fullIncluded := dirname + "/" + includeFile + if trace.Tracing { + trace.Step1("Including %q.", fullIncluded) + } + fullIncluding := ifelseStr(incBase == "Makefile.common" && incDir != "", fname, "") + innerExists, innerResult := pkg.readMakefile(fullIncluded, mainLines, allLines, fullIncluding) + if !innerExists { if fileMklines.indentation.IsCheckedFile(includeFile) { return true // See https://github.com/rillig/pkglint/issues/1 + } - } else if dirname != pkg.File(".") { // Prevent unnecessary syscalls - dirname = pkg.File(".") - if !fileExists(dirname + "/" + includeFile) { - mkline.Errorf("Cannot read %q.", dirname+"/"+includeFile) - result = false - return false - } + // Only look in the directory relative to the + // current file and in the package directory. + // Make(1) has a list of include directories, but pkgsrc + // doesn't make use of that, so pkglint also doesn't + // need this extra complexity. + pkgBasedir := pkg.File(".") + if dirname != pkgBasedir { // Prevent unnecessary syscalls + dirname = pkgBasedir + + fullIncludedFallback := dirname + "/" + includeFile + innerExists, innerResult = pkg.readMakefile(fullIncludedFallback, mainLines, allLines, fullIncluding) } - } - if trace.Tracing { - trace.Step1("Including %q.", dirname+"/"+includeFile) + if !innerExists { + mkline.Errorf("Cannot read %q.", includeFile) + } } - absIncluding := ifelseStr(incBase == "Makefile.common" && incDir != "", fname, "") - absIncluded := dirname + "/" + includeFile - if !pkg.readMakefile(absIncluded, mainLines, allLines, absIncluding) { + + if !innerResult { result = false return false } @@ -369,11 +385,11 @@ func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkL if mkline.IsVarassign() { varname, op, value := mkline.Varname(), mkline.Op(), mkline.Value() - if op != opAssignDefault || !G.Pkg.vars.Defined(varname) { + if op != opAssignDefault || !pkg.vars.Defined(varname) { if trace.Tracing { trace.Stepf("varassign(%q, %q, %q)", varname, op, value) } - G.Pkg.vars.Define(varname, mkline) + pkg.vars.Define(varname, mkline) } } return true @@ -385,7 +401,7 @@ func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkL fileMklines.checkForUsedComment(G.Pkgsrc.ToRel(includingFnameForUsedCheck)) } - return result + return } func (pkg *Package) checkfilePackageMakefile(fname string, mklines *MkLines) { @@ -413,28 +429,14 @@ func (pkg *Package) checkfilePackageMakefile(fname string, mklines *MkLines) { } if perlLine, noconfLine := vars.FirstDefinition("REPLACE_PERL"), vars.FirstDefinition("NO_CONFIGURE"); perlLine != nil && noconfLine != nil { - perlLine.Warnf("REPLACE_PERL is ignored when NO_CONFIGURE is set (in %s)", noconfLine.ReferenceFrom(perlLine.Line)) + perlLine.Warnf("REPLACE_PERL is ignored when NO_CONFIGURE is set (in %s).", noconfLine.ReferenceFrom(perlLine.Line)) } if !vars.Defined("LICENSE") && !vars.Defined("META_PACKAGE") && pkg.once.FirstTime("LICENSE") { NewLineWhole(fname).Errorf("Each package must define its LICENSE.") } - if gnuLine, useLine := vars.FirstDefinition("GNU_CONFIGURE"), vars.FirstDefinition("USE_LANGUAGES"); gnuLine != nil && useLine != nil { - if matches(useLine.VarassignComment(), `(?-i)\b(?:c|empty|none)\b`) { - // Don't emit a warning, since the comment - // probably contains a statement that C is - // really not needed. - - } else if !G.Infrastructure && useLine.Filename == "../../mk/compiler.mk" { - // Ignore this one - - } else if !matches(useLine.Value(), `(?:^|\s+)(?:c|c99|objc)(?:\s+|$)`) { - gnuLine.Warnf("GNU_CONFIGURE almost always needs a C compiler, but \"c\" is not added to USE_LANGUAGES in %s.", - useLine.ReferenceFrom(gnuLine.Line)) - } - } - + pkg.checkGnuConfigureUseLanguages() pkg.determineEffectivePkgVars() pkg.checkPossibleDowngrade() @@ -454,6 +456,22 @@ func (pkg *Package) checkfilePackageMakefile(fname string, mklines *MkLines) { SaveAutofixChanges(mklines.lines) } +func (pkg *Package) checkGnuConfigureUseLanguages() { + vars := pkg.vars + + if gnuLine, useLine := vars.FirstDefinition("GNU_CONFIGURE"), vars.FirstDefinition("USE_LANGUAGES"); gnuLine != nil && useLine != nil { + if matches(useLine.VarassignComment(), `(?-i)\b(?:c|empty|none)\b`) { + // Don't emit a warning, since the comment + // probably contains a statement that C is + // really not needed. + + } else if !matches(useLine.Value(), `(?:^|\s+)(?:c|c99|objc)(?:\s+|$)`) { + gnuLine.Warnf("GNU_CONFIGURE almost always needs a C compiler, but \"c\" is not added to USE_LANGUAGES in %s.", + useLine.ReferenceFrom(gnuLine.Line)) + } + } +} + func (pkg *Package) getNbpart() string { pkgrevision, _ := pkg.vars.Value("PKGREVISION") if rev, err := strconv.Atoi(pkgrevision); err == nil { @@ -514,19 +532,17 @@ func (pkg *Package) determineEffectivePkgVars() { func (pkg *Package) pkgnameFromDistname(pkgname, distname string) string { tokens := NewMkParser(dummyLine, pkgname, false).MkTokens() - subst := func(str, smod string) (result string) { - if trace.Tracing { - defer trace.Call(str, smod, trace.Result(&result))() - } + // Example: + // subst("distname-1.0", "S,name,file,g") => "distfile-1.0" + subst := func(str, smod string) string { qsep := regexp.QuoteMeta(smod[1:2]) - if m, left, from, right, to, flags := regex.Match5(smod, regex.Pattern(`^S`+qsep+`(\^?)([^:]*?)(\$?)`+qsep+`([^:]*)`+qsep+`([1g]*)$`)); m { - result := mkopSubst(str, left != "", from, right != "", to, flags) - if trace.Tracing { - trace.Stepf("subst %q %q => %q", str, smod, result) - } - return result + m, left, from, right, to, flags := match5(smod, regex.Pattern(`^S`+qsep+`(\^?)([^:]*?)(\$?)`+qsep+`([^:]*)`+qsep+`([1g]*)$`)) + G.Assertf(m, "pkgnameFromDistname %q", smod) + result := mkopSubst(str, left != "", from, right != "", to, flags) + if trace.Tracing && result != str { + trace.Stepf("pkgnameFromDistname.subst: %q %q => %q", str, smod, result) } - return str + return result } result := "" @@ -551,23 +567,6 @@ func (pkg *Package) pkgnameFromDistname(pkgname, distname string) string { return result } -func (pkg *Package) expandVariableWithDefault(varname, defaultValue string) string { - mkline := G.Pkg.vars.FirstDefinition(varname) - if mkline == nil { - return defaultValue - } - - value := mkline.Value() - value = mkline.ResolveVarsInRelativePath(value, true) - if containsVarRef(value) { - value = resolveVariableRefs(value) - } - if trace.Tracing { - trace.Step2("Expanded %q to %q", varname, value) - } - return value -} - func (pkg *Package) checkUpdate() { if pkg.EffectivePkgbase != "" { for _, sugg := range G.Pkgsrc.GetSuggestedPackageUpdates() { @@ -848,10 +847,7 @@ func (pkg *Package) checkLocallyModified(fname string) { owner, _ := pkg.vars.Value("OWNER") maintainer, _ := pkg.vars.Value("MAINTAINER") - if containsVarRef(owner) { - owner = "" - } - if containsVarRef(maintainer) || maintainer == "pkgsrc-users@NetBSD.org" { + if maintainer == "pkgsrc-users@NetBSD.org" { maintainer = "" } if owner == "" && maintainer == "" { diff --git a/pkgtools/pkglint/files/package_test.go b/pkgtools/pkglint/files/package_test.go index 1ce7f6a41a0..0477e347fe2 100644 --- a/pkgtools/pkglint/files/package_test.go +++ b/pkgtools/pkglint/files/package_test.go @@ -2,6 +2,40 @@ package main import "gopkg.in/check.v1" +func (s *Suite) Test_Package_checklinesBuildlink3Inclusion__file_but_not_package(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("category/dependency/buildlink3.mk") + G.Pkg = NewPackage(t.File("category/package")) + mklines := t.NewMkLines("category/package/buildlink3.mk", + MkRcsID, + "", + ".include \"../../category/dependency/buildlink3.mk\"") + + G.Pkg.checklinesBuildlink3Inclusion(mklines) + + t.CheckOutputLines( + "WARN: category/package/buildlink3.mk:3: category/dependency/buildlink3.mk is included by this file but not by the package.") +} + +func (s *Suite) Test_Package_checklinesBuildlink3Inclusion__package_but_not_file(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("category/dependency/buildlink3.mk") + G.Pkg = NewPackage(t.File("category/package")) + G.Pkg.bl3["../../category/dependency/buildlink3.mk"] = t.NewLine("fname", 1, "") + mklines := t.NewMkLines("category/package/buildlink3.mk", + MkRcsID) + + t.EnableTracingToLog() + G.Pkg.checklinesBuildlink3Inclusion(mklines) + + t.CheckOutputLines( + "TRACE: + (*Package).checklinesBuildlink3Inclusion()", + "TRACE: 1 ../../category/dependency/buildlink3.mk/buildlink3.mk is included by the package but not by the buildlink3.mk file.", + "TRACE: - (*Package).checklinesBuildlink3Inclusion()") +} + func (s *Suite) Test_Package_pkgnameFromDistname(c *check.C) { t := s.Init(c) @@ -18,6 +52,9 @@ func (s *Suite) Test_Package_pkgnameFromDistname(c *check.C) { c.Check(pkg.pkgnameFromDistname("${DISTNAME:C/beta/.0./}", "fspanel-0.8beta1"), equals, "${DISTNAME:C/beta/.0./}") c.Check(pkg.pkgnameFromDistname("${DISTNAME:S/-0$/.0/1}", "aspell-af-0.50-0"), equals, "aspell-af-0.50.0") + // FIXME: Should produce a parse error since the :S modifier is malformed; see Test_MkParser_MkTokens. + c.Check(pkg.pkgnameFromDistname("${DISTNAME:S,a,b,c,d}", "aspell-af-0.50-0"), equals, "bspell-af-0.50-0") + t.CheckOutputEmpty() } @@ -87,7 +124,7 @@ func (s *Suite) Test_Package_CheckVarorder__comments_are_ignored(c *check.C) { "DISTNAME=\tdistname-1.0", "CATEGORIES=\tsysutils", "", - "MAINTAINER=\tpkgsrc-users@pkgsrc.org", + "MAINTAINER=\tpkgsrc-users@NetBSD.org", "# comment", "COMMENT=\tComment", "LICENSE=\tgnu-gpl-v2")) @@ -109,7 +146,7 @@ func (s *Suite) Test_Package_CheckVarorder__skip_if_there_are_directives(c *chec "CATEGORIES=\tsysutils", "", ".if ${DISTNAME:Mdistname-*}", - "MAINTAINER=\tpkgsrc-users@pkgsrc.org", + "MAINTAINER=\tpkgsrc-users@NetBSD.org", ".endif", "LICENSE=\tgnu-gpl-v2")) @@ -118,7 +155,7 @@ func (s *Suite) Test_Package_CheckVarorder__skip_if_there_are_directives(c *chec t.CheckOutputEmpty() } -func (s *Suite) Test_Package_CheckVarorder_GitHub(c *check.C) { +func (s *Suite) Test_Package_CheckVarorder__GitHub(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Worder") @@ -139,7 +176,7 @@ func (s *Suite) Test_Package_CheckVarorder_GitHub(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_Package_varorder_license(c *check.C) { +func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Worder") @@ -276,6 +313,35 @@ func (s *Suite) Test_Package_determineEffectivePkgVars__precedence(c *check.C) { c.Check(pkg.EffectivePkgversion, equals, "1.0") } +func (s *Suite) Test_Package_determineEffectivePkgVars__same(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-order") + pkg := t.SetupPackage("category/package", + "DISTNAME=\tdistname-1.0", + "PKGNAME=\tdistname-1.0") + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "NOTE: ~/category/package/Makefile:20: " + + "PKGNAME is ${DISTNAME} by default. You probably don't need to define PKGNAME.") +} + +func (s *Suite) Test_Package_determineEffectivePkgVars__invalid_DISTNAME(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-order") + pkg := t.SetupPackage("category/package", + "DISTNAME=\tpkgname-version") + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:3: " + + "As DISTNAME is not a valid package name, please define the PKGNAME explicitly.") +} + func (s *Suite) Test_Package_checkPossibleDowngrade(c *check.C) { t := s.Init(c) @@ -300,11 +366,48 @@ func (s *Suite) Test_Package_checkPossibleDowngrade(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_checkdirPackage(c *check.C) { +func (s *Suite) Test_Package_loadPackageMakefile__dump(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("--dumpmakefile") + t.SetupVartypes() + t.SetupPkgsrc() + t.CreateFileLines("category/Makefile") + t.CreateFileLines("category/package/PLIST", + PlistRcsID, + "bin/program") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "SHA1 (distfile-1.0.tar.gz) = 12341234...", + "RMD160 (distfile-1.0.tar.gz) = 12341234...", + "SHA512 (distfile-1.0.tar.gz) = 12341234...", + "Size (distfile-1.0.tar.gz) = 12341234...") + t.CreateFileLines("category/package/Makefile", + MkRcsID, + "", + "CATEGORIES=category", + "", + "COMMENT=\tComment", + "LICENSE=\t2-clause-bsd") + + G.checkdirPackage(t.File("category/package")) + + t.CheckOutputLines( + "Whole Makefile (with all included files) follows:", + "~/category/package/Makefile:1: # $NetBSD: package_test.go,v 1.30 2018/10/03 22:27:53 rillig Exp $", + "~/category/package/Makefile:2: ", + "~/category/package/Makefile:3: CATEGORIES=category", + "~/category/package/Makefile:4: ", + "~/category/package/Makefile:5: COMMENT=\tComment", + "~/category/package/Makefile:6: LICENSE=\t2-clause-bsd") +} + +func (s *Suite) Test_Pkglint_checkdirPackage(c *check.C) { t := s.Init(c) t.Chdir("category/package") - t.SetupFileLines("Makefile", + t.CreateFileLines("Makefile", MkRcsID) G.checkdirPackage(".") @@ -316,6 +419,56 @@ func (s *Suite) Test_checkdirPackage(c *check.C) { "WARN: Makefile: No COMMENT given.") } +func (s *Suite) Test_Pkglint_checkdirPackage__PKGDIR(c *check.C) { + t := s.Init(c) + + t.SetupVartypes() + t.SetupPkgsrc() + t.CreateFileLines("category/Makefile") + t.CreateFileLines("other/package/Makefile", + MkRcsID) + t.CreateFileLines("other/package/PLIST", + PlistRcsID, + "bin/program") + t.CreateFileLines("other/package/distinfo", + RcsID, + "", + "SHA1 (patch-aa) = da39a3ee5e6b4b0d3255bfef95601890afd80709") + t.CreateFileLines("category/package/patches/patch-aa", + RcsID) + t.Chdir("category/package") + t.CreateFileLines("Makefile", + MkRcsID, + "", + "CATEGORIES=category", + "", + "COMMENT=\tComment", + "LICENSE=\t2-clause-bsd", + "PKGDIR=\t../../other/package") + + // DISTINFO_FILE is resolved relative to PKGDIR, the other places + // are resolved relative to the package base directory. + G.checkdirPackage(".") + + t.CheckOutputLines( + "ERROR: patches/patch-aa:1: Patch files must not be empty.") +} + +func (s *Suite) Test_Pkglint_checkdirPackage__patch_without_distinfo(c *check.C) { + t := s.Init(c) + + pkg := t.SetupPackage("category/package") + t.CreateFileDummyPatch("category/package/patches/patch-aa") + t.Remove("category/package/distinfo") + + G.CheckDirent(pkg) + + // FIXME: One of the below warnings is redundant. + t.CheckOutputLines( + "WARN: ~/category/package/distinfo: File not found. Please run \""+confMake+" makesum\" or define NO_CHECKSUM=yes in the package Makefile.", + "WARN: ~/category/package/distinfo: File not found. Please run \""+confMake+" makepatchsum\".") +} + func (s *Suite) Test_Pkglint_checkdirPackage__meta_package_without_license(c *check.C) { t := s.Init(c) @@ -414,7 +567,7 @@ func (s *Suite) Test_Package__varuse_at_load_time(c *check.C) { func (s *Suite) Test_Package_loadPackageMakefile(c *check.C) { t := s.Init(c) - t.SetupFileLines("category/package/Makefile", + t.CreateFileLines("category/package/Makefile", MkRcsID, "", "PKGNAME=pkgname-1.67", @@ -433,7 +586,34 @@ func (s *Suite) Test_Package_loadPackageMakefile(c *check.C) { "NOTE: ~/category/package/Makefile:4: Definition of DISTNAME is redundant because of Makefile:4.") } -func (s *Suite) Test_Package_conditionalAndUnconditionalInclude(c *check.C) { +func (s *Suite) Test_Package_loadPackageMakefile__PECL_VERSION(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("lang/php/ext.mk", + MkRcsID, + "", + "PHPEXT_MK= # defined", + "PHPPKGSRCDIR= ../../lang/php72", + "LICENSE?= unknown-license", + "COMMENT?= Some PHP package", + "GENERATE_PLIST+=# none", + "", + ".if !defined(PECL_VERSION)", + "DISTINFO_FILE= ${.CURDIR}/${PHPPKGSRCDIR}/distinfo", + ".endif", + ".if defined(USE_PHP_EXT_PATCHES)", + "PATCHDIR= ${.CURDIR}/${PHPPKGSRCDIR}/patches", + ".endif") + pkg := t.SetupPackage("category/package", + "PECL_VERSION=\t1.1.2", + ".include \"../../lang/php/ext.mk\"") + + G.CheckDirent(pkg) + + t.CheckOutputLines() +} + +func (s *Suite) Test_Package_CheckInclude__conditional_and_unconditional_include(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -479,7 +659,7 @@ func (s *Suite) Test_Package_conditionalAndUnconditionalInclude(c *check.C) { } // See https://github.com/rillig/pkglint/issues/1 -func (s *Suite) Test_Package_includeWithoutExists(c *check.C) { +func (s *Suite) Test_Package__include_without_exists(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -494,11 +674,11 @@ func (s *Suite) Test_Package_includeWithoutExists(c *check.C) { G.checkdirPackage(t.File("category/package")) t.CheckOutputLines( - "ERROR: ~/category/package/options.mk: Cannot be read.") + "ERROR: ~/category/package/Makefile:3: Cannot read \"options.mk\".") } // See https://github.com/rillig/pkglint/issues/1 -func (s *Suite) Test_Package_includeAfterExists(c *check.C) { +func (s *Suite) Test_Package__include_after_exists(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -524,7 +704,7 @@ func (s *Suite) Test_Package_includeAfterExists(c *check.C) { } // See https://github.com/rillig/pkglint/issues/1 -func (s *Suite) Test_Package_includeOtherAfterExists(c *check.C) { +func (s *Suite) Test_Package_readMakefile__include_other_after_exists(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -541,7 +721,7 @@ func (s *Suite) Test_Package_includeOtherAfterExists(c *check.C) { G.checkdirPackage(t.File("category/package")) t.CheckOutputLines( - "ERROR: ~/category/package/another.mk: Cannot be read.") + "ERROR: ~/category/package/Makefile:4: Cannot read \"another.mk\".") } // See https://mail-index.netbsd.org/tech-pkg/2018/07/22/msg020092.html @@ -650,3 +830,232 @@ func (s *Suite) Test_NewPackage(c *check.C) { check.PanicMatches, `Package directory "category" must be two subdirectories below the pkgsrc root ".*".`) } + +// Before 2018-09-09, the .CURDIR variable did not have a fallback value. +// When resolving the relative path x11/gst-x11/${.CURDIR}/../../multimedia/gst-base/distinfo, +// "gst-x11/${.CURDIR}" was interpreted as "category/package", and the whole +// path was resolved to "x11/multimedia/gst-base/distinfo, which of course +// could not be found. +func (s *Suite) Test__distinfo_from_other_package(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-space") + t.SetupPkgsrc() + t.Chdir(".") + t.CreateFileLines("x11/gst-x11/Makefile", + MkRcsID, + ".include \"../../multimedia/gst-base/Makefile.common\"", + ".include \"../../mk/bsd.pkg.mk\"") + t.CreateFileLines("multimedia/gst-base/Makefile.common", + MkRcsID, + ".include \"plugins.mk\"") + t.CreateFileLines("multimedia/gst-base/plugins.mk", + MkRcsID, + "DISTINFO_FILE=\t${.CURDIR}/../../multimedia/gst-base/distinfo") + t.CreateFileLines("multimedia/gst-base/distinfo", + RcsID, + "", + "SHA1 (patch-aa) = 1234") + + G.CheckDirent("x11/gst-x11") + + t.CheckOutputLines( + "WARN: x11/gst-x11/Makefile: Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset.", + "ERROR: x11/gst-x11/Makefile: Each package must define its LICENSE.", + "WARN: x11/gst-x11/Makefile: No COMMENT given.", + "WARN: x11/gst-x11/../../multimedia/gst-base/distinfo:3: Patch file \"patch-aa\" does not exist in directory \"../../x11/gst-x11/patches\".") +} + +func (s *Suite) Test_Package_checkfilePackageMakefile__GNU_CONFIGURE(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-space") + pkg := t.SetupPackage("category/package", + "GNU_CONFIGURE=\tyes", + "USE_LANGUAGES=\t#") + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:20: GNU_CONFIGURE almost always needs a C compiler, but \"c\" is not added to USE_LANGUAGES in line 21.") +} + +func (s *Suite) Test_Package_checkfilePackageMakefile__GNU_CONFIGURE_ok(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-space") + pkg := t.SetupPackage("category/package", + "GNU_CONFIGURE=\tyes", + "USE_LANGUAGES=\t# none, really") + + G.CheckDirent(pkg) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkfilePackageMakefile__REPLACE_PERL(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-space") + pkg := t.SetupPackage("category/package", + "REPLACE_PERL=\t*.pl", + "NO_CONFIGURE=\tyes") + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:20: REPLACE_PERL is ignored when NO_CONFIGURE is set (in line 21).") +} + +func (s *Suite) Test_Package_checkfilePackageMakefile__META_PACKAGE_with_distinfo(c *check.C) { + t := s.Init(c) + + pkg := t.SetupPackage("category/package", + "META_PACKAGE=\tyes") + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "WARN: ~/category/package/distinfo: " + + "This file should not exist if NO_CHECKSUM or META_PACKAGE is set.") +} + +func (s *Suite) Test_Package_checkfilePackageMakefile__USE_IMAKE_and_USE_X11(c *check.C) { + t := s.Init(c) + + pkg := t.SetupPackage("category/package", + "USE_X11=\tyes", + "USE_IMAKE=\tyes") + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "NOTE: ~/category/package/Makefile:21: USE_IMAKE makes USE_X11 in line 20 superfluous.") +} + +func (s *Suite) Test_Package_readMakefile__skipping(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-space") + pkg := t.SetupPackage("category/package", + ".include \"${MYSQL_PKGSRCDIR:S/-client$/-server/}/buildlink3.mk\"") + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "NOTE: ~/category/package/Makefile:20: " + + "Skipping include file \"${MYSQL_PKGSRCDIR:S/-client$/-server/}/buildlink3.mk\". " + + "This may result in false warnings.") +} + +func (s *Suite) Test_Package_readMakefile__not_found(c *check.C) { + t := s.Init(c) + + pkg := t.SetupPackage("category/package", + ".include \"../../devel/zlib/buildlink3.mk\"") + t.CreateFileLines("devel/zlib/buildlink3.mk", + ".include \"../../enoent/enoent/buildlink3.mk\"") + + G.checkdirPackage(pkg) + + t.CheckOutputLines( + "ERROR: ~/devel/zlib/buildlink3.mk:1: Cannot read \"../../enoent/enoent/buildlink3.mk\".") +} + +func (s *Suite) Test_Package_readMakefile__relative(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("category/package/extra.mk", + MkRcsID) + pkg := t.SetupPackage("category/package", + ".include \"../package/extra.mk\"") + + G.CheckDirent(pkg) + + // FIXME: One of the below warnings is redundant. + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:20: References to other packages should look like \"../../category/package\", not \"../package\".", + "WARN: ~/category/package/Makefile:20: Invalid relative path \"../package/extra.mk\".") +} + +func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-order") + G.CurrentUsername = "example-user" + t.CreateFileLines("category/package/CVS/Entries", + "/Makefile//modified//") + + // Since MAINTAINER= pkgsrc-users@NetBSD.org, everyone may commit changes. + + pkg := t.SetupPackage("category/package") + + G.CheckDirent(pkg) + + t.CheckOutputEmpty() + + // A package with a MAINTAINER may be edited with care. + + t.CreateFileLines("category/package/Makefile", + MkRcsID, + "", + "DISTNAME=\tdistname-1.0", + "CATEGORIES=\tcategory", + "MASTER_SITES=\t# none", + "", + "MAINTAINER=\tmaintainer@example.org", // Different from default value + "HOMEPAGE=\t# none", + "COMMENT=\tDummy package", + "LICENSE=\t2-clause-bsd", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "NOTE: ~/category/package/Makefile: " + + "Please only commit changes that maintainer@example.org would approve.") + + // A package with an OWNER may NOT be edited by others. + + pkg = t.SetupPackage("category/package", + "OWNER=\towner@example.org") + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile: " + + "Don't commit changes to this file without asking the OWNER, owner@example.org.") + + // ... unless you are the owner, of course. + + G.CurrentUsername = "owner" + + G.CheckDirent(pkg) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Pkglint_checkdirPackage__filename_with_variable(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-order") + pkg := t.SetupPackage("category/package", + ".include \"../../mk/bsd.prefs.mk\"", + "", + "RUBY_VERSIONS_ACCEPTED=\t22 23 24 25", // As of 2018. + ".for rv in ${RUBY_VERSIONS_ACCEPTED}", + "RUBY_VER?=\t\t${rv}", + ".endfor", + "", + "RUBY_PKGDIR=\t../../lang/ruby-${RUBY_VER}-base", + "DISTINFO_FILE=\t${RUBY_PKGDIR}/distinfo") + + // Pkglint cannot currently resolve the location of DISTINFO_FILE completely + // because the variable \"rv\" comes from a .for loop. + // + // TODO: resolve variables in simple .for loops like the above. + G.CheckDirent(pkg) + + t.CheckOutputEmpty() +} diff --git a/pkgtools/pkglint/files/parser.go b/pkgtools/pkglint/files/parser.go index c74ec401ea2..bef30e0ee47 100644 --- a/pkgtools/pkglint/files/parser.go +++ b/pkgtools/pkglint/files/parser.go @@ -12,7 +12,7 @@ type Parser struct { } func NewParser(line Line, s string, emitWarnings bool) *Parser { - return &Parser{line, textproc.NewPrefixReplacer(s), emitWarnings} + return &Parser{line, G.NewPrefixReplacer(s), emitWarnings} } func (p *Parser) EOF() bool { diff --git a/pkgtools/pkglint/files/patches_test.go b/pkgtools/pkglint/files/patches_test.go index c1ec4b14454..6f6ded8e407 100644 --- a/pkgtools/pkglint/files/patches_test.go +++ b/pkgtools/pkglint/files/patches_test.go @@ -42,7 +42,7 @@ func (s *Suite) Test_ChecklinesPatch__without_empty_line__autofix(c *check.C) { "-old line", "+new line", " context after") - t.SetupFileLines("distinfo", + t.CreateFileLines("distinfo", RcsID, "", // The hash is taken from a breakpoint at the beginning of AutofixDistinfo, oldSha1 @@ -691,8 +691,8 @@ func (s *Suite) Test_PatchChecker_checktextRcsid(c *check.C) { ChecklinesPatch(lines) t.CheckOutputLines( - "WARN: ~/patch-aa:7: Found RCS tag \"$Id: patches_test.go,v 1.20 2018/09/05 17:56:22 rillig Exp $\". Please remove it.", - "WARN: ~/patch-aa:8: Found RCS tag \"$Id: patches_test.go,v 1.20 2018/09/05 17:56:22 rillig Exp $\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".", + "WARN: ~/patch-aa:7: Found RCS tag \"$Id: patches_test.go,v 1.21 2018/10/03 22:27:53 rillig Exp $\". Please remove it.", + "WARN: ~/patch-aa:8: Found RCS tag \"$Id: patches_test.go,v 1.21 2018/10/03 22:27:53 rillig Exp $\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".", "WARN: ~/patch-aa:11: Found RCS tag \"$Author: rillig $\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".") } diff --git a/pkgtools/pkglint/files/pkglint.go b/pkgtools/pkglint/files/pkglint.go index 73e25e275b0..4cfe04a7526 100644 --- a/pkgtools/pkglint/files/pkglint.go +++ b/pkgtools/pkglint/files/pkglint.go @@ -5,6 +5,7 @@ import ( "netbsd.org/pkglint/getopt" "netbsd.org/pkglint/histogram" "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/textproc" "netbsd.org/pkglint/trace" "os" "os/user" @@ -49,8 +50,16 @@ type Pkglint struct { logOut *SeparatorWriter logErr *SeparatorWriter - loghisto *histogram.Histogram - loaded *histogram.Histogram + loghisto *histogram.Histogram + loaded *histogram.Histogram + res regex.Registry + fileCache *FileCache +} + +func NewPkglint() Pkglint { + return Pkglint{ + res: regex.NewRegistry(), + fileCache: NewFileCache(100)} } type CmdOpts struct { @@ -106,13 +115,15 @@ type Hash struct { // G is the abbreviation for "global state"; // it is the only global variable in this Go package -var G Pkglint +var G = NewPkglint() + +var exit = os.Exit // Indirect access, to allow main() to be tested. func main() { G.logOut = NewSeparatorWriter(os.Stdout) G.logErr = NewSeparatorWriter(os.Stderr) trace.Out = os.Stdout - os.Exit(G.Main(os.Args...)) + exit(G.Main(os.Args...)) } // Main runs the main program with the given arguments. @@ -139,21 +150,22 @@ func (pkglint *Pkglint) Main(argv ...string) (exitcode int) { if pkglint.opts.Profiling { f, err := os.Create("pkglint.pprof") if err != nil { - dummyLine.Fatalf("Cannot create profiling file: %s", err) + G.Panicf("Cannot create profiling file: %s", err) } defer f.Close() pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() - regex.Profiling = true + pkglint.res.Profiling() pkglint.loghisto = histogram.New() pkglint.loaded = histogram.New() defer func() { pkglint.logOut.Write("") pkglint.loghisto.PrintStats("loghisto", pkglint.logOut.out, -1) - regex.PrintStats(pkglint.logOut.out) + G.res.PrintStats(pkglint.logOut.out) pkglint.loaded.PrintStats("loaded", pkglint.logOut.out, 10) + pkglint.logOut.WriteLine(fmt.Sprintf("fileCache: %d hits, %d misses", pkglint.fileCache.hits, pkglint.fileCache.misses)) }() } @@ -170,7 +182,7 @@ func (pkglint *Pkglint) Main(argv ...string) (exitcode int) { } relTopdir := findPkgsrcTopdir(firstArg) if relTopdir == "" { - dummyLine.Fatalf("%q is not inside a pkgsrc tree.", firstArg) + G.Panicf("%q is not inside a pkgsrc tree.", firstArg) } pkglint.Pkgsrc = NewPkgsrc(firstArg + "/" + relTopdir) @@ -179,7 +191,7 @@ func (pkglint *Pkglint) Main(argv ...string) (exitcode int) { currentUser, err := user.Current() if err == nil { // On Windows, this is `Computername\Username`. - pkglint.CurrentUsername = regex.Compile(`^.*\\`).ReplaceAllString(currentUser.Username, "") + pkglint.CurrentUsername = replaceAll(currentUser.Username, `^.*\\`, "") } for len(pkglint.Todo) != 0 { @@ -332,6 +344,27 @@ func (pkglint *Pkglint) CheckDirent(fname string) { } } +func (pkglint *Pkglint) Panicf(format string, args ...interface{}) { + prefix := ifelseStr(G.opts.GccOutput, llFatal.GccName, llFatal.TraditionalName) + pkglint.logErr.Write(prefix + ": " + fmt.Sprintf(format, args...) + "\n") + panic(pkglintFatal{}) +} + +// Assertf checks that the condition is true. Otherwise it terminates the +// process with a fatal error message, prefixed with "Pkglint internal error". +// +// This method must only be used for programming errors. +// For runtime errors, use Panicf. +func (pkglint *Pkglint) Assertf(cond bool, format string, args ...interface{}) { + if !cond { + pkglint.Panicf("Pkglint internal error: "+format, args...) + } +} + +func (pkglint *Pkglint) NewPrefixReplacer(s string) *textproc.PrefixReplacer { + return textproc.NewPrefixReplacer(s, &pkglint.res) +} + // Returns the pkgsrc top-level directory, relative to the given file or directory. func findPkgsrcTopdir(fname string) string { for _, dir := range [...]string{".", "..", "../..", "../../.."} { @@ -342,9 +375,9 @@ func findPkgsrcTopdir(fname string) string { return "" } -func resolveVariableRefs(text string) string { - if trace.Tracing { - defer trace.Call1(text)() +func resolveVariableRefs(text string) (resolved string) { + if !contains(text, "${") { + return text } visited := make(map[string]bool) // To prevent endless loops @@ -354,14 +387,8 @@ func resolveVariableRefs(text string) string { if !visited[varname] { visited[varname] = true if G.Pkg != nil { - switch varname { - case "KRB5_TYPE": - return "heimdal" - case "PGSQL_VERSION": - return "95" - } - if mkline := G.Pkg.vars.FirstDefinition(varname); mkline != nil { - return mkline.Value() + if value, ok := G.Pkg.vars.Value(varname); ok { + return value } } if G.Mk != nil { @@ -375,8 +402,11 @@ func resolveVariableRefs(text string) string { str := text for { - replaced := regex.Compile(`\$\{([\w.]+)\}`).ReplaceAllStringFunc(str, replacer) + replaced := replaceAllFunc(str, `\$\{([\w.]+)\}`, replacer) if replaced == str { + if trace.Tracing && str != text { + trace.Stepf("resolveVariableRefs %q => %q", text, replaced) + } return replaced } str = replaced @@ -401,7 +431,7 @@ func ChecklinesDescr(lines []Line) { for _, line := range lines { CheckLineLength(line, 80) CheckLineTrailingWhitespace(line) - CheckLineValidCharacters(line, `[\t -~]`) + CheckLineValidCharacters(line) if contains(line.Text, "${") { line.Notef("Variables are not expanded in the DESCR file.") } @@ -453,7 +483,7 @@ func ChecklinesMessage(lines []Line) { for _, line := range lines { CheckLineLength(line, 80) CheckLineTrailingWhitespace(line) - CheckLineValidCharacters(line, `[\t -~]`) + CheckLineValidCharacters(line) } if lastLine := lines[len(lines)-1]; lastLine.Text != hline { fix := lastLine.Autofix() @@ -516,10 +546,16 @@ func (pkglint *Pkglint) Checkfile(fname string) { return } - pkglint.checkExecutable(st, fname) + pkglint.checkExecutable(st) + pkglint.checkMode(fname, st.Mode()) +} +// checkMode checks a directory entry based on its file name and its mode +// (regular file, directory, symlink). +func (pkglint *Pkglint) checkMode(fname string, mode os.FileMode) { + basename := path.Base(fname) switch { - case st.Mode().IsDir(): + case mode.IsDir(): switch { case basename == "files" || basename == "patches" || isIgnoredFilename(basename): // Ok @@ -529,12 +565,12 @@ func (pkglint *Pkglint) Checkfile(fname string) { NewLineWhole(fname).Warnf("Unknown directory name.") } - case st.Mode()&os.ModeSymlink != 0: + case mode&os.ModeSymlink != 0: if !hasPrefix(basename, "work") { NewLineWhole(fname).Warnf("Unknown symlink name.") } - case !st.Mode().IsRegular(): + case !mode.IsRegular(): NewLineWhole(fname).Errorf("Only files and directories are allowed in pkgsrc.") case basename == "ALTERNATIVES": @@ -609,9 +645,6 @@ func (pkglint *Pkglint) Checkfile(fname string) { } } - case basename == "TODO" || basename == "README": - // Ok - case hasPrefix(basename, "CHANGES-"): // This only checks the file, but doesn't register the changes globally. _ = pkglint.Pkgsrc.loadDocChangesFromFile(fname) @@ -620,7 +653,9 @@ func (pkglint *Pkglint) Checkfile(fname string) { // Skip case basename == "spec": - // Ok in regression tests + if !hasPrefix(G.Pkgsrc.ToRel(fname), "regress/") { + NewLineWhole(fname).Warnf("Only packages in regress/ may have spec files.") + } default: NewLineWhole(fname).Warnf("Unexpected file found.") @@ -630,7 +665,8 @@ func (pkglint *Pkglint) Checkfile(fname string) { } } -func (pkglint *Pkglint) checkExecutable(st os.FileInfo, fname string) { +func (pkglint *Pkglint) checkExecutable(st os.FileInfo) { + fname := st.Name() if st.Mode().IsRegular() && st.Mode().Perm()&0111 != 0 && !isCommitted(fname) { line := NewLine(fname, 0, "", nil) fix := line.Autofix() @@ -643,7 +679,7 @@ func (pkglint *Pkglint) checkExecutable(st os.FileInfo, fname string) { fix.Custom(func(printAutofix, autofix bool) { fix.Describef(0, "Clearing executable bits") if autofix { - if err := os.Chmod(line.Filename, st.Mode()&^0111); err != nil { + if err := os.Chmod(fname, st.Mode()&^0111); err != nil { line.Errorf("Cannot clear executable bits: %s", err) } } @@ -714,6 +750,11 @@ func (pkglint *Pkglint) Tool(command string, time ToolTime) (tool *Tool, usable return } +// ToolByVarname looks up the tool by its variable name, e.g. "SED". +// +// The returned tool may come either from the current Makefile or the +// current package. It is not guaranteed to be usable; that must be +// checked by the calling code. func (pkglint *Pkglint) ToolByVarname(varname string, time ToolTime) *Tool { var tool *Tool diff --git a/pkgtools/pkglint/files/pkglint_test.go b/pkgtools/pkglint/files/pkglint_test.go index e3459207519..6424673a25b 100644 --- a/pkgtools/pkglint/files/pkglint_test.go +++ b/pkgtools/pkglint/files/pkglint_test.go @@ -3,13 +3,14 @@ package main import ( "io/ioutil" "strings" + "time" "gopkg.in/check.v1" "netbsd.org/pkglint/trace" "os" ) -func (s *Suite) Test_Pkglint_Main_help(c *check.C) { +func (s *Suite) Test_Pkglint_Main__help(c *check.C) { t := s.Init(c) exitcode := G.Main("pkglint", "-h") @@ -18,7 +19,7 @@ func (s *Suite) Test_Pkglint_Main_help(c *check.C) { c.Check(t.Output(), check.Matches, `^\Qusage: pkglint [options] dir...\E\n(?s).+`) } -func (s *Suite) Test_Pkglint_Main_version(c *check.C) { +func (s *Suite) Test_Pkglint_Main__version(c *check.C) { t := s.Init(c) exitcode := G.Main("pkglint", "--version") @@ -28,7 +29,7 @@ func (s *Suite) Test_Pkglint_Main_version(c *check.C) { confVersion) } -func (s *Suite) Test_Pkglint_Main_no_args(c *check.C) { +func (s *Suite) Test_Pkglint_Main__no_args(c *check.C) { t := s.Init(c) exitcode := G.Main("pkglint") @@ -115,6 +116,18 @@ func (s *Suite) Test_Pkglint_Main__unknown_option(c *check.C) { " (Prefix a flag with \"no-\" to disable it.)") } +func (s *Suite) Test_Pkglint_Main__panic(c *check.C) { + t := s.Init(c) + + pkg := t.SetupPackage("category/package") + + G.logOut = nil // Force an error that cannot happen in practice. + + c.Check( + func() { G.Main("pkglint", pkg) }, + check.PanicMatches, `(?s).*\bnil pointer\b.*`) +} + // Demonstrates which infrastructure files are necessary to actually run // pkglint in a realistic scenario. // For most tests, this setup is too much work, therefore they @@ -129,7 +142,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { // FIXME: pkglint should warn that the latest version in this file // (1.10) doesn't match the current version in the package (1.11). - t.SetupFileLines("doc/CHANGES-2018", + t.CreateFileLines("doc/CHANGES-2018", RcsID, "", "Changes to the packages collection and infrastructure in 2018:", @@ -137,7 +150,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { "\tUpdated sysutils/checkperms to 1.10 [rillig 2018-01-05]") // See Pkgsrc.loadSuggestedUpdates. - t.SetupFileLines("doc/TODO", + t.CreateFileLines("doc/TODO", RcsID, "", "Suggested package updates", @@ -145,19 +158,19 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { "\to checkperms-1.13 [supports more file formats]") // The LICENSE in the package Makefile is searched here. - t.SetupFileLines("licenses/bsd-2", + t.CreateFileLines("licenses/bsd-2", "# dummy") // The MASTER_SITES in the package Makefile are searched here. // See Pkgsrc.loadMasterSites. - t.SetupFileMkLines("mk/fetch/sites.mk", + t.CreateFileLines("mk/fetch/sites.mk", MkRcsID, "", "MASTER_SITE_GITHUB+=\thttps://github.com/") // The existence of this file makes the category "sysutils" valid. // The category "tools" on the other hand is not valid. - t.SetupFileMkLines("sysutils/Makefile", + t.CreateFileLines("sysutils/Makefile", MkRcsID) // The package Makefile is quite simple, containing just the @@ -165,21 +178,21 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { // values is partly defined in the pkgsrc infrastructure files // (as defined in the previous lines), and partly in the pkglint // code directly. Many details can be found in vartypecheck.go. - t.SetupFileMkLines("sysutils/checkperms/Makefile", + t.CreateFileLines("sysutils/checkperms/Makefile", MkRcsID, "", "DISTNAME=\tcheckperms-1.11", "CATEGORIES=\tsysutils tools", "MASTER_SITES=\t${MASTER_SITE_GITHUB:=rillig/}", "", - "MAINTAINER=\tpkgsrc-users@pkgsrc.org", + "MAINTAINER=\tpkgsrc-users@NetBSD.org", "HOMEPAGE=\thttps://github.com/rillig/checkperms/", "COMMENT=\tCheck file permissions", "LICENSE=\tbsd-2", "", ".include \"../../mk/bsd.pkg.mk\"") - t.SetupFileLines("sysutils/checkperms/MESSAGE", + t.CreateFileLines("sysutils/checkperms/MESSAGE", "===========================================================================", RcsID, "", @@ -187,18 +200,18 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { "", "===========================================================================") - t.SetupFileLines("sysutils/checkperms/PLIST", + t.CreateFileLines("sysutils/checkperms/PLIST", PlistRcsID, "bin/checkperms", "man/man1/checkperms.1") - t.SetupFileLines("sysutils/checkperms/README", + t.CreateFileLines("sysutils/checkperms/README", "When updating this package, test the pkgsrc bootstrap.") - t.SetupFileLines("sysutils/checkperms/TODO", + t.CreateFileLines("sysutils/checkperms/TODO", "Make the package work on MS-DOS") - t.SetupFileLines("sysutils/checkperms/patches/patch-checkperms.c", + t.CreateFileLines("sysutils/checkperms/patches/patch-checkperms.c", RcsID, "", "A simple patch demonstrating that pkglint checks for missing", @@ -211,7 +224,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { "+// Header 1", "+// Header 2", "+// Header 3") - t.SetupFileLines("sysutils/checkperms/distinfo", + t.CreateFileLines("sysutils/checkperms/distinfo", RcsID, "", "SHA1 (checkperms-1.12.tar.gz) = 34c084b4d06bcd7a8bba922ff57677e651eeced5", @@ -242,7 +255,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { // pkgsrcdir=... // env PKGLINT_TESTCMDLINE="$pkgsrcdir -r" ./pkglint.test -test.coverprofile pkglint.cov // go tool cover -html=pkglint.cov -o coverage.html -func (s *Suite) Test_Pkglint_coverage(c *check.C) { +func (s *Suite) Test_Pkglint__coverage(c *check.C) { cmdline := os.Getenv("PKGLINT_TESTCMDLINE") if cmdline != "" { G.logOut, G.logErr, trace.Out = NewSeparatorWriter(os.Stdout), NewSeparatorWriter(os.Stderr), os.Stdout @@ -253,7 +266,7 @@ func (s *Suite) Test_Pkglint_coverage(c *check.C) { func (s *Suite) Test_Pkglint_CheckDirent__outside(c *check.C) { t := s.Init(c) - t.SetupFileLines("empty") + t.CreateFileLines("empty") G.CheckDirent(t.File(".")) @@ -261,13 +274,55 @@ func (s *Suite) Test_Pkglint_CheckDirent__outside(c *check.C) { "ERROR: ~: Cannot determine the pkgsrc root directory for \"~\".") } +func (s *Suite) Test_Pkglint_CheckDirent__empty_directory(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.CreateFileLines("category/package/CVS/Entries") + + G.CheckDirent(t.File("category/package")) + + // Empty directories are silently skipped. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Pkglint_CheckDirent__files_directory(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.CreateFileLines("category/package/files/README.md") + + G.CheckDirent(t.File("category/package/files")) + + // This diagnostic is not really correct, but it's an edge case anyway. + t.CheckOutputLines( + "ERROR: ~/category/package/files: Cannot check directories outside a pkgsrc tree.") +} + +func (s *Suite) Test_Pkglint_CheckDirent__manual_patch(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.CreateFileLines("category/package/patches/manual-configure") + t.CreateFileLines("category/package/Makefile", + MkRcsID) + + G.CheckDirent(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile: Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset.", + "WARN: ~/category/package/distinfo: File not found. Please run \""+confMake+" makesum\" or define NO_CHECKSUM=yes in the package Makefile.", + "ERROR: ~/category/package/Makefile: Each package must define its LICENSE.", + "WARN: ~/category/package/Makefile: No COMMENT given.") +} + func (s *Suite) Test_Pkglint_CheckDirent(c *check.C) { t := s.Init(c) - t.SetupFileLines("mk/bsd.pkg.mk") - t.SetupFileLines("category/package/Makefile") - t.SetupFileLines("category/Makefile") - t.SetupFileLines("Makefile") + t.CreateFileLines("mk/bsd.pkg.mk") + t.CreateFileLines("category/package/Makefile") + t.CreateFileLines("category/Makefile") + t.CreateFileLines("Makefile") G.CheckDirent(t.File(".")) @@ -434,10 +489,11 @@ func (s *Suite) Test_Pkglint__profiling(c *check.C) { t := s.Init(c) t.SetupPkgsrc() - G.Main("pkglint", "--profiling", t.File(".")) + t.Chdir(".") + + G.Main("pkglint", "--profiling") // Pkglint always writes the profiling data into the current directory. - // Luckily, this directory is usually writable. c.Check(fileExists("pkglint.pprof"), equals, true) err := os.Remove("pkglint.pprof") @@ -447,7 +503,20 @@ func (s *Suite) Test_Pkglint__profiling(c *check.C) { // or not interesting enough, since that info includes the exact timing // that the top time-consuming regular expressions took. firstOutput := strings.Split(t.Output(), "\n")[0] - c.Check(firstOutput, equals, "ERROR: ~/Makefile: Cannot be read.") + c.Check(firstOutput, equals, "ERROR: Makefile: Cannot be read.") +} + +func (s *Suite) Test_Pkglint__profiling_error(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.Chdir(".") + t.CreateFileLines("pkglint.pprof/file") + + exitcode := G.Main("pkglint", "--profiling") + + c.Check(exitcode, equals, 1) + c.Check(t.Output(), check.Matches, `^FATAL: Cannot create profiling file: open pkglint\.pprof: .*\n$`) } func (s *Suite) Test_Pkglint_Checkfile__in_current_working_directory(c *check.C) { @@ -594,35 +663,17 @@ func (s *Suite) Test_Pkglint_ToolByVarname__fallback(c *check.C) { c.Check(G.ToolByVarname("TOOL", RunTime), equals, global) } -func (s *Suite) Test_Pkglint_Checkfile__CheckExtra(c *check.C) { +func (s *Suite) Test_CheckfileExtra(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Call", "-Wall,no-space") - t.SetupPkgsrc() - G.Pkgsrc.LoadInfrastructure() - t.CreateFileLines("licenses/gnu-gpl-2.0") - t.CreateFileLines("category/Makefile") - t.CreateFileLines("category/package/Makefile", - MkRcsID, - "", - "DISTNAME= pkgname-1.0", - "CATEGORIES= category", - "", - "COMMENT= Comment", - "LICENSE= gnu-gpl-2.0", - "", - "NO_CHECKSUM= yes", - "", - ".include \"../../mk/bsd.pkg.mk\"") - t.CreateFileLines("category/package/PLIST", - PlistRcsID, - "bin/program") + pkg := t.SetupPackage("category/package") t.CreateFileLines("category/package/INSTALL", "#! /bin/sh") t.CreateFileLines("category/package/DEINSTALL", "#! /bin/sh") - G.CheckDirent(t.File("category/package")) + G.CheckDirent(pkg) t.CheckOutputEmpty() } @@ -631,31 +682,13 @@ func (s *Suite) Test_Pkglint_Checkfile__before_import(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Call", "-Wall,no-space", "--import") - t.SetupPkgsrc() - G.Pkgsrc.LoadInfrastructure() - t.CreateFileLines("licenses/gnu-gpl-2.0") - t.CreateFileLines("category/Makefile") - t.CreateFileLines("category/package/Makefile", - MkRcsID, - "", - "DISTNAME= pkgname-1.0", - "CATEGORIES= category", - "", - "COMMENT= Comment", - "LICENSE= gnu-gpl-2.0", - "", - "NO_CHECKSUM= yes", - "", - ".include \"../../mk/bsd.pkg.mk\"") - t.CreateFileLines("category/package/PLIST", - PlistRcsID, - "bin/program") + pkg := t.SetupPackage("category/package") t.CreateFileLines("category/package/work/log") t.CreateFileLines("category/package/Makefile~") t.CreateFileLines("category/package/Makefile.orig") t.CreateFileLines("category/package/Makefile.rej") - G.CheckDirent(t.File("category/package")) + G.CheckDirent(pkg) t.CheckOutputLines( "ERROR: ~/category/package/Makefile.orig: Must be cleaned up before committing the package.", @@ -775,3 +808,145 @@ func (s *Suite) Test_Pkglint_Checkfile__readme_and_todo(c *check.C) { "ERROR: wip/package/TODO: Must be cleaned up before committing the package.", "4 errors and 0 warnings found.") } + +func (s *Suite) Test_Pkglint_Checkfile__unknown_file_in_patches(c *check.C) { + t := s.Init(c) + + t.CreateFileDummyPatch("category/Makefile/patches/index") + + G.Checkfile(t.File("category/Makefile/patches/index")) + + t.CheckOutputLines( + "WARN: ~/category/Makefile/patches/index: " + + "Patch files should be named \"patch-\", followed by letters, '-', '_', '.', and digits only.") +} + +func (s *Suite) Test_Pkglint_Checkfile__file_in_files(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("category/package/files/index") + + G.Checkfile(t.File("category/package/files/index")) + + // These files are ignored since they could contain anything. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Pkglint_Checkfile__spec(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("category/package/spec") + t.CreateFileLines("regress/package/spec") + + G.Checkfile(t.File("category/package/spec")) + G.Checkfile(t.File("regress/package/spec")) + + t.CheckOutputLines( + "WARN: ~/category/package/spec: Only packages in regress/ may have spec files.") +} + +func (s *Suite) Test_Pkglint_checkMode__skipped(c *check.C) { + t := s.Init(c) + + G.checkMode("work", os.ModeSymlink) + G.checkMode("work.i386", os.ModeSymlink) + G.checkMode("work.hostname", os.ModeSymlink) + G.checkMode("other", os.ModeSymlink) + + G.checkMode("device", os.ModeDevice) + + t.CheckOutputLines( + "WARN: other: Unknown symlink name.", + "ERROR: device: Only files and directories are allowed in pkgsrc.") +} + +func (s *Suite) Test_Pkglint_checkdirPackage__ALTERNATIVES(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall,no-space") + pkg := t.SetupPackage("category/package") + t.CreateFileLines("category/package/ALTERNATIVES", + "bin/wrapper bin/wrapper-impl") + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "ERROR: ~/category/package/ALTERNATIVES:1: " + + "Alternative implementation \"bin/wrapper-impl\" must appear in the PLIST.") +} + +func (s *Suite) Test_CheckfileMk__enoent(c *check.C) { + t := s.Init(c) + + CheckfileMk(t.File("fname.mk")) + + t.CheckOutputLines( + "ERROR: ~/fname.mk: Cannot be read.") +} + +func (s *Suite) Test_Pkglint_checkExecutable(c *check.C) { + t := s.Init(c) + + G.checkExecutable(ExecutableFileInfo{t.File("fname.mk")}) + + t.CheckOutputLines( + "WARN: ~/fname.mk: Should not be executable.") + + t.SetupCommandLine("--autofix") + + G.checkExecutable(ExecutableFileInfo{t.File("fname.mk")}) + + // FIXME: The error message "Cannot clear executable bits" is swallowed. + t.CheckOutputLines( + "AUTOFIX: ~/fname.mk: Clearing executable bits") +} + +func (s *Suite) Test_main(c *check.C) { + t := s.Init(c) + + out, err := os.Create(t.CreateFileLines("out")) + c.Check(err, check.IsNil) + + pkg := t.SetupPackage("category/package") + + func() { + args := os.Args + stdout := os.Stdout + stderr := os.Stderr + prevExit := exit + defer func() { + os.Stderr = stderr + os.Stdout = stdout + os.Args = args + exit = prevExit + }() + os.Args = []string{"pkglint", pkg} + os.Stdout = out + os.Stderr = out + exit = func(code int) { + c.Check(code, equals, 0) + } + + main() + }() + + err = out.Close() + c.Check(err, check.IsNil) + + t.CheckOutputEmpty() + t.CheckFileLines("out", + "Looks fine.") +} + +// ExecutableFileInfo mocks a FileInfo because on Windows, +// regular files don't have the executable bit. +type ExecutableFileInfo struct { + name string +} + +func (i ExecutableFileInfo) Name() string { return i.name } +func (i ExecutableFileInfo) Size() int64 { return 13 } +func (i ExecutableFileInfo) Mode() os.FileMode { return 0777 } +func (i ExecutableFileInfo) ModTime() time.Time { return time.Unix(0, 0) } +func (i ExecutableFileInfo) IsDir() bool { return false } +func (i ExecutableFileInfo) Sys() interface{} { return nil } diff --git a/pkgtools/pkglint/files/pkgsrc.go b/pkgtools/pkglint/files/pkgsrc.go index aa3488fbe2d..00b8a50188b 100644 --- a/pkgtools/pkglint/files/pkgsrc.go +++ b/pkgtools/pkglint/files/pkgsrc.go @@ -5,6 +5,7 @@ import ( "netbsd.org/pkglint/regex" "netbsd.org/pkglint/trace" "sort" + "strconv" "strings" ) @@ -138,37 +139,64 @@ func (src *Pkgsrc) LoadInfrastructure() { // Example: // Latest("lang", `^php[0-9]+$`, "../../lang/$0") => "../../lang/php72" func (src *Pkgsrc) Latest(category string, re regex.Pattern, repl string) string { - key := category + "/" + string(re) + " => " + repl - if latest, found := src.latest[key]; found { + if G.Testing { + G.Assertf( + hasPrefix(string(re), "^") && hasSuffix(string(re), "$"), + "Regular expression %q must be anchored at both ends.", re) + } + + cacheKey := category + "/" + string(re) + " => " + repl + if latest, found := src.latest[cacheKey]; found { return latest } categoryDir := src.File(category) error := func() string { dummyLine.Errorf("Cannot find latest version of %q in %q.", re, categoryDir) - src.latest[key] = "" + src.latest[cacheKey] = "" return "" } - all, err := ioutil.ReadDir(categoryDir) - sort.SliceStable(all, func(i, j int) bool { - return naturalLess(all[i].Name(), all[j].Name()) - }) + fileInfos, err := ioutil.ReadDir(categoryDir) if err != nil { return error() } - latest := "" - for _, fileInfo := range all { - if matches(fileInfo.Name(), re) { - latest = regex.Compile(re).ReplaceAllString(fileInfo.Name(), repl) + var names []string + for _, fileInfo := range fileInfos { + name := fileInfo.Name() + if matches(name, re) { + names = append(names, name) + } + } + + keys := make(map[string]int) + for _, name := range names { + if m, pkgbase, versionStr := match2(name, `^(\D+)(\d+)$`); m { + version, _ := strconv.Atoi(versionStr) + if pkgbase == "postgresql" && version < 60 { + version = 10 * version + } + keys[name] = version } } + + sort.SliceStable(names, func(i, j int) bool { + if keyI, keyJ := keys[names[i]], keys[names[j]]; keyI != 0 && keyJ != 0 { + return keyI < keyJ + } + return naturalLess(names[i], names[j]) + }) + + latest := "" + for _, name := range names { + latest = replaceAll(name, re, repl) + } if latest == "" { return error() } - src.latest[key] = latest + src.latest[cacheKey] = latest return latest } @@ -346,7 +374,7 @@ func (src *Pkgsrc) loadDocChangesFromFile(fname string) []*Change { } } } else if text := line.Text; len(text) >= 2 && text[0] == '\t' && 'A' <= text[1] && text[1] <= 'Z' { - line.Warnf("Unknown doc/CHANGES line: %q", text) + line.Warnf("Unknown doc/CHANGES line: %s", text) Explain("See mk/misc/developer.mk for the rules.") } } @@ -549,6 +577,9 @@ func (src *Pkgsrc) initDeprecatedVars() { "SVR4_PKGNAME": "Just remove it.", "PKG_INSTALLATION_TYPES": "Just remove it.", + // November 2015, commit abccb56 + "EVAL_PREFIX": "All packages are installed in PREFIX now.", + // January 2016 "SUBST_POSTCMD.*": "Has been removed, as it seemed unused.", @@ -596,7 +627,7 @@ func (src *Pkgsrc) IsBuildDef(varname string) bool { func (src *Pkgsrc) loadMasterSites() { mklines := src.LoadMk("mk/fetch/sites.mk", MustSucceed|NotEmpty) - nameToUrl := src.MasterSiteVarToURL + nameToURL := src.MasterSiteVarToURL urlToName := src.MasterSiteURLToVar for _, mkline := range mklines.mklines { if mkline.IsVarassign() { @@ -604,8 +635,8 @@ func (src *Pkgsrc) loadMasterSites() { if hasPrefix(varname, "MASTER_SITE_") && varname != "MASTER_SITE_BACKUP" { for _, url := range splitOnSpace(mkline.Value()) { if matches(url, `^(?:http://|https://|ftp://)`) { - if nameToUrl[varname] == "" { - nameToUrl[varname] = url + if nameToURL[varname] == "" { + nameToURL[varname] = url } urlToName[url] = varname } @@ -615,7 +646,7 @@ func (src *Pkgsrc) loadMasterSites() { } // Explicitly allowed, although not defined in mk/fetch/sites.mk. - nameToUrl["MASTER_SITE_LOCAL"] = "ftp://ftp.NetBSD.org/pub/pkgsrc/distfiles/LOCAL_PORTS/" + nameToURL["MASTER_SITE_LOCAL"] = "ftp://ftp.NetBSD.org/pub/pkgsrc/distfiles/LOCAL_PORTS/" if trace.Tracing { trace.Stepf("Loaded %d MASTER_SITE_* URLs.", len(urlToName)) @@ -629,7 +660,7 @@ func (src *Pkgsrc) loadPkgOptions() { if m, optname, optdescr := match2(line.Text, `^([-0-9a-z_+]+)(?:\s+(.*))?$`); m { src.PkgOptions[optname] = optdescr } else { - line.Fatalf("Unknown line format.") + line.Fatalf("Unknown line format: %s", line.Text) } } } diff --git a/pkgtools/pkglint/files/pkgsrc_test.go b/pkgtools/pkglint/files/pkgsrc_test.go index 0db9d07ca6e..f9ad0035ee8 100644 --- a/pkgtools/pkglint/files/pkgsrc_test.go +++ b/pkgtools/pkglint/files/pkgsrc_test.go @@ -61,22 +61,22 @@ func (s *Suite) Test_Pkgsrc_parseSuggestedUpdates(c *check.C) { func (s *Suite) Test_Pkgsrc_loadTools(c *check.C) { t := s.Init(c) - t.SetupFileLines("mk/tools/bsd.tools.mk", + t.CreateFileLines("mk/tools/bsd.tools.mk", ".include \"flex.mk\"", ".include \"gettext.mk\"", ".include \"strip.mk\"", ".include \"replace.mk\"") - t.SetupFileLines("mk/tools/defaults.mk", + t.CreateFileLines("mk/tools/defaults.mk", "_TOOLS_VARNAME.chown=CHOWN", "_TOOLS_VARNAME.gawk=AWK", "_TOOLS_VARNAME.mv=MV", "_TOOLS_VARNAME.pwd=PWD") - t.SetupFileLines("mk/tools/flex.mk", + t.CreateFileLines("mk/tools/flex.mk", "# empty") - t.SetupFileLines("mk/tools/gettext.mk", + t.CreateFileLines("mk/tools/gettext.mk", "USE_TOOLS+=msgfmt", "TOOLS_CREATE+=msgfmt") - t.SetupFileLines("mk/tools/strip.mk", + t.CreateFileLines("mk/tools/strip.mk", ".if defined(_INSTALL_UNSTRIPPED) || !defined(TOOLS_PLATFORM.strip)", "TOOLS_NOOP+= strip", ".else", @@ -84,14 +84,14 @@ func (s *Suite) Test_Pkgsrc_loadTools(c *check.C) { "TOOLS_PATH.strip= ${TOOLS_PLATFORM.strip}", ".endif", "STRIP?= strip") - t.SetupFileLines("mk/tools/replace.mk", + t.CreateFileLines("mk/tools/replace.mk", "_TOOLS.bzip2=\tbzip2 bzcat", "#TOOLS_CREATE+=commented out", "_UNRELATED_VAR=\t# empty") - t.SetupFileLines("mk/bsd.prefs.mk", + t.CreateFileLines("mk/bsd.prefs.mk", "USE_TOOLS+=\tpwd", "USE_TOOLS+=\tm4:pkgsrc") - t.SetupFileLines("mk/bsd.pkg.mk", + t.CreateFileLines("mk/bsd.pkg.mk", "USE_TOOLS+=\tmv") G.Pkgsrc.loadTools() @@ -119,17 +119,46 @@ func (s *Suite) Test_Pkgsrc_loadTools(c *check.C) { "TRACE: - (*Tools).Trace(\"Pkgsrc\")") } +// As a side-benefit, loadTools also loads the _BUILD_DEFS. +func (s *Suite) Test_Pkgsrc_loadTools__BUILD_DEFS(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + t.SetupToolUsable("echo", "ECHO") + pkg := t.SetupPackage("category/package", + "pre-configure:", + "\t@${ECHO} ${PKG_SYSCONFDIR} ${VARBASE}") + t.CreateFileLines("mk/bsd.pkg.mk", + MkRcsID, + "_BUILD_DEFS+=\tPKG_SYSCONFBASEDIR PKG_SYSCONFDIR") + G.Pkgsrc.LoadInfrastructure() + + G.CheckDirent(pkg) + + c.Check(G.Pkgsrc.IsBuildDef("PKG_SYSCONFDIR"), equals, true) + c.Check(G.Pkgsrc.IsBuildDef("VARBASE"), equals, false) + + // FIXME: There should be a warning for VARBASE, but G.Pkgsrc.UserDefinedVars + // does not contain anything at mklinechecker.go:/UserDefinedVars/. + t.CheckOutputLines() +} + func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile(c *check.C) { t := s.Init(c) - t.SetupFileLines("doc/CHANGES-2018", + t.CreateFileLines("doc/CHANGES-2018", "\tAdded category/package version 1.0 [author1 2015-01-01]", // Wrong year "\tUpdated category/package to 1.5 [author2 2018-01-02]", "\tRenamed category/package to category/pkg [author3 2018-01-03]", "\tMoved category/package to other/package [author4 2018-01-04]", "\tRemoved category/package [author5 2018-01-09]", // Too far in the future "\tRemoved category/package successor category/package2 [author6 2018-01-06]", - "\tDowngraded category/package to 1.2 [author7 2018-01-07]") + "\tDowngraded category/package to 1.2 [author7 2018-01-07]", + "", + "\ttoo few fields", + "\ttoo many many many many many fields", + "\tmissing brackets around author", + "\tAdded another [new package]") changes := G.Pkgsrc.loadDocChangesFromFile(t.File("doc/CHANGES-2018")) @@ -144,22 +173,68 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile(c *check.C) { t.CheckOutputLines( "WARN: ~/doc/CHANGES-2018:1: Year 2015 for category/package does not match the file name ~/doc/CHANGES-2018.", - "WARN: ~/doc/CHANGES-2018:6: Date 2018-01-06 for category/package is earlier than 2018-01-09 for category/package.") + "WARN: ~/doc/CHANGES-2018:6: Date 2018-01-06 for category/package is earlier than 2018-01-09 for category/package.", + "WARN: ~/doc/CHANGES-2018:12: Unknown doc/CHANGES line: \tAdded another [new package]") +} + +func (s *Suite) Test_Pkgsrc_loadDocChanges__not_found(c *check.C) { + t := s.Init(c) + + t.SetupPkgsrc() + t.Remove("doc/CHANGES-2018") + t.Remove("doc/TODO") + t.Remove("doc") + + t.ExpectFatal( + G.Pkgsrc.loadDocChanges, + "FATAL: ~/doc: Cannot be read.") } -func (s *Suite) Test_Pkgsrc_deprecated(c *check.C) { +func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__not_found(c *check.C) { + t := s.Init(c) + + t.ExpectFatal( + func() { G.Pkgsrc.loadDocChangesFromFile(t.File("doc/CHANGES-2018")) }, + "FATAL: ~/doc/CHANGES-2018: Cannot be read.") +} + +func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__wip(c *check.C) { + t := s.Init(c) + + pkg := t.SetupPackage("wip/package", + "DISTNAME=\tpackage-1.11", + "CATEGORIES=\tlocal") + t.CreateFileLines("wip/TODO", + RcsID, + "", + "Suggested package updates", + "", + "\to package-1.13 [cool new features]") + G.Pkgsrc.LoadInfrastructure() + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "WARN: ~/wip/package/Makefile:3: This package should be updated to 1.13 ([cool new features]).") +} + +func (s *Suite) Test_Pkgsrc__deprecated(c *check.C) { t := s.Init(c) G.Pkgsrc.initDeprecatedVars() - mkline := t.NewMkLine("Makefile", 5, "USE_PERL5=\tyes") + mklines := t.NewMkLines("Makefile", + "USE_PERL5=\tyes", + "SUBST_POSTCMD.class=${ECHO}") - MkLineChecker{mkline}.checkVarassign() + MkLineChecker{mklines.mklines[0]}.checkVarassign() + MkLineChecker{mklines.mklines[1]}.checkVarassign() t.CheckOutputLines( - "WARN: Makefile:5: Definition of USE_PERL5 is deprecated. Use USE_TOOLS+=perl or USE_TOOLS+=perl:run instead.") + "WARN: Makefile:1: Definition of USE_PERL5 is deprecated. Use USE_TOOLS+=perl or USE_TOOLS+=perl:run instead.", + "WARN: Makefile:2: Definition of SUBST_POSTCMD.class is deprecated. Has been removed, as it seemed unused.") } -func (s *Suite) Test_Pkgsrc_Latest_no_basedir(c *check.C) { +func (s *Suite) Test_Pkgsrc_Latest__no_basedir(c *check.C) { t := s.Init(c) latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") @@ -169,10 +244,10 @@ func (s *Suite) Test_Pkgsrc_Latest_no_basedir(c *check.C) { "ERROR: Cannot find latest version of \"^python[0-9]+$\" in \"~/lang\".") } -func (s *Suite) Test_Pkgsrc_Latest_no_subdirs(c *check.C) { +func (s *Suite) Test_Pkgsrc_Latest__no_subdirs(c *check.C) { t := s.Init(c) - t.SetupFileLines("lang/Makefile") + t.CreateFileLines("lang/Makefile") latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") @@ -181,46 +256,88 @@ func (s *Suite) Test_Pkgsrc_Latest_no_subdirs(c *check.C) { "ERROR: Cannot find latest version of \"^python[0-9]+$\" in \"~/lang\".") } -func (s *Suite) Test_Pkgsrc_Latest_single(c *check.C) { +func (s *Suite) Test_Pkgsrc_Latest__single(c *check.C) { t := s.Init(c) - t.SetupFileLines("lang/Makefile") - t.SetupFileLines("lang/python27/Makefile") + t.CreateFileLines("lang/Makefile") + t.CreateFileLines("lang/python27/Makefile") latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") c.Check(latest, equals, "../../lang/python27") + + cached := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") + + c.Check(cached, equals, "../../lang/python27") } -func (s *Suite) Test_Pkgsrc_Latest_multi(c *check.C) { +func (s *Suite) Test_Pkgsrc_Latest__multi(c *check.C) { t := s.Init(c) - t.SetupFileLines("lang/Makefile") - t.SetupFileLines("lang/python27/Makefile") - t.SetupFileLines("lang/python35/Makefile") + t.CreateFileLines("lang/Makefile") + t.CreateFileLines("lang/python27/Makefile") + t.CreateFileLines("lang/python35/Makefile") latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") c.Check(latest, equals, "../../lang/python35") } -func (s *Suite) Test_Pkgsrc_Latest_numeric(c *check.C) { +func (s *Suite) Test_Pkgsrc_Latest__numeric(c *check.C) { t := s.Init(c) - t.SetupFileLines("databases/postgresql95/Makefile") - t.SetupFileLines("databases/postgresql97/Makefile") - t.SetupFileLines("databases/postgresql100/Makefile") - t.SetupFileLines("databases/postgresql104/Makefile") + t.CreateFileLines("databases/postgresql95/Makefile") + t.CreateFileLines("databases/postgresql97/Makefile") + t.CreateFileLines("databases/postgresql100/Makefile") + t.CreateFileLines("databases/postgresql104/Makefile") latest := G.Pkgsrc.Latest("databases", `^postgresql[0-9]+$`, "$0") c.Check(latest, equals, "postgresql104") } +func (s *Suite) Test_Pkgsrc_Latest__numeric_multiple_numbers(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("emulators/suse_131_32_gtk2/Makefile") + t.CreateFileLines("emulators/suse_131_32_qt5/Makefile") + t.CreateFileLines("emulators/suse_131_gtk2/Makefile") + t.CreateFileLines("emulators/suse_131_qt5/Makefile") + + latest := G.Pkgsrc.Latest("emulators", `^suse_(\d+).*$`, "$1") + + c.Check(latest, equals, "131") +} + +// In 2017, PostgreSQL changed their versioning scheme to SemVer, +// and since the pkgsrc directory contains the major version, +// without any separating dots, the case of version 10 being +// later than 95 needs to be handled specially. +func (s *Suite) Test_Pkgsrc_Latest__postgresql(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("databases/postgresql95/Makefile") + t.CreateFileLines("databases/postgresql97/Makefile") + t.CreateFileLines("databases/postgresql10/Makefile") + t.CreateFileLines("databases/postgresql11/Makefile") + + latest := G.Pkgsrc.Latest("databases", `^postgresql[0-9]+$`, "$0") + + c.Check(latest, equals, "postgresql11") +} + +func (s *Suite) Test_Pkgsrc_Latest__invalid_argument(c *check.C) { + t := s.Init(c) + + t.ExpectFatal( + func() { G.Pkgsrc.Latest("databases", `postgresql[0-9]+`, "$0") }, + "FATAL: Pkglint internal error: Regular expression \"postgresql[0-9]+\" must be anchored at both ends.") +} + func (s *Suite) Test_Pkgsrc_loadPkgOptions(c *check.C) { t := s.Init(c) - t.SetupFileLines("mk/defaults/options.description", + t.CreateFileLines("mk/defaults/options.description", "option-name Description of the option", "<<<<< Merge conflict", "===== Merge conflict", @@ -228,7 +345,7 @@ func (s *Suite) Test_Pkgsrc_loadPkgOptions(c *check.C) { t.ExpectFatal( G.Pkgsrc.loadPkgOptions, - "FATAL: ~/mk/defaults/options.description:2: Unknown line format.") + "FATAL: ~/mk/defaults/options.description:2: Unknown line format: <<<<< Merge conflict") } func (s *Suite) Test_Pkgsrc_loadTools__no_tools_found(c *check.C) { diff --git a/pkgtools/pkglint/files/plist.go b/pkgtools/pkglint/files/plist.go index 86bfed7f2c3..fe2417119fb 100644 --- a/pkgtools/pkglint/files/plist.go +++ b/pkgtools/pkglint/files/plist.go @@ -1,7 +1,6 @@ package main import ( - "netbsd.org/pkglint/regex" "netbsd.org/pkglint/trace" "path" "sort" @@ -300,7 +299,7 @@ func (ck *PlistChecker) checkpathLib(pline *PlistLine, dirname, basename string) } func (ck *PlistChecker) checkpathMan(pline *PlistLine) { - m, catOrMan, section, manpage, ext, gz := regex.Match5(pline.text, `^man/(cat|man)(\w+)/(.*?)\.(\w+)(\.gz)?$`) + m, catOrMan, section, manpage, ext, gz := match5(pline.text, `^man/(cat|man)(\w+)/(.*?)\.(\w+)(\.gz)?$`) if !m { // maybe: line.Warnf("Invalid filename %q for manual page.", text) return diff --git a/pkgtools/pkglint/files/plist_test.go b/pkgtools/pkglint/files/plist_test.go index 0873a2f014e..162ba327697 100644 --- a/pkgtools/pkglint/files/plist_test.go +++ b/pkgtools/pkglint/files/plist_test.go @@ -62,10 +62,10 @@ func (s *Suite) Test_ChecklinesPlist__empty(c *check.C) { "WARN: PLIST:1: PLIST files shouldn't be empty.") } -func (s *Suite) Test_ChecklinesPlist__commonEnd(c *check.C) { +func (s *Suite) Test_ChecklinesPlist__common_end(c *check.C) { t := s.Init(c) - t.SetupFileLines("PLIST.common", + t.CreateFileLines("PLIST.common", PlistRcsID, "bin/common") lines := t.SetupFileLines("PLIST.common_end", @@ -81,7 +81,6 @@ func (s *Suite) Test_ChecklinesPlist__condition(c *check.C) { t := s.Init(c) G.Pkg = NewPackage(t.File("category/pkgbase")) - G.Pkg.plistSubstCond["PLIST.bincmds"] = true lines := t.NewLines("PLIST", PlistRcsID, "${PLIST.bincmds}bin/subdir/command") @@ -111,7 +110,7 @@ func (s *Suite) Test_ChecklinesPlist__sorting(c *check.C) { "WARN: PLIST:6: \"bin/cat\" should be sorted before \"bin/otherprogram\".") } -func (s *Suite) Test_PlistLineSorter_Sort(c *check.C) { +func (s *Suite) Test_plistLineSorter_Sort(c *check.C) { t := s.Init(c) t.SetupCommandLine("--autofix") @@ -167,7 +166,7 @@ func (s *Suite) Test_PlistLineSorter_Sort(c *check.C) { "@exec echo \"after lib/after.la\"") // The footer starts here } -func (s *Suite) Test_PlistChecker_checkpathMan_gz(c *check.C) { +func (s *Suite) Test_PlistChecker_checkpathMan__gz(c *check.C) { t := s.Init(c) G.Pkg = NewPackage(t.File("category/pkgbase")) @@ -181,7 +180,7 @@ func (s *Suite) Test_PlistChecker_checkpathMan_gz(c *check.C) { "NOTE: PLIST:2: The .gz extension is unnecessary for manual pages.") } -func (s *Suite) TestPlistChecker_checkpath__PKGMANDIR(c *check.C) { +func (s *Suite) Test_PlistChecker_checkpath__PKGMANDIR(c *check.C) { t := s.Init(c) lines := t.NewLines("PLIST", @@ -194,7 +193,7 @@ func (s *Suite) TestPlistChecker_checkpath__PKGMANDIR(c *check.C) { "NOTE: PLIST:2: PLIST files should mention \"man/\" instead of \"${PKGMANDIR}\".") } -func (s *Suite) TestPlistChecker_checkpath__python_egg(c *check.C) { +func (s *Suite) Test_PlistChecker_checkpath__python_egg(c *check.C) { t := s.Init(c) lines := t.NewLines("PLIST", diff --git a/pkgtools/pkglint/files/regex/regex.go b/pkgtools/pkglint/files/regex/regex.go index 4c0c8886500..4f8067ef1a5 100644 --- a/pkgtools/pkglint/files/regex/regex.go +++ b/pkgtools/pkglint/files/regex/regex.go @@ -14,97 +14,107 @@ import ( type Pattern string -var ( - Profiling bool -) +type Registry struct { + res map[Pattern]*regexp.Regexp + rematch *histogram.Histogram + renomatch *histogram.Histogram + retime *histogram.Histogram + profiling bool +} -var ( - res = make(map[Pattern]*regexp.Regexp) - rematch = histogram.New() - renomatch = histogram.New() - retime = histogram.New() -) +func NewRegistry() Registry { + return Registry{make(map[Pattern]*regexp.Regexp), nil, nil, nil, false} +} + +func (r *Registry) Profiling() { + if !r.profiling { + r.rematch = histogram.New() + r.renomatch = histogram.New() + r.retime = histogram.New() + r.profiling = true + } +} -func Compile(re Pattern) *regexp.Regexp { - cre := res[re] +func (r *Registry) Compile(re Pattern) *regexp.Regexp { + cre := r.res[re] if cre == nil { cre = regexp.MustCompile(string(re)) - res[re] = cre + r.res[re] = cre } return cre } -func Match(s string, re Pattern) []string { - if !Profiling { - return Compile(re).FindStringSubmatch(s) +func (r *Registry) Match(s string, re Pattern) []string { + if !r.profiling { + return r.Compile(re).FindStringSubmatch(s) } before := time.Now() immediatelyBefore := time.Now() - m := Compile(re).FindStringSubmatch(s) + m := r.Compile(re).FindStringSubmatch(s) after := time.Now() delay := immediatelyBefore.UnixNano() - before.UnixNano() timeTaken := after.UnixNano() - immediatelyBefore.UnixNano() - delay - retime.Add(string(re), int(timeTaken)) + r.retime.Add(string(re), int(timeTaken)) if m != nil { - rematch.Add(string(re), 1) + r.rematch.Add(string(re), 1) } else { - renomatch.Add(string(re), 1) + r.renomatch.Add(string(re), 1) } return m } -func Matches(s string, re Pattern) bool { - matches := Compile(re).MatchString(s) - if Profiling { +func (r *Registry) Matches(s string, re Pattern) bool { + matches := r.Compile(re).MatchString(s) + if r.profiling { if matches { - rematch.Add(string(re), 1) + r.rematch.Add(string(re), 1) } else { - renomatch.Add(string(re), 1) + r.renomatch.Add(string(re), 1) } } return matches } -func Match1(s string, re Pattern) (matched bool, m1 string) { - if m := matchn(s, re, 1); m != nil { +func (r *Registry) Match1(s string, re Pattern) (matched bool, m1 string) { + if m := r.matchn(s, re, 1); m != nil { return true, m[1] } return } -func Match2(s string, re Pattern) (matched bool, m1, m2 string) { - if m := matchn(s, re, 2); m != nil { +func (r *Registry) Match2(s string, re Pattern) (matched bool, m1, m2 string) { + if m := r.matchn(s, re, 2); m != nil { return true, m[1], m[2] } return } -func Match3(s string, re Pattern) (matched bool, m1, m2, m3 string) { - if m := matchn(s, re, 3); m != nil { +func (r *Registry) Match3(s string, re Pattern) (matched bool, m1, m2, m3 string) { + if m := r.matchn(s, re, 3); m != nil { return true, m[1], m[2], m[3] } return } -func Match4(s string, re Pattern) (matched bool, m1, m2, m3, m4 string) { - if m := matchn(s, re, 4); m != nil { +func (r *Registry) Match4(s string, re Pattern) (matched bool, m1, m2, m3, m4 string) { + if m := r.matchn(s, re, 4); m != nil { return true, m[1], m[2], m[3], m[4] } return } -func Match5(s string, re Pattern) (matched bool, m1, m2, m3, m4, m5 string) { - if m := matchn(s, re, 5); m != nil { +func (r *Registry) Match5(s string, re Pattern) (matched bool, m1, m2, m3, m4, m5 string) { + if m := r.matchn(s, re, 5); m != nil { return true, m[1], m[2], m[3], m[4], m[5] } return } -func ReplaceFirst(s string, re Pattern, replacement string) ([]string, string) { - if m := Compile(re).FindStringSubmatchIndex(s); m != nil { +func (r *Registry) ReplaceFirst(s string, re Pattern, replacement string) ([]string, string) { + if m := r.Compile(re).FindStringSubmatchIndex(s); m != nil { replaced := s[:m[0]] + replacement + s[m[1]:] mm := make([]string, len(m)/2) for i := 0; i < len(m); i += 2 { @@ -115,16 +125,16 @@ func ReplaceFirst(s string, re Pattern, replacement string) ([]string, string) { return nil, s } -func PrintStats(out io.Writer) { - if Profiling { - rematch.PrintStats("rematch", out, 10) - renomatch.PrintStats("renomatch", out, 10) - retime.PrintStats("retime", out, 10) +func (r *Registry) PrintStats(out io.Writer) { + if r.profiling { + r.rematch.PrintStats("rematch", out, 10) + r.renomatch.PrintStats("renomatch", out, 10) + r.retime.PrintStats("retime", out, 10) } } -func matchn(s string, re Pattern, n int) []string { - if m := Match(s, re); m != nil { +func (r *Registry) matchn(s string, re Pattern, n int) []string { + if m := r.Match(s, re); m != nil { if len(m) != 1+n { panic(fmt.Sprintf("expected match%d, got match%d for %q", len(m)-1, n, re)) } diff --git a/pkgtools/pkglint/files/shell.go b/pkgtools/pkglint/files/shell.go index 1346cb3e406..11dddfb10e6 100644 --- a/pkgtools/pkglint/files/shell.go +++ b/pkgtools/pkglint/files/shell.go @@ -79,8 +79,9 @@ outer: case quoting == shqPlain: switch { - case repl.AdvanceRegexp(`^[!#\%&\(\)*+,\-.\/0-9:;<=>?@A-Z\[\]^_a-z{|}~]+`), - repl.AdvanceRegexp(`^\\(?:[ !"#'\(\)*./;?\\^{|}]|\$\$)`): + // FIXME: These regular expressions don't belong here, they are the job of the tokenizer. + case repl.AdvanceRegexp(`^[!#%&()*+,\-./0-9:;<=>?@A-Z\[\]^_a-z{|}~]+`), + repl.AdvanceRegexp(`^\\(?:[ !"#'()*./;?\\^{|}]|\$\$)`): case repl.AdvanceStr("'"): quoting = shqSquot case repl.AdvanceStr("\""): @@ -192,10 +193,10 @@ func (shline *ShellLine) checkVaruseToken(parser *MkParser, quoting ShQuoting) b switch { case quoting == shqPlain && varuse.IsQ(): // Fine. - case quoting == shqBackt: - // Don't check anything here, to avoid false positives for tool names. + case (quoting == shqSquot || quoting == shqDquot) && matches(varname, `^(?:.*DIR|.*FILE|.*PATH|.*_VAR|PREFIX|.*BASE|PKGNAME)$`): // This is ok if we don't allow these variables to have embedded [\$\\\"\'\`]. + case quoting == shqDquot && varuse.IsQ(): shline.mkline.Warnf("Please don't use the :Q operator in double quotes.") Explain( @@ -249,14 +250,12 @@ func (shline *ShellLine) unescapeBackticks(shellword string, repl *textproc.Pref "", "To avoid this uncertainty, escape the double quotes using \\\".") - case repl.AdvanceRegexp("^([^\\\\`]+)"): - unescaped += repl.Group(1) - default: - line.Errorf("Internal pkglint error in ShellLine.unescapeBackticks at %q (rest=%q).", shellword, repl.AdvanceRest()) + G.Assertf(repl.AdvanceRegexp("^([^\\\\`]+)"), "incomplete switch") + unescaped += repl.Group(1) } } - line.Errorf("Unfinished backquotes: rest=%q", repl.Rest()) + line.Errorf("Unfinished backquotes: %s", repl.Rest()) return unescaped, quoting } @@ -304,7 +303,7 @@ func (shline *ShellLine) CheckShellCommandLine(shelltext string) { line.Notef("You don't need to use \"-\" before %q.", cmd) } - repl := textproc.NewPrefixReplacer(shelltext) + repl := G.NewPrefixReplacer(shelltext) repl.AdvanceRegexp(`^\s+`) if repl.AdvanceRegexp(`^[-@]+`) { shline.checkHiddenAndSuppress(repl.Group(0), repl.Rest()) @@ -338,25 +337,29 @@ func (shline *ShellLine) CheckShellCommand(shellcmd string, pSetE *bool, time To spc := &ShellProgramChecker{shline} spc.checkConditionalCd(program) - callback := NewMkShWalkCallback() - callback.SimpleCommand = func(command *MkShSimpleCommand) { + walker := NewMkShWalker() + walker.Callback.SimpleCommand = func(command *MkShSimpleCommand) { scc := NewSimpleCommandChecker(shline, command, time) scc.Check() if scc.strcmd.Name == "set" && scc.strcmd.AnyArgMatches(`^-.*e`) { *pSetE = true } } - callback.List = func(list *MkShList) { - spc.checkSetE(list, pSetE) + walker.Callback.AndOr = func(andor *MkShAndOr) { + if G.opts.WarnExtra && !*pSetE && walker.Current().Index != 0 { + spc.checkSetE(walker.Parent(1).(*MkShList), walker.Current().Index, andor) + } } - callback.Pipeline = func(pipeline *MkShPipeline) { + walker.Callback.Pipeline = func(pipeline *MkShPipeline) { spc.checkPipeExitcode(line, pipeline) } - callback.Word = func(word *ShToken) { + walker.Callback.Word = func(word *ShToken) { + // TODO: Try to replace false with true here; it had been set to false + // TODO: in 2016 for no apparent reason. spc.checkWord(word, false, time) } - NewMkShWalker().Walk(program, callback) + walker.Walk(program) } func (shline *ShellLine) CheckShellCommands(shellcmds string, time ToolTime) { @@ -436,7 +439,7 @@ func (scc *SimpleCommandChecker) Check() { } scc.checkCommandStart() - scc.checkAbsolutePathnames() + scc.checkRegexReplace() scc.checkAutoMkdirs() scc.checkInstallMulti() scc.checkPaxPe() @@ -587,7 +590,7 @@ func (scc *SimpleCommandChecker) handleComment() bool { return true } -func (scc *SimpleCommandChecker) checkAbsolutePathnames() { +func (scc *SimpleCommandChecker) checkRegexReplace() { if trace.Tracing { defer trace.Call()() } @@ -598,7 +601,7 @@ func (scc *SimpleCommandChecker) checkAbsolutePathnames() { if !isSubst { CheckLineAbsolutePathname(scc.shline.mkline.Line, arg) } - if false && isSubst && !matches(arg, `"^[\"\'].*[\"\']$`) { + if G.Testing && isSubst && !matches(arg, `"^[\"\'].*[\"\']$`) { scc.shline.mkline.Warnf("Substitution commands like %q should always be quoted.", arg) Explain( "Usually these substitution commands contain characters like '*' or", @@ -739,20 +742,20 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { } } - callback := NewMkShWalkCallback() - callback.If = func(ifClause *MkShIfClause) { + walker := NewMkShWalker() + walker.Callback.If = func(ifClause *MkShIfClause) { for _, cond := range ifClause.Conds { if simple := getSimple(cond); simple != nil { checkConditionalCd(simple) } } } - callback.Loop = func(loop *MkShLoopClause) { + walker.Callback.Loop = func(loop *MkShLoopClause) { if simple := getSimple(loop.Cond); simple != nil { checkConditionalCd(simple) } } - callback.Pipeline = func(pipeline *MkShPipeline) { + walker.Callback.Pipeline = func(pipeline *MkShPipeline) { if pipeline.Negated { spc.shline.mkline.Warnf("The Solaris /bin/sh does not support negation of shell commands.") Explain( @@ -761,17 +764,7 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { "https://www.gnu.org/software/autoconf/manual/autoconf.html#Limitations-of-Builtins") } } - NewMkShWalker().Walk(list, callback) -} - -func (spc *ShellProgramChecker) checkWords(words []*ShToken, checkQuoting bool, time ToolTime) { - if trace.Tracing { - defer trace.Call()() - } - - for _, word := range words { - spc.checkWord(word, checkQuoting, time) - } + walker.Walk(list) } func (spc *ShellProgramChecker) checkWord(word *ShToken, checkQuoting bool, time ToolTime) { @@ -787,41 +780,14 @@ func (spc *ShellProgramChecker) checkPipeExitcode(line Line, pipeline *MkShPipel defer trace.Call()() } - oneOf := func(s string, others ...string) bool { - for _, other := range others { - if s == other { - return true - } - } - return false - } - - // canFail tests whether one of the left-hand side commands of a - // shell pipeline can fail. - // - // Examples: - // echo "hello" | sed 's,$, world,,' => cannot fail - // find . -print | xargs cat | wc -l => can fail canFail := func() (bool, string) { for _, cmd := range pipeline.Cmds[:len(pipeline.Cmds)-1] { - simple := cmd.Simple - if simple == nil { + if spc.canFail(cmd) { + if cmd.Simple != nil && cmd.Simple.Name != nil { + return true, cmd.Simple.Name.MkText + } return true, "" } - commandName := simple.Name.MkText - if len(simple.Redirections) != 0 { - return true, commandName - } - tool, _ := G.Tool(commandName, RunTime) - switch { - case tool == nil: - return true, commandName - case oneOf(tool.Name, "echo", "printf"): - case oneOf(tool.Name, "sed", "gsed", "grep", "ggrep") && len(simple.Args) == 1: - break - default: - return true, commandName - } } return false, "" } @@ -844,29 +810,110 @@ func (spc *ShellProgramChecker) checkPipeExitcode(line Line, pipeline *MkShPipel } } -func (spc *ShellProgramChecker) checkSetE(list *MkShList, eflag *bool) { +// canFail returns true if the given shell command can fail. +// Most shell commands can fail for various reasons, such as missing +// files or invalid arguments. +// +// Commands that can fail: +// echo "hello" > file +// sed 's,$, world,,' < input > output +// find . -print +// wc -l * +// +// Commands that cannot fail: +// echo "hello" +// sed 's,$, world,,' +// wc -l +func (spc *ShellProgramChecker) canFail(cmd *MkShCommand) bool { + simple := cmd.Simple + if simple == nil { + return true + } + + if simple.Name == nil { + for _, assignment := range simple.Assignments { + if contains(assignment.MkText, "`") || contains(assignment.MkText, "$(") { + if !contains(assignment.MkText, "|| ${TRUE}") { + return true + } + } + } + return false + } + + for _, redirect := range simple.Redirections { + if redirect.Target != nil && !hasSuffix(redirect.Op, "&") { + return true + } + } + + cmdName := simple.Name.MkText + switch cmdName { + case "${ECHO_MSG}", "${PHASE_MSG}", "${STEP_MSG}", + "${INFO_MSG}", "${WARNING_MSG}", "${ERROR_MSG}", + "${WARNING_CAT}", "${ERROR_CAT}", + "${DO_NADA}": + return false + case "${FAIL_MSG}": + return true + case "set": + } + + tool, _ := G.Tool(cmdName, RunTime) + if tool == nil { + return true + } + + toolName := tool.Name + args := simple.Args + argc := len(args) + switch toolName { + case "echo", "printf", "tr": + return false + case "sed", "gsed": + if argc == 2 && args[0].MkText == "-e" { + return false + } + return argc != 1 + case "grep", "ggrep": + return argc != 1 + } + + return true +} + +func (spc *ShellProgramChecker) checkSetE(list *MkShList, index int, andor *MkShAndOr) { if trace.Tracing { defer trace.Call()() } - // Disabled until the shell parser can recognize "command || exit 1" reliably. - if false && G.opts.WarnExtra && !*eflag && "the current token" == ";" { - *eflag = true - spc.shline.mkline.Warnf("Please switch to \"set -e\" mode before using a semicolon (the one after %q) to separate commands.", "previous token") - Explain( - "Normally, when a shell command fails (returns non-zero), the", - "remaining commands are still executed. For example, the following", - "commands would remove all files from the HOME directory:", - "", - "\tcd \"$HOME\"; cd /nonexistent; rm -rf *", - "", - "To fix this warning, you can:", - "", - "* insert ${RUN} at the beginning of the line", - " (which among other things does \"set -e\")", - "* insert \"set -e\" explicitly at the beginning of the line", - "* use \"&&\" instead of \";\" to separate the commands") + command := list.AndOrs[index-1].Pipes[0].Cmds[0] + if command.Simple == nil || !spc.canFail(command) { + return } + + line := spc.shline.mkline.Line + if !line.FirstTime("switch to set -e") { + return + } + + line.Warnf("Please switch to \"set -e\" mode before using a semicolon (after %q) to separate commands.", + NewStrCommand(command.Simple).String()) + Explain( + "Normally, when a shell command fails (returns non-zero), the", + "remaining commands are still executed. For example, the following", + "commands would remove all files from the HOME directory:", + "", + "\tcd \"$HOME\"; cd /nonexistent; rm -rf *", + "", + "In \"set -e\" mode, the shell stops when a command fails.", + "", + "To fix this warning, you can:", + "", + "* insert ${RUN} at the beginning of the line", + " (which among other things does \"set -e\")", + "* insert \"set -e\" explicitly at the beginning of the line", + "* use \"&&\" instead of \";\" to separate the commands") } // Some shell commands should not be used in the install phase. @@ -920,6 +967,7 @@ func splitIntoShellTokens(line Line, text string) (tokens []string, rest string) } word := "" + rest = text p := NewShTokenizer(line, text, false) emit := func() { if word != "" { diff --git a/pkgtools/pkglint/files/shell_test.go b/pkgtools/pkglint/files/shell_test.go index 5cd2ae90c4c..ed1926eea9c 100644 --- a/pkgtools/pkglint/files/shell_test.go +++ b/pkgtools/pkglint/files/shell_test.go @@ -2,7 +2,6 @@ package main import ( "gopkg.in/check.v1" - "netbsd.org/pkglint/textproc" ) func (s *Suite) Test_splitIntoShellTokens__line_continuation(c *check.C) { @@ -27,7 +26,7 @@ func (s *Suite) Test_splitIntoShellTokens__dollar_slash(c *check.C) { func (s *Suite) Test_splitIntoShellTokens__dollar_subshell(c *check.C) { words, rest := splitIntoShellTokens(dummyLine, "id=$$(${AWK} '{print}' < ${WRKSRC}/idfile) && echo \"$$id\"") - c.Check(words, deepEquals, []string{"id=", "$$(", "${AWK}", "'{print}'", "<", "${WRKSRC}/idfile", ")", "&&", "echo", "\"$$id\""}) + c.Check(words, deepEquals, []string{"id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)", "&&", "echo", "\"$$id\""}) c.Check(rest, equals, "") } @@ -138,6 +137,10 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { t.SetupCommandLine("-Wall") t.SetupVartypes() + t.SetupToolUsable("awk", "AWK") + t.SetupToolUsable("cp", "CP") + t.SetupToolUsable("mkdir", "MKDIR") // This is actually "mkdir -p". + t.SetupToolUsable("unzip", "UNZIP_CMD") checkShellCommandLine := func(shellCommand string) { G.Mk = t.NewMkLines("fname", @@ -157,6 +160,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { t.CheckOutputLines( "WARN: fname:1: Unknown shell command \"uname\".", + "WARN: fname:1: Please switch to \"set -e\" mode before using a semicolon (after \"uname=`uname`\") to separate commands.", "WARN: fname:1: Unknown shell command \"echo\".", "WARN: fname:1: Unknown shell command \"echo\".") @@ -216,9 +220,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { checkShellCommandLine("${RUN} subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\"") t.CheckOutputLines( - "WARN: fname:1: The exitcode of \"unzip\" at the left of the | operator is ignored.", - "WARN: fname:1: Unknown shell command \"unzip\".", - "WARN: fname:1: Unknown shell command \"awk\".") + "WARN: fname:1: The exitcode of \"unzip\" at the left of the | operator is ignored.") // From mail/thunderbird/Makefile, rev. 1.159 checkShellCommandLine("" + @@ -232,12 +234,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { t.CheckOutputLines( "WARN: fname:1: XPI_FILES is used but not defined.", - "WARN: fname:1: The exitcode of \"${UNZIP_CMD}\" at the left of the | operator is ignored.", - "WARN: fname:1: UNZIP_CMD is used but not defined.", - "WARN: fname:1: Unknown shell command \"awk\".", - "WARN: fname:1: Unknown shell command \"${MKDIR}\".", - "WARN: fname:1: MKDIR is used but not defined.", - "WARN: fname:1: UNZIP_CMD is used but not defined.") + "WARN: fname:1: The exitcode of \"${UNZIP_CMD}\" at the left of the | operator is ignored.") // From x11/wxGTK28/Makefile checkShellCommandLine("" + @@ -255,8 +252,18 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { checkShellCommandLine("@cp from to") t.CheckOutputLines( - "WARN: fname:1: The shell command \"cp\" should not be hidden.", - "WARN: fname:1: Unknown shell command \"cp\".") + "WARN: fname:1: The shell command \"cp\" should not be hidden.") + + checkShellCommandLine("-cp from to") + + t.CheckOutputLines( + "WARN: fname:1: Using a leading \"-\" to suppress errors is deprecated.") + + checkShellCommandLine("-${MKDIR} deeply/nested/subdir") + + t.CheckOutputLines( + "NOTE: fname:1: You don't need to use \"-\" before \"${MKDIR} deeply/nested/subdir\".", + "WARN: fname:1: Using a leading \"-\" to suppress errors is deprecated.") G.Pkg = NewPackage(t.File("category/pkgbase")) G.Pkg.PlistDirs["share/pkgbase"] = true @@ -283,7 +290,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { t.CheckOutputEmpty() // No warning about missing error checking. } -func (s *Suite) Test_ShellLine_CheckShellCommandLine_strip(c *check.C) { +func (s *Suite) Test_ShellLine_CheckShellCommandLine__strip(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -344,8 +351,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__show_autofix(c *check.C) { "AUTOFIX: Makefile:1: Replacing \"${PKGNAME:Q}\" with \"${PKGNAME}\".") } -// Simple commands like echo(1) or printf(1) are assumed to never fail. -func (s *Suite) Test_ShellLine_CheckShellCommandLine__exitcode(c *check.C) { +func (s *Suite) Test_ShellProgramChecker_checkPipeExitcode(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall") @@ -363,7 +369,10 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__exitcode(c *check.C) { "\t cat | echo | right-side", "\t echo | cat | right-side", "\t sed s,s,s, filename | right-side", - "\t sed s,s,s < input | right-side") + "\t sed s,s,s < input | right-side", + "\t ./unknown | right-side", + "\t var=value | right-side", + "\t if :; then :; fi | right-side") for _, mkline := range G.Mk.mklines { shline := NewShellLine(mkline) @@ -375,7 +384,9 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__exitcode(c *check.C) { "WARN: Makefile:5: The exitcode of \"cat\" at the left of the | operator is ignored.", "WARN: Makefile:6: The exitcode of \"cat\" at the left of the | operator is ignored.", "WARN: Makefile:7: The exitcode of \"sed\" at the left of the | operator is ignored.", - "WARN: Makefile:8: The exitcode of \"sed\" at the left of the | operator is ignored.") + "WARN: Makefile:8: The exitcode of \"sed\" at the left of the | operator is ignored.", + "WARN: Makefile:9: The exitcode of \"./unknown\" at the left of the | operator is ignored.", + "WARN: Makefile:11: The exitcode of the command at the left of the | operator is ignored.") } func (s *Suite) Test_ShellLine_CheckShellCommandLine__autofix(c *check.C) { @@ -423,7 +434,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__implementation(c *check.C) "WARN: fname:1: Unknown shell command \"echo\".") } -func (s *Suite) Test_ShellLine_CheckShelltext__dollar_without_variable(c *check.C) { +func (s *Suite) Test_ShellLine_CheckShellCommandLine__dollar_without_variable(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -511,6 +522,44 @@ func (s *Suite) Test_ShellLine_CheckWord__dollar_without_variable(c *check.C) { t.CheckOutputEmpty() } +func (s *Suite) Test_ShellLine_CheckWord__backslash_plus(c *check.C) { + t := s.Init(c) + + shline := t.NewShellLine("fname", 1, "\tfind . -exec rm -rf {} \\+") + + shline.CheckShellCommandLine(shline.mkline.ShellCommand()) + + // FIXME: A backslash before any other character than "\` keeps its original meaning. + t.CheckOutputLines( + "WARN: fname:1: Pkglint parse error in ShellLine.CheckWord at \"\\\\+\" (quoting=plain), rest: \\+") +} + +func (s *Suite) Test_ShellLine_CheckWord__squot_dollar(c *check.C) { + t := s.Init(c) + + shline := t.NewShellLine("fname", 1, "\t'$") + + shline.CheckWord(shline.mkline.ShellCommand(), false, RunTime) + + // FIXME: Should be parsed correctly. Make passes the dollar through (probably), + // and the shell parser should complain about the unfinished string literal. + t.CheckOutputLines( + "WARN: fname:1: Pkglint parse error in ShellLine.CheckWord at \"'$\" (quoting=s), rest: $") +} + +func (s *Suite) Test_ShellLine_CheckWord__dquot_dollar(c *check.C) { + t := s.Init(c) + + shline := t.NewShellLine("fname", 1, "\t\"$") + + shline.CheckWord(shline.mkline.ShellCommand(), false, RunTime) + + // FIXME: Should be parsed correctly. Make passes the dollar through (probably), + // and the shell parser should complain about the unfinished string literal. + t.CheckOutputLines( + "WARN: fname:1: Pkglint parse error in ShellLine.CheckWord at \"\\\"$\" (quoting=d), rest: $") +} + func (s *Suite) Test_ShellLine_CheckWord__dollar_subshell(c *check.C) { t := s.Init(c) @@ -522,6 +571,67 @@ func (s *Suite) Test_ShellLine_CheckWord__dollar_subshell(c *check.C) { "WARN: fname:1: Invoking subshells via $(...) is not portable enough.") } +func (s *Suite) Test_ShellLine_unescapeBackticks__unfinished(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + mklines := t.NewMkLines("fname.mk", + MkRcsID, + "", + "pre-configure:", + "\t`${VAR}", // Error in first shell word + "\techo `${VAR}") // Error after first shell word + + // Breakpoint in ShellLine.CheckShellCommand + // Breakpoint in ShellLine.CheckToken + // Breakpoint in ShellLine.unescapeBackticks + mklines.Check() + + // FIXME: Mention the unfinished backquote. + t.CheckOutputLines( + "WARN: fname.mk:4: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", + "WARN: fname.mk:5: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"echo\"}") +} + +func (s *Suite) Test_ShellLine_unescapeBackticks__unfinished_direct(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + + // This call is unrealistic. It doesn't happen in practice, and this + // direct, forcing test is only to reach the code coverage. + NewShellLine(dummyMkLine).unescapeBackticks( + "dummy", + G.NewPrefixReplacer(""), + shqBackt) + + t.CheckOutputLines( + "ERROR: Unfinished backquotes: ") +} + +func (s *Suite) Test_ShellLine_variableNeedsQuoting(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + t.SetupVartypes() + t.SetupToolUsable("cp", "") + mklines := t.NewMkLines("fname.mk", + MkRcsID, + "", + // It's a bit silly to use shell variables in CONFIGURE_ARGS, + // but currently that's the only way to run ShellLine.variableNeedsQuoting. + "CONFIGURE_ARGS+=\t; cp $$dir $$\\# $$target", + "pre-configure:", + "\tcp $$dir $$\\# $$target") + + mklines.Check() + + // Quoting check is currently disabled for real shell commands. + // See ShellLine.CheckShellCommand, spc.checkWord. + t.CheckOutputLines( + "WARN: fname.mk:3: Unquoted shell variable \"target\".") +} + func (s *Suite) Test_ShellLine_CheckShellCommandLine__echo(c *check.C) { t := s.Init(c) @@ -689,7 +799,7 @@ func (s *Suite) Test_ShellLine_unescapeBackticks(c *check.C) { shline := t.NewShellLine("dummy.mk", 13, "# dummy") // foobar="`echo \"foo bar\" "\ " "three"`" text := "foobar=\"`echo \\\"foo bar\\\" \"\\ \" \"three\"`\"" - repl := textproc.NewPrefixReplacer(text) + repl := G.NewPrefixReplacer(text) repl.AdvanceStr("foobar=\"`") backtCommand, newQuoting := shline.unescapeBackticks(text, repl, shqDquotBackt) @@ -761,6 +871,64 @@ func (s *Suite) Test_ShellLine_CheckShellCommand__negated_pipe(c *check.C) { "WARN: Makefile:3: Found absolute pathname: /etc/passwd") } +func (s *Suite) Test_ShellLine_CheckShellCommand__subshell(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", + MkRcsID, + "", + "pre-configure:", + "\t@(echo ok)", + "\techo $$(uname -r); echo $$(expr 4 '*' $$(echo 1024))", + "\t@(echo nb$$(uname -r) $$(${EXPR} 4 \\* $$(echo 1024)))") + + mklines.Check() + + // FIXME: Fix the parse errors (nested subshells). + // FIXME: Fix the duplicate diagnostic in line 6. + // FIXME: "(" is not a shell command, it's an operator. + t.CheckOutputLines( + "WARN: Makefile:4: The shell command \"(\" should not be hidden.", + "WARN: Makefile:5: Pkglint parse error in ShTokenizer.ShAtom at \"$$(echo 1024))\" (quoting=S).", + "WARN: Makefile:5: Invoking subshells via $(...) is not portable enough.", + "WARN: Makefile:6: Pkglint parse error in ShTokenizer.ShAtom at \"$$(echo 1024)))\" (quoting=S).", + "WARN: Makefile:6: The shell command \"(\" should not be hidden.", + "WARN: Makefile:6: Pkglint parse error in ShTokenizer.ShAtom at \"$$(echo 1024)))\" (quoting=S).", + "WARN: Makefile:6: Invoking subshells via $(...) is not portable enough.") +} + +func (s *Suite) Test_ShellLine_CheckShellCommand__case_patterns_from_variable(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", + MkRcsID, + "", + "pre-configure:", + "\tcase $$file in ${CHECK_PERMS_SKIP:@pattern@${pattern}) ;;@} *) continue; esac") + + mklines.Check() + + // FIXME: Support the above variable expansion. + t.CheckOutputLines( + "WARN: Makefile:4: Pkglint ShellLine.CheckShellCommand: " + + "parse error at []string{\"*\", \")\", \"continue\", \";\", \"esac\"}") +} + +func (s *Suite) Test_ShellLine_checkHiddenAndSuppress(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", + MkRcsID, + "", + "show-all-targets: .PHONY", + "\t@echo 'hello'", + "\t@ls -l") + + mklines.Check() + + t.CheckOutputEmpty() +} + func (s *Suite) Test_SimpleCommandChecker_handleForbiddenCommand(c *check.C) { t := s.Init(c) @@ -779,6 +947,66 @@ func (s *Suite) Test_SimpleCommandChecker_handleForbiddenCommand(c *check.C) { "ERROR: Makefile:3: \"truss\" must not be used in Makefiles.") } +func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable(c *check.C) { + t := s.Init(c) + + t.SetupToolUsable("perl", "PERL5") + t.SetupTool("perl6", "PERL6") + mklines := t.NewMkLines("Makefile", + MkRcsID, + "", + "PERL5_VARS_CMD=\t${PERL5:Q}", + "PERL5_VARS_CMD=\t${PERL6:Q}") + + mklines.Check() + + // FIXME: Warn about using _PERL5_VARS because it starts with an underscore. + // FIXME: Omit the redundant PERL5_VARS_CMD warning in line 4. + // FIXME: In PERL5:Q and PERL6:Q, the :Q is wrong. + t.CheckOutputLines( + "WARN: Makefile:3: PERL5_VARS_CMD is defined but not used.", + "WARN: Makefile:4: The \"perl6\" tool is used but not added to USE_TOOLS.", + "WARN: Makefile:4: PERL5_VARS_CMD is defined but not used.") +} + +// The package Makefile and other .mk files in a package directory +// may use any shell commands defined by any included files. +// But only if the package is checked as a whole. +// +// On the contrary, when pkglint checks a single .mk file, these +// commands are not guaranteed to be defined, not even when the +// .mk file includes the file defining the command. +// FIXME: This paragraph sounds wrong. All commands from included files should be valid. +// +// The variable must not be called *_CMD, or another code path is taken. +func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__from_package(c *check.C) { + t := s.Init(c) + + pkg := t.SetupPackage("category/package", + "post-install:", + "\t${PYTHON_BIN}", + "", + ".include \"extra.mk\"") + t.CreateFileLines("category/package/extra.mk", + MkRcsID, + "PYTHON_BIN= my_cmd") + + G.CheckDirent(pkg) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_SimpleCommandChecker_handleComment(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("file.mk", 3, "\t# comment; continuation") + + MkLineChecker{mkline}.Check() + + t.CheckOutputLines( + "WARN: file.mk:3: A shell comment should not contain semicolons.") +} + func (s *Suite) Test_SimpleCommandChecker_checkPaxPe(c *check.C) { t := s.Init(c) @@ -813,16 +1041,122 @@ func (s *Suite) Test_SimpleCommandChecker_checkEchoN(c *check.C) { "WARN: Makefile:4: Please use ${ECHO_N} instead of \"echo -n\".") } -func (s *Suite) Test_SimpleCommandChecker_checkConditionalCd(c *check.C) { +func (s *Suite) Test_ShellProgramChecker_checkConditionalCd(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", + MkRcsID, + "pre-configure:", + "\t${RUN} while cd ..; do printf .; done", + // TODO: "\t${RUN} if ls | tr -d $$; then :; fi", + "\t${RUN} if ls | tr -d shell$$; then :; fi") + + mklines.Check() + + // FIXME: Fix the parse error. + t.CheckOutputLines( + "ERROR: Makefile:3: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", + "WARN: Pkglint parse error in ShTokenizer.ShAtom at \"$$\" (quoting=plain).") +} + +func (s *Suite) Test_SimpleCommandChecker_checkRegexReplace(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("Makefile", MkRcsID, "pre-configure:", - "\t${RUN} while cd ..; do printf .; done") + "\t${PAX} -s s,.*,, src dst", + "\tpax -s s,.*,, src dst", + "\t${SED} -e s,.*,, src dst", + "\tsed -e s,.*,, src dst", + "\tpax -s s,\\.orig,, src dst", + "\tsed -e s,a,b,g src dst") + + mklines.Check() + + // FIXME: warn for "pax -s". + // FIXME: warn for "sed -e". + // TODO: don't warn for "pax .orig". + // TODO: don't warn for "s,a,b,g". + t.CheckOutputLines( + "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.", + "WARN: Makefile:5: Substitution commands like \"s,.*,,\" should always be quoted.") + +} + +func (s *Suite) Test_ShellProgramChecker_checkSetE__simple_commands(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + t.SetupToolUsable("echo", "") + t.SetupToolUsable("rm", "") + t.SetupToolUsable("touch", "") + mklines := t.NewMkLines("Makefile", + MkRcsID, + "pre-configure:", + "\techo 1; echo 2; echo 3", + "\techo 1; touch file; rm file", + "\techo 1; var=value; echo 3") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:4: Please switch to \"set -e\" mode before using a semicolon (after \"touch file\") to separate commands.") +} + +func (s *Suite) Test_ShellProgramChecker_checkSetE__compound_commands(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + t.SetupToolUsable("echo", "") + t.SetupToolUsable("touch", "") + mklines := t.NewMkLines("Makefile", + MkRcsID, + "pre-configure:", + "\ttouch file; for f in file; do echo \"$$f\"; done", + "\tfor f in file; do echo \"$$f\"; done; touch file") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon (after \"touch file\") to separate commands.") +} + +func (s *Suite) Test_ShellProgramChecker_canFail(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wall") + t.SetupVartypes() + t.SetupToolUsable("echo", "") + t.SetupToolUsable("grep", "GREP") + t.SetupToolUsable("sed", "") + t.SetupToolUsable("touch", "") + t.SetupToolUsable("tr", "tr") + t.SetupToolUsable("true", "TRUE") + mklines := t.NewMkLines("Makefile", + MkRcsID, + "pre-configure:", + "\tsocklen=`${GREP} 'expr' ${WRKSRC}/config.h`; echo 'done.'", + "\tsocklen=`${GREP} 'expr' ${WRKSRC}/config.h || ${TRUE}`; echo 'done.'", + "\t${ECHO_MSG} \"Message\"; echo 'done.'", + "\t${FAIL_MSG} \"Failure\"; echo 'done.'", + "\tset -x; echo 'done.'", + "\techo 'input' | sed -e s,in,out,; echo 'done.'", + "\tsed -e s,in,out,; echo 'done.'", + "\tsed s,in,out,; echo 'done.'", + "\tgrep input; echo 'done.'", + "\ttouch file; echo 'done.'", + "\techo 'starting'; echo 'done.'", + "\techo 'logging' > log; echo 'done.'", + "\techo 'to stderr' 1>&2; echo 'done.'", + "\techo 'hello' | tr -d 'aeiou'") mklines.Check() t.CheckOutputLines( - "ERROR: Makefile:3: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.") + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon (after \"socklen=`${GREP} 'expr' ${WRKSRC}/config.h`\") to separate commands.", + "WARN: Makefile:6: Please switch to \"set -e\" mode before using a semicolon (after \"${FAIL_MSG} \\\"Failure\\\"\") to separate commands.", + "WARN: Makefile:7: Please switch to \"set -e\" mode before using a semicolon (after \"set -x\") to separate commands.", + "WARN: Makefile:12: Please switch to \"set -e\" mode before using a semicolon (after \"touch file\") to separate commands.", + "WARN: Makefile:14: Please switch to \"set -e\" mode before using a semicolon (after \"echo 'logging'\") to separate commands.") } diff --git a/pkgtools/pkglint/files/shtokenizer.go b/pkgtools/pkglint/files/shtokenizer.go index acb1b6b8505..e4eef3fd7f9 100644 --- a/pkgtools/pkglint/files/shtokenizer.go +++ b/pkgtools/pkglint/files/shtokenizer.go @@ -12,7 +12,8 @@ func NewShTokenizer(line Line, text string, emitWarnings bool) *ShTokenizer { } // ShAtom parses a basic building block of a shell program. -// Examples for such atoms are: variable reference, operator, text, quote, space. +// Examples for such atoms are: variable reference (both make and shell), +// operator, text, quote, space. // // See ShQuote.Feed func (p *ShTokenizer) ShAtom(quoting ShQuoting) *ShAtom { @@ -45,6 +46,8 @@ func (p *ShTokenizer) ShAtom(quoting ShQuoting) *ShAtom { atom = p.shAtomBacktDquot() case shqBacktSquot: atom = p.shAtomBacktSquot() + case shqSubshDquot: + atom = p.shAtomSubshDquot() case shqSubshSquot: atom = p.shAtomSubshSquot() case shqDquotBacktDquot: @@ -82,7 +85,7 @@ func (p *ShTokenizer) shAtomPlain() *ShAtom { case repl.AdvanceRegexp(`^#.*`): return &ShAtom{shtComment, repl.Group(0), q, nil} case repl.AdvanceStr("$$("): - return &ShAtom{shtSubshell, repl.Str(), q, nil} + return &ShAtom{shtSubshell, repl.Str(), shqSubsh, nil} } return p.shAtomInternal(q, false, false) @@ -133,31 +136,24 @@ func (p *ShTokenizer) shAtomBackt() *ShAtom { // compatibility with /bin/sh from Solaris 7. func (p *ShTokenizer) shAtomSubsh() *ShAtom { const q = shqSubsh - if op := p.shOperator(q); op != nil { - return op - } repl := p.parser.repl - mark := repl.Mark() - atom := func(typ ShAtomType) *ShAtom { - return &ShAtom{typ, repl.Since(mark), shqSubsh, nil} - } switch { case repl.AdvanceHspace(): - return atom(shtSpace) + return &ShAtom{shtSpace, repl.Str(), q, nil} case repl.AdvanceStr("\""): - //return &ShAtom{shtWord, repl.Str(), shqDquot, nil} + return &ShAtom{shtWord, repl.Str(), shqSubshDquot, nil} case repl.AdvanceStr("'"): return &ShAtom{shtWord, repl.Str(), shqSubshSquot, nil} case repl.AdvanceStr("`"): - //return &ShAtom{shtWord, repl.Str(), shqBackt, nil} - case repl.AdvanceRegexp(`^#.*`): + // FIXME: return &ShAtom{shtWord, repl.Str(), shqBackt, nil} + case repl.AdvanceRegexp(`^#[^)]*`): return &ShAtom{shtComment, repl.Group(0), q, nil} case repl.AdvanceStr(")"): return &ShAtom{shtWord, repl.Str(), shqPlain, nil} case repl.AdvanceRegexp(`^(?:[!#%*+,\-./0-9:=?@A-Z\[\]^_a-z{}~]+|\\[^$]|` + reShDollar + `)+`): return &ShAtom{shtWord, repl.Group(0), q, nil} } - return nil + return p.shOperator(q) } func (p *ShTokenizer) shAtomDquotBackt() *ShAtom { @@ -206,6 +202,17 @@ func (p *ShTokenizer) shAtomBacktSquot() *ShAtom { return nil } +func (p *ShTokenizer) shAtomSubshDquot() *ShAtom { + repl := p.parser.repl + switch { + case repl.AdvanceStr("\""): + return &ShAtom{shtWord, repl.Str(), shqSubsh, nil} + case repl.AdvanceRegexp(`^(?:[\t !%&()*+,\-./0-9:;<=>?@A-Z\[\]^_a-z{|}~]+|\\[^$]|` + reShDollar + `)+`): + return &ShAtom{shtWord, repl.Group(0), shqSubshDquot, nil} + } + return nil +} + func (p *ShTokenizer) shAtomSubshSquot() *ShAtom { const q = shqSubshSquot repl := p.parser.repl @@ -317,6 +324,7 @@ func (p *ShTokenizer) ShAtoms() []*ShAtom { func (p *ShTokenizer) ShToken() *ShToken { var curr *ShAtom q := shqPlain + peek := func() *ShAtom { if curr == nil { curr = p.ShAtom(q) @@ -356,9 +364,7 @@ nextAtom: } repl.Reset(mark) - if len(atoms) == 0 { - return nil - } + G.Assertf(len(atoms) != 0, "ShTokenizer.ShToken") return NewShToken(repl.Since(initialMark), atoms...) } diff --git a/pkgtools/pkglint/files/shtokenizer_test.go b/pkgtools/pkglint/files/shtokenizer_test.go index 950ecce8f99..aee08e1b88a 100644 --- a/pkgtools/pkglint/files/shtokenizer_test.go +++ b/pkgtools/pkglint/files/shtokenizer_test.go @@ -1,10 +1,15 @@ package main -import "gopkg.in/check.v1" +import ( + "gopkg.in/check.v1" + "strings" +) func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { t := s.Init(c) + // checkRest ensures that the given string is parsed to the expected + // atoms, and returns the remaining text. checkRest := func(s string, expected ...*ShAtom) string { p := NewShTokenizer(dummyLine, s, false) q := shqPlain @@ -14,328 +19,371 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { } return p.Rest() } + + // check ensures that the given string is parsed to the expected + // atoms, and that the text is completely consumed by the parser. check := func(str string, expected ...*ShAtom) { rest := checkRest(str, expected...) c.Check(rest, equals, "") t.CheckOutputEmpty() } - token := func(typ ShAtomType, text string, quoting ShQuoting) *ShAtom { - return &ShAtom{typ, text, quoting, nil} + atom := func(typ ShAtomType, text string) *ShAtom { + return &ShAtom{typ, text, shqPlain, nil} } - word := func(s string) *ShAtom { return token(shtWord, s, shqPlain) } - dquot := func(s string) *ShAtom { return token(shtWord, s, shqDquot) } - squot := func(s string) *ShAtom { return token(shtWord, s, shqSquot) } - backt := func(s string) *ShAtom { return token(shtWord, s, shqBackt) } - operator := func(s string) *ShAtom { return token(shtOperator, s, shqPlain) } - varuse := func(varname string, modifiers ...string) *ShAtom { + + operator := func(s string) *ShAtom { return atom(shtOperator, s) } + comment := func(s string) *ShAtom { return atom(shtComment, s) } + mkvar := func(varname string, modifiers ...string) *ShAtom { text := "${" + varname for _, modifier := range modifiers { - text += ":" + modifier + text += ":" + strings.Replace(strings.Replace(modifier, "\\", "\\\\", -1), ":", "\\:", -1) } text += "}" varuse := &MkVarUse{varname: varname, modifiers: modifiers} return &ShAtom{shtVaruse, text, shqPlain, varuse} } - q := func(q ShQuoting, token *ShAtom) *ShAtom { - return &ShAtom{token.Type, token.MkText, q, token.Data} - } - whitespace := func(s string) *ShAtom { return token(shtSpace, s, shqPlain) } - space := token(shtSpace, " ", shqPlain) + text := func(s string) *ShAtom { return atom(shtWord, s) } + whitespace := func(s string) *ShAtom { return atom(shtSpace, s) } + + space := whitespace(" ") semicolon := operator(";") pipe := operator("|") + subshell := atom(shtSubshell, "$$(") + + q := func(q ShQuoting, atom *ShAtom) *ShAtom { + return &ShAtom{atom.Type, atom.MkText, q, atom.data} + } + backt := func(atom *ShAtom) *ShAtom { return q(shqBackt, atom) } + dquot := func(atom *ShAtom) *ShAtom { return q(shqDquot, atom) } + squot := func(atom *ShAtom) *ShAtom { return q(shqSquot, atom) } + subsh := func(atom *ShAtom) *ShAtom { return q(shqSubsh, atom) } + backtDquot := func(atom *ShAtom) *ShAtom { return q(shqBacktDquot, atom) } + backtSquot := func(atom *ShAtom) *ShAtom { return q(shqBacktSquot, atom) } + dquotBackt := func(atom *ShAtom) *ShAtom { return q(shqDquotBackt, atom) } + subshDquot := func(atom *ShAtom) *ShAtom { return q(shqSubshDquot, atom) } + subshSquot := func(atom *ShAtom) *ShAtom { return q(shqSubshSquot, atom) } + dquotBacktDquot := func(atom *ShAtom) *ShAtom { return q(shqDquotBacktDquot, atom) } + dquotBacktSquot := func(atom *ShAtom) *ShAtom { return q(shqDquotBacktSquot, atom) } + + // Ignore unused functions; useful for deleting some of the tests during debugging. + use := func(args ...interface{}) {} + use(checkRest, check) + use(operator, comment, mkvar, text, whitespace) + use(space, semicolon, pipe, subshell) + use(backt, dquot, squot, subsh) + use(backtDquot, backtSquot, dquotBackt, subshDquot, subshSquot) + use(dquotBacktDquot, dquotBacktSquot) check("" /* none */) check("$$var", - word("$$var")) + text("$$var")) check("$$var$$var", - word("$$var$$var")) + text("$$var$$var")) check("$$var;;", - word("$$var"), + text("$$var"), operator(";;")) check("'single-quoted'", - q(shqSquot, word("'")), - q(shqSquot, word("single-quoted")), - q(shqPlain, word("'"))) + squot(text("'")), + squot(text("single-quoted")), + text("'")) rest := checkRest("\"" /* none */) c.Check(rest, equals, "\"") check("$${file%.c}.o", - word("$${file%.c}.o")) + text("$${file%.c}.o")) check("hello", - word("hello")) + text("hello")) check("hello, world", - word("hello,"), + text("hello,"), space, - word("world")) + text("world")) check("\"", - dquot("\"")) + dquot(text("\""))) check("`", - backt("`")) + backt(text("`"))) check("`cat fname`", - backt("`"), - backt("cat"), - token(shtSpace, " ", shqBackt), - backt("fname"), - word("`")) + backt(text("`")), + backt(text("cat")), + backt(space), + backt(text("fname")), + text("`")) check("hello, \"world\"", - word("hello,"), + text("hello,"), space, - dquot("\""), - dquot("world"), - word("\"")) + dquot(text("\"")), + dquot(text("world")), + text("\"")) check("set -e;", - word("set"), + text("set"), space, - word("-e"), + text("-e"), semicolon) check("cd ${WRKSRC}/doc/man/man3; PAGES=\"`ls -1 | ${SED} -e 's,3qt$$,3,'`\";", - word("cd"), + text("cd"), space, - varuse("WRKSRC"), - word("/doc/man/man3"), + mkvar("WRKSRC"), + text("/doc/man/man3"), semicolon, space, - word("PAGES="), - dquot("\""), - q(shqDquotBackt, word("`")), - q(shqDquotBackt, word("ls")), - q(shqDquotBackt, space), - q(shqDquotBackt, word("-1")), - q(shqDquotBackt, space), - q(shqDquotBackt, operator("|")), - q(shqDquotBackt, space), - q(shqDquotBackt, varuse("SED")), - q(shqDquotBackt, space), - q(shqDquotBackt, word("-e")), - q(shqDquotBackt, space), - q(shqDquotBacktSquot, word("'")), - q(shqDquotBacktSquot, word("s,3qt$$,3,")), - q(shqDquotBackt, word("'")), - q(shqDquot, word("`")), - q(shqPlain, word("\"")), + text("PAGES="), + dquot(text("\"")), + dquotBackt(text("`")), + dquotBackt(text("ls")), + dquotBackt(space), + dquotBackt(text("-1")), + dquotBackt(space), + dquotBackt(operator("|")), + dquotBackt(space), + dquotBackt(mkvar("SED")), + dquotBackt(space), + dquotBackt(text("-e")), + dquotBackt(space), + dquotBacktSquot(text("'")), + dquotBacktSquot(text("s,3qt$$,3,")), + dquotBackt(text("'")), + dquot(text("`")), + text("\""), semicolon) check("ls -1 | ${SED} -e 's,3qt$$,3,'", - word("ls"), space, word("-1"), space, + text("ls"), space, text("-1"), space, pipe, space, - varuse("SED"), space, word("-e"), space, - squot("'"), squot("s,3qt$$,3,"), word("'")) + mkvar("SED"), space, text("-e"), space, + squot(text("'")), squot(text("s,3qt$$,3,")), text("'")) check("(for PAGE in $$PAGES; do ", - &ShAtom{shtOperator, "(", shqPlain, nil}, - word("for"), + operator("("), + text("for"), space, - word("PAGE"), + text("PAGE"), space, - word("in"), + text("in"), space, - word("$$PAGES"), + text("$$PAGES"), semicolon, space, - word("do"), + text("do"), space) check(" ${ECHO} installing ${DESTDIR}${QTPREFIX}/man/man3/$${PAGE}; ", whitespace(" "), - varuse("ECHO"), + mkvar("ECHO"), space, - word("installing"), + text("installing"), space, - varuse("DESTDIR"), - varuse("QTPREFIX"), - word("/man/man3/$${PAGE}"), + mkvar("DESTDIR"), + mkvar("QTPREFIX"), + text("/man/man3/$${PAGE}"), semicolon, space) check(" set - X `head -1 $${PAGE}qt`; ", whitespace(" "), - word("set"), + text("set"), space, - word("-"), + text("-"), space, - word("X"), + text("X"), space, - backt("`"), - backt("head"), - q(shqBackt, space), - backt("-1"), - q(shqBackt, space), - backt("$${PAGE}qt"), - word("`"), + backt(text("`")), + backt(text("head")), + backt(space), + backt(text("-1")), + backt(space), + backt(text("$${PAGE}qt")), + text("`"), semicolon, space) check("`\"one word\"`", - backt("`"), - q(shqBacktDquot, word("\"")), - q(shqBacktDquot, word("one word")), - q(shqBackt, word("\"")), - word("`")) + backt(text("`")), + backtDquot(text("\"")), + backtDquot(text("one word")), + backt(text("\"")), + text("`")) check("$$var \"$$var\" '$$var' `$$var`", - word("$$var"), + text("$$var"), space, - dquot("\""), - dquot("$$var"), - word("\""), + dquot(text("\"")), + dquot(text("$$var")), + text("\""), space, - squot("'"), - squot("$$var"), - word("'"), + squot(text("'")), + squot(text("$$var")), + text("'"), space, - backt("`"), - backt("$$var"), - word("`")) + backt(text("`")), + backt(text("$$var")), + text("`")) check("\"`'echo;echo'`\"", - q(shqDquot, word("\"")), - q(shqDquotBackt, word("`")), - q(shqDquotBacktSquot, word("'")), - q(shqDquotBacktSquot, word("echo;echo")), - q(shqDquotBackt, word("'")), - q(shqDquot, word("`")), - q(shqPlain, word("\""))) + dquot(text("\"")), + dquotBackt(text("`")), + dquotBacktSquot(text("'")), + dquotBacktSquot(text("echo;echo")), + dquotBackt(text("'")), + dquot(text("`")), + text("\"")) check("cat<file", - word("cat"), + text("cat"), operator("<"), - word("file")) + text("file")) check("-e \"s,\\$$sysconfdir/jabberd,\\$$sysconfdir,g\"", - word("-e"), + text("-e"), space, - dquot("\""), - dquot("s,\\$$sysconfdir/jabberd,\\$$sysconfdir,g"), - word("\"")) + dquot(text("\"")), + dquot(text("s,\\$$sysconfdir/jabberd,\\$$sysconfdir,g")), + text("\"")) check("echo $$, $$- $$/ $$; $$| $$,$$/$$;$$-", - word("echo"), + text("echo"), space, - word("$$,"), + text("$$,"), space, - word("$$-"), + text("$$-"), space, - word("$$/"), + text("$$/"), space, - word("$$"), + text("$$"), semicolon, space, - word("$$"), + text("$$"), pipe, space, - word("$$,$$/$$"), + text("$$,$$/$$"), semicolon, - word("$$-")) + text("$$-")) rest = checkRest("COMMENT=\t\\Make $$$$ fast\"", - word("COMMENT="), + text("COMMENT="), whitespace("\t"), - word("\\Make"), + text("\\Make"), space, - word("$$$$"), + text("$$$$"), space, - word("fast")) + text("fast")) c.Check(rest, equals, "\"") check("var=`echo;echo|echo&echo||echo&&echo>echo`", - q(shqPlain, word("var=")), - q(shqBackt, word("`")), - q(shqBackt, word("echo")), - q(shqBackt, semicolon), - q(shqBackt, word("echo")), - q(shqBackt, operator("|")), - q(shqBackt, word("echo")), - q(shqBackt, operator("&")), - q(shqBackt, word("echo")), - q(shqBackt, operator("||")), - q(shqBackt, word("echo")), - q(shqBackt, operator("&&")), - q(shqBackt, word("echo")), - q(shqBackt, operator(">")), - q(shqBackt, word("echo")), - q(shqPlain, word("`"))) + text("var="), + backt(text("`")), + backt(text("echo")), + backt(semicolon), + backt(text("echo")), + backt(operator("|")), + backt(text("echo")), + backt(operator("&")), + backt(text("echo")), + backt(operator("||")), + backt(text("echo")), + backt(operator("&&")), + backt(text("echo")), + backt(operator(">")), + backt(text("echo")), + text("`")) check("# comment", - token(shtComment, "# comment", shqPlain)) + comment("# comment")) check("no#comment", - word("no#comment")) + text("no#comment")) check("`# comment`continue", - token(shtWord, "`", shqBackt), - token(shtComment, "# comment", shqBackt), - token(shtWord, "`", shqPlain), - token(shtWord, "continue", shqPlain)) + backt(text("`")), + backt(comment("# comment")), + text("`"), + text("continue")) check("`no#comment`continue", - token(shtWord, "`", shqBackt), - token(shtWord, "no#comment", shqBackt), - token(shtWord, "`", shqPlain), - token(shtWord, "continue", shqPlain)) + backt(text("`")), + backt(text("no#comment")), + text("`"), + text("continue")) check("var=`tr 'A-Z' 'a-z'`", - token(shtWord, "var=", shqPlain), - token(shtWord, "`", shqBackt), - token(shtWord, "tr", shqBackt), - token(shtSpace, " ", shqBackt), - token(shtWord, "'", shqBacktSquot), - token(shtWord, "A-Z", shqBacktSquot), - token(shtWord, "'", shqBackt), - token(shtSpace, " ", shqBackt), - token(shtWord, "'", shqBacktSquot), - token(shtWord, "a-z", shqBacktSquot), - token(shtWord, "'", shqBackt), - token(shtWord, "`", shqPlain)) + text("var="), + backt(text("`")), + backt(text("tr")), + backt(space), + backtSquot(text("'")), + backtSquot(text("A-Z")), + backt(text("'")), + backt(space), + backtSquot(text("'")), + backtSquot(text("a-z")), + backt(text("'")), + text("`")) check("var=\"`echo \"\\`echo foo\\`\"`\"", - token(shtWord, "var=", shqPlain), - token(shtWord, "\"", shqDquot), - token(shtWord, "`", shqDquotBackt), - token(shtWord, "echo", shqDquotBackt), - token(shtSpace, " ", shqDquotBackt), - token(shtWord, "\"", shqDquotBacktDquot), - token(shtWord, "\\`echo foo\\`", shqDquotBacktDquot), // One token, since it doesn't influence parsing. - token(shtWord, "\"", shqDquotBackt), - token(shtWord, "`", shqDquot), - token(shtWord, "\"", shqPlain)) + text("var="), + dquot(text("\"")), + dquotBackt(text("`")), + dquotBackt(text("echo")), + dquotBackt(space), + dquotBacktDquot(text("\"")), + dquotBacktDquot(text("\\`echo foo\\`")), // One atom, since it doesn't influence parsing. + dquotBackt(text("\"")), + dquot(text("`")), + text("\"")) check("if cond1; then action1; elif cond2; then action2; else action3; fi", - word("if"), space, word("cond1"), semicolon, space, - word("then"), space, word("action1"), semicolon, space, - word("elif"), space, word("cond2"), semicolon, space, - word("then"), space, word("action2"), semicolon, space, - word("else"), space, word("action3"), semicolon, space, - word("fi")) - - if false { - check("$$(cat)", - token(shtWord, "$$(", shqSubsh), - token(shtWord, "cat", shqSubsh), - token(shtWord, ")", shqPlain)) - - check("$$(cat 'file')", - token(shtWord, "$$(", shqSubsh), - token(shtWord, "cat", shqSubsh), - token(shtSpace, " ", shqSubsh), - token(shtWord, "'", shqSubshSquot), - token(shtWord, "file", shqSubshSquot), - token(shtWord, "'", shqSubsh), - token(shtWord, ")", shqPlain)) - } + text("if"), space, text("cond1"), semicolon, space, + text("then"), space, text("action1"), semicolon, space, + text("elif"), space, text("cond2"), semicolon, space, + text("then"), space, text("action2"), semicolon, space, + text("else"), space, text("action3"), semicolon, space, + text("fi")) + + check("$$(cat)", + subsh(subshell), + subsh(text("cat")), + text(")")) + + check("$$(cat 'file')", + subsh(subshell), + subsh(text("cat")), + subsh(space), + subshSquot(text("'")), + subshSquot(text("file")), + subsh(text("'")), + text(")")) + + check("$$(# comment) arg", + subsh(subshell), + subsh(comment("# comment")), + text(")"), + space, + text("arg")) + + check("$$(echo \"first\" 'second')", + subsh(subshell), + subsh(text("echo")), + subsh(space), + subshDquot(text("\"")), + subshDquot(text("first")), + subsh(text("\"")), + subsh(space), + subshSquot(text("'")), + subshSquot(text("second")), + subsh(text("'")), + text(")")) } -func (s *Suite) Test_Shtokenizer_ShAtom__quoting(c *check.C) { +func (s *Suite) Test_ShTokenizer_ShAtom__quoting(c *check.C) { checkQuotingChange := func(input, expectedOutput string) { p := NewShTokenizer(dummyLine, input, false) q := shqPlain @@ -372,105 +420,158 @@ func (s *Suite) Test_Shtokenizer_ShAtom__quoting(c *check.C) { func (s *Suite) Test_ShTokenizer_ShToken(c *check.C) { t := s.Init(c) - check := func(str string, expected ...*ShToken) { + // testRest ensures that the given string is parsed to the expected + // tokens, and returns the remaining text. + testRest := func(str string, expected ...string) string { p := NewShTokenizer(dummyLine, str, false) for _, exp := range expected { - c.Check(p.ShToken(), deepEquals, exp) + c.Check(p.ShToken().MkText, equals, exp) } + return p.Rest() + } + test := func(str string, expected ...string) { + p := NewShTokenizer(dummyLine, str, false) + for _, exp := range expected { + c.Check(p.ShToken().MkText, equals, exp) + } + c.Check(p.Rest(), equals, "") + t.CheckOutputEmpty() + } + checkNil := func(str string) { + p := NewShTokenizer(dummyLine, str, false) + c.Check(p.ShToken(), check.IsNil) c.Check(p.Rest(), equals, "") t.CheckOutputEmpty() } - check("", - nil) - - check("echo", - NewShToken("echo", - NewShAtom(shtWord, "echo", shqPlain))) - - check("`cat file`", - NewShToken("`cat file`", - NewShAtom(shtWord, "`", shqBackt), - NewShAtom(shtWord, "cat", shqBackt), - NewShAtom(shtSpace, " ", shqBackt), - NewShAtom(shtWord, "file", shqBackt), - NewShAtom(shtWord, "`", shqPlain))) - - check("PAGES=\"`ls -1 | ${SED} -e 's,3qt$$,3,'`\"", - NewShToken("PAGES=\"`ls -1 | ${SED} -e 's,3qt$$,3,'`\"", - NewShAtom(shtWord, "PAGES=", shqPlain), - NewShAtom(shtWord, "\"", shqDquot), - NewShAtom(shtWord, "`", shqDquotBackt), - NewShAtom(shtWord, "ls", shqDquotBackt), - NewShAtom(shtSpace, " ", shqDquotBackt), - NewShAtom(shtWord, "-1", shqDquotBackt), - NewShAtom(shtSpace, " ", shqDquotBackt), - NewShAtom(shtOperator, "|", shqDquotBackt), - NewShAtom(shtSpace, " ", shqDquotBackt), - NewShAtomVaruse("${SED}", shqDquotBackt, "SED"), - NewShAtom(shtSpace, " ", shqDquotBackt), - NewShAtom(shtWord, "-e", shqDquotBackt), - NewShAtom(shtSpace, " ", shqDquotBackt), - NewShAtom(shtWord, "'", shqDquotBacktSquot), - NewShAtom(shtWord, "s,3qt$$,3,", shqDquotBacktSquot), - NewShAtom(shtWord, "'", shqDquotBackt), - NewShAtom(shtWord, "`", shqDquot), - NewShAtom(shtWord, "\"", shqPlain))) - - check("echo hello, world", - NewShToken("echo", - NewShAtom(shtWord, "echo", shqPlain)), - NewShToken("hello,", - NewShAtom(shtWord, "hello,", shqPlain)), - NewShToken("world", - NewShAtom(shtWord, "world", shqPlain))) + checkNil("") + checkNil(" ") + rest := testRest("\t\t\t\n\n\n\n\t ", + "\n", + "\n", // TODO: Why three separators? One should be enough. What does the grammar say? + "\n") + c.Check(rest, equals, "\n\t ") // TODO: Why is the newline still here? + + test("echo", + "echo") + + test("`cat file`", + "`cat file`") + + test("PAGES=\"`ls -1 | ${SED} -e 's,3qt$$,3,'`\"", + "PAGES=\"`ls -1 | ${SED} -e 's,3qt$$,3,'`\"") + + test("echo hello, world", + "echo", + "hello,", + "world") + + test("if cond1; then action1; elif cond2; then action2; else action3; fi", + "if", "cond1", ";", "then", + "action1", ";", + "elif", "cond2", ";", "then", + "action2", ";", + "else", "action3", ";", + "fi") + + test("PATH=/nonexistent env PATH=${PATH:Q} true", + "PATH=/nonexistent", + "env", + "PATH=${PATH:Q}", + "true") + + test("id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)", + "id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)") + + test("id=`${AWK} '{print}' < ${WRKSRC}/idfile`", + "id=`${AWK} '{print}' < ${WRKSRC}/idfile`") +} - check("if cond1; then action1; elif cond2; then action2; else action3; fi", - NewShToken("if", NewShAtom(shtWord, "if", shqPlain)), - NewShToken("cond1", NewShAtom(shtWord, "cond1", shqPlain)), - NewShToken(";", NewShAtom(shtOperator, ";", shqPlain)), - NewShToken("then", NewShAtom(shtWord, "then", shqPlain)), - NewShToken("action1", NewShAtom(shtWord, "action1", shqPlain)), - NewShToken(";", NewShAtom(shtOperator, ";", shqPlain)), - NewShToken("elif", NewShAtom(shtWord, "elif", shqPlain)), - NewShToken("cond2", NewShAtom(shtWord, "cond2", shqPlain)), - NewShToken(";", NewShAtom(shtOperator, ";", shqPlain)), - NewShToken("then", NewShAtom(shtWord, "then", shqPlain)), - NewShToken("action2", NewShAtom(shtWord, "action2", shqPlain)), - NewShToken(";", NewShAtom(shtOperator, ";", shqPlain)), - NewShToken("else", NewShAtom(shtWord, "else", shqPlain)), - NewShToken("action3", NewShAtom(shtWord, "action3", shqPlain)), - NewShToken(";", NewShAtom(shtOperator, ";", shqPlain)), - NewShToken("fi", NewShAtom(shtWord, "fi", shqPlain))) - - check("PATH=/nonexistent env PATH=${PATH:Q} true", - NewShToken("PATH=/nonexistent", NewShAtom(shtWord, "PATH=/nonexistent", shqPlain)), - NewShToken("env", NewShAtom(shtWord, "env", shqPlain)), - NewShToken("PATH=${PATH:Q}", - NewShAtom(shtWord, "PATH=", shqPlain), - NewShAtomVaruse("${PATH:Q}", shqPlain, "PATH", "Q")), - NewShToken("true", NewShAtom(shtWord, "true", shqPlain))) - - if false { // Don't know how to tokenize this correctly. - check("id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)", - NewShToken("id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)", - NewShAtom(shtWord, "id=", shqPlain), - NewShAtom(shtWord, "$$(", shqPlain), - NewShAtomVaruse("${AWK}", shqPlain, "AWK"))) +func (s *Suite) Test_ShTokenizer__examples_from_fuzzing(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("fuzzing.mk", + MkRcsID, + "", + "pre-configure:", + + // Covers shAtomBacktDquot: return nil. + // These are nested backticks with double quotes, + // which should be avoided since POSIX marks them as unspecified. + "\t"+"`\"`", + + // Covers shAtomBacktSquot: return nil + "\t"+"`'$`", + + // Covers shAtomDquotBacktSquot: return nil + "\t"+"\"`'`y", + + // Covers shAtomDquotBackt: return nil + // FIXME: Pkglint must parse unescpaed dollar in the same way, everywhere. + "\t"+"\"`$|", + + // Covers shAtomDquotBacktDquot: return nil + // FIXME: Pkglint must support unlimited nesting. + "\t"+"\"`\"`", + + // Covers shAtomSubshDquot: return nil + "\t"+"$$(\"'", + + // Covers shAtomSubsh: case repl.AdvanceStr("`") + "\t"+"$$(`", + + // Covers shAtomSubshSquot: return nil + "\t"+"$$('$)", + + // Covers shAtomDquotBackt: case repl.AdvanceRegexp("^#[^`]*") + "\t"+"\"`# comment") + + mklines.Check() + + // Just good that these redundant error messages don't occur every day. + t.CheckOutputLines( + "WARN: fuzzing.mk:4: Pkglint parse error in ShTokenizer.ShAtom at \"`\" (quoting=bd).", + "WARN: fuzzing.mk:4: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", + + "WARN: fuzzing.mk:5: Pkglint parse error in ShTokenizer.ShAtom at \"$`\" (quoting=bs).", + "WARN: fuzzing.mk:5: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", + "WARN: fuzzing.mk:5: Pkglint parse error in MkLine.Tokenize at \"$`\".", + + "WARN: fuzzing.mk:6: Pkglint parse error in ShTokenizer.ShAtom at \"`y\" (quoting=dbs).", + "WARN: fuzzing.mk:6: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", + + "WARN: fuzzing.mk:7: Pkglint parse error in ShTokenizer.ShAtom at \"$|\" (quoting=db).", + "WARN: fuzzing.mk:7: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", + "WARN: fuzzing.mk:7: Pkglint parse error in MkLine.Tokenize at \"$|\".", + + "WARN: fuzzing.mk:8: Pkglint parse error in ShTokenizer.ShAtom at \"`\" (quoting=dbd).", + "WARN: fuzzing.mk:8: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", + + "WARN: fuzzing.mk:9: Pkglint parse error in ShTokenizer.ShAtom at \"'\" (quoting=Sd).", + "WARN: fuzzing.mk:9: Invoking subshells via $(...) is not portable enough.", + + "WARN: fuzzing.mk:10: Pkglint parse error in ShTokenizer.ShAtom at \"`\" (quoting=S).", + "WARN: fuzzing.mk:10: Invoking subshells via $(...) is not portable enough.", + + "WARN: fuzzing.mk:11: Pkglint parse error in ShTokenizer.ShAtom at \"$)\" (quoting=Ss).", + "WARN: fuzzing.mk:11: Invoking subshells via $(...) is not portable enough.", + "WARN: fuzzing.mk:11: Pkglint parse error in MkLine.Tokenize at \"$)\".", + + "WARN: fuzzing.mk:12: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}") +} + +func (s *Suite) Test_ShTokenizer__fuzzing(c *check.C) { + t := s.Init(c) + + fuzzer := NewFuzzer() + fuzzer.Char("\"'`$();|_#", 10) + fuzzer.Range('a', 'z', 5) + + defer fuzzer.CheckOk() + for i := 0; i < 1000; i++ { + tokenizer := NewShTokenizer(dummyLine, fuzzer.Generate(50), false) + tokenizer.ShAtoms() + t.Output() // Discard the output, only react on fatal errors. } - check("id=`${AWK} '{print}' < ${WRKSRC}/idfile`", - NewShToken("id=`${AWK} '{print}' < ${WRKSRC}/idfile`", - NewShAtom(shtWord, "id=", shqPlain), - NewShAtom(shtWord, "`", shqBackt), - NewShAtomVaruse("${AWK}", shqBackt, "AWK"), - NewShAtom(shtSpace, " ", shqBackt), - NewShAtom(shtWord, "'", shqBacktSquot), - NewShAtom(shtWord, "{print}", shqBacktSquot), - NewShAtom(shtWord, "'", shqBackt), - NewShAtom(shtSpace, " ", shqBackt), - NewShAtom(shtOperator, "<", shqBackt), - NewShAtom(shtSpace, " ", shqBackt), - NewShAtomVaruse("${WRKSRC}", shqBackt, "WRKSRC"), - NewShAtom(shtWord, "/idfile", shqBackt), - NewShAtom(shtWord, "`", shqPlain))) + fuzzer.Ok() } diff --git a/pkgtools/pkglint/files/shtypes.go b/pkgtools/pkglint/files/shtypes.go index 3268e27edc4..36a010e8d4e 100644 --- a/pkgtools/pkglint/files/shtypes.go +++ b/pkgtools/pkglint/files/shtypes.go @@ -40,15 +40,15 @@ type ShAtom struct { Type ShAtomType MkText string Quoting ShQuoting // The quoting state at the end of the token - Data interface{} + data interface{} } func (atom *ShAtom) String() string { - if atom.Type == shtWord && atom.Quoting == shqPlain && atom.Data == nil { + if atom.Type == shtWord && atom.Quoting == shqPlain && atom.data == nil { return fmt.Sprintf("%q", atom.MkText) } if atom.Type == shtVaruse { - varuse := atom.Data.(*MkVarUse) + varuse := atom.VarUse() return fmt.Sprintf("varuse(%q)", varuse.varname+varuse.Mod()) } return fmt.Sprintf("ShAtom(%v, %q, %s)", atom.Type, atom.MkText, atom.Quoting) @@ -57,7 +57,7 @@ func (atom *ShAtom) String() string { // VarUse returns a read access to a Makefile variable, or nil for plain shell tokens. func (atom *ShAtom) VarUse() *MkVarUse { if atom.Type == shtVaruse { - return atom.Data.(*MkVarUse) + return atom.data.(*MkVarUse) } return nil } @@ -67,24 +67,25 @@ func (atom *ShAtom) VarUse() *MkVarUse { type ShQuoting uint8 const ( - shqPlain ShQuoting = iota - shqDquot - shqSquot - shqBackt - shqSubsh - shqDquotBackt - shqBacktDquot - shqBacktSquot - shqSubshSquot - shqDquotBacktDquot - shqDquotBacktSquot + shqPlain ShQuoting = iota // e.g. word + shqDquot // e.g. "word" + shqSquot // e.g. 'word' + shqBackt // e.g. `word` + shqSubsh // e.g. $(word) + shqDquotBackt // e.g. "`word`" + shqBacktDquot // e.g. `"word"` + shqBacktSquot // e.g. `'word'` + shqSubshDquot // e.g. $("word") + shqSubshSquot // e.g. $('word') + shqDquotBacktDquot // e.g. "`"word"`" + shqDquotBacktSquot // e.g. "`'word'`" ) func (q ShQuoting) String() string { return [...]string{ "plain", "d", "s", "b", "S", - "db", "bd", "bs", "Ss", + "db", "bd", "bs", "Sd", "Ss", "dbd", "dbs", }[q] } @@ -103,6 +104,13 @@ func (q ShQuoting) ToVarUseContext() vucQuoting { return vucQuotUnknown } +// ShToken is an operator or a keyword or some text intermingled with variables. +// +// Examples: +// ; +// then +// "The number of pkgsrc packages in ${PREFIX} is $$packages." +// // See http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_10_02 type ShToken struct { MkText string // The text as it appeared in the Makefile, after replacing `\#` with `#` diff --git a/pkgtools/pkglint/files/substcontext_test.go b/pkgtools/pkglint/files/substcontext_test.go index 2dde1c83278..bd886618d0c 100644 --- a/pkgtools/pkglint/files/substcontext_test.go +++ b/pkgtools/pkglint/files/substcontext_test.go @@ -241,37 +241,18 @@ func (s *Suite) Test_SubstContext__pre_configure_with_NO_CONFIGURE(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall,no-space") - t.SetupPkgsrc() - G.Pkgsrc.LoadInfrastructure() - t.CreateFileLines("category/Makefile") - t.CreateFileLines("licenses/2-clause-bsd") - - t.Chdir("category/package") - t.CreateFileLines("PLIST", - PlistRcsID, - "bin/program") - t.CreateFileLines("Makefile", - MkRcsID, - "", - "CATEGORIES= category", - "", - "COMMENT= Comment", - "LICENSE= 2-clause-bsd", - "", + pkg := t.SetupPackage("category/package", "SUBST_CLASSES+= os", "SUBST_STAGE.os= pre-configure", "SUBST_FILES.os= guess-os.h", "SUBST_SED.os= -e s,@OPSYS@,Darwin,", "", - "NO_CHECKSUM= yes", - "NO_CONFIGURE= yes", - "", - ".include \"../../mk/bsd.pkg.mk\"") + "NO_CONFIGURE= yes") - G.checkdirPackage(".") + G.CheckDirent(pkg) t.CheckOutputLines( - "WARN: Makefile:9: SUBST_STAGE pre-configure has no effect when NO_CONFIGURE is set (in line 14).") + "WARN: ~/category/package/Makefile:21: SUBST_STAGE pre-configure has no effect when NO_CONFIGURE is set (in line 25).") } func (s *Suite) Test_SubstContext__adjacent(c *check.C) { diff --git a/pkgtools/pkglint/files/testnames_test.go b/pkgtools/pkglint/files/testnames_test.go new file mode 100644 index 00000000000..d79ed4dc299 --- /dev/null +++ b/pkgtools/pkglint/files/testnames_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "go/ast" + "go/parser" + "go/token" + "gopkg.in/check.v1" + "os" + "sort" + "strings" +) + +// Ensures that all test names follow a common naming scheme: +// +// Test_${Type}_${Method}__${description_using_underscores} +func (s *Suite) Test__test_names(c *check.C) { + + // addTestee adds a single type or function declaration + // to the testees. + addTestee := func(testees *[]string, decl ast.Decl) { + switch decl := decl.(type) { + + case *ast.GenDecl: + for _, spec := range decl.Specs { + switch spec := spec.(type) { + case *ast.TypeSpec: + *testees = append(*testees, spec.Name.Name) + } + } + + case *ast.FuncDecl: + typePrefix := "" + if decl.Recv != nil { + typeExpr := decl.Recv.List[0].Type.(ast.Expr) + var typeName string + if star, ok := typeExpr.(*ast.StarExpr); ok { + typeName = star.X.(*ast.Ident).Name + } else { + typeName = typeExpr.(*ast.Ident).Name + } + typePrefix = strings.TrimSuffix(typeName, "Impl") + "." + } + *testees = append(*testees, typePrefix+decl.Name.Name) + } + } + + // loadAllTestees returns all type, function and method names + // from the current package, in the form FunctionName or + // TypeName.MethodName (omitting the * from the type name). + loadAllTestees := func() []string { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", func(fi os.FileInfo) bool { return true }, 0) + if err != nil { + panic(err) + } + + var typesAndFunctions []string + for _, file := range pkgs["main"].Files { + for _, decl := range file.Decls { + addTestee(&typesAndFunctions, decl) + } + } + + sort.Strings(typesAndFunctions) + return typesAndFunctions + } + + generateAllowedPrefixes := func(typesAndFunctions []string) map[string]bool { + prefixes := make(map[string]bool) + for _, funcName := range typesAndFunctions { + prefix := strings.Replace(funcName, ".", "_", 1) + prefixes[prefix] = true + } + + // Allow some special test name prefixes. + prefixes["Varalign"] = true + prefixes["ShellParser"] = true + return prefixes + } + + checkTestName := func(fullTestMethod string, testee string, descr string, prefixes map[string]bool) { + if !prefixes[testee] { + c.Errorf("%s: Testee %q not found.\n", fullTestMethod, testee) + } + if matches(descr, `\p{Ll}\p{Lu}`) { + switch descr { + case "comparing_YesNo_variable_to_string", + "GitHub", + "enumFrom", + "dquotBacktDquot", + "and_getSubdirs": + // These exceptions are ok. + + default: + c.Errorf("%s: Test description must not use CamelCase.\n", fullTestMethod) + } + } + } + + checkAll := func(typesAndFunctions []string, prefixes map[string]bool) { + for _, funcName := range typesAndFunctions { + typeAndMethod := strings.SplitN(funcName, ".", 2) + if len(typeAndMethod) == 2 { + method := typeAndMethod[1] + switch { + case !hasPrefix(method, "Test"): + // Ignore + + case hasPrefix(method, "Test__"): + // OK + + case hasPrefix(method, "Test_"): + refAndDescr := strings.SplitN(method[5:], "__", 2) + descr := "" + if len(refAndDescr) > 1 { + descr = refAndDescr[1] + } + checkTestName(funcName, refAndDescr[0], descr, prefixes) + + default: + c.Errorf("%s: Missing underscore.\n", funcName) + } + } + } + } + + testees := loadAllTestees() + prefixes := generateAllowedPrefixes(testees) + checkAll(testees, prefixes) +} diff --git a/pkgtools/pkglint/files/textproc/prefixreplacer.go b/pkgtools/pkglint/files/textproc/prefixreplacer.go index 5ee33012cfa..41ec82a6d64 100644 --- a/pkgtools/pkglint/files/textproc/prefixreplacer.go +++ b/pkgtools/pkglint/files/textproc/prefixreplacer.go @@ -17,10 +17,11 @@ type PrefixReplacer struct { rest string s string m []string + res *regex.Registry } -func NewPrefixReplacer(s string) *PrefixReplacer { - return &PrefixReplacer{s, "", nil} +func NewPrefixReplacer(s string, res *regex.Registry) *PrefixReplacer { + return &PrefixReplacer{s, "", nil, res} } func (pr *PrefixReplacer) EOF() bool { @@ -98,10 +99,10 @@ func (pr *PrefixReplacer) AdvanceRegexp(re regex.Pattern) bool { if !strings.HasPrefix(string(re), "^") { panic(fmt.Sprintf("PrefixReplacer.AdvanceRegexp: regular expression %q must have prefix %q.", re, "^")) } - if Testing && regex.Matches("", re) { + if Testing && pr.res.Matches("", re) { panic(fmt.Sprintf("PrefixReplacer.AdvanceRegexp: the empty string must not match the regular expression %q.", re)) } - if m := regex.Match(pr.rest, re); m != nil { + if m := pr.res.Match(pr.rest, re); m != nil { if trace.Tracing { trace.Stepf("PrefixReplacer.AdvanceRegexp(%q, %q, %q)", pr.rest, re, m[0]) } @@ -153,5 +154,5 @@ func (pr *PrefixReplacer) HasPrefix(str string) bool { } func (pr *PrefixReplacer) HasPrefixRegexp(re regex.Pattern) bool { - return regex.Matches(pr.rest, re) + return pr.res.Matches(pr.rest, re) } diff --git a/pkgtools/pkglint/files/tools.go b/pkgtools/pkglint/files/tools.go index 501d6919582..1f92dfe279a 100644 --- a/pkgtools/pkglint/files/tools.go +++ b/pkgtools/pkglint/files/tools.go @@ -279,11 +279,30 @@ func (tr *Tools) Usable(tool *Tool, time ToolTime) bool { } func (tr *Tools) AddAll(other Tools) { - if trace.Tracing { - defer trace.Call(other.TraceName, "to", tr.TraceName)() + if trace.Tracing && len(other.byName) != 0 { + defer trace.Call(other.TraceName+" to "+tr.TraceName, len(other.byName))() + } + + // Same as the code below, just a little faster. + if !trace.Tracing { + for _, otherTool := range other.byName { + tool := tr.def(otherTool.Name, otherTool.Varname, nil) + tool.MustUseVarForm = tool.MustUseVarForm || otherTool.MustUseVarForm + if otherTool.Validity > tool.Validity { + tool.SetValidity(otherTool.Validity, tr.TraceName) + } + } + return + } + + var names []string + for name := range other.byName { + names = append(names, name) } + sort.Strings(names) - for _, otherTool := range other.byName { + for _, name := range names { + otherTool := other.byName[name] if trace.Tracing { trace.Stepf("Tools.AddAll %+v", *otherTool) } diff --git a/pkgtools/pkglint/files/tools_test.go b/pkgtools/pkglint/files/tools_test.go index 5a994f3f63a..1e5168c2f73 100644 --- a/pkgtools/pkglint/files/tools_test.go +++ b/pkgtools/pkglint/files/tools_test.go @@ -7,7 +7,7 @@ func (s *Suite) Test_Tools_ParseToolLine(c *check.C) { t.SetupToolUsable("tool1", "") t.SetupVartypes() - t.SetupFileLines("Makefile", + t.CreateFileLines("Makefile", MkRcsID, "", "USE_TOOLS.NetBSD+=\ttool1") @@ -18,7 +18,7 @@ func (s *Suite) Test_Tools_ParseToolLine(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_Tools_validateToolName__invalid(c *check.C) { +func (s *Suite) Test_Tools_def__invalid_tool_name(c *check.C) { t := s.Init(c) reg := NewTools("") @@ -54,7 +54,7 @@ func (s *Suite) Test_Tools__USE_TOOLS_predefined_sed(c *check.C) { "USE_TOOLS+=\tsed:pkgsrc") t.CreateFileLines("mk/tools/defaults.mk", "_TOOLS_VARNAME.sed=\tSED") - t.SetupFileMkLines("module.mk", + t.CreateFileLines("module.mk", MkRcsID, "", "do-build:", diff --git a/pkgtools/pkglint/files/toplevel_test.go b/pkgtools/pkglint/files/toplevel_test.go index 1d9114cc146..24306ca66bf 100644 --- a/pkgtools/pkglint/files/toplevel_test.go +++ b/pkgtools/pkglint/files/toplevel_test.go @@ -5,7 +5,7 @@ import "gopkg.in/check.v1" func (s *Suite) Test_CheckdirToplevel(c *check.C) { t := s.Init(c) - t.SetupFileLines("Makefile", + t.CreateFileLines("Makefile", MkRcsID, "", "SUBDIR+= x11", @@ -15,10 +15,10 @@ func (s *Suite) Test_CheckdirToplevel(c *check.C) { "#SUBDIR+=\tignoreme", "SUBDIR+=\tnonexisting", // This doesn't happen in practice, therefore no warning. "SUBDIR+=\tbbb") - t.SetupFileLines("archivers/Makefile") - t.SetupFileLines("bbb/Makefile") - t.SetupFileLines("ccc/Makefile") - t.SetupFileLines("x11/Makefile") + t.CreateFileLines("archivers/Makefile") + t.CreateFileLines("bbb/Makefile") + t.CreateFileLines("ccc/Makefile") + t.CreateFileLines("x11/Makefile") t.SetupVartypes() CheckdirToplevel(t.File(".")) diff --git a/pkgtools/pkglint/files/util.go b/pkgtools/pkglint/files/util.go index 634aa6550cb..ff156bdc061 100644 --- a/pkgtools/pkglint/files/util.go +++ b/pkgtools/pkglint/files/util.go @@ -13,6 +13,7 @@ import ( "strconv" "strings" "time" + "unicode" ) type YesNoUnknown uint8 @@ -38,19 +39,32 @@ func hasSuffix(s, suffix string) bool { return strings.HasSuffix(s, suffix) } func matches(s string, re regex.Pattern) bool { - return regex.Matches(s, re) + return G.res.Matches(s, re) } func match1(s string, re regex.Pattern) (matched bool, m1 string) { - return regex.Match1(s, re) + return G.res.Match1(s, re) } func match2(s string, re regex.Pattern) (matched bool, m1, m2 string) { - return regex.Match2(s, re) + return G.res.Match2(s, re) } func match3(s string, re regex.Pattern) (matched bool, m1, m2, m3 string) { - return regex.Match3(s, re) + return G.res.Match3(s, re) } func match4(s string, re regex.Pattern) (matched bool, m1, m2, m3, m4 string) { - return regex.Match4(s, re) + return G.res.Match4(s, re) +} +func match5(s string, re regex.Pattern) (matched bool, m1, m2, m3, m4, m5 string) { + return G.res.Match5(s, re) +} +func replaceFirst(s string, re regex.Pattern, repl string) string { + m, replaced := G.res.ReplaceFirst(s, re, repl) + return ifelseStr(m != nil, replaced, s) +} +func replaceAll(s string, re regex.Pattern, repl string) string { + return G.res.Compile(re).ReplaceAllString(s, repl) +} +func replaceAllFunc(s string, re regex.Pattern, repl func(string) string) string { + return G.res.Compile(re).ReplaceAllStringFunc(s, repl) } func ifelseStr(cond bool, a, b string) string { @@ -77,7 +91,7 @@ func imax(a, b int) int { } func mustMatch(s string, re regex.Pattern) []string { - if m := regex.Match(s, re); m != nil { + if m := G.res.Match(s, re); m != nil { return m } panic(fmt.Sprintf("mustMatch %q %q", s, re)) @@ -138,12 +152,9 @@ func isCommitted(fname string) bool { } func isLocallyModified(fname string) bool { - lines := loadCvsEntries(fname) - if len(lines) == 0 { - return false - } - baseName := path.Base(fname) + + lines := loadCvsEntries(fname) for _, line := range lines { fields := strings.Split(line.Text, "/") if 3 < len(fields) && fields[1] == baseName { @@ -156,7 +167,7 @@ func isLocallyModified(fname string) bool { cvsModTime := fields[3] fsModTime := st.ModTime().Format(time.ANSIC) if trace.Tracing { - trace.Stepf("cvs.time=%q fs.time=%q", cvsModTime, st.ModTime()) + trace.Stepf("cvs.time=%q fs.time=%q", cvsModTime, fsModTime) } return cvsModTime != fsModTime @@ -220,21 +231,21 @@ func shorten(s string, maxChars int) string { func varnameBase(varname string) string { dot := strings.IndexByte(varname, '.') - if dot != -1 { + if dot > 0 { return varname[:dot] } return varname } func varnameCanon(varname string) string { dot := strings.IndexByte(varname, '.') - if dot != -1 { + if dot > 0 { return varname[:dot] + ".*" } return varname } func varnameParam(varname string) string { dot := strings.IndexByte(varname, '.') - if dot != -1 { + if dot > 0 { return varname[dot+1:] } return "" @@ -267,7 +278,28 @@ func varIsUsed(varname string) bool { } func splitOnSpace(s string) []string { - return regex.Compile(`\S+`).FindAllString(s, -1) + i := 0 + n := len(s) + + for i < n && unicode.IsSpace(rune(s[i])) { + i++ + } + + var parts []string + for i < n { + start := i + for i < n && !unicode.IsSpace(rune(s[i])) { + i++ + } + if start != i { + parts = append(parts, s[start:i]) + } + for i < n && unicode.IsSpace(rune(s[i])) { + i++ + } + } + + return parts } func fileExists(fname string) bool { @@ -317,7 +349,7 @@ func mkopSubst(s string, left bool, from string, right bool, to string, flags st re := regex.Pattern(ifelseStr(left, "^", "") + regexp.QuoteMeta(from) + ifelseStr(right, "$", "")) done := false gflag := contains(flags, "g") - return regex.Compile(re).ReplaceAllStringFunc(s, func(match string) string { + return replaceAllFunc(s, re, func(match string) string { if gflag || !done { done = !gflag return to @@ -328,13 +360,10 @@ func mkopSubst(s string, left bool, from string, right bool, to string, flags st // relpath returns the relative path from `from` to `to`. func relpath(from, to string) string { - absFrom, err1 := filepath.Abs(from) - absTo, err2 := filepath.Abs(to) - rel, err3 := filepath.Rel(absFrom, absTo) - if err1 != nil || err2 != nil || err3 != nil { - trace.Stepf("relpath.panic", from, to, err1, err2, err3) - panic(fmt.Sprintf("relpath %q, %q", from, to)) - } + absFrom := abspath(from) + absTo := abspath(to) + rel, err := filepath.Rel(absFrom, absTo) + G.Assertf(err == nil, "relpath %q %q.", from, to) result := filepath.ToSlash(rel) if trace.Tracing { trace.Stepf("relpath from %q to %q = %q", from, to, result) @@ -344,9 +373,7 @@ func relpath(from, to string) string { func abspath(fname string) string { abs, err := filepath.Abs(fname) - if err != nil { - NewLineWhole(fname).Fatalf("Cannot determine absolute path.") - } + G.Assertf(err == nil, "abspath %q.", fname) return filepath.ToSlash(abs) } @@ -367,7 +394,24 @@ func cleanpath(fname string) string { for contains(tmp, "//") { tmp = strings.Replace(tmp, "//", "/", -1) } - tmp = reReplaceRepeatedly(tmp, `/[^.][^/]*/[^.][^/]*/\.\./\.\./`, "/") + + // Repeatedly replace `/[^.][^/]*/[^.][^/]*/\.\./\.\./` with "/" +again: + slash0 := -1 + slash1 := -1 + slash2 := -1 + for i, ch := range []byte(tmp) { + if ch == '/' { + slash0 = slash1 + slash1 = slash2 + slash2 = i + if slash0 != -1 && tmp[slash0+1:slash1] != ".." && tmp[slash1+1:slash2] != ".." && hasPrefix(tmp[i:], "/../../") { + tmp = tmp[:slash0] + tmp[i+6:] + goto again + } + } + } + tmp = strings.TrimSuffix(tmp, "/") return tmp } @@ -376,14 +420,6 @@ func containsVarRef(s string) bool { return contains(s, "${") } -func reReplaceRepeatedly(from string, re regex.Pattern, to string) string { - replaced := regex.Compile(re).ReplaceAllString(from, to) - if replaced != from { - return reReplaceRepeatedly(replaced, re, to) - } - return replaced -} - func hasAlnumPrefix(s string) bool { if s == "" { return false @@ -412,12 +448,13 @@ func (o *Once) FirstTime(what string) bool { // Scope remembers which variables are defined and which are used // in a certain scope, such as a package or a file. type Scope struct { - defined map[string]MkLine - used map[string]MkLine + defined map[string]MkLine + fallback map[string]string + used map[string]MkLine } func NewScope() Scope { - return Scope{make(map[string]MkLine), make(map[string]MkLine)} + return Scope{make(map[string]MkLine), make(map[string]string), make(map[string]MkLine)} } // Define marks the variable and its canonicalized form as defined. @@ -437,6 +474,10 @@ func (s *Scope) Define(varname string, mkline MkLine) { } } +func (s *Scope) Fallback(varname string, value string) { + s.fallback[varname] = value +} + // Use marks the variable and its canonicalized form as used. func (s *Scope) Use(varname string, line MkLine) { if s.used[varname] == nil { @@ -514,12 +555,21 @@ func (s *Scope) Value(varname string) (value string, found bool) { if mkline != nil { return mkline.Value(), true } + if fallback, ok := s.fallback[varname]; ok { + return fallback, true + } return "", false } func (s *Scope) DefineAll(other Scope) { - for varname, mkline := range other.defined { - s.Define(varname, mkline) + var varnames []string + for varname := range other.defined { + varnames = append(varnames, varname) + } + sort.Strings(varnames) + + for _, varname := range varnames { + s.Define(varname, other.defined[varname]) } } @@ -690,3 +740,102 @@ func IsPrefs(fileName string) bool { } return false } + +func isalnum(s string) bool { + for _, ch := range []byte(s) { + if !('0' <= ch && ch <= '9' || 'A' <= ch && ch <= 'Z' || ch == '_' || 'a' <= ch && ch <= 'z') { + return false + } + } + return true +} + +// FileCache reduces the IO load for commonly loaded files by about 50%, +// especially for buildlink3.mk and *.buildlink3.mk files. +type FileCache struct { + table []*fileCacheEntry + mapping map[string]*fileCacheEntry // Pointers into FileCache.table + hits int + misses int +} + +type fileCacheEntry struct { + count int + fileName string + options LoadOptions + lines []Line +} + +func NewFileCache(size int) *FileCache { + return &FileCache{ + make([]*fileCacheEntry, 0, size), + make(map[string]*fileCacheEntry), + 0, + 0} +} + +func (c *FileCache) Put(fileName string, options LoadOptions, lines []Line) { + key := c.key(fileName) + + entry := c.mapping[key] + if entry == nil { + if len(c.table) == cap(c.table) { + sort.Slice(c.table, func(i, j int) bool { return c.table[j].count < c.table[i].count }) + minCount := c.table[len(c.table)-1].count + newLen := len(c.table) + for newLen > 0 && c.table[newLen-1].count == minCount { + delete(c.mapping, c.key(c.table[newLen-1].fileName)) + newLen-- + } + c.table = c.table[0:newLen] + + // To avoid files from getting stuck in the cache. + for _, e := range c.table { + e.count /= 2 + } + } + + entry = &fileCacheEntry{0, fileName, options, lines} + c.table = append(c.table, entry) + c.mapping[key] = entry + } + + entry.count = 0 + entry.fileName = fileName + entry.options = options + entry.lines = lines +} + +func (c *FileCache) Get(fileName string, options LoadOptions) []Line { + key := c.key(fileName) + entry, found := c.mapping[key] + if found && entry.options == options { + c.hits++ + entry.count++ + + lines := make([]Line, len(entry.lines)) + for i, line := range entry.lines { + lines[i] = NewLineMulti(fileName, int(line.firstLine), int(line.lastLine), line.Text, line.raw) + } + return lines + } + c.misses++ + return nil +} + +func (c *FileCache) Evict(fileName string) { + key := c.key(fileName) + entry, found := c.mapping[key] + if found { + delete(c.mapping, key) + + sort.Slice(c.table, func(i, j int) bool { + return c.table[j] == entry && c.table[i] != entry + }) + c.table = c.table[0 : len(c.table)-1] + } +} + +func (c *FileCache) key(fileName string) string { + return path.Clean(fileName) +} diff --git a/pkgtools/pkglint/files/util_test.go b/pkgtools/pkglint/files/util_test.go index 37ff637c262..fb1fd51c4a3 100644 --- a/pkgtools/pkglint/files/util_test.go +++ b/pkgtools/pkglint/files/util_test.go @@ -2,8 +2,6 @@ package main import ( "gopkg.in/check.v1" - "netbsd.org/pkglint/regex" - "netbsd.org/pkglint/textproc" "os" "runtime" "testing" @@ -16,36 +14,36 @@ func (s *Suite) Test_YesNoUnknown_String(c *check.C) { c.Check(unknown.String(), equals, "unknown") } -func (s *Suite) Test_MkopSubst__middle(c *check.C) { +func (s *Suite) Test_mkopSubst__middle(c *check.C) { c.Check(mkopSubst("pkgname", false, "kgna", false, "ri", ""), equals, "prime") c.Check(mkopSubst("pkgname", false, "pkgname", false, "replacement", ""), equals, "replacement") c.Check(mkopSubst("aaaaaaa", false, "a", false, "b", ""), equals, "baaaaaa") } -func (s *Suite) Test_MkopSubst__left(c *check.C) { +func (s *Suite) Test_mkopSubst__left(c *check.C) { c.Check(mkopSubst("pkgname", true, "kgna", false, "ri", ""), equals, "pkgname") c.Check(mkopSubst("pkgname", true, "pkgname", false, "replacement", ""), equals, "replacement") } -func (s *Suite) Test_MkopSubst__right(c *check.C) { +func (s *Suite) Test_mkopSubst__right(c *check.C) { c.Check(mkopSubst("pkgname", false, "kgna", true, "ri", ""), equals, "pkgname") c.Check(mkopSubst("pkgname", false, "pkgname", true, "replacement", ""), equals, "replacement") } -func (s *Suite) Test_MkopSubst__leftRight(c *check.C) { +func (s *Suite) Test_mkopSubst__left_and_right(c *check.C) { c.Check(mkopSubst("pkgname", true, "kgna", true, "ri", ""), equals, "pkgname") c.Check(mkopSubst("pkgname", false, "pkgname", false, "replacement", ""), equals, "replacement") } -func (s *Suite) Test_MkopSubst__gflag(c *check.C) { +func (s *Suite) Test_mkopSubst__gflag(c *check.C) { c.Check(mkopSubst("aaaaa", false, "a", false, "b", "g"), equals, "bbbbb") c.Check(mkopSubst("aaaaa", true, "a", false, "b", "g"), equals, "baaaa") c.Check(mkopSubst("aaaaa", false, "a", true, "b", "g"), equals, "aaaab") c.Check(mkopSubst("aaaaa", true, "a", true, "b", "g"), equals, "aaaaa") } -func (s *Suite) Test_replaceFirst(c *check.C) { - m, rest := regex.ReplaceFirst("a+b+c+d", `(\w)(.)(\w)`, "X") +func (s *Suite) Test__regex_ReplaceFirst(c *check.C) { + m, rest := G.res.ReplaceFirst("a+b+c+d", `(\w)(.)(\w)`, "X") c.Assert(m, check.NotNil) c.Check(m, check.DeepEquals, []string{"a+b", "a", "+", "b"}) @@ -65,7 +63,7 @@ func (s *Suite) Test_shorten(c *check.C) { c.Check(shorten("aaa", 5), equals, "aaa") } -func (s *Suite) Test_tabLength(c *check.C) { +func (s *Suite) Test_tabWidth(c *check.C) { c.Check(tabWidth("12345"), equals, 5) c.Check(tabWidth("\t"), equals, 8) c.Check(tabWidth("123\t"), equals, 8) @@ -81,13 +79,18 @@ func (s *Suite) Test_cleanpath(c *check.C) { c.Check(cleanpath("dir/../dir/../dir/../dir/subdir/../../Makefile"), equals, "dir/../dir/../dir/../Makefile") c.Check(cleanpath("dir/multi/././/file"), equals, "dir/multi/file") c.Check(cleanpath("111/222/../../333/444/../../555/666/../../777/888/9"), equals, "111/222/../../777/888/9") + c.Check(cleanpath("1/2/3/../../4/5/6/../../7/8/9/../../../../10"), equals, "1/10") c.Check(cleanpath("cat/pkg.v1/../../cat/pkg.v2/Makefile"), equals, "cat/pkg.v1/../../cat/pkg.v2/Makefile") c.Check(cleanpath("dir/"), equals, "dir") } func (s *Suite) Test_relpath(c *check.C) { + t := s.Init(c) + if runtime.GOOS == "windows" { - c.Check(func() { relpath("c:/", "d:/") }, check.Panics, "relpath \"c:/\", \"d:/\"") + t.ExpectFatal( + func() { relpath("c:/", "d:/") }, + "FATAL: Pkglint internal error: relpath \"c:/\" \"d:/\".") } } @@ -97,21 +100,21 @@ func (s *Suite) Test_abspath(c *check.C) { if runtime.GOOS == "windows" { t.ExpectFatal( func() { abspath("file\u0000name") }, - "FATAL: file\x00name: Cannot determine absolute path.") + "FATAL: Pkglint internal error: abspath \"file\\x00name\".") } } -func (s *Suite) Test_isEmptyDir_and_getSubdirs(c *check.C) { +func (s *Suite) Test_isEmptyDir__and_getSubdirs(c *check.C) { t := s.Init(c) - t.SetupFileLines("CVS/Entries", + t.CreateFileLines("CVS/Entries", "dummy") if dir := t.File("."); true { c.Check(isEmptyDir(dir), equals, true) c.Check(getSubdirs(dir), check.DeepEquals, []string(nil)) - t.SetupFileLines("somedir/file") + t.CreateFileLines("somedir/file") c.Check(isEmptyDir(dir), equals, false) c.Check(getSubdirs(dir), check.DeepEquals, []string{"somedir"}) @@ -130,7 +133,7 @@ func (s *Suite) Test_isEmptyDir_and_getSubdirs(c *check.C) { func (s *Suite) Test_isEmptyDir__empty_subdir(c *check.C) { t := s.Init(c) - t.SetupFileLines("CVS/Entries", + t.CreateFileLines("CVS/Entries", "dummy") t.CreateFileLines("subdir/CVS/Entries", "dummy") @@ -138,8 +141,8 @@ func (s *Suite) Test_isEmptyDir__empty_subdir(c *check.C) { c.Check(isEmptyDir(t.File(".")), equals, true) } -func (s *Suite) Test_PrefixReplacer_Since(c *check.C) { - repl := textproc.NewPrefixReplacer("hello, world") +func (s *Suite) Test__PrefixReplacer_Since(c *check.C) { + repl := G.NewPrefixReplacer("hello, world") mark := repl.Mark() repl.AdvanceRegexp(`^\w+`) c.Check(repl.Since(mark), equals, "hello") @@ -168,7 +171,7 @@ func Benchmark_match3_bsd_pkg_mk(b *testing.B) { } } -func Benchmark_match3_samedir(b *testing.B) { +func Benchmark_match3_same_dir(b *testing.B) { for i := 0; i < b.N; i++ { match3(".include \"options.mk\"", reMkIncludeBenchmark) } @@ -192,7 +195,7 @@ func Benchmark_match3_bsd_pkg_mk_positive(b *testing.B) { } } -func Benchmark_match3_samedir_positive(b *testing.B) { +func Benchmark_match3_same_dir_positive(b *testing.B) { for i := 0; i < b.N; i++ { match3(".include \"options.mk\"", reMkIncludeBenchmarkPositive) } @@ -305,4 +308,95 @@ func (s *Suite) Test_naturalLess(c *check.C) { c.Check(naturalLess("000", "0000"), equals, true) c.Check(naturalLess("0000", "000"), equals, false) c.Check(naturalLess("000", "000"), equals, false) + c.Check(naturalLess("00011", "000111"), equals, true) + c.Check(naturalLess("00011", "00012"), equals, true) +} + +func (s *Suite) Test_varnameBase(c *check.C) { + c.Check(varnameBase("VAR"), equals, "VAR") + c.Check(varnameBase("VAR.param"), equals, "VAR") + c.Check(varnameBase(".CURDIR"), equals, ".CURDIR") +} + +func (s *Suite) Test_varnameParam(c *check.C) { + c.Check(varnameParam("VAR"), equals, "") + c.Check(varnameParam("VAR.param"), equals, "param") + c.Check(varnameParam(".CURDIR"), equals, "") +} + +func (s *Suite) Test_varnameCanon(c *check.C) { + c.Check(varnameCanon("VAR"), equals, "VAR") + c.Check(varnameCanon("VAR.param"), equals, "VAR.*") + c.Check(varnameCanon(".CURDIR"), equals, ".CURDIR") +} + +func (s *Suite) Test_isalnum(c *check.C) { + c.Check(isalnum(""), equals, true) + c.Check(isalnum("/"), equals, false) + c.Check(isalnum("0"), equals, true) + c.Check(isalnum("9"), equals, true) + c.Check(isalnum(":"), equals, false) + c.Check(isalnum("@"), equals, false) + c.Check(isalnum("A"), equals, true) + c.Check(isalnum("Z"), equals, true) + c.Check(isalnum("["), equals, false) + c.Check(isalnum("_"), equals, true) + c.Check(isalnum("`"), equals, false) + c.Check(isalnum("a"), equals, true) + c.Check(isalnum("z"), equals, true) + c.Check(isalnum("{"), equals, false) + c.Check(isalnum("Hello_world005"), equals, true) + c.Check(isalnum("Hello,world005"), equals, false) +} + +func (s *Suite) Test_FileCache(c *check.C) { + t := s.Init(c) + + cache := NewFileCache(3) + + lines := t.NewLines("Makefile", + MkRcsID, + "# line 2") + + c.Check(cache.Get("Makefile", 0), check.IsNil) + c.Check(cache.hits, equals, 0) + c.Check(cache.misses, equals, 1) + + cache.Put("Makefile", 0, lines) + c.Check(cache.Get("Makefile", MustSucceed|LogErrors), check.IsNil) // Wrong LoadOptions. + + linesFromCache := cache.Get("Makefile", 0) + c.Check(linesFromCache, check.HasLen, 2) + c.Check(linesFromCache[1].Filename, equals, "Makefile") + + // Cache keys are normalized using path.Clean. + linesFromCache2 := cache.Get("./Makefile", 0) + c.Check(linesFromCache2, check.HasLen, 2) + c.Check(linesFromCache2[1].Filename, equals, "./Makefile") + + cache.Put("file1.mk", 0, lines) + cache.Put("file2.mk", 0, lines) + + // Now the cache is full. All three entries can be retrieved. + c.Check(cache.Get("Makefile", 0), check.NotNil) + c.Check(cache.Get("file1.mk", 0), check.NotNil) + c.Check(cache.Get("file2.mk", 0), check.NotNil) + + // Adding another entry removes all entries with minimum count, + // which currently are file1.mk and file2.mk. + // Makefile is still in the cache because it was accessed once. + cache.Put("file3.mk", 0, lines) + + c.Check(cache.Get("Makefile", 0), check.NotNil) + c.Check(cache.Get("file1.mk", 0), check.IsNil) + c.Check(cache.Get("file2.mk", 0), check.IsNil) + c.Check(cache.Get("file3.mk", 0), check.NotNil) + + cache.Evict("Makefile") + + c.Check(cache.Get("Makefile", 0), check.IsNil) + c.Check(cache.table, check.HasLen, 1) + c.Check(cache.mapping, check.HasLen, 1) + c.Check(cache.hits, equals, 7) + c.Check(cache.misses, equals, 5) } diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go index de6fd241934..4fea03798f6 100644 --- a/pkgtools/pkglint/files/vardefs.go +++ b/pkgtools/pkglint/files/vardefs.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "netbsd.org/pkglint/trace" "path" "strings" @@ -26,9 +25,6 @@ func (src *Pkgsrc) InitVartypes() { vartype := &Vartype{kindOfList, checker, parseACLEntries(varname, aclEntries), false} - if src.vartypes == nil { - src.vartypes = make(map[string]*Vartype) - } if varparam == "" || varparam == "*" { src.vartypes[varbase] = vartype } @@ -247,203 +243,209 @@ func (src *Pkgsrc) InitVartypes() { usr("BIN_INSTALL_FLAGS", lkShell, BtShellWord) usr("LOCALPATCHES", lkNone, BtPathname) - if false { - // The remaining variables from mk/defaults/mk.conf may be overridden by packages. - // Therefore they need a separate definition of "user-settable". - usr := func(varname string, kindOfList KindOfList, checker *BasicType) { - acl(varname, kindOfList, checker, ""+ - "Makefile: set, use; "+ - "buildlink3.mk, builtin.mk:; "+ - "Makefile.*, *.mk: default, set, use; "+ - "*: use-loadtime, use") - } - usr("ACROREAD_FONTPATH", lkNone, BtPathlist) - usr("AMANDA_USER", lkNone, BtUserGroupName) - usr("AMANDA_TMP", lkNone, BtPathname) - usr("AMANDA_VAR", lkNone, BtPathname) - usr("APACHE_USER", lkNone, BtUserGroupName) - usr("APACHE_GROUP", lkNone, BtUserGroupName) - usr("APACHE_SUEXEC_CONFIGURE_ARGS", lkShell, BtShellWord) - usr("APACHE_SUEXEC_DOCROOT", lkShell, BtPathname) - usr("ARLA_CACHE", lkNone, BtPathname) - usr("BIND_DIR", lkNone, BtPathname) - usr("BIND_GROUP", lkNone, BtUserGroupName) - usr("BIND_USER", lkNone, BtUserGroupName) - usr("CACTI_GROUP", lkNone, BtUserGroupName) - usr("CACTI_USER", lkNone, BtUserGroupName) - usr("CANNA_GROUP", lkNone, BtUserGroupName) - usr("CANNA_USER", lkNone, BtUserGroupName) - usr("CDRECORD_CONF", lkNone, BtPathname) - usr("CLAMAV_GROUP", lkNone, BtUserGroupName) - usr("CLAMAV_USER", lkNone, BtUserGroupName) - usr("CLAMAV_DBDIR", lkNone, BtPathname) - usr("CONSERVER_DEFAULTHOST", lkNone, BtIdentifier) - usr("CONSERVER_DEFAULTPORT", lkNone, BtInteger) - usr("CUPS_GROUP", lkNone, BtUserGroupName) - usr("CUPS_USER", lkNone, BtUserGroupName) - usr("CUPS_SYSTEM_GROUPS", lkShell, BtUserGroupName) - usr("CYRUS_IDLE", lkNone, enum("poll idled no")) - usr("CYRUS_GROUP", lkNone, BtUserGroupName) - usr("CYRUS_USER", lkNone, BtUserGroupName) - usr("DBUS_GROUP", lkNone, BtUserGroupName) - usr("DBUS_USER", lkNone, BtUserGroupName) - usr("DEFANG_GROUP", lkNone, BtUserGroupName) - usr("DEFANG_USER", lkNone, BtUserGroupName) - usr("DEFANG_SPOOLDIR", lkNone, BtPathname) - usr("DEFAULT_IRC_SERVER", lkNone, BtIdentifier) - usr("DEFAULT_SERIAL_DEVICE", lkNone, BtPathname) - usr("DIALER_GROUP", lkNone, BtUserGroupName) - usr("DT_LAYOUT", lkNone, enum("US FI FR GER DV")) - usr("ELK_GUI", lkShell, enum("none xaw motif")) - usr("EMACS_TYPE", lkNone, enum("emacs25 emacs25nox emacs21 emacs21nox emacs20 xemacs214 xemacs215")) - usr("EXIM_GROUP", lkNone, BtUserGroupName) - usr("EXIM_USER", lkNone, BtUserGroupName) - usr("FLUXBOX_USE_XINERAMA", lkNone, enum("YES NO")) - usr("FLUXBOX_USE_KDE", lkNone, enum("YES NO")) - usr("FLUXBOX_USE_GNOME", lkNone, enum("YES NO")) - usr("FLUXBOX_USE_XFT", lkNone, enum("YES NO")) - usr("FOX_USE_XUNICODE", lkNone, enum("YES NO")) - usr("FREEWNN_USER", lkNone, BtUserGroupName) - usr("FREEWNN_GROUP", lkNone, BtUserGroupName) - usr("GAMES_USER", lkNone, BtUserGroupName) - usr("GAMES_GROUP", lkNone, BtUserGroupName) - usr("GAMEMODE", lkNone, BtFileMode) - usr("GAMEDIRMODE", lkNone, BtFileMode) - usr("GAMEDATAMODE", lkNone, BtFileMode) - usr("GAMEGRP", lkNone, BtUserGroupName) - usr("GAMEOWN", lkNone, BtUserGroupName) - usr("GRUB_NETWORK_CARDS", lkNone, BtIdentifier) - usr("GRUB_PRESET_COMMAND", lkNone, enum("bootp dhcp rarp")) - usr("GRUB_SCAN_ARGS", lkShell, BtShellWord) - usr("HASKELL_COMPILER", lkNone, enum("ghc")) - usr("HOWL_GROUP", lkNone, BtUserGroupName) - usr("HOWL_USER", lkNone, BtUserGroupName) - usr("ICECAST_CHROOTDIR", lkNone, BtPathname) - usr("ICECAST_CHUNKLEN", lkNone, BtInteger) - usr("ICECAST_SOURCE_BUFFSIZE", lkNone, BtInteger) - usr("IMAP_UW_CCLIENT_MBOX_FMT", lkNone, enum("mbox mbx mh mmdf mtx mx news phile tenex unix")) - usr("IMAP_UW_MAILSPOOLHOME", lkNone, BtFilename) - usr("IMDICTDIR", lkNone, BtPathname) - usr("INN_DATA_DIR", lkNone, BtPathname) - usr("INN_USER", lkNone, BtUserGroupName) - usr("INN_GROUP", lkNone, BtUserGroupName) - usr("IRCD_HYBRID_NICLEN", lkNone, BtInteger) - usr("IRCD_HYBRID_TOPICLEN", lkNone, BtInteger) - usr("IRCD_HYBRID_SYSLOG_EVENTS", lkNone, BtUnknown) - usr("IRCD_HYBRID_SYSLOG_FACILITY", lkNone, BtIdentifier) - usr("IRCD_HYBRID_MAXCONN", lkNone, BtInteger) - usr("IRCD_HYBRID_IRC_USER", lkNone, BtUserGroupName) - usr("IRCD_HYBRID_IRC_GROUP", lkNone, BtUserGroupName) - usr("IRRD_USE_PGP", lkNone, enum("5 2")) - usr("JABBERD_USER", lkNone, BtUserGroupName) - usr("JABBERD_GROUP", lkNone, BtUserGroupName) - usr("JABBERD_LOGDIR", lkNone, BtPathname) - usr("JABBERD_SPOOLDIR", lkNone, BtPathname) - usr("JABBERD_PIDDIR", lkNone, BtPathname) - usr("JAKARTA_HOME", lkNone, BtPathname) - usr("KERBEROS", lkNone, BtYes) - usr("KERMIT_SUID_UUCP", lkNone, BtYes) - usr("KJS_USE_PCRE", lkNone, BtYes) - usr("KNEWS_DOMAIN_FILE", lkNone, BtPathname) - usr("KNEWS_DOMAIN_NAME", lkNone, BtIdentifier) - usr("LIBDVDCSS_HOMEPAGE", lkNone, BtHomepage) - usr("LIBDVDCSS_MASTER_SITES", lkShell, BtFetchURL) - usr("LATEX2HTML_ICONPATH", lkNone, BtURL) - usr("LEAFNODE_DATA_DIR", lkNone, BtPathname) - usr("LEAFNODE_USER", lkNone, BtUserGroupName) - usr("LEAFNODE_GROUP", lkNone, BtUserGroupName) - usr("LINUX_LOCALES", lkShell, BtIdentifier) - usr("MAILAGENT_DOMAIN", lkNone, BtIdentifier) - usr("MAILAGENT_EMAIL", lkNone, BtMailAddress) - usr("MAILAGENT_FQDN", lkNone, BtIdentifier) - usr("MAILAGENT_ORGANIZATION", lkNone, BtUnknown) - usr("MAJORDOMO_HOMEDIR", lkNone, BtPathname) - usr("MAKEINFO_ARGS", lkShell, BtShellWord) - usr("MECAB_CHARSET", lkNone, BtIdentifier) - usr("MEDIATOMB_GROUP", lkNone, BtUserGroupName) - usr("MEDIATOMB_USER", lkNone, BtUserGroupName) - usr("MLDONKEY_GROUP", lkNone, BtUserGroupName) - usr("MLDONKEY_HOME", lkNone, BtPathname) - usr("MLDONKEY_USER", lkNone, BtUserGroupName) - usr("MONOTONE_GROUP", lkNone, BtUserGroupName) - usr("MONOTONE_USER", lkNone, BtUserGroupName) - usr("MOTIF_TYPE", lkNone, enum("motif openmotif lesstif dt")) - usr("MOTIF_TYPE_DEFAULT", lkNone, enum("motif openmotif lesstif dt")) - usr("MTOOLS_ENABLE_FLOPPYD", lkNone, BtYesNo) - usr("MYSQL_USER", lkNone, BtUserGroupName) - usr("MYSQL_GROUP", lkNone, BtUserGroupName) - usr("MYSQL_DATADIR", lkNone, BtPathname) - usr("MYSQL_CHARSET", lkNone, BtIdentifier) - usr("MYSQL_EXTRA_CHARSET", lkShell, BtIdentifier) - usr("NAGIOS_GROUP", lkNone, BtUserGroupName) - usr("NAGIOS_USER", lkNone, BtUserGroupName) - usr("NAGIOSCMD_GROUP", lkNone, BtUserGroupName) - usr("NAGIOSDIR", lkNone, BtPathname) - usr("NBPAX_PROGRAM_PREFIX", lkNone, BtIdentifier) - usr("NMH_EDITOR", lkNone, BtIdentifier) - usr("NMH_MTA", lkNone, enum("smtp sendmail")) - usr("NMH_PAGER", lkNone, BtIdentifier) - usr("NS_PREFERRED", lkNone, enum("communicator navigator mozilla")) - usr("OPENSSH_CHROOT", lkNone, BtPathname) - usr("OPENSSH_USER", lkNone, BtUserGroupName) - usr("OPENSSH_GROUP", lkNone, BtUserGroupName) - usr("P4USER", lkNone, BtUserGroupName) - usr("P4GROUP", lkNone, BtUserGroupName) - usr("P4ROOT", lkNone, BtPathname) - usr("P4PORT", lkNone, BtInteger) - usr("PALMOS_DEFAULT_SDK", lkNone, enum("1 2 3.1 3.5")) - usr("PAPERSIZE", lkNone, enum("A4 Letter")) - usr("PGGROUP", lkNone, BtUserGroupName) - usr("PGUSER", lkNone, BtUserGroupName) - usr("PGHOME", lkNone, BtPathname) - usr("PILRC_USE_GTK", lkNone, BtYesNo) - usr("PKG_JVM_DEFAULT", lkNone, jvms) - usr("POPTOP_USE_MPPE", lkNone, BtYes) - usr("PROCMAIL_MAILSPOOLHOME", lkNone, BtFilename) - usr("PROCMAIL_TRUSTED_IDS", lkShell, BtIdentifier) - usr("PVM_SSH", lkNone, BtPathname) - usr("QMAILDIR", lkNone, BtPathname) - usr("QMAIL_QFILTER_TMPDIR", lkNone, BtPathname) - usr("QMAIL_QUEUE_DIR", lkNone, BtPathname) - usr("QMAIL_QUEUE_EXTRA", lkNone, BtMailAddress) - usr("QPOPPER_FAC", lkNone, BtIdentifier) - usr("QPOPPER_USER", lkNone, BtUserGroupName) - usr("QPOPPER_SPOOL_DIR", lkNone, BtPathname) - usr("RASMOL_DEPTH", lkNone, enum("8 16 32")) - usr("RELAY_CTRL_DIR", lkNone, BtPathname) - usr("RPM_DB_PREFIX", lkNone, BtPathname) - usr("RSSH_SCP_PATH", lkNone, BtPathname) - usr("RSSH_SFTP_SERVER_PATH", lkNone, BtPathname) - usr("RSSH_CVS_PATH", lkNone, BtPathname) - usr("RSSH_RDIST_PATH", lkNone, BtPathname) - usr("RSSH_RSYNC_PATH", lkNone, BtPathname) - usr("SAWFISH_THEMES", lkShell, BtFilename) - usr("SCREWS_GROUP", lkNone, BtUserGroupName) - usr("SCREWS_USER", lkNone, BtUserGroupName) - usr("SDIST_PAWD", lkNone, enum("pawd pwd")) - usr("SERIAL_DEVICES", lkShell, BtPathname) - usr("SILC_CLIENT_WITH_PERL", lkNone, BtYesNo) - usr("SSH_SUID", lkNone, BtYesNo) - usr("SSYNC_PAWD", lkNone, enum("pawd pwd")) - usr("SUSE_PREFER", lkNone, enum("13.1 12.1 10.0")) - usr("TEXMFSITE", lkNone, BtPathname) - usr("THTTPD_LOG_FACILITY", lkNone, BtIdentifier) - usr("UNPRIVILEGED", lkNone, BtYesNo) - usr("USE_CROSS_COMPILE", lkNone, BtYesNo) - usr("USERPPP_GROUP", lkNone, BtUserGroupName) - usr("UUCP_GROUP", lkNone, BtUserGroupName) - usr("UUCP_USER", lkNone, BtUserGroupName) - usr("VIM_EXTRA_OPTS", lkShell, BtShellWord) - usr("WCALC_HTMLDIR", lkNone, BtPathname) - usr("WCALC_HTMLPATH", lkNone, BtPathname) // URL path - usr("WCALC_CGIDIR", lkNone, BtPrefixPathname) - usr("WCALC_CGIPATH", lkNone, BtPathname) // URL path - usr("WDM_MANAGERS", lkShell, BtIdentifier) - usr("X10_PORT", lkNone, BtPathname) - usr("XAW_TYPE", lkNone, enum("standard 3d xpm neXtaw")) - usr("XLOCK_DEFAULT_MODE", lkNone, BtIdentifier) - usr("ZSH_STATIC", lkNone, BtYes) + // The remaining variables from mk/defaults/mk.conf may be overridden by packages. + // Therefore they need a separate definition of "user-settable". + // + // It is debatable whether packages should be allowed to override these + // variables at all since then there are two competing sources for the + // default values. Current practice is to have exactly this ambiguity, + // combined with some package Makefiles including bsd.prefs.mk and others + // omitting this necessary inclusion. + // + // TODO: parse all the below information directly from mk/defaults/mk.conf. + usrpkg := func(varname string, kindOfList KindOfList, checker *BasicType) { + acl(varname, kindOfList, checker, ""+ + "Makefile: default, set, use, use-loadtime; "+ + "buildlink3.mk, builtin.mk:; "+ + "Makefile.*, *.mk: default, set, use, use-loadtime; "+ + "*: use-loadtime, use") } + usrpkg("ACROREAD_FONTPATH", lkNone, BtPathlist) + usrpkg("AMANDA_USER", lkNone, BtUserGroupName) + usrpkg("AMANDA_TMP", lkNone, BtPathname) + usrpkg("AMANDA_VAR", lkNone, BtPathname) + usrpkg("APACHE_USER", lkNone, BtUserGroupName) + usrpkg("APACHE_GROUP", lkNone, BtUserGroupName) + usrpkg("APACHE_SUEXEC_CONFIGURE_ARGS", lkShell, BtShellWord) + usrpkg("APACHE_SUEXEC_DOCROOT", lkShell, BtPathname) + usrpkg("ARLA_CACHE", lkNone, BtPathname) + usrpkg("BIND_DIR", lkNone, BtPathname) + usrpkg("BIND_GROUP", lkNone, BtUserGroupName) + usrpkg("BIND_USER", lkNone, BtUserGroupName) + usrpkg("CACTI_GROUP", lkNone, BtUserGroupName) + usrpkg("CACTI_USER", lkNone, BtUserGroupName) + usrpkg("CANNA_GROUP", lkNone, BtUserGroupName) + usrpkg("CANNA_USER", lkNone, BtUserGroupName) + usrpkg("CDRECORD_CONF", lkNone, BtPathname) + usrpkg("CLAMAV_GROUP", lkNone, BtUserGroupName) + usrpkg("CLAMAV_USER", lkNone, BtUserGroupName) + usrpkg("CLAMAV_DBDIR", lkNone, BtPathname) + usrpkg("CONSERVER_DEFAULTHOST", lkNone, BtIdentifier) + usrpkg("CONSERVER_DEFAULTPORT", lkNone, BtInteger) + usrpkg("CUPS_GROUP", lkNone, BtUserGroupName) + usrpkg("CUPS_USER", lkNone, BtUserGroupName) + usrpkg("CUPS_SYSTEM_GROUPS", lkShell, BtUserGroupName) + usrpkg("CYRUS_IDLE", lkNone, enum("poll idled no")) + usrpkg("CYRUS_GROUP", lkNone, BtUserGroupName) + usrpkg("CYRUS_USER", lkNone, BtUserGroupName) + usrpkg("DBUS_GROUP", lkNone, BtUserGroupName) + usrpkg("DBUS_USER", lkNone, BtUserGroupName) + usrpkg("DEFANG_GROUP", lkNone, BtUserGroupName) + usrpkg("DEFANG_USER", lkNone, BtUserGroupName) + usrpkg("DEFANG_SPOOLDIR", lkNone, BtPathname) + usrpkg("DEFAULT_IRC_SERVER", lkNone, BtIdentifier) + usrpkg("DEFAULT_SERIAL_DEVICE", lkNone, BtPathname) + usrpkg("DIALER_GROUP", lkNone, BtUserGroupName) + usrpkg("DT_LAYOUT", lkNone, enum("US FI FR GER DV")) + usrpkg("ELK_GUI", lkShell, enum("none xaw motif")) + usrpkg("EMACS_TYPE", lkNone, emacsVersions) + usrpkg("EXIM_GROUP", lkNone, BtUserGroupName) + usrpkg("EXIM_USER", lkNone, BtUserGroupName) + usrpkg("FLUXBOX_USE_XINERAMA", lkNone, enum("YES NO")) + usrpkg("FLUXBOX_USE_KDE", lkNone, enum("YES NO")) + usrpkg("FLUXBOX_USE_GNOME", lkNone, enum("YES NO")) + usrpkg("FLUXBOX_USE_XFT", lkNone, enum("YES NO")) + usrpkg("FOX_USE_XUNICODE", lkNone, enum("YES NO")) + usrpkg("FREEWNN_USER", lkNone, BtUserGroupName) + usrpkg("FREEWNN_GROUP", lkNone, BtUserGroupName) + usrpkg("GAMES_USER", lkNone, BtUserGroupName) + usrpkg("GAMES_GROUP", lkNone, BtUserGroupName) + usrpkg("GAMEMODE", lkNone, BtFileMode) + usrpkg("GAMEDIRMODE", lkNone, BtFileMode) + usrpkg("GAMEDATAMODE", lkNone, BtFileMode) + usrpkg("GAMEGRP", lkNone, BtUserGroupName) + usrpkg("GAMEOWN", lkNone, BtUserGroupName) + usrpkg("GRUB_NETWORK_CARDS", lkNone, BtIdentifier) + usrpkg("GRUB_PRESET_COMMAND", lkNone, enum("bootp dhcp rarp")) + usrpkg("GRUB_SCAN_ARGS", lkShell, BtShellWord) + usrpkg("HASKELL_COMPILER", lkNone, enum("ghc")) + usrpkg("HOWL_GROUP", lkNone, BtUserGroupName) + usrpkg("HOWL_USER", lkNone, BtUserGroupName) + usrpkg("ICECAST_CHROOTDIR", lkNone, BtPathname) + usrpkg("ICECAST_CHUNKLEN", lkNone, BtInteger) + usrpkg("ICECAST_SOURCE_BUFFSIZE", lkNone, BtInteger) + usrpkg("IMAP_UW_CCLIENT_MBOX_FMT", lkNone, enum("mbox mbx mh mmdf mtx mx news phile tenex unix")) + usrpkg("IMAP_UW_MAILSPOOLHOME", lkNone, BtFilename) + usrpkg("IMDICTDIR", lkNone, BtPathname) + usrpkg("INN_DATA_DIR", lkNone, BtPathname) + usrpkg("INN_USER", lkNone, BtUserGroupName) + usrpkg("INN_GROUP", lkNone, BtUserGroupName) + usrpkg("IRCD_HYBRID_NICLEN", lkNone, BtInteger) + usrpkg("IRCD_HYBRID_TOPICLEN", lkNone, BtInteger) + usrpkg("IRCD_HYBRID_SYSLOG_EVENTS", lkNone, BtUnknown) + usrpkg("IRCD_HYBRID_SYSLOG_FACILITY", lkNone, BtIdentifier) + usrpkg("IRCD_HYBRID_MAXCONN", lkNone, BtInteger) + usrpkg("IRCD_HYBRID_IRC_USER", lkNone, BtUserGroupName) + usrpkg("IRCD_HYBRID_IRC_GROUP", lkNone, BtUserGroupName) + usrpkg("IRRD_USE_PGP", lkNone, enum("5 2")) + usrpkg("JABBERD_USER", lkNone, BtUserGroupName) + usrpkg("JABBERD_GROUP", lkNone, BtUserGroupName) + usrpkg("JABBERD_LOGDIR", lkNone, BtPathname) + usrpkg("JABBERD_SPOOLDIR", lkNone, BtPathname) + usrpkg("JABBERD_PIDDIR", lkNone, BtPathname) + usrpkg("JAKARTA_HOME", lkNone, BtPathname) + usrpkg("KERBEROS", lkNone, BtYes) + usrpkg("KERMIT_SUID_UUCP", lkNone, BtYes) + usrpkg("KJS_USE_PCRE", lkNone, BtYes) + usrpkg("KNEWS_DOMAIN_FILE", lkNone, BtPathname) + usrpkg("KNEWS_DOMAIN_NAME", lkNone, BtIdentifier) + usrpkg("LIBDVDCSS_HOMEPAGE", lkNone, BtHomepage) + usrpkg("LIBDVDCSS_MASTER_SITES", lkShell, BtFetchURL) + usrpkg("LATEX2HTML_ICONPATH", lkNone, BtURL) + usrpkg("LEAFNODE_DATA_DIR", lkNone, BtPathname) + usrpkg("LEAFNODE_USER", lkNone, BtUserGroupName) + usrpkg("LEAFNODE_GROUP", lkNone, BtUserGroupName) + usrpkg("LINUX_LOCALES", lkShell, BtIdentifier) + usrpkg("MAILAGENT_DOMAIN", lkNone, BtIdentifier) + usrpkg("MAILAGENT_EMAIL", lkNone, BtMailAddress) + usrpkg("MAILAGENT_FQDN", lkNone, BtIdentifier) + usrpkg("MAILAGENT_ORGANIZATION", lkNone, BtUnknown) + usrpkg("MAJORDOMO_HOMEDIR", lkNone, BtPathname) + usrpkg("MAKEINFO_ARGS", lkShell, BtShellWord) + usrpkg("MECAB_CHARSET", lkNone, BtIdentifier) + usrpkg("MEDIATOMB_GROUP", lkNone, BtUserGroupName) + usrpkg("MEDIATOMB_USER", lkNone, BtUserGroupName) + usrpkg("MLDONKEY_GROUP", lkNone, BtUserGroupName) + usrpkg("MLDONKEY_HOME", lkNone, BtPathname) + usrpkg("MLDONKEY_USER", lkNone, BtUserGroupName) + usrpkg("MONOTONE_GROUP", lkNone, BtUserGroupName) + usrpkg("MONOTONE_USER", lkNone, BtUserGroupName) + usrpkg("MOTIF_TYPE", lkNone, enum("motif openmotif lesstif dt")) + usrpkg("MOTIF_TYPE_DEFAULT", lkNone, enum("motif openmotif lesstif dt")) + usrpkg("MTOOLS_ENABLE_FLOPPYD", lkNone, BtYesNo) + usrpkg("MYSQL_USER", lkNone, BtUserGroupName) + usrpkg("MYSQL_GROUP", lkNone, BtUserGroupName) + usrpkg("MYSQL_DATADIR", lkNone, BtPathname) + usrpkg("MYSQL_CHARSET", lkNone, BtIdentifier) + usrpkg("MYSQL_EXTRA_CHARSET", lkShell, BtIdentifier) + usrpkg("NAGIOS_GROUP", lkNone, BtUserGroupName) + usrpkg("NAGIOS_USER", lkNone, BtUserGroupName) + usrpkg("NAGIOSCMD_GROUP", lkNone, BtUserGroupName) + usrpkg("NAGIOSDIR", lkNone, BtPathname) + usrpkg("NBPAX_PROGRAM_PREFIX", lkNone, BtUnknown) + usrpkg("NMH_EDITOR", lkNone, BtIdentifier) + usrpkg("NMH_MTA", lkNone, enum("smtp sendmail")) + usrpkg("NMH_PAGER", lkNone, BtIdentifier) + usrpkg("NS_PREFERRED", lkNone, enum("communicator navigator mozilla")) + usrpkg("OPENSSH_CHROOT", lkNone, BtPathname) + usrpkg("OPENSSH_USER", lkNone, BtUserGroupName) + usrpkg("OPENSSH_GROUP", lkNone, BtUserGroupName) + usrpkg("P4USER", lkNone, BtUserGroupName) + usrpkg("P4GROUP", lkNone, BtUserGroupName) + usrpkg("P4ROOT", lkNone, BtPathname) + usrpkg("P4PORT", lkNone, BtInteger) + usrpkg("PALMOS_DEFAULT_SDK", lkNone, enum("1 2 3.1 3.5")) + usrpkg("PAPERSIZE", lkNone, enum("A4 Letter")) + usrpkg("PGGROUP", lkNone, BtUserGroupName) + usrpkg("PGUSER", lkNone, BtUserGroupName) + usrpkg("PGHOME", lkNone, BtPathname) + usrpkg("PILRC_USE_GTK", lkNone, BtYesNo) + usrpkg("PKG_JVM_DEFAULT", lkNone, jvms) + usrpkg("POPTOP_USE_MPPE", lkNone, BtYes) + usrpkg("PROCMAIL_MAILSPOOLHOME", lkNone, BtFilename) + usrpkg("PROCMAIL_TRUSTED_IDS", lkShell, BtUnknown) // Comma-separated list of string or integer literals. + usrpkg("PVM_SSH", lkNone, BtPathname) + usrpkg("QMAILDIR", lkNone, BtPathname) + usrpkg("QMAIL_QFILTER_TMPDIR", lkNone, BtPathname) + usrpkg("QMAIL_QUEUE_DIR", lkNone, BtPathname) + usrpkg("QMAIL_QUEUE_EXTRA", lkNone, BtMailAddress) + usrpkg("QPOPPER_FAC", lkNone, BtIdentifier) + usrpkg("QPOPPER_USER", lkNone, BtUserGroupName) + usrpkg("QPOPPER_SPOOL_DIR", lkNone, BtPathname) + usrpkg("RASMOL_DEPTH", lkNone, enum("8 16 32")) + usrpkg("RELAY_CTRL_DIR", lkNone, BtPathname) + usrpkg("RPM_DB_PREFIX", lkNone, BtPathname) + usrpkg("RSSH_SCP_PATH", lkNone, BtPathname) + usrpkg("RSSH_SFTP_SERVER_PATH", lkNone, BtPathname) + usrpkg("RSSH_CVS_PATH", lkNone, BtPathname) + usrpkg("RSSH_RDIST_PATH", lkNone, BtPathname) + usrpkg("RSSH_RSYNC_PATH", lkNone, BtPathname) + usrpkg("SAWFISH_THEMES", lkShell, BtFilename) + usrpkg("SCREWS_GROUP", lkNone, BtUserGroupName) + usrpkg("SCREWS_USER", lkNone, BtUserGroupName) + usrpkg("SDIST_PAWD", lkNone, enum("pawd pwd")) + usrpkg("SERIAL_DEVICES", lkShell, BtPathname) + usrpkg("SILC_CLIENT_WITH_PERL", lkNone, BtYesNo) + usrpkg("SSH_SUID", lkNone, BtYesNo) + usrpkg("SSYNC_PAWD", lkNone, enum("pawd pwd")) + usrpkg("SUSE_PREFER", lkNone, enum("13.1 12.1 10.0")) + usrpkg("TEXMFSITE", lkNone, BtPathname) + usrpkg("THTTPD_LOG_FACILITY", lkNone, BtIdentifier) + usrpkg("UNPRIVILEGED", lkNone, BtYesNo) + usrpkg("USE_CROSS_COMPILE", lkNone, BtYesNo) + usrpkg("USERPPP_GROUP", lkNone, BtUserGroupName) + usrpkg("UUCP_GROUP", lkNone, BtUserGroupName) + usrpkg("UUCP_USER", lkNone, BtUserGroupName) + usrpkg("VIM_EXTRA_OPTS", lkShell, BtShellWord) + usrpkg("WCALC_HTMLDIR", lkNone, BtPathname) + usrpkg("WCALC_HTMLPATH", lkNone, BtPathname) // URL path + usrpkg("WCALC_CGIDIR", lkNone, BtPrefixPathname) + usrpkg("WCALC_CGIPATH", lkNone, BtPathname) // URL path + usrpkg("WDM_MANAGERS", lkShell, BtIdentifier) + usrpkg("X10_PORT", lkNone, BtPathname) + usrpkg("XAW_TYPE", lkNone, enum("standard 3d xpm neXtaw")) + usrpkg("XLOCK_DEFAULT_MODE", lkNone, BtIdentifier) + usrpkg("ZSH_STATIC", lkNone, BtYes) // some other variables, sorted alphabetically @@ -654,7 +656,6 @@ func (src *Pkgsrc) InitVartypes() { usr("EMUL_TYPE.*", lkNone, enum("native builtin suse suse-9.1 suse-9.x suse-10.0 suse-10.x")) sys("ERROR_CAT", lkNone, BtShellCommand) sys("ERROR_MSG", lkNone, BtShellCommand) - acl("EVAL_PREFIX", lkSpace, BtShellWord, "Makefile, Makefile.common: append") // XXX: Combining ShellWord with lkSpace looks weird. sys("EXPORT_SYMBOLS_LDFLAGS", lkShell, BtLdFlag) sys("EXTRACT_CMD", lkNone, BtShellCommand) pkg("EXTRACT_DIR", lkNone, BtPathname) @@ -956,7 +957,7 @@ func (src *Pkgsrc) InitVartypes() { acl("PKG_SYSCONFVAR", lkNone, BtIdentifier, "") acl("PKG_UID", lkNone, BtInteger, "Makefile: set") acl("PKG_USERS", lkShell, BtShellWord, "Makefile: set, append") - pkg("PKG_USERS_VARS", lkShell, BtVariableName) + pkglist("PKG_USERS_VARS", lkShell, BtVariableName) acl("PKG_USE_KERBEROS", lkNone, BtYes, "Makefile, Makefile.common: set") pkg("PLIST.*", lkNone, BtYes) pkglist("PLIST_VARS", lkShell, BtIdentifier) @@ -978,8 +979,8 @@ func (src *Pkgsrc) InitVartypes() { acl("PYPKGPREFIX", lkNone, enum("py27 py34 py35 py36"), "pyversion.mk: set; *: use-loadtime, use") pkg("PYTHON_FOR_BUILD_ONLY", lkNone, BtYes) pkglist("REPLACE_PYTHON", lkShell, BtPathmask) - pkg("PYTHON_VERSIONS_ACCEPTED", lkShell, BtVersion) - pkg("PYTHON_VERSIONS_INCOMPATIBLE", lkShell, BtVersion) + pkglist("PYTHON_VERSIONS_ACCEPTED", lkShell, BtVersion) + pkglist("PYTHON_VERSIONS_INCOMPATIBLE", lkShell, BtVersion) usr("PYTHON_VERSION_DEFAULT", lkNone, BtVersion) usr("PYTHON_VERSION_REQD", lkNone, BtVersion) pkglist("PYTHON_VERSIONED_DEPENDENCIES", lkShell, BtPythonDependency) @@ -1139,9 +1140,10 @@ func parseACLEntries(varname string, aclEntries string) []ACLEntry { globs = strings.TrimSuffix(arg, ":") } if perms == prevperms { - fmt.Printf("Repeated permissions for %s: %s\n", varname, perms) + G.Panicf("Repeated permissions %q for %q.", perms, varname) } prevperms = perms + var permissions ACLPermissions for _, perm := range strings.Split(perms, ", ") { switch perm { @@ -1158,9 +1160,10 @@ func parseACLEntries(varname string, aclEntries string) []ACLEntry { case "": break default: - print(fmt.Sprintf("Invalid ACL permission %q for varname %q.\n", perm, varname)) + G.Panicf("Invalid ACL permission %q for %q.", perm, varname) } } + for _, glob := range strings.Split(globs, ", ") { switch glob { case "*", @@ -1169,11 +1172,11 @@ func parseACLEntries(varname string, aclEntries string) []ACLEntry { "bsd.options.mk", "pkgconfig-builtin.mk", "pyversion.mk": break default: - print(fmt.Sprintf("Invalid ACL glob %q for varname %q.\n", glob, varname)) + G.Panicf("Invalid ACL glob %q for %q.", glob, varname) } for _, prev := range result { if matched, err := path.Match(prev.glob, glob); err != nil || matched { - print(fmt.Sprintf("Ineffective ACL glob %q for varname %q.\n", glob, varname)) + G.Panicf("Ineffective ACL glob %q for %q.", glob, varname) } } result = append(result, ACLEntry{glob, permissions}) diff --git a/pkgtools/pkglint/files/vardefs_test.go b/pkgtools/pkglint/files/vardefs_test.go index df7f932561d..23a0aa3d1e2 100644 --- a/pkgtools/pkglint/files/vardefs_test.go +++ b/pkgtools/pkglint/files/vardefs_test.go @@ -2,21 +2,21 @@ package main import "gopkg.in/check.v1" -func (s *Suite) Test_InitVartypes__enumFrom(c *check.C) { +func (s *Suite) Test_Pkgsrc_InitVartypes__enumFrom(c *check.C) { t := s.Init(c) - t.SetupFileMkLines("editors/emacs/modules.mk", + t.CreateFileLines("editors/emacs/modules.mk", MkRcsID, "", "_EMACS_VERSIONS_ALL= emacs31", "_EMACS_VERSIONS_ALL+= emacs29") - t.SetupFileMkLines("mk/java-vm.mk", + t.CreateFileLines("mk/java-vm.mk", MkRcsID, "", "_PKG_JVMS.8= openjdk8 oracle-jdk8", "_PKG_JVMS.7= ${_PKG_JVMS.8} openjdk7 sun-jdk7", "_PKG_JVMS.6= ${_PKG_JVMS.7} sun-jdk6 jdk16") - t.SetupFileMkLines("mk/compiler.mk", + t.CreateFileLines("mk/compiler.mk", MkRcsID, "", "_COMPILERS= gcc ido mipspro-ucode \\", @@ -41,3 +41,23 @@ func (s *Suite) Test_InitVartypes__enumFrom(c *check.C) { checkEnumValues("USE_LANGUAGES", "ShellList of enum: ada c c++ c++03 c++0x c++11 c++14 c99 fortran fortran77 gnu++03 gnu++0x gnu++11 gnu++14 java obj-c++ objc ") checkEnumValues("PKGSRC_COMPILER", "ShellList of enum: ccache distcc f2c g95 gcc ido mipspro-ucode sunpro ") } + +func (s *Suite) Test_parseACLEntries(c *check.C) { + t := s.Init(c) + + t.ExpectFatal( + func() { parseACLEntries("VARNAME", "buildlink3.mk: *; *: *") }, + "FATAL: Invalid ACL permission \"*\" for \"VARNAME\".") + + t.ExpectFatal( + func() { parseACLEntries("VARNAME", "buildlink3.mk: use; *: use") }, + "FATAL: Repeated permissions \"use\" for \"VARNAME\".") + + t.ExpectFatal( + func() { parseACLEntries("VARNAME", "*.txt: use") }, + "FATAL: Invalid ACL glob \"*.txt\" for \"VARNAME\".") + + t.ExpectFatal( + func() { parseACLEntries("VARNAME", "*.mk: use; buildlink3.mk: append") }, + "FATAL: Ineffective ACL glob \"buildlink3.mk\" for \"VARNAME\".") +} diff --git a/pkgtools/pkglint/files/vartype.go b/pkgtools/pkglint/files/vartype.go index 5f7387768ff..ff93c5142ce 100644 --- a/pkgtools/pkglint/files/vartype.go +++ b/pkgtools/pkglint/files/vartype.go @@ -232,7 +232,7 @@ var ( BtPathname = &BasicType{"Pathname", (*VartypeCheck).Pathname} BtPerl5Packlist = &BasicType{"Perl5Packlist", (*VartypeCheck).Perl5Packlist} BtPerms = &BasicType{"Perms", (*VartypeCheck).Perms} - BtPkgName = &BasicType{"PkgName", (*VartypeCheck).PkgName} + BtPkgName = &BasicType{"Pkgname", (*VartypeCheck).Pkgname} BtPkgPath = &BasicType{"PkgPath", (*VartypeCheck).PkgPath} BtPkgOptionsVar = &BasicType{"PkgOptionsVar", (*VartypeCheck).PkgOptionsVar} BtPkgRevision = &BasicType{"PkgRevision", (*VartypeCheck).PkgRevision} diff --git a/pkgtools/pkglint/files/vartype_test.go b/pkgtools/pkglint/files/vartype_test.go index d83f085ad15..e39e68a9c9c 100644 --- a/pkgtools/pkglint/files/vartype_test.go +++ b/pkgtools/pkglint/files/vartype_test.go @@ -23,7 +23,7 @@ func (s *Suite) Test_Vartype_EffectivePermissions(c *check.C) { } } -func (s *Suite) Test_VarChecker_HasEnum(c *check.C) { +func (s *Suite) Test_BasicType_HasEnum(c *check.C) { vc := enum("catinstall middle maninstall") c.Check(vc.HasEnum("catinstall"), equals, true) @@ -31,7 +31,7 @@ func (s *Suite) Test_VarChecker_HasEnum(c *check.C) { c.Check(vc.HasEnum("maninstall"), equals, true) } -func (s *Suite) Test_AclPermissions_Contains(c *check.C) { +func (s *Suite) Test_ACLPermissions_Contains(c *check.C) { perms := aclpAllRuntime c.Check(perms.Contains(aclpAllRuntime), equals, true) @@ -39,7 +39,7 @@ func (s *Suite) Test_AclPermissions_Contains(c *check.C) { c.Check(perms.Contains(aclpUseLoadtime), equals, false) } -func (s *Suite) Test_AclPermissions_String(c *check.C) { +func (s *Suite) Test_ACLPermissions_String(c *check.C) { c.Check(ACLPermissions(0).String(), equals, "none") c.Check(aclpAll.String(), equals, "set, set-default, append, use-loadtime, use") c.Check(aclpUnknown.String(), equals, "unknown") diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go index 3bb57b1da27..74a5315d823 100644 --- a/pkgtools/pkglint/files/vartypecheck.go +++ b/pkgtools/pkglint/files/vartypecheck.go @@ -1,7 +1,6 @@ package main import ( - "netbsd.org/pkglint/regex" "netbsd.org/pkglint/trace" "path" "sort" @@ -277,7 +276,8 @@ func (cv *VartypeCheck) Dependency() { "foo-* matches foo-1.2, but also foo-client-1.2 and foo-server-1.2.") } - if nocclasses := regex.Compile(`\[[\d-]+\]`).ReplaceAllString(wildcard, ""); contains(nocclasses, "-") { + withoutCharClasses := replaceAll(wildcard, `\[[\d-]+\]`, "") + if contains(withoutCharClasses, "-") { line.Warnf("The version pattern %q should not contain a hyphen.", wildcard) Explain( "Pkgsrc interprets package names with version numbers like this:", @@ -730,7 +730,7 @@ func (cv *VartypeCheck) Perms() { } } -func (cv *VartypeCheck) PkgName() { +func (cv *VartypeCheck) Pkgname() { if cv.Op != opUseMatch && cv.Value == cv.ValueNoVar && !matches(cv.Value, rePkgname) { cv.Line.Warnf("%q is not a valid package name. A valid package name has the form packagename-version, where version consists only of digits, letters and dots.", cv.Value) } @@ -792,25 +792,34 @@ func (cv *VartypeCheck) MachinePlatformPattern() { pattern += "-*" } - if m, opsysPattern, _, archPattern := match3(pattern, reTriple); m { + if m, opsysPattern, versionPattern, archPattern := match3(pattern, reTriple); m { opsysCv := &VartypeCheck{ cv.MkLine, cv.Line, "the operating system part of " + cv.Varname, - opUseMatch, // Always allow patterns, since this is a PlatformPattern. + opUseMatch, // Always allow patterns, since this is a platform pattern. opsysPattern, opsysPattern, cv.MkComment, cv.Guessed} enumMachineOpsys.checker(opsysCv) - // no check for os_version + versionCv := &VartypeCheck{ + cv.MkLine, + cv.Line, + "the version part of " + cv.Varname, + opUseMatch, // Always allow patterns, since this is a platform pattern. + versionPattern, + versionPattern, + cv.MkComment, + cv.Guessed} + versionCv.Version() archCv := &VartypeCheck{ cv.MkLine, cv.Line, "the hardware architecture part of " + cv.Varname, - opUseMatch, // Always allow patterns, since this is a PlatformPattern. + opUseMatch, // Always allow patterns, since this is a platform pattern. archPattern, archPattern, cv.MkComment, @@ -976,6 +985,7 @@ func (cv *VartypeCheck) Tool() { } } +// Unknown doesn't check for anything. func (cv *VartypeCheck) Unknown() { // Do nothing. } @@ -1035,12 +1045,30 @@ func (cv *VartypeCheck) VariableName() { } func (cv *VartypeCheck) Version() { + line := cv.Line + value := cv.Value + if cv.Op == opUseMatch { - if !matches(cv.Value, `^[\d?\[][\w\-.*?\[\]]+$`) { - cv.Line.Warnf("Invalid version number pattern %q.", cv.Value) + if value != "*" && !matches(value, `^[\d?\[][\w\-.*?\[\]]+$`) { + line.Warnf("Invalid version number pattern %q.", value) + return + } + + const digit = `(?:\d|\[[\d-]+\])` + const alnum = `(?:\w|\[[\d-]+\])` + if m, ver, suffix := match2(value, `^(`+digit+alnum+`*(?:\.`+alnum+`+)*)(\.\*|\*|)$`); m { + if suffix == "*" && ver != "[0-9]" { + line.Warnf("Please use %q instead of %q as the version pattern.", ver+".*", ver+"*") + Explain( + "For example, the version \"1*\" also matches \"10.0.0\", which is", + "probably not intended.") + } } - } else if cv.Value == cv.ValueNoVar && !matches(cv.Value, `^\d[\w.]*$`) { - cv.Line.Warnf("Invalid version number %q.", cv.Value) + return + } + + if value == cv.ValueNoVar && !matches(value, `^\d[\w.]*$`) { + line.Warnf("Invalid version number %q.", value) } } diff --git a/pkgtools/pkglint/files/vartypecheck_test.go b/pkgtools/pkglint/files/vartypecheck_test.go index b11625586c8..7f3db87b21f 100644 --- a/pkgtools/pkglint/files/vartypecheck_test.go +++ b/pkgtools/pkglint/files/vartypecheck_test.go @@ -48,9 +48,9 @@ func (s *Suite) Test_VartypeCheck_Category(c *check.C) { t := s.Init(c) vt := NewVartypeCheckTester(t, (*VartypeCheck).Category) - t.SetupFileLines("filesyscategory/Makefile", + t.CreateFileLines("filesyscategory/Makefile", "# empty") - t.SetupFileLines("wip/Makefile", + t.CreateFileLines("wip/Makefile", "# empty") vt.Varname("CATEGORIES") @@ -669,7 +669,7 @@ func (s *Suite) Test_VartypeCheck_Perms(c *check.C) { } func (s *Suite) Test_VartypeCheck_Pkgname(c *check.C) { - vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).PkgName) + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Pkgname) vt.Varname("PKGNAME") vt.Values( @@ -754,7 +754,8 @@ func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { "NetBSD-1.6.2-i386", "FreeBSD*", "FreeBSD-*", - "${LINUX}") + "${LINUX}", + "NetBSD-[0-1]*-*") vt.Output( "WARN: fname:1: \"linux-i386\" is not a valid platform pattern.", @@ -780,7 +781,8 @@ func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { "i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 "+ "mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ "} for the hardware architecture part of ONLY_FOR_PLATFORM.", - "WARN: fname:5: \"FreeBSD*\" is not a valid platform pattern.") + "WARN: fname:5: \"FreeBSD*\" is not a valid platform pattern.", + "WARN: fname:8: Please use \"[0-1].*\" instead of \"[0-1]*\" as the version pattern.") } func (s *Suite) Test_VartypeCheck_PythonDependency(c *check.C) { @@ -1015,10 +1017,16 @@ func (s *Suite) Test_VartypeCheck_Version(c *check.C) { vt.Values( "a*", "1.2/456", + "4*", + "?.??", + "1.[234]*", + "1.[2-7].*", "[0-9]*") vt.Output( "WARN: fname:11: Invalid version number pattern \"a*\".", - "WARN: fname:12: Invalid version number pattern \"1.2/456\".") + "WARN: fname:12: Invalid version number pattern \"1.2/456\".", + "WARN: fname:13: Please use \"4.*\" instead of \"4*\" as the version pattern.", + "WARN: fname:15: Please use \"1.[234].*\" instead of \"1.[234]*\" as the version pattern.") } func (s *Suite) Test_VartypeCheck_WrapperReorder(c *check.C) { diff --git a/pkgtools/pkglint/select.mk b/pkgtools/pkglint/select.mk index 56bc9577908..cc448607347 100644 --- a/pkgtools/pkglint/select.mk +++ b/pkgtools/pkglint/select.mk @@ -1,4 +1,4 @@ -# $NetBSD: select.mk,v 1.4 2016/03/23 16:36:53 gdt Exp $ +# $NetBSD: select.mk,v 1.5 2018/10/03 22:27:53 rillig Exp $ # # Selects the proper version of pkglint, depending on whether the # platform supports the Go programming language. @@ -8,7 +8,7 @@ # See lang/go/version.mk # While it's wrong in the above, go14 does not build on NetBSD 5. -.if ${MACHINE_ARCH:Ni386:Nx86_64:Nevbarm} || ${MACHINE_PLATFORM:MSunOS-*-i386} || ${MACHINE_PLATFORM:MNetBSD-[1-5]*-*} +.if ${MACHINE_ARCH:Ni386:Nx86_64:Nevbarm} || ${MACHINE_PLATFORM:MSunOS-*-i386} || ${MACHINE_PLATFORM:MNetBSD-[1-5].*-*} DEPENDS+= pkglint4>=4.82<5:../../pkgtools/pkglint4 .else DEPENDS+= pkglint>=5:../../pkgtools/pkglint |