diff options
| author | rillig <rillig@pkgsrc.org> | 2019-06-30 20:56:18 +0000 |
|---|---|---|
| committer | rillig <rillig@pkgsrc.org> | 2019-06-30 20:56:18 +0000 |
| commit | 705b6a55cc98cda8d732ca886796e48b180d4258 (patch) | |
| tree | d6c03aee392418755fbd7a5a77d29baff57855d6 /pkgtools | |
| parent | 5ae793651d2eac119a606ed67f0a6b98429e38bd (diff) | |
| download | pkgsrc-705b6a55cc98cda8d732ca886796e48b180d4258.tar.gz | |
pkgtools/pkglint: update to 5.7.14
Changes since 5.7.13:
- Removed the -Cextra command line option since it didn't produce useful
warnings.
- Removed unwarranted warnings about _WRAP_EXTRA_ARGS.CC being used in
packages.
- Cleaned up the canonical order of variables in package Makefiles.
- Added a few commands to those that cannot fail, to reduce the number of
"at the left of the | operator" in shell programs.
- Fixed warnings about "-ggdb" being an unknown shell command.
- Reduced number of warnings about lists being used where a single value
is expected.
- Replaced unreliable check for invalid CFLAGS and LDFLAGS with a more
practical check.
- Renamed "RCS tag" to "CVS tag" to make the diagnostics more modern.
- Added warning when PKGNAME or PKGVERSION is used in MASTER_SITES.
- Reworded warning for missing or superfluous PLIST files.
- Lots of other detail changes, refactorings and automatic tests.
Diffstat (limited to 'pkgtools')
82 files changed, 6134 insertions, 2785 deletions
diff --git a/pkgtools/pkglint/Makefile b/pkgtools/pkglint/Makefile index 9ec9dc61fa0..808bc09cfd2 100644 --- a/pkgtools/pkglint/Makefile +++ b/pkgtools/pkglint/Makefile @@ -1,6 +1,6 @@ -# $NetBSD: Makefile,v 1.585 2019/06/10 19:51:57 rillig Exp $ +# $NetBSD: Makefile,v 1.586 2019/06/30 20:56:18 rillig Exp $ -PKGNAME= pkglint-5.7.13 +PKGNAME= pkglint-5.7.14 CATEGORIES= pkgtools DISTNAME= tools MASTER_SITES= ${MASTER_SITE_GITHUB:=golang/} diff --git a/pkgtools/pkglint/files/alternatives.go b/pkgtools/pkglint/files/alternatives.go index 7818dc2a02d..4fb88b2247e 100644 --- a/pkgtools/pkglint/files/alternatives.go +++ b/pkgtools/pkglint/files/alternatives.go @@ -16,13 +16,13 @@ func CheckFileAlternatives(filename string) { plist = G.Pkg.Plist } - checkPlistWrapper := func(line Line, wrapper string) { + checkPlistWrapper := func(line *Line, wrapper string) { if plist.Files[wrapper] { line.Errorf("Alternative wrapper %q must not appear in the PLIST.", wrapper) } } - checkPlistAlternative := func(line Line, alternative string) { + checkPlistAlternative := func(line *Line, alternative string) { relImplementation := strings.Replace(alternative, "@PREFIX@/", "", 1) plistName := replaceAll(relImplementation, `@(\w+)@`, "${$1}") if plist.Files[plistName] || G.Pkg.vars.Defined("ALTERNATIVES_SRC") { diff --git a/pkgtools/pkglint/files/alternatives_test.go b/pkgtools/pkglint/files/alternatives_test.go index 596f99fb6ab..62ddfb5ef03 100644 --- a/pkgtools/pkglint/files/alternatives_test.go +++ b/pkgtools/pkglint/files/alternatives_test.go @@ -17,7 +17,7 @@ func (s *Suite) Test_CheckFileAlternatives__PLIST(c *check.C) { "highscores @VARBASE@/game/scores", "sbin/init /sbin/init") t.CreateFileLines("PLIST", - PlistRcsID, + PlistCvsID, "bin/echo", "bin/vim", "sbin/sendmail.exim${EXIMVER}") diff --git a/pkgtools/pkglint/files/autofix.go b/pkgtools/pkglint/files/autofix.go index 9996e9c3fb6..11ea21fb9a6 100644 --- a/pkgtools/pkglint/files/autofix.go +++ b/pkgtools/pkglint/files/autofix.go @@ -13,7 +13,7 @@ import ( // The modifications are kept in memory only, // until they are written to disk by SaveAutofixChanges. type Autofix struct { - line Line + line *Line linesBefore []string // Newly inserted lines, including \n linesAfter []string // Newly inserted lines, including \n // Whether an actual fix has been applied (or, without --show-autofix, @@ -51,7 +51,10 @@ const SilentAutofixFormat = "SilentAutofixFormat" // Since these are not really diagnostics, duplicates are not suppressed. const AutofixFormat = "AutofixFormat" -func NewAutofix(line Line) *Autofix { +func NewAutofix(line *Line) *Autofix { + // FIXME: replacing the returned value with + // &Autofix{line: line, autofixShortTerm: autofixShortTerm{anyway: true}} + // makes some tests output source code without diagnostic. return &Autofix{line: line} } @@ -75,9 +78,7 @@ func (fix *Autofix) Explain(explanation ...string) { // Since a silent fix doesn't have a diagnostic, its explanation would // not provide any clue as to what diagnostic it belongs. That would // be confusing, therefore this case is not allowed. - assertf( - fix.diagFormat != SilentAutofixFormat, - "Autofix: Silent fixes cannot have an explanation.") + assert(fix.diagFormat != SilentAutofixFormat) fix.explanation = explanation } @@ -95,8 +96,19 @@ func (fix *Autofix) ReplaceAfter(prefix, from string, to string) { return } + prefixFrom := prefix + from + prefixTo := prefix + to + + n := 0 for _, rawLine := range fix.line.raw { - replaced := strings.Replace(rawLine.textnl, prefix+from, prefix+to, 1) + n += strings.Count(rawLine.textnl, prefixFrom) + } + if n != 1 { + return + } + + for _, rawLine := range fix.line.raw { + replaced := strings.Replace(rawLine.textnl, prefixFrom, prefixTo, 1) if replaced != rawLine.textnl { if G.Logger.IsAutofix() { rawLine.textnl = replaced @@ -108,7 +120,7 @@ func (fix *Autofix) ReplaceAfter(prefix, from string, to string) { // TODO: Do this properly by parsing the whole line again, // and ideally everything that depends on the parsed line. // This probably requires a generic notification mechanism. - fix.line.Text = strings.Replace(fix.line.Text, prefix+from, prefix+to, 1) + fix.line.Text = strings.Replace(fix.line.Text, prefixFrom, prefixTo, 1) } fix.Describef(rawLine.Lineno, "Replacing %q with %q.", from, to) return @@ -274,11 +286,10 @@ func (fix *Autofix) Anyway() { func (fix *Autofix) Apply() { line := fix.line + // Each autofix must have a log level and a diagnostic. // To fix this assertion, call one of Autofix.Errorf, Autofix.Warnf // or Autofix.Notef before calling Apply. - assertf( - fix.level != nil, - "Each autofix must have a log level and a diagnostic.") + assert(fix.level != nil) reset := func() { if len(fix.actions) > 0 { @@ -339,14 +350,14 @@ func (fix *Autofix) Apply() { reset() } -func (fix *Autofix) Realign(mkline MkLine, newWidth int) { +func (fix *Autofix) Realign(mkline *MkLine, newWidth int) { // XXX: Check whether this method can be implemented as Custom fix. // This complicated code should not be in the Autofix type. fix.assertRealLine() - assertf(mkline.IsMultiline(), "Line must be a multiline.") - assertf(mkline.IsVarassign() || mkline.IsCommentedVarassign(), "Line must be a variable assignment.") + assert(mkline.IsMultiline()) + assert(mkline.IsVarassign() || mkline.IsCommentedVarassign()) if fix.skip() { return @@ -409,8 +420,8 @@ func (fix *Autofix) setDiag(level *LogLevel, format string, args []interface{}) "Autofix: format %q must end with a period.", format) } - assertf(fix.level == nil, "Autofix can only have a single diagnostic.") - assertf(fix.diagFormat == "", "Autofix can only have a single diagnostic.") + assert(fix.level == nil) // Autofix can only have a single diagnostic. + assert(fix.diagFormat == "") // Autofix can only have a single diagnostic. fix.level = level fix.diagFormat = format @@ -418,22 +429,21 @@ func (fix *Autofix) setDiag(level *LogLevel, format string, args []interface{}) } func (fix *Autofix) skip() bool { - assertf( - fix.diagFormat != "", - "Autofix: The diagnostic must be given before the action.") + assert(fix.diagFormat != "") // The diagnostic must be given before the action. + // This check is necessary for the --only command line option. return !G.Logger.shallBeLogged(fix.diagFormat) } func (fix *Autofix) assertRealLine() { - assertf(fix.line.firstLine >= 1, "Cannot autofix this line since it is not a real line.") + assert(fix.line.firstLine >= 1) // Cannot autofix this line since it is not a real line. } // SaveAutofixChanges writes the given lines back into their files, // applying the autofix changes. // The lines may come from different files. // Only files that actually have changed lines are saved. -func SaveAutofixChanges(lines Lines) (autofixed bool) { +func SaveAutofixChanges(lines *Lines) (autofixed bool) { if trace.Tracing { defer trace.Call0()() } diff --git a/pkgtools/pkglint/files/autofix_test.go b/pkgtools/pkglint/files/autofix_test.go index 2f54fb1ca5e..993ce219e77 100644 --- a/pkgtools/pkglint/files/autofix_test.go +++ b/pkgtools/pkglint/files/autofix_test.go @@ -14,9 +14,7 @@ func (s *Suite) Test_Autofix_Warnf__duplicate(c *check.C) { fix := line.Autofix() fix.Warnf("Warning 1.") - t.ExpectPanic( - func() { fix.Warnf("Warning 2.") }, - "Pkglint internal error: Autofix can only have a single diagnostic.") + t.ExpectAssert(func() { fix.Warnf("Warning 2.") }) } func (s *Suite) Test_Autofix__default_leaves_line_unchanged(c *check.C) { @@ -58,7 +56,7 @@ func (s *Suite) Test_Autofix__show_autofix_modifies_line(c *check.C) { fix := line.Autofix() fix.Warnf("Row should be replaced with line.") - fix.ReplaceAfter("", "row", "line") + fix.ReplaceAfter("", "# row", "# line") fix.ReplaceRegex(`row \d+`, "the above line", -1) fix.InsertBefore("above") fix.InsertAfter("below") @@ -70,7 +68,7 @@ func (s *Suite) Test_Autofix__show_autofix_modifies_line(c *check.C) { "below\n") t.CheckOutputLines( "WARN: ~/Makefile:1--2: Row should be replaced with line.", - "AUTOFIX: ~/Makefile:1: Replacing \"row\" with \"line\".", + "AUTOFIX: ~/Makefile:1: Replacing \"# row\" with \"# line\".", "AUTOFIX: ~/Makefile:2: Replacing \"row 1\" with \"the above line\".", "AUTOFIX: ~/Makefile:1: Inserting a line \"above\" before this line.", "AUTOFIX: ~/Makefile:2: Inserting a line \"below\" after this line.", @@ -83,7 +81,7 @@ func (s *Suite) Test_Autofix__show_autofix_modifies_line(c *check.C) { c.Check(fix.modified, equals, true) } -func (s *Suite) Test_Autofix_ReplaceAfter__autofix(c *check.C) { +func (s *Suite) Test_Autofix_ReplaceAfter__autofix_in_continuation_line(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--autofix", "--source") @@ -93,18 +91,53 @@ func (s *Suite) Test_Autofix_ReplaceAfter__autofix(c *check.C) { "continuation 2") fix := mklines.lines.Lines[0].Autofix() - fix.Warnf("N should be replaced with V.") - fix.ReplaceAfter("", "n", "v") + fix.Warnf("Line should be replaced with Row.") + fix.ReplaceAfter("", "line", "row") fix.Apply() t.CheckOutputLines( - "AUTOFIX: ~/Makefile:1: Replacing \"n\" with \"v\".", + "AUTOFIX: ~/Makefile:1: Replacing \"line\" with \"row\".", "-\t# line 1 \\", - "+\t# live 1 \\", + "+\t# row 1 \\", "\tcontinuation 1 \\", "\tcontinuation 2") } +func (s *Suite) Test_Autofix_ReplaceAfter__autofix_several_times_in_continuation_line(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--autofix", "--source") + mklines := t.SetUpFileMkLines("Makefile", + "# line 1 \\", + "continuation 1 \\", + "continuation 2") + + fix := mklines.lines.Lines[0].Autofix() + fix.Warnf("N should be replaced with V.") + fix.ReplaceAfter("", "n", "v") + fix.Apply() + + // Nothing is logged or fixed because the "n" appears more than once, + // and as of June 2019, pkglint doesn't know which occurrence is the + // correct one. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Autofix_ReplaceAfter__autofix_one_time(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--autofix", "--source") + mklines := t.SetUpFileMkLines("Makefile", + MkCvsID, + "VAR=\t$$(var) $(var)") + + mklines.Check() + + // Nothing is replaced since, as of June 2019, pkglint doesn't + // know which of the two "$(var)" should be replaced. + t.CheckOutputEmpty() +} + func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix(c *check.C) { t := s.Init(c) @@ -157,19 +190,19 @@ func (s *Suite) Test_Autofix_ReplaceRegex__autofix(c *check.C) { // After calling fix.Apply above, the autofix is ready to be used again. fix.Warnf("Use Y instead of X.") - fix.Replace("X", "Y") + fix.Replace("XXX", "YYY") fix.Apply() t.CheckOutputLines( - "AUTOFIX: ~/Makefile:2: Replacing \"X\" with \"Y\".", + "AUTOFIX: ~/Makefile:2: Replacing \"XXX\" with \"YYY\".", "-\tline2", - "+\tYXXe2") + "+\tYYYe2") SaveAutofixChanges(lines) t.CheckFileLines("Makefile", "line1", - "YXXe2", + "YYYe2", "line3") } @@ -188,7 +221,7 @@ func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix_and_source(c *check.C) { fix.Apply() fix.Warnf("Use Y instead of X.") - fix.Replace("X", "Y") + fix.Replace("XXXXX", "YYYYY") fix.Apply() SaveAutofixChanges(lines) @@ -204,9 +237,9 @@ func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix_and_source(c *check.C) { "+\tXXXXX", "", "WARN: ~/Makefile:2: Use Y instead of X.", - "AUTOFIX: ~/Makefile:2: Replacing \"X\" with \"Y\".", + "AUTOFIX: ~/Makefile:2: Replacing \"XXXXX\" with \"YYYYY\".", "-\tline2", - "+\tYXXXX") + "+\tYYYYY") } // When an autofix replaces text, it does not touch those @@ -309,7 +342,7 @@ func (s *Suite) Test_Autofix__multiple_fixes(c *check.C) { { fix := line.Autofix() fix.Warnf(SilentAutofixFormat) - fix.Replace("i", "u") + fix.Replace("ig", "ug") fix.Apply() } @@ -317,7 +350,7 @@ func (s *Suite) Test_Autofix__multiple_fixes(c *check.C) { c.Check(line.raw, check.DeepEquals, t.NewRawLines(1, "original\n", "lruginao\n")) c.Check(line.raw[0].textnl, equals, "lruginao\n") t.CheckOutputLines( - "AUTOFIX: filename:1: Replacing \"i\" with \"u\".") + "AUTOFIX: filename:1: Replacing \"ig\" with \"ug\".") { fix := line.Autofix() @@ -429,9 +462,7 @@ func (s *Suite) Test_Autofix_Explain__SilentAutofixFormat(c *check.C) { fix := line.Autofix() fix.Warnf(SilentAutofixFormat) - t.ExpectPanic( - func() { fix.Explain("Explanation for inserting a line before.") }, - "Pkglint internal error: Autofix: Silent fixes cannot have an explanation.") + t.ExpectAssert(func() { fix.Explain("Explanation for inserting a line before.") }) } // To combine a silent diagnostic with an explanation, two separate autofixes @@ -468,7 +499,7 @@ func (s *Suite) Test_Autofix__show_autofix_and_source_continuation_line(c *check t.SetUpCommandLine("--show-autofix", "--source") mklines := t.SetUpFileMkLines("Makefile", - MkRcsID, + MkCvsID, "# before \\", "The old song \\", "after") @@ -534,7 +565,7 @@ func (s *Suite) Test_Autofix_Delete__continuation_line(c *check.C) { t.SetUpCommandLine("--show-autofix", "--source") mklines := t.SetUpFileMkLines("Makefile", - MkRcsID, + MkCvsID, "# line 1 \\", "continued") line := mklines.lines.Lines[1] @@ -713,7 +744,7 @@ func (s *Suite) Test_Autofix_Custom__in_memory(c *check.C) { "line2", "line3") - doFix := func(line Line) { + doFix := func(line *Line) { fix := line.Autofix() fix.Warnf("Please write in ALL-UPPERCASE.") fix.Custom(func(showAutofix, autofix bool) { @@ -816,20 +847,18 @@ func (s *Suite) Test_Autofix_Apply__panic(c *check.C) { line := t.NewLine("filename", 123, "text") - t.ExpectPanic( + t.ExpectAssert( func() { fix := line.Autofix() fix.Apply() - }, - "Pkglint internal error: Each autofix must have a log level and a diagnostic.") + }) - t.ExpectPanic( + t.ExpectAssert( func() { fix := line.Autofix() fix.Replace("from", "to") fix.Apply() - }, - "Pkglint internal error: Autofix: The diagnostic must be given before the action.") + }) t.ExpectPanic( func() { @@ -936,6 +965,29 @@ func (s *Suite) Test_Autofix_Apply__autofix_and_show_autofix_options(c *check.C) "AUTOFIX: filename:5: Replacing \"text\" with \"replacement\".") } +// In --autofix mode or --show-autofix mode, the fix.Anyway doesn't +// have any effect, therefore the errors from such autofixes are +// not counted, and the exitcode stays at 0. +func (s *Suite) Test_Autofix_Apply__anyway_error(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--autofix") + mklines := t.SetUpFileMkLines("filename.mk", + MkCvsID, + "VAR=\tvalue") + + fix := mklines.mklines[1].Autofix() + fix.Errorf("From must be To.") + fix.Replace("from", "to") + fix.Anyway() + fix.Apply() + + mklines.SaveAutofixChanges() + + t.Check(G.Logger.errors, equals, 0) + t.CheckOutputEmpty() +} + // Ensures that without explanations, the separator between the individual // diagnostics are generated. func (s *Suite) Test_Autofix_Apply__source_without_explain(c *check.C) { @@ -1017,16 +1069,14 @@ func (s *Suite) Test_Autofix_Realign__wrong_line_type(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("file.mk", - MkRcsID, + MkCvsID, ".if \\", "${PKGSRC_RUN_TESTS}") mkline := mklines.mklines[1] fix := mkline.Autofix() - t.ExpectPanic( - func() { fix.Realign(mkline, 16) }, - "Pkglint internal error: Line must be a variable assignment.") + t.ExpectAssert(func() { fix.Realign(mkline, 16) }) } func (s *Suite) Test_Autofix_Realign__short_continuation_line(c *check.C) { @@ -1034,7 +1084,7 @@ func (s *Suite) Test_Autofix_Realign__short_continuation_line(c *check.C) { t.SetUpCommandLine("--autofix") mklines := t.SetUpFileMkLines("file.mk", - MkRcsID, + MkCvsID, "BUILD_DIRS= \\", "\tdir \\", "") @@ -1050,7 +1100,7 @@ func (s *Suite) Test_Autofix_Realign__short_continuation_line(c *check.C) { t.CheckOutputEmpty() t.CheckFileLines("file.mk", - MkRcsID, + MkCvsID, "BUILD_DIRS= \\", "\tdir \\", "") @@ -1061,7 +1111,7 @@ func (s *Suite) Test_Autofix_Realign__multiline_indented_with_spaces(c *check.C) t.SetUpCommandLine("--autofix") mklines := t.SetUpFileMkLines("file.mk", - MkRcsID, + MkCvsID, "BUILD_DIRS= \\", "\t dir1 \\", "\t\tdir2 \\", @@ -1079,7 +1129,7 @@ func (s *Suite) Test_Autofix_Realign__multiline_indented_with_spaces(c *check.C) t.CheckOutputLines( "AUTOFIX: ~/file.mk:3: Replacing indentation \"\\t \" with \"\\t\\t\".") t.CheckFileLines("file.mk", - MkRcsID, + MkCvsID, "BUILD_DIRS= \\", "\t\tdir1 \\", "\t\tdir2 \\", @@ -1109,14 +1159,10 @@ func (s *Suite) Test_Autofix_setDiag__bad_call_sequence(c *check.C) { fix := line.Autofix() fix.Notef("Note.") - t.ExpectPanic( - func() { fix.Notef("Note 2.") }, - "Pkglint internal error: Autofix can only have a single diagnostic.") + t.ExpectAssert(func() { fix.Notef("Note 2.") }) fix.level = nil // To cover the second assertion. - t.ExpectPanic( - func() { fix.Notef("Note 2.") }, - "Pkglint internal error: Autofix can only have a single diagnostic.") + t.ExpectAssert(func() { fix.Notef("Note 2.") }) } func (s *Suite) Test_Autofix_assertRealLine(c *check.C) { @@ -1126,9 +1172,7 @@ func (s *Suite) Test_Autofix_assertRealLine(c *check.C) { fix := line.Autofix() fix.Warnf("Warning.") - t.ExpectPanic( - func() { fix.Replace("from", "to") }, - "Pkglint internal error: Cannot autofix this line since it is not a real line.") + t.ExpectAssert(func() { fix.Replace("from", "to") }) } func (s *Suite) Test_SaveAutofixChanges__file_removed(c *check.C) { @@ -1164,7 +1208,7 @@ func (s *Suite) Test_SaveAutofixChanges__file_busy_Windows(c *check.C) { // As long as the file is kept open, it cannot be overwritten or deleted. openFile, err := os.OpenFile(t.File("subdir/file.txt"), 0, 0666) - defer openFile.Close() + defer func() { assertNil(openFile.Close(), "") }() c.Check(err, check.IsNil) fix := lines.Lines[0].Autofix() @@ -1220,7 +1264,7 @@ func (s *Suite) Test_Autofix__lonely_source(c *check.C) { "DISTNAME=\txorgproto-1.0") t.CreateFileDummyBuildlink3("x11/xorgproto/buildlink3.mk") t.CreateFileLines("x11/xorgproto/builtin.mk", - MkRcsID, + MkCvsID, "", "BUILTIN_PKG:=\txorgproto", "", diff --git a/pkgtools/pkglint/files/buildlink3.go b/pkgtools/pkglint/files/buildlink3.go index c396a124d07..f87acee5232 100644 --- a/pkgtools/pkglint/files/buildlink3.go +++ b/pkgtools/pkglint/files/buildlink3.go @@ -7,28 +7,28 @@ import ( ) type Buildlink3Checker struct { - mklines MkLines + mklines *MkLines pkgbase string - pkgbaseLine MkLine - abiLine, apiLine MkLine + pkgbaseLine *MkLine + abiLine, apiLine *MkLine abi, api *DependencyPattern } -func CheckLinesBuildlink3Mk(mklines MkLines) { +func CheckLinesBuildlink3Mk(mklines *MkLines) { (&Buildlink3Checker{mklines: mklines}).Check() } func (ck *Buildlink3Checker) Check() { mklines := ck.mklines if trace.Tracing { - defer trace.Call1(mklines.lines.FileName)() + defer trace.Call1(mklines.lines.Filename)() } mklines.Check() llex := NewMkLinesLexer(mklines) - for llex.SkipIf(MkLine.IsComment) { + for llex.SkipIf((*MkLine).IsComment) { line := llex.PreviousLine() // See pkgtools/createbuildlink/files/createbuildlink if hasPrefix(line.Text, "# XXX This file was created automatically") { @@ -94,7 +94,7 @@ func (ck *Buildlink3Checker) checkFirstParagraph(mlex *MkLinesLexer) bool { return true } -func (ck *Buildlink3Checker) checkUniquePkgbase(pkgbase string, mkline MkLine) { +func (ck *Buildlink3Checker) checkUniquePkgbase(pkgbase string, mkline *MkLine) { prev := G.InterPackage.Bl3(pkgbase, &mkline.Location) if prev == nil { return @@ -182,7 +182,7 @@ func (ck *Buildlink3Checker) checkMainPart(mlex *MkLinesLexer) bool { return true } -func (ck *Buildlink3Checker) checkVarassign(mlex *MkLinesLexer, mkline MkLine, pkgbase string) { +func (ck *Buildlink3Checker) checkVarassign(mlex *MkLinesLexer, mkline *MkLine, pkgbase string) { varname, value := mkline.Varname(), mkline.Value() doCheck := false @@ -229,7 +229,7 @@ func (ck *Buildlink3Checker) checkVarassign(mlex *MkLinesLexer, mkline MkLine, p } } -func (ck *Buildlink3Checker) checkVaruseInPkgbase(pkgbase string, pkgbaseLine MkLine) { +func (ck *Buildlink3Checker) checkVaruseInPkgbase(pkgbase string, pkgbaseLine *MkLine) { tokens, _ := pkgbaseLine.ValueTokens() for _, token := range tokens { if token.Varuse == nil { diff --git a/pkgtools/pkglint/files/buildlink3_test.go b/pkgtools/pkglint/files/buildlink3_test.go index 5d1a50daf90..17d7d86cce8 100644 --- a/pkgtools/pkglint/files/buildlink3_test.go +++ b/pkgtools/pkglint/files/buildlink3_test.go @@ -9,12 +9,10 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__package(c *check.C) { t := s.Init(c) t.CreateFileLines("category/dependency1/buildlink3.mk", - MkRcsID) + MkCvsID) t.CreateFileLines("category/dependency2/buildlink3.mk", - MkRcsID) + MkCvsID) t.SetUpPackage("category/package", - "PKGNAME=\tpackage-1.0", - "", ".include \"../../category/dependency1/buildlink3.mk\"") t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", @@ -36,7 +34,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__unfinished_url2pkg(c *check.C) { t.CreateFileLines("x11/Xbae/Makefile") t.CreateFileLines("mk/motif.buildlink3.mk") mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", - MkRcsID, + MkCvsID, "# XXX This file was created automatically using createbuildlink-@PKGVERSION@", "", "BUILDLINK_TREE+=\tXbae", @@ -73,7 +71,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_Haskell_incomplete(c "DISTNAME=\tX11-1.0") t.Chdir("x11/hs-X11") t.CreateFileLines("buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\ths-X11", "", @@ -105,7 +103,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_Haskell_complete(c *c t := s.Init(c) t.CreateFileLines("mk/haskell.mk", - MkRcsID, + MkCvsID, "PKGNAME?=\ths-${DISTNAME}") t.SetUpPackage("x11/hs-X11", "DISTNAME=\tX11-1.0", @@ -113,7 +111,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_Haskell_complete(c *c ".include \"../../mk/haskell.mk\"") t.Chdir("x11/hs-X11") t.CreateFileLines("buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\ths-X11", "", @@ -140,7 +138,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch__Perl(c *check.C) { "DISTNAME=\tGtk2-1.0", "PKGNAME=\t${DISTNAME:C:Gtk2:p5-gtk2:}") t.CreateFileLines("x11/p5-gtk2/buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\tp5-gtk2", "", @@ -171,7 +169,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_multiple_inclusion(c t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\tpkgbase1", "", @@ -195,7 +193,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_abi_api(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\ths-X11", "", @@ -222,7 +220,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__abi_api_versions(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\ths-X11", "", @@ -252,7 +250,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_api_versions_brace(c t.SetUpVartypes() t.CreateFileLines("multimedia/totem/Makefile") mklines := t.SetUpFileMkLines("multimedia/totem/buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\ttotem", "", @@ -279,7 +277,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__missing_BUILDLINK_TREE_at_beginning t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "", ".if !defined(HS_X11_BUILDLINK3_MK)", "HS_X11_BUILDLINK3_MK:=", @@ -298,7 +296,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__missing_BUILDLINK_TREE_at_end(c *ch t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\ths-X11", "", @@ -324,7 +322,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__DEPMETHOD_placement(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_DEPMETHOD.hs-X11?=\tfull", "", @@ -351,7 +349,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__multiple_inclusion_wrong(c *check.C t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\ths-X11", "", @@ -372,7 +370,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__missing_endif(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\tpkgbase1", "", @@ -391,7 +389,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__invalid_dependency_patterns(c *chec t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\ths-X11", "", @@ -418,7 +416,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_variable(c *check.C) { t.SetUpVartypes() mklinesPhp := t.NewMkLines("x11/php-wxwidgets/buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\t${PHP_PKG_PREFIX}-wxWidgets", "", @@ -432,7 +430,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_variable(c *check.C) { "", "BUILDLINK_TREE+=\t-${PHP_PKG_PREFIX}-wxWidgets") mklinesPy := t.NewMkLines("x11/py-wxwidgets/buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\t${PYPKGPREFIX}-wxWidgets", "", @@ -446,7 +444,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_variable(c *check.C) { "", "BUILDLINK_TREE+=\t-${PYPKGPREFIX}-wxWidgets") mklinesRuby1 := t.NewMkLines("x11/ruby1-wxwidgets/buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\t${RUBY_BASE}-wxWidgets", "", @@ -460,7 +458,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_variable(c *check.C) { "", "BUILDLINK_TREE+=\t-${RUBY_BASE}-wxWidgets") mklinesRuby2 := t.NewMkLines("x11/ruby2-wxwidgets/buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\t${RUBY_PKGPREFIX}-wxWidgets", "", @@ -491,7 +489,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_unknown_variable(c *ch t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\t${LICENSE}-wxWidgets", "", @@ -576,8 +574,7 @@ func (s *Suite) Test_Buildlink3Checker_checkUniquePkgbase(c *check.C) { func (s *Suite) Test_Buildlink3Checker_checkMainPart__if_else_endif(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "PKGNAME=\tpackage-1.0") + t.SetUpPackage("category/package") t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", ".if ${X11_TYPE} == modular", ".else", @@ -592,8 +589,7 @@ func (s *Suite) Test_Buildlink3Checker_checkMainPart__if_else_endif(c *check.C) func (s *Suite) Test_Buildlink3Checker_checkVarassign__dependencies_with_path(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "PKGNAME=\tpackage-1.0") + t.SetUpPackage("category/package") t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", "BUILDLINK_ABI_DEPENDS.package+=\tpackage>=1.0:../../category/package", "BUILDLINK_API_DEPENDS.package+=\tpackage>=1.5:../../category/package") @@ -613,11 +609,10 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__dependencies_with_path(c func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_without_api(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "PKGNAME=\tpackage-1.0") + t.SetUpPackage("category/package") // t.CreateFileDummyBuildlink3() cannot be used here since it always adds an API line. t.CreateFileLines("category/package/buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\tpackage", "", @@ -643,8 +638,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_without_api(c *check. func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_and_api_with_variables(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "PKGNAME=\tpackage-1.0") + t.SetUpPackage("category/package") t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", "BUILDLINK_ABI_DEPENDS.package+=\tpackage>=${ABI_VERSION}", "BUILDLINK_API_DEPENDS.package+=\tpackage>=${API_VERSION}", @@ -662,8 +656,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_and_api_with_variable func (s *Suite) Test_Buildlink3Checker_checkVarassign__api_with_variable(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "PKGNAME=\tpackage-1.0") + t.SetUpPackage("category/package") t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", "BUILDLINK_ABI_DEPENDS.package+=\tpackage>=1.0", "BUILDLINK_API_DEPENDS.package+=\tpackage>=${API_VERSION}", @@ -680,8 +673,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__api_with_variable(c *chec func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_and_api_with_pattern(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "PKGNAME=\tpackage-1.0") + t.SetUpPackage("category/package") t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", "BUILDLINK_ABI_DEPENDS.package+=\tpackage-1.*", "BUILDLINK_API_DEPENDS.package+=\tpackage-2.*") @@ -697,8 +689,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_and_api_with_pattern( func (s *Suite) Test_Buildlink3Checker_checkVarassign__api_with_pattern(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "PKGNAME=\tpackage-1.0") + t.SetUpPackage("category/package") t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", "BUILDLINK_ABI_DEPENDS.package+=\tpackage>=1", "BUILDLINK_API_DEPENDS.package+=\tpackage-1.*") @@ -714,8 +705,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__api_with_pattern(c *check func (s *Suite) Test_Buildlink3Checker_checkVarassign__other_variables(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "PKGNAME=\tpackage-1.0") + t.SetUpPackage("category/package") t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", "BUILDLINK_TREE+=\tmistake", // Wrong, but doesn't happen in practice. "", @@ -762,6 +752,8 @@ func (s *Suite) Test_Buildlink3Checker_checkSecondParagraph__missing_mkbase(c *c // There is no warning from buildlink3.mk about mismatched package names // since that is only a follow-up error of being unable to parse the pkgbase. t.CheckOutputLines( + "WARN: ~/category/package/Makefile:3: As DISTNAME is not a valid package name, "+ + "please define the PKGNAME explicitly.", "WARN: ~/category/package/Makefile:4: \"\" is not a valid package name.") } @@ -772,7 +764,7 @@ func (s *Suite) Test_Buildlink3Checker_checkMainPart__nested_if(c *check.C) { t.SetUpVartypes() mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\ths-X11", "", @@ -799,7 +791,7 @@ func (s *Suite) Test_Buildlink3Checker_checkMainPart__comment_at_end_of_file(c * t.SetUpVartypes() mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", - MkRcsID, + MkCvsID, "", "BUILDLINK_TREE+=\ths-X11", "", diff --git a/pkgtools/pkglint/files/category.go b/pkgtools/pkglint/files/category.go index 46a0c2a0a61..4519df9c604 100644 --- a/pkgtools/pkglint/files/category.go +++ b/pkgtools/pkglint/files/category.go @@ -23,7 +23,7 @@ func CheckdirCategory(dir string) { } mlex.SkipEmptyOrNote() - if mlex.SkipIf(func(mkline MkLine) bool { return mkline.IsVarassign() && mkline.Varname() == "COMMENT" }) { + if mlex.SkipIf(func(mkline *MkLine) bool { return mkline.IsVarassign() && mkline.Varname() == "COMMENT" }) { mkline := mlex.PreviousMkLine() lex := textproc.NewLexer(mkline.Value()) @@ -50,7 +50,7 @@ func CheckdirCategory(dir string) { type subdir struct { name string - line MkLine + line *MkLine } // And now to the most complicated part of the category Makefiles, @@ -60,7 +60,7 @@ func CheckdirCategory(dir string) { fSubdirs := getSubdirs(dir) var mSubdirs []subdir - seen := make(map[string]MkLine) + seen := make(map[string]*MkLine) for !mlex.EOF() { mkline := mlex.CurrentMkLine() @@ -113,7 +113,7 @@ func CheckdirCategory(dir string) { if len(fRest) > 0 && (len(mRest) == 0 || fRest[0] < mRest[0].name) { fCurrent := fRest[0] if !mCheck[fCurrent] { - var line Line + var line *Line if len(mRest) > 0 { line = mRest[0].line.Line } else { diff --git a/pkgtools/pkglint/files/category_test.go b/pkgtools/pkglint/files/category_test.go index c92769fffd4..ee49f398c7b 100644 --- a/pkgtools/pkglint/files/category_test.go +++ b/pkgtools/pkglint/files/category_test.go @@ -41,7 +41,7 @@ func (s *Suite) Test_CheckdirCategory__invalid_comment(c *check.C) { t.SetUpVartypes() t.CreateFileLines("archivers/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\t\\Make $$$$ fast\"", "", @@ -72,7 +72,7 @@ func (s *Suite) Test_CheckdirCategory__wip(c *check.C) { t.CreateFileLines("mk/misc/category.mk") t.CreateFileLines("wip/package/Makefile") t.CreateFileLines("wip/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tCategory comment", "", @@ -102,7 +102,7 @@ func (s *Suite) Test_CheckdirCategory__subdirs(c *check.C) { t.CreateFileLines("category/commented-mk-and-fs/Makefile") t.CreateFileLines("category/commented-without-reason/Makefile") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tCategory comment", "", @@ -137,7 +137,7 @@ func (s *Suite) Test_CheckdirCategory__only_in_Makefile(c *check.C) { t.CreateFileLines("mk/misc/category.mk") t.CreateFileLines("category/both/Makefile") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tCategory comment", "", @@ -166,7 +166,7 @@ func (s *Suite) Test_CheckdirCategory__only_in_file_system(c *check.C) { t.CreateFileLines("category/both/Makefile") t.CreateFileLines("category/only-in-fs/Makefile") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tCategory comment", "", @@ -193,7 +193,7 @@ func (s *Suite) Test_CheckdirCategory__recursive(c *check.C) { t.CreateFileLines("category/commented/Makefile") t.CreateFileLines("category/package/Makefile") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tCategory comment", "", @@ -204,13 +204,22 @@ func (s *Suite) Test_CheckdirCategory__recursive(c *check.C) { t.Chdir("category") t.FinishSetUp() + // The default argument "." is added when parsing the command line. + // It is only removed in Pkglint.Main, therefore it stays there even + // after the call to CheckdirCategory. This is a bit unrealistic, + // but close enough for this test. + t.Check( + G.Todo, + deepEquals, + []string{"."}) + CheckdirCategory(".") t.CheckOutputEmpty() t.Check( G.Todo, deepEquals, - []string{"./package"}) + []string{"./package", "."}) } // Ensures that a directory in the file system can be added at the very @@ -230,7 +239,7 @@ func (s *Suite) Test_CheckdirCategory__subdirs_file_system_at_the_bottom(c *chec t.CreateFileLines("category/mk-and-fs/Makefile") t.CreateFileLines("category/zzz-fs-only/Makefile") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tCategory comment", "", @@ -254,7 +263,7 @@ func (s *Suite) Test_CheckdirCategory__indentation(c *check.C) { t.CreateFileLines("category/package1/Makefile") t.CreateFileLines("category/package2/Makefile") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tCategory comment", "", @@ -277,7 +286,7 @@ func (s *Suite) Test_CheckdirCategory__comment_at_the_top(c *check.C) { t.CreateFileLines("mk/misc/category.mk") t.CreateFileLines("category/package/Makefile") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "", "# This category collects all programs that don't fit anywhere else.", "", @@ -312,7 +321,7 @@ func (s *Suite) Test_CheckdirCategory__unexpected_EOF_while_reading_SUBDIR(c *ch t.CreateFileLines("mk/misc/category.mk") t.CreateFileLines("category/package/Makefile") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tCategory comment", "", diff --git a/pkgtools/pkglint/files/check_test.go b/pkgtools/pkglint/files/check_test.go index 32e247f227f..493084c1182 100644 --- a/pkgtools/pkglint/files/check_test.go +++ b/pkgtools/pkglint/files/check_test.go @@ -19,9 +19,9 @@ import ( var equals = check.Equals var deepEquals = check.DeepEquals -const RcsID = "$" + "NetBSD$" -const MkRcsID = "# $" + "NetBSD$" -const PlistRcsID = "@comment $" + "NetBSD$" +const CvsID = "$" + "NetBSD$" +const MkCvsID = "# $" + "NetBSD$" +const PlistCvsID = "@comment $" + "NetBSD$" type Suite struct { Tester *Tester @@ -72,29 +72,28 @@ func (s *Suite) SetUpTest(c *check.C) { t.c = c t.SetUpCommandLine("-Wall") // To catch duplicate warnings - t.c = nil // To improve code coverage and ensure that trace.Result works // in all cases. The latter cannot be ensured at compile time. t.EnableSilentTracing() prevdir, err := os.Getwd() - if err != nil { - c.Fatalf("Cannot get current working directory: %s", err) - } + assertNil(err, "Cannot get current working directory: %s", err) t.prevdir = prevdir + + // No longer usable; see https://github.com/go-check/check/issues/22 + t.c = nil } func (s *Suite) TearDownTest(c *check.C) { t := s.Tester t.c = nil // No longer usable; see https://github.com/go-check/check/issues/22 - if err := os.Chdir(t.prevdir); err != nil { - t.Errorf("Cannot chdir back to previous dir: %s", err) - } + err := os.Chdir(t.prevdir) + assertNil(err, "Cannot chdir back to previous dir: %s", err) if t.seenSetupPkgsrc > 0 && !t.seenFinish && !t.seenMain { - t.Errorf("After t.SetupPkgsrc(), t.FinishSetUp() or t.Main() must be called.") + t.Errorf("After t.SetupPkgsrc(), either t.FinishSetUp() or t.Main() must be called.") } if out := t.Output(); out != "" { @@ -202,7 +201,7 @@ func (t *Tester) SetUpTool(name, varname string, validity Validity) *Tool { // The file is then read in, without interpreting line continuations. // // See SetUpFileMkLines for loading a Makefile fragment. -func (t *Tester) SetUpFileLines(relativeFileName string, lines ...string) Lines { +func (t *Tester) SetUpFileLines(relativeFileName string, lines ...string) *Lines { filename := t.CreateFileLines(relativeFileName, lines...) return Load(filename, MustSucceed) } @@ -211,7 +210,7 @@ func (t *Tester) SetUpFileLines(relativeFileName string, lines ...string) Lines // The file is then read in, handling line continuations for Makefiles. // // See SetUpFileLines for loading an ordinary file. -func (t *Tester) SetUpFileMkLines(relativeFileName string, lines ...string) MkLines { +func (t *Tester) SetUpFileMkLines(relativeFileName string, lines ...string) *MkLines { filename := t.CreateFileLines(relativeFileName, lines...) return LoadMk(filename, MustSucceed) } @@ -220,8 +219,8 @@ func (t *Tester) SetUpFileMkLines(relativeFileName string, lines ...string) MkLi // merging all the lines into a single MkLines object. // // This is useful for testing code related to Package.readMakefile. -func (t *Tester) LoadMkInclude(relativeFileName string) MkLines { - var lines []Line +func (t *Tester) LoadMkInclude(relativeFileName string) *MkLines { + var lines []*Line // TODO: Include files with multiple-inclusion guard only once. // TODO: Include files without multiple-inclusion guard as often as needed. @@ -251,23 +250,20 @@ func (t *Tester) LoadMkInclude(relativeFileName string) MkLines { // Individual files may be overwritten by calling other SetUp* methods. // // This setup is especially interesting for testing Pkglint.Main. -// -// If the test works on a lower level than Pkglint.Main, -// LoadInfrastructure must be called to actually load the infrastructure files. func (t *Tester) SetUpPkgsrc() { // This file is needed to locate the pkgsrc root directory. // See findPkgsrcTopdir. t.CreateFileLines("mk/bsd.pkg.mk", - MkRcsID) + MkCvsID) // See Pkgsrc.loadDocChanges. t.CreateFileLines("doc/CHANGES-2018", - RcsID) + CvsID) // See Pkgsrc.loadSuggestedUpdates. t.CreateFileLines("doc/TODO", - RcsID) + CvsID) // Some example licenses so that the tests for whole packages // don't need to define them on their own. @@ -283,7 +279,7 @@ func (t *Tester) SetUpPkgsrc() { // // See Pkgsrc.loadMasterSites. t.CreateFileLines("mk/fetch/sites.mk", - MkRcsID) + MkCvsID) // The options for the PKG_OPTIONS framework are defined here. // @@ -295,7 +291,7 @@ func (t *Tester) SetUpPkgsrc() { // The user-defined variables are read in to check for missing // BUILD_DEFS declarations in the package Makefile. t.CreateFileLines("mk/defaults/mk.conf", - MkRcsID) + MkCvsID) // The tool definitions are defined in various files in mk/tools/. // The relevant files are listed in bsd.tools.mk. @@ -303,14 +299,14 @@ func (t *Tester) SetUpPkgsrc() { t.CreateFileLines("mk/tools/bsd.tools.mk", ".include \"defaults.mk\"") t.CreateFileLines("mk/tools/defaults.mk", - MkRcsID) + MkCvsID) // Those tools that are added to USE_TOOLS in bsd.prefs.mk may be // used at load time by packages. t.CreateFileLines("mk/bsd.prefs.mk", - MkRcsID) + MkCvsID) t.CreateFileLines("mk/bsd.fast.prefs.mk", - MkRcsID) + MkCvsID) // Category Makefiles require this file for the common definitions. t.CreateFileLines("mk/misc/category.mk") @@ -321,11 +317,11 @@ func (t *Tester) SetUpPkgsrc() { // SetUpCategory makes the given category valid by creating a dummy Makefile. // After that, it can be mentioned in the CATEGORIES variable of a package. func (t *Tester) SetUpCategory(name string) { - assertf(!contains(name, "/"), "Category must not contain a slash.") + assert(!contains(name, "/")) // Category must not contain a slash. if _, err := os.Stat(t.File(name + "/Makefile")); os.IsNotExist(err) { t.CreateFileLines(name+"/Makefile", - MkRcsID) + MkCvsID) } } @@ -341,7 +337,9 @@ func (t *Tester) SetUpCategory(name string) { // At the end of the setup phase, t.FinishSetUp() must be called to load all // the files. func (t *Tester) SetUpPackage(pkgpath string, makefileLines ...string) string { + assertf(matches(pkgpath, `^[^/]+/[^/]+$`), "pkgpath %q must have the form \"category/package\"", pkgpath) + distname := path.Base(pkgpath) category := path.Dir(pkgpath) if category == "wip" { // To avoid boilerplate CATEGORIES definitions for wip packages. @@ -354,7 +352,7 @@ func (t *Tester) SetUpPackage(pkgpath string, makefileLines ...string) string { t.CreateFileLines(pkgpath+"/DESCR", "Package description") t.CreateFileLines(pkgpath+"/PLIST", - PlistRcsID, + PlistCvsID, "bin/program") // Because the package Makefile includes this file, the check for the @@ -364,12 +362,12 @@ func (t *Tester) SetUpPackage(pkgpath string, makefileLines ...string) string { // unrelated warnings about the variable order, that check is suppressed // here. t.CreateFileLines(pkgpath+"/suppress-varorder.mk", - MkRcsID) + MkCvsID) // This distinfo file contains dummy hashes since pkglint cannot check the // distfiles hashes anyway. It can only check the hashes for the patches. t.CreateFileLines(pkgpath+"/distinfo", - RcsID, + CvsID, "", "SHA1 (distfile-1.0.tar.gz) = 12341234", "RMD160 (distfile-1.0.tar.gz) = 12341234", @@ -377,9 +375,9 @@ func (t *Tester) SetUpPackage(pkgpath string, makefileLines ...string) string { "Size (distfile-1.0.tar.gz) = 12341234") mlines := []string{ - MkRcsID, + MkCvsID, "", - "DISTNAME=\tdistname-1.0", + "DISTNAME=\t" + distname + "-1.0", "#PKGNAME=\tpackage-1.0", "CATEGORIES=\t" + category, "MASTER_SITES=\t# none", @@ -396,6 +394,8 @@ func (t *Tester) SetUpPackage(pkgpath string, makefileLines ...string) string { line: for _, line := range makefileLines { + assert(!hasSuffix(line, "\\")) // Continuation lines are not yet supported. + if m, prefix := match1(line, `^#?(\w+=)`); m { for i, existingLine := range mlines[:19] { if hasPrefix(strings.TrimPrefix(existingLine, "#"), prefix) { @@ -444,7 +444,7 @@ func (t *Tester) CreateFileLines(relativeFileName string, lines ...string) (file // temporary directory. func (t *Tester) CreateFileDummyPatch(relativeFileName string) { t.CreateFileLines(relativeFileName, - RcsID, + CvsID, "", "Documentation", "", @@ -458,7 +458,8 @@ func (t *Tester) CreateFileDummyPatch(relativeFileName string) { func (t *Tester) CreateFileDummyBuildlink3(relativeFileName string, customLines ...string) { dir := path.Dir(relativeFileName) lower := path.Base(dir) - upper := strings.ToUpper(lower) + // see pkgtools/createbuildlink/files/createbuildlink, "package specific variables" + upper := strings.Replace(strings.ToUpper(lower), "-", "_", -1) width := tabWidth(sprintf("BUILDLINK_API_DEPENDS.%s+=\t", lower)) @@ -472,7 +473,7 @@ func (t *Tester) CreateFileDummyBuildlink3(relativeFileName string, customLines var lines []string lines = append(lines, - MkRcsID, + MkCvsID, "", sprintf("BUILDLINK_TREE+=\t%s", lower), "", @@ -508,9 +509,14 @@ func (t *Tester) File(relativeFileName string) string { // Copy copies a file inside the temporary directory. func (t *Tester) Copy(relativeSrc, relativeDst string) { - data, err := ioutil.ReadFile(t.File(relativeSrc)) + src := t.File(relativeSrc) + dst := t.File(relativeDst) + + data, err := ioutil.ReadFile(src) assertNil(err, "Copy.Read") - err = ioutil.WriteFile(t.File(relativeDst), data, 0777) + err = os.MkdirAll(path.Dir(dst), 0777) + assertNil(err, "Copy.MkdirAll") + err = ioutil.WriteFile(dst, data, 0777) assertNil(err, "Copy.Write") } @@ -536,15 +542,14 @@ func (t *Tester) Chdir(relativeDirName string) { } absDirName := t.File(relativeDirName) - _ = os.MkdirAll(absDirName, 0700) - if err := os.Chdir(absDirName); err != nil { - t.c.Fatalf("Cannot chdir: %s", err) - } + assertNil(os.MkdirAll(absDirName, 0700), "MkDirAll") + assertNil(os.Chdir(absDirName), "Chdir") t.relCwd = relativeDirName G.cwd = absDirName } -// Remove removes the file from the temporary directory. The file must exist. +// Remove removes the file or directory from the temporary directory. +// The file or directory must exist. func (t *Tester) Remove(relativeFileName string) { filename := t.File(relativeFileName) err := os.Remove(filename) @@ -577,13 +582,13 @@ func (t *Tester) Remove(relativeFileName string) { // subdir/module.mk includes subdir/version.mk, the include line is just: // .include "version.mk" func (t *Tester) SetUpHierarchy() ( - include func(filename string, args ...interface{}) MkLines, - get func(string) MkLines) { + include func(filename string, args ...interface{}) *MkLines, + get func(string) *MkLines) { - files := map[string]MkLines{} + files := map[string]*MkLines{} - include = func(filename string, args ...interface{}) MkLines { - var lines []Line + include = func(filename string, args ...interface{}) *MkLines { + var lines []*Line lineno := 1 addLine := func(text string) { @@ -595,8 +600,8 @@ func (t *Tester) SetUpHierarchy() ( switch arg := arg.(type) { case string: addLine(arg) - case MkLines: - text := sprintf(".include %q", relpath(path.Dir(filename), arg.lines.FileName)) + case *MkLines: + text := sprintf(".include %q", relpath(path.Dir(filename), arg.lines.Filename)) addLine(text) lines = append(lines, arg.lines.Lines...) default: @@ -610,7 +615,7 @@ func (t *Tester) SetUpHierarchy() ( return mklines } - get = func(filename string) MkLines { + get = func(filename string) *MkLines { assertf(files[filename] != nil, "MkLines with name %q doesn't exist.", filename) return files[filename] } @@ -636,7 +641,7 @@ func (s *Suite) Test_Tester_SetUpHierarchy(c *check.C) { mklines := get("including.mk") - mklines.ForEach(func(mkline MkLine) { mkline.Notef("Text is: %s", mkline.Text) }) + mklines.ForEach(func(mkline *MkLine) { mkline.Notef("Text is: %s", mkline.Text) }) t.CheckOutputLines( "NOTE: including.mk:1: Text is: .include \"other.mk\"", @@ -690,7 +695,7 @@ func (t *Tester) Main(args ...string) int { } } - return G.Main(argv...) + return G.Main(&t.stdout, &t.stderr, argv) } // Check delegates a check to the check.Check function. @@ -751,7 +756,7 @@ func (t *Tester) ExpectFatalMatches(action func(), expected regex.Pattern) { } // ExpectPanic runs the given action and expects that this action calls -// Pkglint.Assertf or uses some other way to panic. +// assertf or uses some other way to panic. // // Usage: // t.ExpectPanic( @@ -761,6 +766,15 @@ func (t *Tester) ExpectPanic(action func(), expectedMessage string) { t.Check(action, check.Panics, expectedMessage) } +// ExpectAssert runs the given action and expects that this action calls assert. +// +// Usage: +// t.ExpectAssert( +// func() { /* do something that panics */ }) +func (t *Tester) ExpectAssert(action func()) { + t.Check(action, check.Panics, "Pkglint internal error") +} + // NewRawLines creates lines from line numbers and raw text, including newlines. // // Arguments are sequences of either (lineno, orignl) or (lineno, orignl, textnl). @@ -788,14 +802,14 @@ func (t *Tester) NewRawLines(args ...interface{}) []*RawLine { // NewLine creates an in-memory line with the given text. // This line does not correspond to any line in a file. -func (t *Tester) NewLine(filename string, lineno int, text string) Line { +func (t *Tester) NewLine(filename string, lineno int, text string) *Line { textnl := text + "\n" rawLine := RawLine{lineno, textnl, textnl} return NewLine(filename, lineno, text, &rawLine) } // NewMkLine creates an in-memory line in the Makefile format with the given text. -func (t *Tester) NewMkLine(filename string, lineno int, text string) MkLine { +func (t *Tester) NewMkLine(filename string, lineno int, text string) *MkLine { basename := path.Base(filename) assertf( hasSuffix(basename, ".mk") || @@ -815,15 +829,15 @@ func (t *Tester) NewShellLineChecker(text string) *ShellLineChecker { // NewLines returns a list of simple lines that belong together. // // To work with line continuations like in Makefiles, use SetUpFileMkLines. -func (t *Tester) NewLines(filename string, lines ...string) Lines { +func (t *Tester) NewLines(filename string, lines ...string) *Lines { return t.NewLinesAt(filename, 1, lines...) } // NewLinesAt returns a list of simple lines that belong together. // // To work with line continuations like in Makefiles, use SetUpFileMkLines. -func (t *Tester) NewLinesAt(filename string, firstLine int, texts ...string) Lines { - lines := make([]Line, len(texts)) +func (t *Tester) NewLinesAt(filename string, firstLine int, texts ...string) *Lines { + lines := make([]*Line, len(texts)) for i, text := range texts { lines[i] = t.NewLine(filename, i+firstLine, text) } @@ -836,7 +850,7 @@ func (t *Tester) NewLinesAt(filename string, firstLine int, texts ...string) Lin // // No actual file is created for the lines; // see SetUpFileMkLines for loading Makefile fragments with line continuations. -func (t *Tester) NewMkLines(filename string, lines ...string) MkLines { +func (t *Tester) NewMkLines(filename string, lines ...string) *MkLines { basename := path.Base(filename) assertf( hasSuffix(basename, ".mk") || basename == "Makefile" || hasPrefix(basename, "Makefile."), @@ -888,6 +902,25 @@ func (t *Tester) CheckOutputLines(expectedLines ...string) { t.CheckOutput(expectedLines) } +// CheckOutputLinesMatching checks that the lines from the output that match +// the given pattern equal the given lines. +// +// After the comparison, the output buffers are cleared so that later +// calls only check against the newly added output. +// +// See CheckOutputEmpty, CheckOutputLinesIgnoreSpace. +func (t *Tester) CheckOutputLinesMatching(pattern regex.Pattern, expectedLines ...string) { + output := t.Output() + var actualLines []string + actualLines = append(actualLines) + for _, line := range strings.Split(strings.TrimSuffix(output, "\n"), "\n") { + if matches(line, pattern) { + actualLines = append(actualLines, line) + } + } + t.Check(emptyToNil(actualLines), deepEquals, emptyToNil(expectedLines)) +} + // CheckOutputLinesIgnoreSpace checks that the output up to now equals the given lines. // During comparison, each run of whitespace (space, tab, newline) is normalized so that // different line breaks are ignored. This is useful for testing line-wrapped explanations. @@ -982,11 +1015,7 @@ func (t *Tester) CheckOutputMatches(expectedLines ...regex.Pattern) { pattern := `^(?:` + string(expectedLine) + `)$` re, err := regexp.Compile(pattern) - if err != nil { - return false - } - - return re.MatchString(actualLine) + return err == nil && re.MatchString(actualLine) } // If a line matches the corresponding pattern, make them equal in the diff --git a/pkgtools/pkglint/files/cmd/pkglint/main.go b/pkgtools/pkglint/files/cmd/pkglint/main.go index 8b4abedc693..fd612ba3965 100644 --- a/pkgtools/pkglint/files/cmd/pkglint/main.go +++ b/pkgtools/pkglint/files/cmd/pkglint/main.go @@ -8,5 +8,5 @@ import ( var exit = os.Exit func main() { - exit(pkglint.Main()) + exit(pkglint.G.Main(os.Stdout, os.Stderr, os.Args)) } diff --git a/pkgtools/pkglint/files/distinfo.go b/pkgtools/pkglint/files/distinfo.go index 0aca9263a6b..e88e7a8157f 100644 --- a/pkgtools/pkglint/files/distinfo.go +++ b/pkgtools/pkglint/files/distinfo.go @@ -13,12 +13,12 @@ import ( "strings" ) -func CheckLinesDistinfo(pkg *Package, lines Lines) { +func CheckLinesDistinfo(pkg *Package, lines *Lines) { if trace.Tracing { - defer trace.Call1(lines.FileName)() + defer trace.Call1(lines.Filename)() } - filename := lines.FileName + filename := lines.Filename patchdir := "patches" if pkg != nil && dirExists(pkg.File(pkg.Patchdir)) { patchdir = pkg.Patchdir @@ -41,7 +41,7 @@ func CheckLinesDistinfo(pkg *Package, lines Lines) { type distinfoLinesChecker struct { pkg *Package - lines Lines + lines *Lines patchdir string // Relative to pkg distinfoIsCommitted bool @@ -53,7 +53,7 @@ func (ck *distinfoLinesChecker) parse() { lines := ck.lines llex := NewLinesLexer(lines) - if lines.CheckRcsID(0, ``, "") { + if lines.CheckCvsID(0, ``, "") { llex.Skip() } llex.SkipEmptyOrNote() @@ -273,7 +273,7 @@ func (ck *distinfoLinesChecker) checkAlgorithmsDistfile(info distinfoFileInfo) { // that the distfile is the expected one. Now generate the missing hashes // and insert them, in the correct order. - var insertion Line + var insertion *Line var remainingHashes = info.hashes for _, alg := range algorithms { if missing[alg] { @@ -313,7 +313,7 @@ func (ck *distinfoLinesChecker) checkUnrecordedPatches() { for _, file := range patchFiles { patchName := file.Name() if file.Mode().IsRegular() && ck.infos[patchName].isPatch != yes && hasPrefix(patchName, "patch-") { - line := NewLineWhole(ck.lines.FileName) + line := NewLineWhole(ck.lines.Filename) line.Errorf("Patch %q is not recorded. Run %q.", line.PathToFile(ck.pkg.File(ck.patchdir+"/"+patchName)), bmake("makepatchsum")) @@ -381,12 +381,14 @@ func (ck *distinfoLinesChecker) checkUncommittedPatch(info distinfoHash) { } } -func (ck *distinfoLinesChecker) checkPatchSha1(line Line, patchFileName, distinfoSha1Hex string) { - fileSha1Hex, err := computePatchSha1Hex(ck.pkg.File(patchFileName)) - if err != nil { +func (ck *distinfoLinesChecker) checkPatchSha1(line *Line, patchFileName, distinfoSha1Hex string) { + lines := Load(ck.pkg.File(patchFileName), 0) + if lines == nil { line.Errorf("Patch %s does not exist.", patchFileName) return } + + fileSha1Hex := computePatchSha1Hex(lines) if distinfoSha1Hex != fileSha1Hex { fix := line.Autofix() fix.Errorf("SHA1 hash of %s differs (distinfo has %s, patch file has %s).", @@ -408,7 +410,7 @@ type distinfoFileInfo struct { } func (info *distinfoFileInfo) filename() string { return info.hashes[0].filename } -func (info *distinfoFileInfo) line() Line { return info.hashes[0].line } +func (info *distinfoFileInfo) line() *Line { return info.hashes[0].line } func (info *distinfoFileInfo) algorithms() string { var algs []string @@ -419,25 +421,24 @@ func (info *distinfoFileInfo) algorithms() string { } type distinfoHash struct { - line Line + line *Line filename string algorithm string hash string } // Same as in mk/checksum/distinfo.awk:/function patchsum/ -func computePatchSha1Hex(patchFilename string) (string, error) { - patchBytes, err := ioutil.ReadFile(patchFilename) - if err != nil { - return "", err - } +func computePatchSha1Hex(lines *Lines) string { hasher := sha1.New() - skipText := []byte("$" + "NetBSD") - for _, patchLine := range bytes.SplitAfter(patchBytes, []byte("\n")) { - if !bytes.Contains(patchLine, skipText) { - _, _ = hasher.Write(patchLine) + skipText := "$" + "NetBSD" + for _, line := range lines.Lines { + for _, raw := range line.raw { + textnl := raw.orignl + if !contains(textnl, skipText) { + _, _ = hasher.Write([]byte(textnl)) + } } } - return sprintf("%x", hasher.Sum(nil)), nil + return sprintf("%x", hasher.Sum(nil)) } diff --git a/pkgtools/pkglint/files/distinfo_test.go b/pkgtools/pkglint/files/distinfo_test.go index 953969df293..0b896932918 100644 --- a/pkgtools/pkglint/files/distinfo_test.go +++ b/pkgtools/pkglint/files/distinfo_test.go @@ -7,12 +7,12 @@ func (s *Suite) Test_CheckLinesDistinfo__parse_errors(c *check.C) { t.Chdir("category/package") t.CreateFileLines("patches/patch-aa", - RcsID+" line is ignored for computing the SHA1 hash", + CvsID+" line is ignored for computing the SHA1 hash", "patch contents") t.CreateFileLines("patches/patch-ab", "patch contents") lines := t.SetUpFileLines("distinfo", - "should be the RCS ID", + "should be the CVS ID", "should be empty", "MD5 (distfile.tar.gz) = 12345678901234567890123456789012", "SHA1 (distfile.tar.gz) = 1234567890123456789012345678901234567890", @@ -28,7 +28,7 @@ func (s *Suite) Test_CheckLinesDistinfo__parse_errors(c *check.C) { t.CheckOutputLines( "ERROR: distinfo:1: Expected \"$"+"NetBSD$\".", "NOTE: distinfo:1: Empty line expected before this line.", - "ERROR: distinfo:1: Invalid line: should be the RCS ID", + "ERROR: distinfo:1: Invalid line: should be the CVS ID", "ERROR: distinfo:2: Invalid line: should be empty", "ERROR: distinfo:8: Invalid line: Another invalid line", "ERROR: distinfo:3: Expected SHA1, RMD160, SHA512, Size checksums for \"distfile.tar.gz\", got MD5, SHA1.", @@ -41,7 +41,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__nonexistent_distfile_ t.Chdir("category/package") lines := t.SetUpFileLines("distinfo", - RcsID, + CvsID, "", "MD5 (patch-5.3.tar.gz) = 12345678901234567890123456789012", "SHA1 (patch-5.3.tar.gz) = 1234567890123456789012345678901234567890") @@ -62,7 +62,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__wrong_distfile_algori t.Chdir("category/package") lines := t.SetUpFileLines("distinfo", - RcsID, + CvsID, "", "MD5 (distfile.tar.gz) = 12345678901234567890123456789012", "SHA1 (distfile.tar.gz) = 1234567890123456789012345678901234567890") @@ -86,7 +86,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__ambiguous_distfile(c t.SetUpCommandLine("--explain") t.Chdir("category/package") lines := t.SetUpFileLines("distinfo", - RcsID, + CvsID, "", "MD5 (patch-4.2.tar.gz) = 12345678901234567890123456789012") @@ -109,7 +109,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__wrong_patch_algorithm t.Chdir("category/package") t.CreateFileDummyPatch("patches/patch-aa") t.SetUpFileLines("distinfo", - RcsID, + CvsID, "", "MD5 (patch-aa) = 12345678901234567890123456789012", "SHA1 (patch-aa) = 1234567890123456789012345678901234567890") @@ -128,7 +128,7 @@ func (s *Suite) Test_distinfoLinesChecker_parse__empty(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("distinfo", - RcsID, + CvsID, "") CheckLinesDistinfo(nil, lines) @@ -150,25 +150,25 @@ func (s *Suite) Test_distinfoLinesChecker_checkGlobalDistfileMismatch(c *check.C t.SetUpPackage("category/package1") t.SetUpPackage("category/package2") t.CreateFileLines("category/package1/distinfo", - RcsID, + CvsID, "", "SHA512 (distfile-1.0.tar.gz) = 1234567811111111", "SHA512 (distfile-1.1.tar.gz) = 1111111111111111", "SHA512 (patch-4.2.tar.gz) = 1234567812345678") t.CreateFileLines("category/package2/distinfo", - RcsID, + CvsID, "", "SHA512 (distfile-1.0.tar.gz) = 1234567822222222", "SHA512 (distfile-1.1.tar.gz) = 1111111111111111", "SHA512 (encoding-error.tar.gz) = 12345678abcdefgh") t.CreateFileLines("Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tThis is pkgsrc", "", "SUBDIR+=\tcategory") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tUseful programs", "", @@ -212,7 +212,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__missing_patch_with_di t := s.Init(c) lines := t.SetUpFileLines("distinfo", - RcsID, + CvsID, "", "SHA1 (patch-aa) = ...", "RMD160 (patch-aa) = ...", @@ -234,7 +234,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__existing_patch_with_d t.SetUpPackage("category/package") t.CreateFileLines("category/package/distinfo", - RcsID, + CvsID, "", "SHA1 (patch-aa) = ...", "RMD160 (patch-aa) = ...", @@ -264,7 +264,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__missing_patch_with_wr t.SetUpPackage("category/package") t.SetUpFileLines("category/package/distinfo", - RcsID, + CvsID, "", "RMD160 (patch-aa) = ...") t.FinishSetUp() @@ -286,9 +286,9 @@ func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch__bad(c *check.C) t.Chdir("category/package") t.CreateFileDummyPatch("patches/patch-aa") t.CreateFileLines("CVS/Entries", - "/distinfo/...") + "/distinfo//modified//") t.SetUpFileLines("distinfo", - RcsID, + CvsID, "", "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") t.FinishSetUp() @@ -306,11 +306,11 @@ func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch__good(c *check.C t.Chdir("category/package") t.CreateFileDummyPatch("patches/patch-aa") t.CreateFileLines("CVS/Entries", - "/distinfo/...") + "/distinfo//modified//") t.CreateFileLines("patches/CVS/Entries", - "/patch-aa/...") + "/patch-aa//modified//") t.SetUpFileLines("distinfo", - RcsID, + CvsID, "", "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") t.FinishSetUp() @@ -329,7 +329,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkUnrecordedPatches(c *check.C) { t.CreateFileDummyPatch("patches/patch-aa") t.CreateFileDummyPatch("patches/patch-src-Makefile") t.SetUpFileLines("distinfo", - RcsID, + CvsID, "", "SHA1 (distfile.tar.gz) = ...", "RMD160 (distfile.tar.gz) = ...", @@ -358,7 +358,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1__relative_path_in_disti t.CreateFileDummyPatch("devel/patches/patches/patch-aa") t.CreateFileDummyPatch("devel/patches/patches/patch-only-in-patches") t.SetUpFileLines("other/common/distinfo", - RcsID, + CvsID, "", "SHA1 (patch-aa) = ...", "SHA1 (patch-only-in-distinfo) = ...") @@ -389,7 +389,7 @@ func (s *Suite) Test_CheckLinesDistinfo__distinfo_and_patches_in_separate_direct t.CreateFileDummyPatch("other/common/patches/patch-aa") t.CreateFileDummyPatch("other/common/patches/patch-only-in-patches") t.SetUpFileLines("other/common/distinfo", - RcsID, + CvsID, "", "SHA1 (patch-aa) = ...", "SHA1 (patch-only-in-distinfo) = ...") @@ -413,7 +413,7 @@ func (s *Suite) Test_CheckLinesDistinfo__manual_patches(c *check.C) { t.Chdir("category/package") t.CreateFileLines("patches/manual-libtool.m4") lines := t.SetUpFileLines("distinfo", - RcsID, + CvsID, "", "SHA1 (patch-aa) = ...") @@ -446,7 +446,7 @@ func (s *Suite) Test_CheckLinesDistinfo__missing_php_patches(c *check.C) { t.SetUpCommandLine("-Wall,no-space") t.CreateFileLines("licenses/unknown-license") t.CreateFileLines("lang/php/ext.mk", - MkRcsID, + MkCvsID, "", "PHPEXT_MK= # defined", "PHPPKGSRCDIR= ../../lang/php72", @@ -462,12 +462,12 @@ func (s *Suite) Test_CheckLinesDistinfo__missing_php_patches(c *check.C) { ".endif") t.CreateFileDummyPatch("lang/php72/patches/patch-php72") t.CreateFileLines("lang/php72/distinfo", - RcsID, + CvsID, "", "SHA1 (patch-php72) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") t.CreateFileLines("archivers/php-bz2/Makefile", - MkRcsID, + MkCvsID, "", "USE_PHP_EXT_PATCHES= yes", "", @@ -478,7 +478,7 @@ func (s *Suite) Test_CheckLinesDistinfo__missing_php_patches(c *check.C) { G.Check(t.File("archivers/php-bz2")) t.CreateFileLines("archivers/php-zlib/Makefile", - MkRcsID, + MkCvsID, "", ".include \"../../lang/php/ext.mk\"", ".include \"../../mk/bsd.pkg.mk\"") @@ -510,7 +510,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__add_missing_h t.SetUpCommandLine("-Wall", "--explain") t.SetUpPackage("category/package") t.CreateFileLines("category/package/distinfo", - RcsID, + CvsID, "", "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", "Size (package-1.0.txt) = 13 bytes", @@ -578,7 +578,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__add_missing_h t.SetUpCommandLine("-Wall", "--explain") t.SetUpPackage("category/package") t.CreateFileLines("category/package/distinfo", - RcsID, + CvsID, "", "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", "Size (package-1.0.txt) = 13 bytes", @@ -622,7 +622,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__wrong_distfil t.SetUpPackage("category/package") t.CreateFileLines("category/package/distinfo", - RcsID, + CvsID, "", "RMD160 (package-1.0.txt) = 1234wrongHash1234") t.CreateFileLines("distfiles/package-1.0.txt", @@ -645,7 +645,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__no_usual_algo t.SetUpPackage("category/package") t.CreateFileLines("category/package/distinfo", - RcsID, + CvsID, "", "MD5 (package-1.0.txt) = 1234wrongHash1234") t.CreateFileLines("distfiles/package-1.0.txt", @@ -665,7 +665,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__top_algorithm t.SetUpPackage("category/package") t.CreateFileLines("category/package/distinfo", - RcsID, + CvsID, "", "SHA512 (package-1.0.txt) = f65f341b35981fda842b09b2c8af9bcdb7602a4c2e6fa1f7"+ "d41f0974d3e3122f268fc79d5a4af66358f5133885cd1c165c916f80ab25e5d8d95db46f803c782c", @@ -689,7 +689,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__bottom_algori t.SetUpPackage("category/package") t.CreateFileLines("category/package/distinfo", - RcsID, + CvsID, "", "SHA1 (package-1.0.txt) = cd50d19784897085a8d0e3e413f8612b097c03f1", "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a") @@ -724,7 +724,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__algorithms_in t.SetUpPackage("category/package") t.CreateFileLines("category/package/distinfo", - RcsID, + CvsID, "", "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", "SHA1 (package-1.0.txt) = cd50d19784897085a8d0e3e413f8612b097c03f1", @@ -750,7 +750,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__some_algorith t.SetUpPackage("category/package") t.CreateFileLines("category/package/distinfo", - RcsID, + CvsID, "", "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", "Size (package-1.0.txt) = 13 bytes", diff --git a/pkgtools/pkglint/files/files.go b/pkgtools/pkglint/files/files.go index 09efcefa92c..76745360d82 100644 --- a/pkgtools/pkglint/files/files.go +++ b/pkgtools/pkglint/files/files.go @@ -16,7 +16,7 @@ const ( LogErrors // ) -func Load(filename string, options LoadOptions) Lines { +func Load(filename string, options LoadOptions) *Lines { if fromCache := G.fileCache.Get(filename, options); fromCache != nil { return fromCache } @@ -54,7 +54,7 @@ func Load(filename string, options LoadOptions) Lines { return result } -func LoadMk(filename string, options LoadOptions) MkLines { +func LoadMk(filename string, options LoadOptions) *MkLines { lines := Load(filename, options|Makefile) if lines == nil { return nil @@ -62,7 +62,7 @@ func LoadMk(filename string, options LoadOptions) MkLines { return NewMkLines(lines) } -func nextLogicalLine(filename string, rawLines []*RawLine, index int) (Line, int) { +func nextLogicalLine(filename string, rawLines []*RawLine, index int) (*Line, int) { { // Handle the common case efficiently rawLine := rawLines[index] textnl := rawLine.textnl @@ -137,7 +137,7 @@ func matchContinuationLine(textnl string) (leadingWhitespace, text, trailingWhit return } -func convertToLogicalLines(filename string, rawText string, joinBackslashLines bool) Lines { +func convertToLogicalLines(filename string, rawText string, joinBackslashLines bool) *Lines { var rawLines []*RawLine for lineno, rawLine := range strings.SplitAfter(rawText, "\n") { if rawLine != "" { @@ -145,7 +145,7 @@ func convertToLogicalLines(filename string, rawText string, joinBackslashLines b } } - var loglines []Line + var loglines []*Line if joinBackslashLines { for lineno := 0; lineno < len(rawLines); { line, nextLineno := nextLogicalLine(filename, rawLines, lineno) diff --git a/pkgtools/pkglint/files/files_test.go b/pkgtools/pkglint/files/files_test.go index 049bd34aab2..f010dd17022 100644 --- a/pkgtools/pkglint/files/files_test.go +++ b/pkgtools/pkglint/files/files_test.go @@ -173,10 +173,40 @@ func (s *Suite) Test_matchContinuationLine(c *check.C) { c.Check(continuation, equals, "\\") } -func (s *Suite) Test_Load__errors(c *check.C) { +func (s *Suite) Test_Load(c *check.C) { t := s.Init(c) - t.CreateFileLines("empty") + nonexistent := t.File("nonexistent") + empty := t.CreateFileLines("empty") + oneLiner := t.CreateFileLines("one-liner", + "hello, world") + + t.Check(Load(nonexistent, 0), check.IsNil) + t.Check(Load(empty, 0).Lines, check.HasLen, 0) + t.Check(Load(oneLiner, 0).Lines[0].Text, equals, "hello, world") + + t.CheckOutputEmpty() + + t.Check(Load(nonexistent, LogErrors), check.IsNil) + t.Check(Load(empty, LogErrors).Lines, check.HasLen, 0) + t.Check(Load(oneLiner, LogErrors).Lines[0].Text, equals, "hello, world") + + t.CheckOutputLines( + "ERROR: ~/nonexistent: Cannot be read.") + + t.Check(Load(nonexistent, NotEmpty), check.IsNil) + t.Check(Load(empty, NotEmpty), check.IsNil) + t.Check(Load(oneLiner, NotEmpty).Lines[0].Text, equals, "hello, world") + + t.CheckOutputEmpty() + + t.Check(Load(nonexistent, NotEmpty|LogErrors), check.IsNil) + t.Check(Load(empty, NotEmpty|LogErrors), check.IsNil) + t.Check(Load(oneLiner, NotEmpty|LogErrors).Lines[0].Text, equals, "hello, world") + + t.CheckOutputLines( + "ERROR: ~/nonexistent: Cannot be read.", + "ERROR: ~/empty: Must not be empty.") t.ExpectFatal( func() { Load(t.File("does-not-exist"), MustSucceed) }, diff --git a/pkgtools/pkglint/files/fuzzer_test.go b/pkgtools/pkglint/files/fuzzer_test.go index df1284c2a16..d35bd2b5e0b 100644 --- a/pkgtools/pkglint/files/fuzzer_test.go +++ b/pkgtools/pkglint/files/fuzzer_test.go @@ -57,7 +57,7 @@ func (f *Fuzzer) addChar(r rune, weight int) { } func (f *Fuzzer) Generate(length int) string { - rs := make([]rune, length, length) + rs := make([]rune, length) for i := 0; i < length; i++ { rs[i] = f.randomChar() } diff --git a/pkgtools/pkglint/files/getopt/getopt_test.go b/pkgtools/pkglint/files/getopt/getopt_test.go index c77f90c3ed5..667b95b0c70 100644 --- a/pkgtools/pkglint/files/getopt/getopt_test.go +++ b/pkgtools/pkglint/files/getopt/getopt_test.go @@ -169,13 +169,17 @@ func (s *Suite) Test_Options_Parse__string_list(c *check.C) { c.Check(includes, check.DeepEquals, []string{"included1", "included2", "included3", "included4"}) c.Check(excludes, check.DeepEquals, []string{"excluded1", "excluded2", "excluded3", "excluded4"}) - args, err = opts.Parse([]string{"progname", "-i"}) + _, err = opts.Parse([]string{"progname", "-i"}) - c.Check(err.Error(), check.Equals, "progname: option requires an argument: -i") + if c.Check(err, check.NotNil) { + c.Check(err.Error(), check.Equals, "progname: option requires an argument: -i") + } - args, err = opts.Parse([]string{"progname", "--include"}) + _, err = opts.Parse([]string{"progname", "--include"}) - c.Check(err.Error(), check.Equals, "progname: option requires an argument: --include") + if c.Check(err, check.NotNil) { + c.Check(err.Error(), check.Equals, "progname: option requires an argument: --include") + } } func (s *Suite) Test_Options_Parse__long_flags(c *check.C) { diff --git a/pkgtools/pkglint/files/intqa/testnames.go b/pkgtools/pkglint/files/intqa/testnames.go index 5ed0683ba3e..02f604739cc 100644 --- a/pkgtools/pkglint/files/intqa/testnames.go +++ b/pkgtools/pkglint/files/intqa/testnames.go @@ -2,13 +2,11 @@ package intqa import ( - "bytes" "fmt" "go/ast" "go/parser" "go/token" "gopkg.in/check.v1" - "io/ioutil" "os" "path/filepath" "sort" @@ -122,35 +120,6 @@ func (ck *TestNameChecker) addElement(elements *[]*testeeElement, decl ast.Decl, } } -// fixTabs replaces literal tabs with proper escape sequences, -// except for the indentation tabs. -// -// It doesn't really belong to this type (TestNameChecker) but -// merely uses its infrastructure. -func (ck *TestNameChecker) fixTabs(filename string) { - if ck.isIgnored(filename) { - return - } - - readBytes, err := ioutil.ReadFile(filename) - ck.c.Assert(err, check.IsNil) - - var fixed bytes.Buffer - for _, line := range strings.SplitAfter(string(readBytes), "\n") { - rest := strings.TrimLeft(line, "\t") - fixed.WriteString(line[:len(line)-len(rest)]) - fixed.WriteString(strings.Replace(rest, "\t", "\\t", -1)) - } - - if fixed.String() != string(readBytes) { - tmpName := filename + ".tmp" - err = ioutil.WriteFile(tmpName, fixed.Bytes(), 0666) - ck.c.Assert(err, check.IsNil) - err = os.Rename(tmpName, filename) - ck.c.Assert(err, check.IsNil) - } -} - // loadAllElements returns all type, function and method names // from the current package, in the form FunctionName or // TypeName.MethodName (omitting the * from the type name). diff --git a/pkgtools/pkglint/files/licenses.go b/pkgtools/pkglint/files/licenses.go index bdf1a7c085c..944ad42d0fa 100644 --- a/pkgtools/pkglint/files/licenses.go +++ b/pkgtools/pkglint/files/licenses.go @@ -3,8 +3,8 @@ package pkglint import "netbsd.org/pkglint/licenses" type LicenseChecker struct { - MkLines MkLines - MkLine MkLine + MkLines *MkLines + MkLine *MkLine } func (lc *LicenseChecker) Check(value string, op MkOperator) { diff --git a/pkgtools/pkglint/files/line.go b/pkgtools/pkglint/files/line.go index cc2c239f089..ed9050ab574 100644 --- a/pkgtools/pkglint/files/line.go +++ b/pkgtools/pkglint/files/line.go @@ -14,6 +14,7 @@ package pkglint // used in the --autofix mode. import ( + "netbsd.org/pkglint/regex" "path" "strconv" ) @@ -60,11 +61,7 @@ func (loc *Location) Linenos() string { } // Line represents a line of text from a file. -// It aliases a pointer type to reduces the number of *Line occurrences in the code. -// Using a type alias is more efficient than an interface type, I guess. -type Line = *LineImpl - -type LineImpl struct { +type Line struct { // TODO: Consider storing pointers to the Filename and Basename instead of strings to save memory. // But first find out where and why pkglint needs so much memory (200 MB for a full recursive run over pkgsrc + wip). Location @@ -82,33 +79,33 @@ type LineImpl struct { // XXX: Filename and Basename could be replaced with a pointer to a Lines object. } -func NewLine(filename string, lineno int, text string, rawLine *RawLine) Line { - assertf(rawLine != nil, "use NewLineMulti for creating a Line with no RawLine attached to it") +func NewLine(filename string, lineno int, text string, rawLine *RawLine) *Line { + assert(rawLine != nil) // Use NewLineMulti for creating a Line with no RawLine attached to it. return NewLineMulti(filename, lineno, lineno, text, []*RawLine{rawLine}) } // NewLineMulti is for logical Makefile lines that end with backslash. -func NewLineMulti(filename string, firstLine, lastLine int, text string, rawLines []*RawLine) Line { - return &LineImpl{NewLocation(filename, firstLine, lastLine), path.Base(filename), text, rawLines, nil, Once{}} +func NewLineMulti(filename string, firstLine, lastLine int, text string, rawLines []*RawLine) *Line { + return &Line{NewLocation(filename, firstLine, lastLine), path.Base(filename), text, rawLines, nil, Once{}} } // NewLineEOF creates a dummy line for logging, with the "line number" EOF. -func NewLineEOF(filename string) Line { +func NewLineEOF(filename string) *Line { return NewLineMulti(filename, -1, 0, "", nil) } // NewLineWhole creates a dummy line for logging messages that affect a file as a whole. -func NewLineWhole(filename string) Line { +func NewLineWhole(filename string) *Line { return NewLineMulti(filename, 0, 0, "", nil) } // RefTo returns a reference to another line, // which can be in the same file or in a different file. -func (line *LineImpl) RefTo(other Line) string { +func (line *Line) RefTo(other *Line) string { return line.RefToLocation(other.Location) } -func (line *LineImpl) RefToLocation(other Location) string { +func (line *Line) RefToLocation(other Location) string { if line.Filename != other.Filename { return line.PathToFile(other.Filename) + ":" + other.Linenos() } @@ -118,15 +115,20 @@ func (line *LineImpl) RefToLocation(other Location) string { // PathToFile returns the relative path from this line to the given file path. // This is typically used for arguments in diagnostics, which should always be // relative to the line with which the diagnostic is associated. -func (line *LineImpl) PathToFile(filePath string) string { +func (line *Line) PathToFile(filePath string) string { return relpath(path.Dir(line.Filename), filePath) } -func (line *LineImpl) IsMultiline() bool { +func (line *Line) IsMultiline() bool { return line.firstLine > 0 && line.firstLine != line.lastLine } -func (line *LineImpl) showSource(out *SeparatorWriter) { +func (line *Line) IsCvsID(prefixRe regex.Pattern) (found bool, expanded bool) { + m, exp := match1(line.Text, `^`+prefixRe+`\$`+`NetBSD(:[^\$]+)?\$$`) + return m, exp != "" +} + +func (line *Line) showSource(out *SeparatorWriter) { if !G.Logger.Opts.ShowSource { return } @@ -172,28 +174,28 @@ func (line *LineImpl) showSource(out *SeparatorWriter) { } } -func (line *LineImpl) Fatalf(format string, args ...interface{}) { +func (line *Line) Fatalf(format string, args ...interface{}) { if trace.Tracing { trace.Stepf("Fatalf: %q, %v", format, args) } G.Logger.Diag(line, Fatal, format, args...) } -func (line *LineImpl) Errorf(format string, args ...interface{}) { +func (line *Line) Errorf(format string, args ...interface{}) { G.Logger.Diag(line, Error, format, args...) } -func (line *LineImpl) Warnf(format string, args ...interface{}) { +func (line *Line) Warnf(format string, args ...interface{}) { G.Logger.Diag(line, Warn, format, args...) } -func (line *LineImpl) Notef(format string, args ...interface{}) { +func (line *Line) Notef(format string, args ...interface{}) { G.Logger.Diag(line, Note, format, args...) } -func (line *LineImpl) Explain(explanation ...string) { G.Logger.Explain(explanation...) } +func (line *Line) Explain(explanation ...string) { G.Logger.Explain(explanation...) } -func (line *LineImpl) String() string { +func (line *Line) String() string { return sprintf("%s:%s: %s", line.Filename, line.Linenos(), line.Text) } @@ -220,7 +222,7 @@ func (line *LineImpl) String() string { // fix.Custom(func(showAutofix, autofix bool) {}) // // fix.Apply() -func (line *LineImpl) Autofix() *Autofix { +func (line *Line) Autofix() *Autofix { if line.autofix == nil { line.autofix = NewAutofix(line) } diff --git a/pkgtools/pkglint/files/line_test.go b/pkgtools/pkglint/files/line_test.go index d0c74fc1152..7a54a6b7b83 100644 --- a/pkgtools/pkglint/files/line_test.go +++ b/pkgtools/pkglint/files/line_test.go @@ -15,9 +15,7 @@ func (s *Suite) Test_RawLine_String(c *check.C) { func (s *Suite) Test_NewLine__assertion(c *check.C) { t := s.Init(c) - t.ExpectPanic( - func() { NewLine("filename", 123, "text", nil) }, - "Pkglint internal error: use NewLineMulti for creating a Line with no RawLine attached to it") + t.ExpectAssert(func() { NewLine("filename", 123, "text", nil) }) } func (s *Suite) Test_Line_IsMultiline(c *check.C) { diff --git a/pkgtools/pkglint/files/linechecker.go b/pkgtools/pkglint/files/linechecker.go index 8fc3b18d6c5..525f26ae079 100644 --- a/pkgtools/pkglint/files/linechecker.go +++ b/pkgtools/pkglint/files/linechecker.go @@ -6,7 +6,7 @@ import ( ) type LineChecker struct { - line Line + line *Line } func (ck LineChecker) CheckLength(maxLength int) { diff --git a/pkgtools/pkglint/files/linelexer.go b/pkgtools/pkglint/files/linelexer.go index b89b5da2131..f74076ed0b6 100644 --- a/pkgtools/pkglint/files/linelexer.go +++ b/pkgtools/pkglint/files/linelexer.go @@ -4,24 +4,24 @@ import "netbsd.org/pkglint/regex" // LinesLexer records the state when checking a list of lines from top to bottom. type LinesLexer struct { - lines Lines + lines *Lines index int } -func NewLinesLexer(lines Lines) *LinesLexer { +func NewLinesLexer(lines *Lines) *LinesLexer { return &LinesLexer{lines, 0} } // CurrentLine returns the line that the lexer is currently looking at. // If it is at the end of file, the line number of the line is EOF. -func (llex *LinesLexer) CurrentLine() Line { +func (llex *LinesLexer) CurrentLine() *Line { if llex.index < llex.lines.Len() { return llex.lines.Lines[llex.index] } - return NewLineEOF(llex.lines.FileName) + return NewLineEOF(llex.lines.Filename) } -func (llex *LinesLexer) PreviousLine() Line { +func (llex *LinesLexer) PreviousLine() *Line { return llex.lines.Lines[llex.index-1] } @@ -115,23 +115,23 @@ func (llex *LinesLexer) SkipContainsOrWarn(text string) bool { // MkLinesLexer records the state when checking a list of Makefile lines from top to bottom. type MkLinesLexer struct { - mklines MkLines + mklines *MkLines LinesLexer } -func NewMkLinesLexer(mklines MkLines) *MkLinesLexer { +func NewMkLinesLexer(mklines *MkLines) *MkLinesLexer { return &MkLinesLexer{mklines, *NewLinesLexer(mklines.lines)} } -func (mlex *MkLinesLexer) PreviousMkLine() MkLine { +func (mlex *MkLinesLexer) PreviousMkLine() *MkLine { return mlex.mklines.mklines[mlex.index-1] } -func (mlex *MkLinesLexer) CurrentMkLine() MkLine { +func (mlex *MkLinesLexer) CurrentMkLine() *MkLine { return mlex.mklines.mklines[mlex.index] } -func (mlex *MkLinesLexer) SkipWhile(pred func(mkline MkLine) bool) { +func (mlex *MkLinesLexer) SkipWhile(pred func(mkline *MkLine) bool) { if trace.Tracing { defer trace.Call(mlex.CurrentMkLine().Text)() } @@ -141,7 +141,7 @@ func (mlex *MkLinesLexer) SkipWhile(pred func(mkline MkLine) bool) { } } -func (mlex *MkLinesLexer) SkipIf(pred func(mkline MkLine) bool) bool { +func (mlex *MkLinesLexer) SkipIf(pred func(mkline *MkLine) bool) bool { if !mlex.EOF() && pred(mlex.CurrentMkLine()) { mlex.Skip() return true diff --git a/pkgtools/pkglint/files/lines.go b/pkgtools/pkglint/files/lines.go index 0d4b9914497..77aef21e675 100644 --- a/pkgtools/pkglint/files/lines.go +++ b/pkgtools/pkglint/files/lines.go @@ -5,46 +5,44 @@ import ( "path" ) -type Lines = *LinesImpl - -type LinesImpl struct { - FileName string +type Lines struct { + Filename string BaseName string - Lines []Line + Lines []*Line } -func NewLines(filename string, lines []Line) Lines { - return &LinesImpl{filename, path.Base(filename), lines} +func NewLines(filename string, lines []*Line) *Lines { + return &Lines{filename, path.Base(filename), lines} } -func (ls *LinesImpl) Len() int { return len(ls.Lines) } +func (ls *Lines) Len() int { return len(ls.Lines) } -func (ls *LinesImpl) LastLine() Line { return ls.Lines[ls.Len()-1] } +func (ls *Lines) LastLine() *Line { return ls.Lines[ls.Len()-1] } -func (ls *LinesImpl) EOFLine() Line { return NewLineMulti(ls.FileName, -1, -1, "", nil) } +func (ls *Lines) EOFLine() *Line { return NewLineMulti(ls.Filename, -1, -1, "", nil) } -func (ls *LinesImpl) Errorf(format string, args ...interface{}) { - NewLineWhole(ls.FileName).Errorf(format, args...) +func (ls *Lines) Errorf(format string, args ...interface{}) { + NewLineWhole(ls.Filename).Errorf(format, args...) } -func (ls *LinesImpl) Warnf(format string, args ...interface{}) { - NewLineWhole(ls.FileName).Warnf(format, args...) +func (ls *Lines) Warnf(format string, args ...interface{}) { + NewLineWhole(ls.Filename).Warnf(format, args...) } -func (ls *LinesImpl) SaveAutofixChanges() bool { +func (ls *Lines) SaveAutofixChanges() bool { return SaveAutofixChanges(ls) } -// CheckRcsID returns true if the expected RCS Id was found. -func (ls *LinesImpl) CheckRcsID(index int, prefixRe regex.Pattern, suggestedPrefix string) bool { +// CheckCvsID returns true if the expected CVS Id was found. +func (ls *Lines) CheckCvsID(index int, prefixRe regex.Pattern, suggestedPrefix string) bool { if trace.Tracing { defer trace.Call(prefixRe, suggestedPrefix)() } line := ls.Lines[index] - if m, expanded := match1(line.Text, `^`+prefixRe+`\$`+`NetBSD(:[^\$]+)?\$$`); m { + if m, expanded := line.IsCvsID(prefixRe); m { - if G.Testing && G.Wip && expanded != "" { + if G.Testing && G.Wip && expanded { fix := line.Autofix() fix.Notef("Expected exactly %q.", suggestedPrefix+"$"+"NetBSD$") fix.Explain( diff --git a/pkgtools/pkglint/files/lines_test.go b/pkgtools/pkglint/files/lines_test.go index 2e758114cb1..3d105d74cd9 100644 --- a/pkgtools/pkglint/files/lines_test.go +++ b/pkgtools/pkglint/files/lines_test.go @@ -2,7 +2,7 @@ package pkglint import "gopkg.in/check.v1" -func (s *Suite) Test_Lines_CheckRcsID(c *check.C) { +func (s *Suite) Test_Lines_CheckCvsID(c *check.C) { t := s.Init(c) lines := t.NewLines("filename", @@ -13,7 +13,7 @@ func (s *Suite) Test_Lines_CheckRcsID(c *check.C) { "$"+"FreeBSD$") for i := range lines.Lines { - lines.CheckRcsID(i, ``, "") + lines.CheckCvsID(i, ``, "") } t.CheckOutputLines( @@ -27,7 +27,7 @@ func (s *Suite) Test_Lines_CheckRcsID(c *check.C) { // "$NetBSD:" is a copy-and-paste mistake rather than an intentional // documentation of the file's history. Therefore, pkgsrc-wip files should // only use the unexpanded form. -func (s *Suite) Test_Lines_CheckRcsID__wip(c *check.C) { +func (s *Suite) Test_Lines_CheckCvsID__wip(c *check.C) { t := s.Init(c) t.SetUpPkgsrc() diff --git a/pkgtools/pkglint/files/logging.go b/pkgtools/pkglint/files/logging.go index 0e727198541..364aae3df85 100644 --- a/pkgtools/pkglint/files/logging.go +++ b/pkgtools/pkglint/files/logging.go @@ -153,7 +153,7 @@ func (l *Logger) shallBeLogged(format string) bool { // and duplicates are suppressed unless the --log-verbose command line option is given. // // See Logf for logging arbitrary messages. -func (l *Logger) Diag(line Line, level *LogLevel, format string, args ...interface{}) { +func (l *Logger) Diag(line *Line, level *LogLevel, format string, args ...interface{}) { if l.Opts.ShowAutofix || l.Opts.Autofix { // In these two cases, the only interesting diagnostics are those that can // be fixed automatically. These are logged by Autofix.Apply. diff --git a/pkgtools/pkglint/files/logging_test.go b/pkgtools/pkglint/files/logging_test.go index f730fef8bdf..737f5936f5d 100644 --- a/pkgtools/pkglint/files/logging_test.go +++ b/pkgtools/pkglint/files/logging_test.go @@ -775,7 +775,7 @@ func (s *Suite) Test_Logger_Diag__source_duplicates(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("category/dependency/patches/patch-aa", - RcsID, + CvsID, "", "--- old file", "+++ new file", diff --git a/pkgtools/pkglint/files/mkline.go b/pkgtools/pkglint/files/mkline.go index c8005c4eaaf..a9f10cc5f38 100644 --- a/pkgtools/pkglint/files/mkline.go +++ b/pkgtools/pkglint/files/mkline.go @@ -12,14 +12,17 @@ import ( // There are several types of lines. // The most common types in pkgsrc are variable assignments, // shell commands and directives like .if and .for. -type MkLine = *MkLineImpl +type MkLine struct { + *Line -type MkLineImpl struct { - Line - data interface{} // One of the following mkLine* types + // One of the following mkLine* types. + // + // For the larger of these types, a pointer is used instead of a direct + // struct because of https://github.com/golang/go/issues/28045. + data interface{} } -type mkLineAssign = *mkLineAssignImpl // See https://github.com/golang/go/issues/28045 -type mkLineAssignImpl struct { + +type mkLineAssign struct { commented bool // Whether the whole variable assignment is commented out varname string // e.g. "HOMEPAGE", "SUBST_SED.perl" varcanon string // e.g. "HOMEPAGE", "SUBST_SED.*" @@ -34,29 +37,33 @@ type mkLineAssignImpl struct { spaceAfterValue string comment string } + type mkLineShell struct { command string } -type mkLineComment struct{} // See mkLineAssignImpl.commented for another type of comment line + +type mkLineComment struct{} // See mkLineAssign.commented for another type of comment line + type mkLineEmpty struct{} -type mkLineDirective = *mkLineDirectiveImpl // See https://github.com/golang/go/issues/28045 -type mkLineDirectiveImpl struct { + +type mkLineDirective struct { indent string // the space between the leading "." and the directive directive string // "if", "else", "for", etc. args string comment string // mainly interesting for .endif and .endfor - elseLine MkLine // for .if (filled in later) - cond MkCond // for .if and .elif (filled in on first access) + elseLine *MkLine // for .if (filled in later) + cond *MkCond // for .if and .elif (filled in on first access) fields []string // the arguments for the .for loop (filled in on first access) } -type mkLineInclude = *mkLineIncludeImpl // See https://github.com/golang/go/issues/28045 -type mkLineIncludeImpl struct { + +type mkLineInclude struct { mustExist bool // for .sinclude, nonexistent files are ignored sys bool // whether the include uses <file.mk> (very rare) instead of "file.mk" indent string // the space between the leading "." and the directive includedFile string // the text between the <brackets> or "quotes" conditionalVars []string // variables on which this inclusion depends (filled in later, as needed) } + type mkLineDependency struct { targets string sources string @@ -68,7 +75,7 @@ type MkLineParser struct{} // it is: variable assignment, include, comment, etc. // // See devel/bmake/parse.c:/^Parse_File/ -func (p MkLineParser) Parse(line Line) *MkLineImpl { +func (p MkLineParser) Parse(line *Line) *MkLine { text := line.Text // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing. @@ -111,10 +118,10 @@ func (p MkLineParser) Parse(line Line) *MkLineImpl { // The %q is deliberate here since it shows possible strange characters. line.Errorf("Unknown Makefile line format: %q.", text) - return &MkLineImpl{line, nil} + return &MkLine{line, nil} } -func (p MkLineParser) parseVarassign(line Line, data mkLineSplitResult) MkLine { +func (p MkLineParser) parseVarassign(line *Line, data mkLineSplitResult) *MkLine { m, a := p.MatchVarassign(line, line.Text, data) if !m { return nil @@ -147,46 +154,46 @@ func (p MkLineParser) parseVarassign(line Line, data mkLineSplitResult) MkLine { "it should be preceded by a space in order to make it more visible.") } - return &MkLineImpl{line, a} + return &MkLine{line, a} } -func (p MkLineParser) parseShellcmd(line Line) MkLine { - return &MkLineImpl{line, mkLineShell{line.Text[1:]}} +func (p MkLineParser) parseShellcmd(line *Line) *MkLine { + return &MkLine{line, mkLineShell{line.Text[1:]}} } -func (p MkLineParser) parseCommentOrEmpty(line Line) MkLine { +func (p MkLineParser) parseCommentOrEmpty(line *Line) *MkLine { trimmedText := trimHspace(line.Text) if strings.HasPrefix(trimmedText, "#") { - return &MkLineImpl{line, mkLineComment{}} + return &MkLine{line, mkLineComment{}} } if trimmedText == "" { - return &MkLineImpl{line, mkLineEmpty{}} + return &MkLine{line, mkLineEmpty{}} } return nil } -func (p MkLineParser) parseInclude(line Line) MkLine { +func (p MkLineParser) parseInclude(line *Line) *MkLine { m, indent, directive, includedFile := MatchMkInclude(line.Text) if !m { return nil } - return &MkLineImpl{line, &mkLineIncludeImpl{directive == "include", false, indent, includedFile, nil}} + return &MkLine{line, &mkLineInclude{directive == "include", false, indent, includedFile, nil}} } -func (p MkLineParser) parseSysinclude(line Line) MkLine { +func (p MkLineParser) parseSysinclude(line *Line) *MkLine { m, indent, directive, includedFile := match3(line.Text, `^\.([\t ]*)(s?include)[\t ]+<([^>]+)>[\t ]*(?:#.*)?$`) if !m { return nil } - return &MkLineImpl{line, &mkLineIncludeImpl{directive == "include", true, indent, includedFile, nil}} + return &MkLine{line, &mkLineInclude{directive == "include", true, indent, includedFile, nil}} } -func (p MkLineParser) parseDependency(line Line) MkLine { +func (p MkLineParser) parseDependency(line *Line) *MkLine { // XXX: Replace this regular expression with proper parsing. // There might be a ${VAR:M*.c} in these variables, which the below regular expression cannot handle. m, targets, whitespace, sources := match3(line.Text, `^([^\t :]+(?:[\t ]*[^\t :]+)*)([\t ]*):[\t ]*([^#]*?)(?:[\t ]*#.*)?$`) @@ -197,35 +204,37 @@ func (p MkLineParser) parseDependency(line Line) MkLine { if whitespace != "" { line.Notef("Space before colon in dependency line.") } - return &MkLineImpl{line, mkLineDependency{targets, sources}} + return &MkLine{line, mkLineDependency{targets, sources}} } -func (p MkLineParser) parseMergeConflict(line Line) MkLine { +func (p MkLineParser) parseMergeConflict(line *Line) *MkLine { if !matches(line.Text, `^(<<<<<<<|=======|>>>>>>>)`) { return nil } - return &MkLineImpl{line, nil} + return &MkLine{line, nil} } // String returns the filename and line numbers. -func (mkline *MkLineImpl) String() string { +func (mkline *MkLine) String() string { return sprintf("%s:%s", mkline.Filename, mkline.Linenos()) } // IsVarassign returns true for variable assignments of the form VAR=value. // // See IsCommentedVarassign. -func (mkline *MkLineImpl) IsVarassign() bool { - data, ok := mkline.data.(mkLineAssign) +func (mkline *MkLine) IsVarassign() bool { + // See https://github.com/golang/go/issues/28045 for the reason why + // a pointer type is used here instead of a direct struct. + data, ok := mkline.data.(*mkLineAssign) return ok && !data.commented } // IsCommentedVarassign returns true for commented-out variable assignments. // In most cases these are treated as ordinary comments, but in some others // they are treated like variable assignments, just inactive ones. -func (mkline *MkLineImpl) IsCommentedVarassign() bool { - data, ok := mkline.data.(mkLineAssign) +func (mkline *MkLine) IsCommentedVarassign() bool { + data, ok := mkline.data.(*mkLineAssign) return ok && data.commented } @@ -234,18 +243,18 @@ func (mkline *MkLineImpl) IsCommentedVarassign() bool { // // pre-configure: # IsDependency // ${ECHO} # IsShellCommand -func (mkline *MkLineImpl) IsShellCommand() bool { +func (mkline *MkLine) IsShellCommand() bool { _, ok := mkline.data.(mkLineShell) return ok } // IsComment returns true for lines that consist entirely of a comment. -func (mkline *MkLineImpl) IsComment() bool { +func (mkline *MkLine) IsComment() bool { _, ok := mkline.data.(mkLineComment) return ok || mkline.IsCommentedVarassign() } -func (mkline *MkLineImpl) IsEmpty() bool { +func (mkline *MkLine) IsEmpty() bool { _, ok := mkline.data.(mkLineEmpty) return ok } @@ -253,29 +262,29 @@ func (mkline *MkLineImpl) IsEmpty() bool { // IsDirective returns true for conditionals (.if/.elif/.else/.if) or loops (.for/.endfor). // // See IsInclude. -func (mkline *MkLineImpl) IsDirective() bool { - _, ok := mkline.data.(mkLineDirective) +func (mkline *MkLine) IsDirective() bool { + _, ok := mkline.data.(*mkLineDirective) return ok } // IsInclude returns true for lines like: .include "other.mk" // // See IsSysinclude for lines like: .include <sys.mk> -func (mkline *MkLineImpl) IsInclude() bool { - incl, ok := mkline.data.(mkLineInclude) +func (mkline *MkLine) IsInclude() bool { + incl, ok := mkline.data.(*mkLineInclude) return ok && !incl.sys } // IsSysinclude returns true for lines like: .include <sys.mk> // // See IsInclude for lines like: .include "other.mk" -func (mkline *MkLineImpl) IsSysinclude() bool { - incl, ok := mkline.data.(mkLineInclude) +func (mkline *MkLine) IsSysinclude() bool { + incl, ok := mkline.data.(*mkLineInclude) return ok && incl.sys } // IsDependency returns true for dependency lines like "target: source". -func (mkline *MkLineImpl) IsDependency() bool { +func (mkline *MkLine) IsDependency() bool { _, ok := mkline.data.(mkLineDependency) return ok } @@ -285,30 +294,30 @@ func (mkline *MkLineImpl) IsDependency() bool { // // Example: // VARNAME.${param}?= value # Varname is "VARNAME.${param}" -func (mkline *MkLineImpl) Varname() string { return mkline.data.(mkLineAssign).varname } +func (mkline *MkLine) Varname() string { return mkline.data.(*mkLineAssign).varname } // Varcanon applies to variable assignments and returns the canonicalized variable name for parameterized variables. // Examples: // HOMEPAGE => "HOMEPAGE" // SUBST_SED.anything => "SUBST_SED.*" // SUBST_SED.${param} => "SUBST_SED.*" -func (mkline *MkLineImpl) Varcanon() string { return mkline.data.(mkLineAssign).varcanon } +func (mkline *MkLine) Varcanon() string { return mkline.data.(*mkLineAssign).varcanon } // Varparam applies to variable assignments and returns the parameter for parameterized variables. // Examples: // HOMEPAGE => "" // SUBST_SED.anything => "anything" // SUBST_SED.${param} => "${param}" -func (mkline *MkLineImpl) Varparam() string { return mkline.data.(mkLineAssign).varparam } +func (mkline *MkLine) Varparam() string { return mkline.data.(*mkLineAssign).varparam } // Op applies to variable assignments and returns the assignment operator. -func (mkline *MkLineImpl) Op() MkOperator { return mkline.data.(mkLineAssign).op } +func (mkline *MkLine) Op() MkOperator { return mkline.data.(*mkLineAssign).op } // ValueAlign applies to variable assignments and returns all the text // before the variable value, e.g. "VARNAME+=\t". -func (mkline *MkLineImpl) ValueAlign() string { return mkline.data.(mkLineAssign).valueAlign } +func (mkline *MkLine) ValueAlign() string { return mkline.data.(*mkLineAssign).valueAlign } -func (mkline *MkLineImpl) Value() string { return mkline.data.(mkLineAssign).value } +func (mkline *MkLine) Value() string { return mkline.data.(*mkLineAssign).value } // VarassignComment applies to variable assignments and returns the comment. // @@ -318,7 +327,7 @@ func (mkline *MkLineImpl) Value() string { return mkline.data.(mkLineAssign).val // In the above line, the comment is "# comment". // // The leading "#" is included so that pkglint can distinguish between no comment at all and an empty comment. -func (mkline *MkLineImpl) VarassignComment() string { return mkline.data.(mkLineAssign).comment } +func (mkline *MkLine) VarassignComment() string { return mkline.data.(*mkLineAssign).comment } // FirstLineContainsValue returns whether the variable assignment of a // multiline contains a textual value in the first line. @@ -327,9 +336,9 @@ func (mkline *MkLineImpl) VarassignComment() string { return mkline.data.(mkLine // starts in first line // NO_VALUE_IN_FIRST_LINE= \ // value starts in second line -func (mkline *MkLineImpl) FirstLineContainsValue() bool { - assertf(mkline.IsVarassign() || mkline.IsCommentedVarassign(), "Line must be a variable assignment.") - assertf(mkline.IsMultiline(), "Line must be multiline.") +func (mkline *MkLine) FirstLineContainsValue() bool { + assert(mkline.IsVarassign() || mkline.IsCommentedVarassign()) + assert(mkline.IsMultiline()) // Parsing the continuation marker as variable value is cheating but works well. text := strings.TrimSuffix(mkline.raw[0].orignl, "\n") @@ -338,64 +347,64 @@ func (mkline *MkLineImpl) FirstLineContainsValue() bool { return a.value != "\\" } -func (mkline *MkLineImpl) ShellCommand() string { return mkline.data.(mkLineShell).command } +func (mkline *MkLine) ShellCommand() string { return mkline.data.(mkLineShell).command } -func (mkline *MkLineImpl) Indent() string { +func (mkline *MkLine) Indent() string { if mkline.IsDirective() { - return mkline.data.(mkLineDirective).indent + return mkline.data.(*mkLineDirective).indent } else { - return mkline.data.(mkLineInclude).indent + return mkline.data.(*mkLineInclude).indent } } // Directive returns the preprocessing directive, like "if", "for", "endfor", etc. // // See matchMkDirective. -func (mkline *MkLineImpl) Directive() string { return mkline.data.(mkLineDirective).directive } +func (mkline *MkLine) Directive() string { return mkline.data.(*mkLineDirective).directive } // Args returns the arguments from an .if, .ifdef, .ifndef, .elif, .for, .undef. -func (mkline *MkLineImpl) Args() string { return mkline.data.(mkLineDirective).args } +func (mkline *MkLine) Args() string { return mkline.data.(*mkLineDirective).args } // Cond applies to an .if or .elif line and returns the parsed condition. // // If a parse error occurs, it is silently swallowed, returning a // best-effort part of the condition, or even nil. -func (mkline *MkLineImpl) Cond() MkCond { - cond := mkline.data.(mkLineDirective).cond +func (mkline *MkLine) Cond() *MkCond { + cond := mkline.data.(*mkLineDirective).cond if cond == nil { cond = NewMkParser(mkline.Line, mkline.Args(), true).MkCond() - mkline.data.(mkLineDirective).cond = cond + mkline.data.(*mkLineDirective).cond = cond } return cond } // DirectiveComment is the trailing end-of-line comment, typically at a deeply nested .endif or .endfor. -func (mkline *MkLineImpl) DirectiveComment() string { return mkline.data.(mkLineDirective).comment } +func (mkline *MkLine) DirectiveComment() string { return mkline.data.(*mkLineDirective).comment } -func (mkline *MkLineImpl) HasElseBranch() bool { return mkline.data.(mkLineDirective).elseLine != nil } +func (mkline *MkLine) HasElseBranch() bool { return mkline.data.(*mkLineDirective).elseLine != nil } -func (mkline *MkLineImpl) SetHasElseBranch(elseLine MkLine) { - data := mkline.data.(mkLineDirective) +func (mkline *MkLine) SetHasElseBranch(elseLine *MkLine) { + data := mkline.data.(*mkLineDirective) data.elseLine = elseLine mkline.data = data } -func (mkline *MkLineImpl) MustExist() bool { return mkline.data.(mkLineInclude).mustExist } +func (mkline *MkLine) MustExist() bool { return mkline.data.(*mkLineInclude).mustExist } -func (mkline *MkLineImpl) IncludedFile() string { return mkline.data.(mkLineInclude).includedFile } +func (mkline *MkLine) IncludedFile() string { return mkline.data.(*mkLineInclude).includedFile } -func (mkline *MkLineImpl) Targets() string { return mkline.data.(mkLineDependency).targets } +func (mkline *MkLine) Targets() string { return mkline.data.(mkLineDependency).targets } -func (mkline *MkLineImpl) Sources() string { return mkline.data.(mkLineDependency).sources } +func (mkline *MkLine) Sources() string { return mkline.data.(mkLineDependency).sources } // ConditionalVars applies to .include lines and is a space-separated // list of those variable names on which the inclusion depends. // It is initialized later, step by step, when parsing other lines. -func (mkline *MkLineImpl) ConditionalVars() []string { - return mkline.data.(mkLineInclude).conditionalVars +func (mkline *MkLine) ConditionalVars() []string { + return mkline.data.(*mkLineInclude).conditionalVars } -func (mkline *MkLineImpl) SetConditionalVars(varnames []string) { - include := mkline.data.(mkLineInclude) +func (mkline *MkLine) SetConditionalVars(varnames []string) { + include := mkline.data.(*mkLineInclude) include.conditionalVars = varnames mkline.data = include } @@ -415,7 +424,7 @@ func (mkline *MkLineImpl) SetConditionalVars(varnames []string) { // output: [MkToken("${PREFIX}", MkVarUse("PREFIX")), MkToken("/bin abc")] // // See ValueTokens, which is the tokenized version of Value. -func (mkline *MkLineImpl) Tokenize(text string, warn bool) []*MkToken { +func (mkline *MkLine) Tokenize(text string, warn bool) []*MkToken { if trace.Tracing { defer trace.Call(mkline, text)() } @@ -449,8 +458,8 @@ func (mkline *MkLineImpl) Tokenize(text string, warn bool) []*MkToken { // at that point since the colon is inside a variable use. // // When several separators are adjacent, this results in empty words in the output. -func (mkline *MkLineImpl) ValueSplit(value string, separator string) []string { - assertf(separator != "", "Separator must not be empty; use ValueFields to split on whitespace") +func (mkline *MkLine) ValueSplit(value string, separator string) []string { + assert(separator != "") // Separator must not be empty; use ValueFields to split on whitespace. tokens := mkline.Tokenize(value, false) var split []string @@ -508,7 +517,7 @@ var notSpace = textproc.Space.Inverse() // Compare devel/bmake/files/str.c, function brk_string. // // TODO: Compare with brk_string from devel/bmake, especially for backticks. -func (mkline *MkLineImpl) ValueFields(value string) []string { +func (mkline *MkLine) ValueFields(value string) []string { if trace.Tracing { defer trace.Call(mkline, value)() } @@ -543,13 +552,13 @@ func (mkline *MkLineImpl) ValueFields(value string) []string { return words } -func (mkline *MkLineImpl) ValueTokens() ([]*MkToken, string) { +func (mkline *MkLine) ValueTokens() ([]*MkToken, string) { value := mkline.Value() if value == "" { return nil, "" } - assign := mkline.data.(mkLineAssign) + assign := mkline.data.(*mkLineAssign) if assign.valueMk != nil || assign.valueMkRest != "" { return assign.valueMk, assign.valueMkRest } @@ -565,14 +574,14 @@ func (mkline *MkLineImpl) ValueTokens() ([]*MkToken, string) { // Fields applies to variable assignments and .for loops. // For variable assignments, it returns the right-hand side, properly split into words. // For .for loops, it returns all arguments (including variable names), properly split into words. -func (mkline *MkLineImpl) Fields() []string { +func (mkline *MkLine) Fields() []string { if mkline.IsVarassign() || mkline.IsCommentedVarassign() { value := mkline.Value() if value == "" { return nil } - assign := mkline.data.(mkLineAssign) + assign := mkline.data.(*mkLineAssign) if assign.fields != nil { return assign.fields } @@ -587,7 +596,7 @@ func (mkline *MkLineImpl) Fields() []string { return nil } - directive := mkline.data.(mkLineDirective) + directive := mkline.data.(*mkLineDirective) if directive.fields != nil { return directive.fields } @@ -597,7 +606,7 @@ func (mkline *MkLineImpl) Fields() []string { } -func (*MkLineImpl) WithoutMakeVariables(value string) string { +func (*MkLine) WithoutMakeVariables(value string) string { var valueNovar strings.Builder for _, token := range NewMkParser(nil, value, false).MkTokens() { if token.Varuse == nil { @@ -607,7 +616,7 @@ func (*MkLineImpl) WithoutMakeVariables(value string) string { return valueNovar.String() } -func (mkline *MkLineImpl) ResolveVarsInRelativePath(relativePath string) string { +func (mkline *MkLine) ResolveVarsInRelativePath(relativePath string) string { if !contains(relativePath, "$") { return cleanpath(relativePath) } @@ -682,7 +691,7 @@ func (mkline *MkLineImpl) ResolveVarsInRelativePath(relativePath string) string return tmp } -func (mkline *MkLineImpl) ExplainRelativeDirs() { +func (mkline *MkLine) ExplainRelativeDirs() { mkline.Explain( "Directories in the form \"../../category/package\" make it easier to", "move a package around in pkgsrc, for example from pkgsrc-wip to the", @@ -694,7 +703,7 @@ func (mkline *MkLineImpl) ExplainRelativeDirs() { // // If there is a type mismatch when calling this function, try to add ".line" to // either the method receiver or the other line. -func (mkline *MkLineImpl) RefTo(other MkLine) string { +func (mkline *MkLine) RefTo(other *MkLine) string { return mkline.Line.RefTo(other.Line) } @@ -753,7 +762,7 @@ again: return main, lexer.Rest() } - assertf(lexer.EOF(), "unescapeComment(%q): sb = %q, rest = %q", text, main, lexer.Rest()) + assert(lexer.EOF()) return main, "" } @@ -775,7 +784,7 @@ type mkLineSplitResult struct { // This applies to all line types except those starting with a tab, which // contain the shell commands to be associated with make targets. These cannot // have comments. -func (p MkLineParser) split(line Line, text string) mkLineSplitResult { +func (p MkLineParser) split(line *Line, text string) mkLineSplitResult { mainWithSpaces, comment := p.unescapeComment(text) @@ -813,7 +822,7 @@ func (p MkLineParser) split(line Line, text string) mkLineSplitResult { tokens = append(tokens, &MkToken{other, nil}) } else { - assertf(lexer.SkipByte('$'), "Parse error for %q.", text) + assert(lexer.SkipByte('$')) tokens = append(tokens, &MkToken{"$", nil}) } } @@ -840,7 +849,7 @@ func (p MkLineParser) split(line Line, text string) mkLineSplitResult { return mkLineSplitResult{mainTrimmed, tokens, spaceBeforeComment, hasComment, comment} } -func (p MkLineParser) parseDirective(line Line, data mkLineSplitResult) MkLine { +func (p MkLineParser) parseDirective(line *Line, data mkLineSplitResult) *MkLine { text := line.Text if !hasPrefix(text, ".") { return nil @@ -871,7 +880,7 @@ func (p MkLineParser) parseDirective(line Line, data mkLineSplitResult) MkLine { // it must be trimmed. trimmedComment := trimHspace(data.comment) - return &MkLineImpl{line, &mkLineDirectiveImpl{indent, directive, args, trimmedComment, nil, nil, nil}} + return &MkLine{line, &mkLineDirective{indent, directive, args, trimmedComment, nil, nil, nil}} } // VariableNeedsQuoting determines whether the given variable needs the :Q operator @@ -880,7 +889,7 @@ func (p MkLineParser) parseDirective(line Line, data mkLineSplitResult) MkLine { // This decision depends on many factors, such as whether the type of the context is // a list of things, whether the variable is a list, whether it can contain only // safe characters, and so on. -func (mkline *MkLineImpl) VariableNeedsQuoting(mklines MkLines, varuse *MkVarUse, vartype *Vartype, vuc *VarUseContext) (needsQuoting YesNoUnknown) { +func (mkline *MkLine) VariableNeedsQuoting(mklines *MkLines, varuse *MkVarUse, vartype *Vartype, vuc *VarUseContext) (needsQuoting YesNoUnknown) { if trace.Tracing { defer trace.Call(varuse, vartype, vuc, trace.Result(&needsQuoting))() } @@ -986,11 +995,11 @@ func (mkline *MkLineImpl) VariableNeedsQuoting(mklines MkLines, varuse *MkVarUse } // ForEachUsed calls the action for each variable that is used in the line. -func (mkline *MkLineImpl) ForEachUsed(action func(varUse *MkVarUse, time vucTime)) { +func (mkline *MkLine) ForEachUsed(action func(varUse *MkVarUse, time VucTime)) { - var searchIn func(text string, time vucTime) // mutually recursive with searchInVarUse + var searchIn func(text string, time VucTime) // mutually recursive with searchInVarUse - searchInVarUse := func(varuse *MkVarUse, time vucTime) { + searchInVarUse := func(varuse *MkVarUse, time VucTime) { varname := varuse.varname if !varuse.IsExpression() { action(varuse, time) @@ -1001,7 +1010,7 @@ func (mkline *MkLineImpl) ForEachUsed(action func(varUse *MkVarUse, time vucTime } } - searchIn = func(text string, time vucTime) { + searchIn = func(text string, time VucTime) { if !contains(text, "$") { return } @@ -1016,31 +1025,31 @@ func (mkline *MkLineImpl) ForEachUsed(action func(varUse *MkVarUse, time vucTime switch { case mkline.IsVarassign(): - searchIn(mkline.Varname(), vucTimeLoad) + searchIn(mkline.Varname(), VucLoadTime) searchIn(mkline.Value(), mkline.Op().Time()) case mkline.IsDirective() && mkline.Directive() == "for": - searchIn(mkline.Args(), vucTimeLoad) + searchIn(mkline.Args(), VucLoadTime) case mkline.IsDirective() && mkline.Cond() != nil: mkline.Cond().Walk(&MkCondCallback{ VarUse: func(varuse *MkVarUse) { - searchInVarUse(varuse, vucTimeLoad) + searchInVarUse(varuse, VucLoadTime) }}) case mkline.IsShellCommand(): - searchIn(mkline.ShellCommand(), vucTimeRun) + searchIn(mkline.ShellCommand(), VucRunTime) case mkline.IsDependency(): - searchIn(mkline.Targets(), vucTimeLoad) - searchIn(mkline.Sources(), vucTimeLoad) + searchIn(mkline.Targets(), VucLoadTime) + searchIn(mkline.Sources(), VucLoadTime) case mkline.IsInclude(): - searchIn(mkline.IncludedFile(), vucTimeLoad) + searchIn(mkline.IncludedFile(), VucLoadTime) } } -func (*MkLineImpl) UnquoteShell(str string) string { +func (*MkLine) UnquoteShell(str string) string { var sb strings.Builder n := len(str) @@ -1115,11 +1124,11 @@ func (op MkOperator) String() string { // Time returns the time at which the right-hand side of the assignment is // evaluated. -func (op MkOperator) Time() vucTime { +func (op MkOperator) Time() VucTime { if op == opAssignShell || op == opAssignEval { - return vucTimeLoad + return VucLoadTime } - return vucTimeRun + return VucRunTime } // VarUseContext defines the context in which a variable is defined @@ -1138,34 +1147,39 @@ func (op MkOperator) Time() vucTime { // x86_64 doesn't make sense. type VarUseContext struct { vartype *Vartype - time vucTime + time VucTime quoting VucQuoting IsWordPart bool // Example: LOCALBASE=${LOCALBASE} } -// vucTime is the time at which a variable is used. +// VucTime is the time at which a variable is used. // // See ToolTime, which is the same except that there is no unknown. -type vucTime uint8 +type VucTime uint8 const ( - vucTimeUnknown vucTime = iota + VucUnknownTime VucTime = iota + // VucLoadTime marks a variable use that happens directly when + // the Makefile fragment is loaded. + // // When Makefiles are loaded, the operators := and != evaluate their // right-hand side, as well as the directives .if, .elif and .for. // During loading, not all variables are available yet. // Variable values are still subject to change, especially lists. - vucTimeLoad + VucLoadTime - // All files have been read, all variables can be referenced. - // Variable values don't change anymore. + // VucRunTime marks a variable use that happens after all files have been loaded. + // + // At this time, all variables can be referenced. // + // At this time, variable values don't change anymore. // Well, except for the ::= modifier. // But that modifier is usually not used in pkgsrc. - vucTimeRun + VucRunTime ) -func (t vucTime) String() string { return [...]string{"unknown", "parse", "run"}[t] } +func (t VucTime) String() string { return [...]string{"unknown", "load", "run"}[t] } // VucQuoting describes in what level of quoting the variable is used. // Depending on this context, the modifiers :Q or :M can be allowed or not. @@ -1198,19 +1212,20 @@ func (vuc *VarUseContext) String() string { // indentation. By convention, each directive is indented by 2 spaces. // An excepting are multiple-inclusion guards, they don't increase the // indentation. +// +// Indentation starts with 0 spaces. +// Each .if or .for indents all inner directives by 2. +// Except for .if with multiple-inclusion guard, which indents all inner directives by 0. +// Each .elif, .else, .endif, .endfor uses the outer indentation instead. type Indentation struct { levels []indentationLevel } -func NewIndentation() *Indentation { - ind := Indentation{} - ind.Push(nil, 0, "") // Dummy - return &ind -} +func NewIndentation() *Indentation { return &Indentation{} } func (ind *Indentation) String() string { var s strings.Builder - for _, level := range ind.levels[1:] { + for _, level := range ind.levels { _, _ = fmt.Fprintf(&s, " %d", level.depth) if len(level.conditionalVars) > 0 { _, _ = fmt.Fprintf(&s, " (%s)", strings.Join(level.conditionalVars, " ")) @@ -1219,13 +1234,13 @@ func (ind *Indentation) String() string { return "[" + trimHspace(s.String()) + "]" } -func (ind *Indentation) RememberUsedVariables(cond MkCond) { +func (ind *Indentation) RememberUsedVariables(cond *MkCond) { cond.Walk(&MkCondCallback{ VarUse: func(varuse *MkVarUse) { ind.AddVar(varuse.varname) }}) } type indentationLevel struct { - mkline MkLine // The line in which the indentation started; the .if/.for + mkline *MkLine // The line in which the indentation started; the .if/.for depth int // Number of space characters; always a multiple of 2 args string // The arguments from the .if or .for, or the latest .elif conditionalVars []string // Variables on which the current path depends @@ -1238,12 +1253,12 @@ type indentationLevel struct { checkedFiles []string } -func (ind *Indentation) Len() int { - return len(ind.levels) +func (ind *Indentation) Empty() bool { + return len(ind.levels) == 0 } func (ind *Indentation) top() *indentationLevel { - return &ind.levels[ind.Len()-1] + return &ind.levels[len(ind.levels)-1] } // Depth returns the number of space characters by which the directive @@ -1252,18 +1267,23 @@ func (ind *Indentation) top() *indentationLevel { // This is typically two more than the surrounding level, except for // multiple-inclusion guards. func (ind *Indentation) Depth(directive string) int { + i := len(ind.levels) - 1 switch directive { - case "if", "elif", "else", "endfor", "endif": - return ind.levels[imax(0, ind.Len()-2)].depth + case "elif", "else", "endfor", "endif": + i-- + } + if i < 0 { + return 0 } - return ind.top().depth + return ind.levels[i].depth } func (ind *Indentation) Pop() { - ind.levels = ind.levels[:ind.Len()-1] + ind.levels = ind.levels[:len(ind.levels)-1] } -func (ind *Indentation) Push(mkline MkLine, indent int, condition string) { +func (ind *Indentation) Push(mkline *MkLine, indent int, condition string) { + assert(mkline.IsDirective()) ind.levels = append(ind.levels, indentationLevel{mkline, indent, condition, nil, nil}) } @@ -1272,7 +1292,7 @@ func (ind *Indentation) Push(mkline MkLine, indent int, condition string) { // // Variables named *_MK are ignored since they are usually not interesting. func (ind *Indentation) AddVar(varname string) { - if hasSuffix(varname, "_MK") { + if hasSuffix(varname, "_MK") || ind.Empty() { return } @@ -1318,9 +1338,9 @@ func (ind *Indentation) Varnames() []string { var varnames []string for _, level := range ind.levels { for _, levelVarname := range level.conditionalVars { - assertf( - !hasSuffix(levelVarname, "_MK"), - "multiple-inclusion guard must be filtered out earlier.") + // multiple-inclusion guard must be filtered out earlier. + assert(!hasSuffix(levelVarname, "_MK")) + varnames = append(varnames, levelVarname) } } @@ -1337,7 +1357,9 @@ func (ind *Indentation) AddCheckedFile(filename string) { top.checkedFiles = append(top.checkedFiles, filename) } -func (ind *Indentation) IsCheckedFile(filename string) bool { +// HasExists returns whether the given filename has been tested in an +// exists(filename) condition and thus may or may not exist. +func (ind *Indentation) HasExists(filename string) bool { for _, level := range ind.levels { for _, levelFilename := range level.checkedFiles { if filename == levelFilename { @@ -1348,7 +1370,7 @@ func (ind *Indentation) IsCheckedFile(filename string) bool { return false } -func (ind *Indentation) TrackBefore(mkline MkLine) { +func (ind *Indentation) TrackBefore(mkline *MkLine) { if !mkline.IsDirective() { return } @@ -1358,11 +1380,11 @@ func (ind *Indentation) TrackBefore(mkline MkLine) { switch mkline.Directive() { case "for", "if", "ifdef", "ifndef": - ind.Push(mkline, ind.top().depth, mkline.Args()) + ind.Push(mkline, ind.Depth(mkline.Directive()), mkline.Args()) } } -func (ind *Indentation) TrackAfter(mkline MkLine) { +func (ind *Indentation) TrackAfter(mkline *MkLine) { if !mkline.IsDirective() { return } @@ -1385,16 +1407,17 @@ func (ind *Indentation) TrackAfter(mkline MkLine) { case "elif": // Handled here instead of TrackBefore to allow the action to access the previous condition. - ind.top().args = args + if !ind.Empty() { + ind.top().args = args + } case "else": - top := ind.top() - if top.mkline != nil { - top.mkline.SetHasElseBranch(mkline) + if !ind.Empty() { + ind.top().mkline.SetHasElseBranch(mkline) } case "endfor", "endif": - if ind.Len() > 1 { // Can only be false in unbalanced files. + if !ind.Empty() { // Can only be false in unbalanced files. ind.Pop() } } @@ -1422,12 +1445,12 @@ func (ind *Indentation) TrackAfter(mkline MkLine) { } func (ind *Indentation) CheckFinish(filename string) { - if ind.Len() <= 1 { + if ind.Empty() { return } eofLine := NewLineEOF(filename) - for ind.Len() > 1 { - openingMkline := ind.levels[ind.Len()-1].mkline + for !ind.Empty() { + openingMkline := ind.top().mkline eofLine.Errorf(".%s from %s must be closed.", openingMkline.Directive(), eofLine.RefTo(openingMkline.Line)) ind.Pop() } @@ -1448,10 +1471,10 @@ func (ind *Indentation) CheckFinish(filename string) { // of the variable. The square bracket is only allowed in the parameter part. var ( VarbaseBytes = textproc.NewByteSet("A-Za-z_0-9+---") - VarparamBytes = textproc.NewByteSet("A-Za-z_0-9#*+---.[") + VarparamBytes = textproc.NewByteSet("A-Za-z_0-9#*+---./[") ) -func (p MkLineParser) MatchVarassign(line Line, text string, asdfData mkLineSplitResult) (m bool, assignment mkLineAssign) { +func (p MkLineParser) MatchVarassign(line *Line, text string, asdfData mkLineSplitResult) (m bool, assignment *mkLineAssign) { // A commented variable assignment does not have leading whitespace. // Otherwise line 1 of almost every Makefile fragment would need to @@ -1518,7 +1541,7 @@ func (p MkLineParser) MatchVarassign(line Line, text string, asdfData mkLineSpli spaceBeforeComment = "" } - return true, &mkLineAssignImpl{ + return true, &mkLineAssign{ commented: commented, varname: varname, varcanon: varnameCanon(varname), diff --git a/pkgtools/pkglint/files/mkline_test.go b/pkgtools/pkglint/files/mkline_test.go index 455e495af36..be5c27aa521 100644 --- a/pkgtools/pkglint/files/mkline_test.go +++ b/pkgtools/pkglint/files/mkline_test.go @@ -159,7 +159,7 @@ func (s *Suite) Test_MkLineParser_Parse__autofix_space_after_varname(c *check.C) t.SetUpCommandLine("-Wspace") filename := t.CreateFileLines("Makefile", - MkRcsID, + MkCvsID, "VARNAME +=\t${VARNAME}", "VARNAME+ =\t${VARNAME+}", "VARNAME+ +=\t${VARNAME+}", @@ -182,7 +182,7 @@ func (s *Suite) Test_MkLineParser_Parse__autofix_space_after_varname(c *check.C) "AUTOFIX: ~/Makefile:2: Replacing \"VARNAME +=\" with \"VARNAME+=\".", "AUTOFIX: ~/Makefile:5: Replacing \"VARNAME+ ?=\" with \"VARNAME+?=\".") t.CheckFileLines("Makefile", - MkRcsID+"", + MkCvsID+"", "VARNAME+=\t${VARNAME}", "VARNAME+ =\t${VARNAME+}", "VARNAME+ +=\t${VARNAME+}", @@ -264,7 +264,7 @@ func (s *Suite) Test_MkLine_FirstLineContainsValue(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "VAR=\tvalue", "VAR= value \\", "\tstarts in first line", @@ -275,13 +275,9 @@ func (s *Suite) Test_MkLine_FirstLineContainsValue(c *check.C) { "#VAR= \\", "\tvalue starts in second line") - t.ExpectPanic( - func() { mklines.mklines[0].FirstLineContainsValue() }, - "Pkglint internal error: Line must be a variable assignment.") + t.ExpectAssert(func() { mklines.mklines[0].FirstLineContainsValue() }) - t.ExpectPanic( - func() { mklines.mklines[1].FirstLineContainsValue() }, - "Pkglint internal error: Line must be multiline.") + t.ExpectAssert(func() { mklines.mklines[1].FirstLineContainsValue() }) t.Check(mklines.mklines[2].FirstLineContainsValue(), equals, true) t.Check(mklines.mklines[3].FirstLineContainsValue(), equals, false) @@ -308,7 +304,7 @@ func (s *Suite) Test_VarUseContext_String(c *check.C) { t.SetUpVartypes() vartype := G.Pkgsrc.VariableType(nil, "PKGNAME") - vuc := VarUseContext{vartype, vucTimeUnknown, VucQuotBackt, false} + vuc := VarUseContext{vartype, VucUnknownTime, VucQuotBackt, false} c.Check(vuc.String(), equals, "(Pkgname (package-settable) time:unknown quoting:backt wordpart:false)") } @@ -363,7 +359,7 @@ func (s *Suite) Test_MkLineParser_Parse__infrastructure(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("infra.mk", - MkRcsID, + MkCvsID, " USE_BUILTIN.${_pkg_:S/^-//}:=no", ".error \"Something went wrong\"", ".export WRKDIR", @@ -399,7 +395,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__unknown_rhs(c *check.C) { mkline := t.NewMkLine("filename.mk", 1, "PKGNAME:= ${UNKNOWN}") t.SetUpVartypes() - vuc := VarUseContext{G.Pkgsrc.VariableType(nil, "PKGNAME"), vucTimeLoad, VucQuotUnknown, false} + vuc := VarUseContext{G.Pkgsrc.VariableType(nil, "PKGNAME"), VucLoadTime, VucQuotUnknown, false} nq := mkline.VariableNeedsQuoting(nil, &MkVarUse{"UNKNOWN", nil}, nil, &vuc) c.Check(nq, equals, unknown) @@ -411,11 +407,11 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__append_URL_to_list_of_URLs(c * t.SetUpVartypes() t.SetUpMasterSite("MASTER_SITE_SOURCEFORGE", "http://downloads.sourceforge.net/sourceforge/") mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "MASTER_SITES=\t${HOMEPAGE}") mkline := mklines.mklines[1] - vuc := VarUseContext{G.Pkgsrc.vartypes.Canon("MASTER_SITES"), vucTimeRun, VucQuotPlain, false} + vuc := VarUseContext{G.Pkgsrc.vartypes.Canon("MASTER_SITES"), VucRunTime, VucQuotPlain, false} nq := mkline.VariableNeedsQuoting(nil, &MkVarUse{"HOMEPAGE", nil}, G.Pkgsrc.vartypes.Canon("HOMEPAGE"), &vuc) c.Check(nq, equals, no) @@ -431,7 +427,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__append_list_to_list(c *check.C t.SetUpVartypes() t.SetUpMasterSite("MASTER_SITE_SOURCEFORGE", "http://downloads.sourceforge.net/sourceforge/") mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "MASTER_SITES=\t${MASTER_SITE_SOURCEFORGE:=squirrel-sql/}") MkLineChecker{mklines, mklines.mklines[1]}.checkVarassign() @@ -445,7 +441,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__eval_shell(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("builtin.mk", - MkRcsID, + MkCvsID, "USE_BUILTIN.Xfixes!=\t${PKG_ADMIN} pmatch 'pkg-[0-9]*' ${BUILTIN_PKG.Xfixes:Q}") MkLineChecker{mklines, mklines.mklines[1]}.checkVarassign() @@ -459,7 +455,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_single_quotes(c *ch t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "SUBST_SED.hpath=\t-e 's|^\\(INSTALL[\t:]*=\\).*|\\1${INSTALL}|'") MkLineChecker{mklines, mklines.mklines[1]}.checkVarassign() @@ -477,7 +473,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_command(c *check.C) t.SetUpTool("sort", "SORT", AtRunTime) G.Pkg = NewPackage(t.File("category/pkgbase")) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "GENERATE_PLIST= cd ${DESTDIR}${PREFIX}; ${FIND} * \\( -type f -or -type l \\) | ${SORT};") mklines.collectDefinedVariables() @@ -492,7 +488,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__word_as_part_of_word(c *check. t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "EGDIR=\t${EGDIR}/${MACHINE_GNU_PLATFORM}") MkLineChecker{mklines, mklines.mklines[1]}.Check() @@ -513,7 +509,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_as_command_argument(c t.SetUpTool("bash", "BASH", AtRunTime) t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "\t${RUN} cd ${WRKSRC} && ( ${ECHO} ${PERL5:Q} ; ${ECHO} ) | ${BASH} ./install", "\t${RUN} cd ${WRKSRC} && ( ${ECHO} ${PERL5} ; ${ECHO} ) | ${BASH} ./install") @@ -530,7 +526,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__URL_as_part_of_word_in_list(c t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "MASTER_SITES=${HOMEPAGE}archive/") MkLineChecker{mklines, mklines.mklines[1]}.Check() @@ -543,7 +539,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__MASTER_SITES_and_HOMEPAGE(c *c t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "MASTER_SITES=\t${HOMEPAGE}", "MASTER_SITES=\t${PATH}", // Some nonsense just for branch coverage. "HOMEPAGE=\t${MASTER_SITES}", @@ -568,7 +564,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_subshell(c *check.C t.SetUpTool("awk", "AWK", AtRunTime) t.SetUpTool("echo", "ECHO", AtRunTime) mklines := t.NewMkLines("xpi.mk", - MkRcsID, + MkCvsID, "\t id=$$(${AWK} '{print}' < ${WRKSRC}/idfile) && echo \"$$id\"", "\t id=`${AWK} '{print}' < ${WRKSRC}/idfile` && echo \"$$id\"") @@ -588,7 +584,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__LDFLAGS_in_single_quotes(c *ch t.SetUpVartypes() mklines := t.NewMkLines("x11/mlterm/Makefile", - MkRcsID, + MkCvsID, "SUBST_SED.link=-e 's|(LIBTOOL_LINK).*(LIBS)|& ${LDFLAGS:M*:Q}|g'", "SUBST_SED.link=-e 's|(LIBTOOL_LINK).*(LIBS)|& '${LDFLAGS:M*:Q}'|g'") @@ -605,7 +601,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__package_options(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "PKG_SUGGESTED_OPTIONS+=\t${PKG_DEFAULT_OPTIONS:Mcdecimal} ${PKG_OPTIONS.py-trytond:Mcdecimal}") MkLineChecker{mklines, mklines.mklines[1]}.Check() @@ -621,7 +617,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__tool_in_quotes_in_subshell_in_ t.SetUpTool("sh", "SH", AtRunTime) t.SetUpVartypes() mklines := t.NewMkLines("x11/labltk/Makefile", - MkRcsID, + MkCvsID, "CONFIGURE_ARGS+=\t-tklibs \"`${SH} -c '${ECHO} $$TK_LD_FLAGS'`\"") MkLineChecker{mklines, mklines.mklines[1]}.Check() @@ -677,7 +673,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__guessed_list_variable_in_quote t.SetUpVartypes() mklines := t.NewMkLines("audio/jack-rack/Makefile", - MkRcsID, + MkCvsID, "LADSPA_PLUGIN_PATH=\t${PREFIX}/lib/ladspa", "CPPFLAGS+=\t\t-DLADSPA_PATH=\"\\\"${LADSPA_PLUGIN_PATH}\\\"\"") @@ -692,7 +688,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__list_in_list(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("x11/eterm/Makefile", - MkRcsID, + MkCvsID, "DISTFILES=\t${DEFAULT_DISTFILES} ${PIXMAP_FILES}") mklines.Check() @@ -708,7 +704,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__PKGNAME_and_URL_list_in_URL_li t.SetUpMasterSite("MASTER_SITE_GNOME", "http://ftp.gnome.org/") t.SetUpVartypes() mklines := t.NewMkLines("x11/gtk3/Makefile", - MkRcsID, + MkCvsID, "MASTER_SITES=\tftp://ftp.gtk.org/${PKGNAME}/ ${MASTER_SITE_GNOME:=subdir/}") MkLineChecker{mklines, mklines.mklines[1]}.checkVarassignRightVaruse() @@ -722,7 +718,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__tool_in_CONFIGURE_ENV(c *check t.SetUpVartypes() t.SetUpTool("tar", "TAR", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "CONFIGURE_ENV+=\tSYS_TAR_COMMAND_PATH=${TOOLS_TAR:Q}") @@ -742,7 +738,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__backticks(c *check.C) { t.SetUpVartypes() t.SetUpTool("cat", "CAT", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "COMPILE_CMD=\tcc `${CAT} ${WRKDIR}/compileflags`", "COMMENT_CMD=\techo `echo ${COMMENT}`") @@ -770,7 +766,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__only_remove_known(c *check.C) t.SetUpVartypes() mklines := t.SetUpFileMkLines("Makefile", - MkRcsID, + MkCvsID, "", "demo: .PHONY", "\t${ECHO} ${WRKSRC:Q}", @@ -781,7 +777,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__only_remove_known(c *check.C) t.CheckOutputLines( "AUTOFIX: ~/Makefile:4: Replacing \"${WRKSRC:Q}\" with \"${WRKSRC}\".") t.CheckFileLines("Makefile", - MkRcsID, + MkCvsID, "", "demo: .PHONY", "\t${ECHO} ${WRKSRC}", @@ -797,7 +793,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__shellword_part(c *check.C) { t.SetUpVartypes() mklines := t.SetUpFileMkLines("Makefile", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+= class", "SUBST_STAGE.class= pre-configure", @@ -820,7 +816,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__tool_in_shell_command(c *check t.SetUpTool("bash", "BASH", AtRunTime) mklines := t.SetUpFileMkLines("Makefile", - MkRcsID, + MkCvsID, "", "CONFIG_SHELL= ${BASH}") @@ -838,7 +834,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__uncovered_cases(c *check.C) { t.SetUpVartypes() mklines := t.SetUpFileMkLines("Makefile", - MkRcsID, + MkCvsID, "", "GO_SRCPATH= ${HOMEPAGE:S,https://,,}", "LINKER_RPATH_FLAG:= ${LINKER_RPATH_FLAG:S/-rpath/& /}", @@ -929,7 +925,7 @@ func (s *Suite) Test_MkLine__shell_varuse_in_backt_dquot(c *check.C) { t.SetUpVartypes() t.SetUpTool("grep", "GREP", AtRunTime) mklines := t.NewMkLines("x11/motif/Makefile", - MkRcsID, + MkCvsID, "post-patch:", "\tfiles=`${GREP} -l \".fB$${name}.fP(3)\" *.3`") @@ -945,7 +941,7 @@ func (s *Suite) Test_MkLine__comment_in_comment(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "COMMENT=\tPKCS#5 v2.0 PBKDF2 Module") mklines.Check() @@ -1018,9 +1014,7 @@ func (s *Suite) Test_MkLine_ValueSplit__invalid_argument(c *check.C) { mkline := t.NewMkLine("filename.mk", 123, "VAR=\tvalue") - t.ExpectPanic( - func() { mkline.ValueSplit("value", "") }, - "Pkglint internal error: Separator must not be empty; use ValueFields to split on whitespace") + t.ExpectAssert(func() { mkline.ValueSplit("value", "") }) } func (s *Suite) Test_MkLine_Fields__varassign(c *check.C) { @@ -1272,7 +1266,7 @@ func (s *Suite) Test_MkLine_ValueTokens__warnings(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "ROUND=\t$(ROUND)") mklines.mklines[1].ValueTokens() @@ -1298,7 +1292,7 @@ func (s *Suite) Test_MkLine_ResolveVarsInRelativePath(c *check.C) { t.CreateFileLines("emulators/suse100_base/Makefile") t.CreateFileLines("lang/python36/Makefile") mklines := t.SetUpFileMkLines("Makefile", - MkRcsID) + MkCvsID) mkline := mklines.mklines[0] test := func(before string, after string) { @@ -1330,7 +1324,7 @@ func (s *Suite) Test_MkLine_ResolveVarsInRelativePath__directory_depth(c *check. t.SetUpVartypes() mklines := t.SetUpFileMkLines("multimedia/totem/bla.mk", - MkRcsID, + MkCvsID, "BUILDLINK_PKGSRCDIR.totem?=\t../../multimedia/totem") mklines.Check() @@ -1349,14 +1343,12 @@ func (s *Suite) Test_MkLine_ResolveVarsInRelativePath__without_tracing(c *check. t.DisableTracing() t.SetUpVartypes() mklines := t.SetUpFileMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "BUILDLINK_PKGSRCDIR.totem?=\t../../${PKGPATH.multimedia/totem}") mklines.Check() t.CheckOutputLines( - // FIXME: It's ok to have variable parameters including a slash. - "WARN: ~/buildlink3.mk:2: Invalid part \"/totem\" after variable name \"PKGPATH.multimedia\".", "WARN: ~/buildlink3.mk:2: PKGPATH.multimedia/totem is used but not defined.") } @@ -1372,7 +1364,7 @@ func (s *Suite) Test_MkLineParser_MatchVarassign(c *check.C) { return } - expected := mkLineAssignImpl{ + expected := mkLineAssign{ commented: commented, varname: varname, varcanon: varnameCanon(varname), @@ -1525,7 +1517,7 @@ func (s *Suite) Test_MkLineParser_MatchVarassign(c *check.C) { testInvalid("# VAR=value") testInvalid("#\tVAR=value") - testInvalid(MkRcsID) + testInvalid(MkCvsID) } func (s *Suite) Test_NewMkOperator(c *check.C) { @@ -1547,7 +1539,7 @@ func (s *Suite) Test_Indentation(c *check.C) { ind.Push(mkline, 2, "") - c.Check(ind.Depth("if"), equals, 0) // Because "if" is handled in MkLines.TrackBefore. + c.Check(ind.Depth("if"), equals, 2) c.Check(ind.Depth("endfor"), equals, 0) ind.AddVar("LEVEL1.VAR1") @@ -1579,12 +1571,75 @@ func (s *Suite) Test_Indentation(c *check.C) { c.Check(ind.String(), equals, "[]") } +func (s *Suite) Test_Indentation__realistic(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + ".if 1", + ". if !defined(GUARD_MK)", + ". for var in 1 2 3", + ". if !defined(GUARD_MK)", + ". if 3", + ". endif", + ". endif", + ". endfor", + ". endif", + ".elif 1", + ". for var in 1 2 3", + ". endfor", + ".else", + ". for var in 1 2 3", + ". endfor", + ".endif") + + t.EnableTracingToLog() + + mklines.ForEach(func(mkline *MkLine) {}) + + t.CheckOutputLinesMatching(`Indentation`, + "TRACE: Indentation before line 3: []", + "TRACE: Indentation after line 3: [2]", + "TRACE: Indentation before line 4: [2]", + "TRACE: Indentation after line 4: [2 2]", + "TRACE: Indentation before line 5: [2 2]", + "TRACE: Indentation after line 5: [2 2 4]", + "TRACE: Indentation before line 6: [2 2 4]", + "TRACE: Indentation after line 6: [2 2 4 4]", + "TRACE: Indentation before line 7: [2 2 4 4]", + "TRACE: Indentation after line 7: [2 2 4 4 6]", + "TRACE: Indentation before line 8: [2 2 4 4 6]", + "TRACE: Indentation after line 8: [2 2 4 4]", + "TRACE: Indentation before line 9: [2 2 4 4]", + "TRACE: Indentation after line 9: [2 2 4]", + "TRACE: Indentation before line 10: [2 2 4]", + "TRACE: Indentation after line 10: [2 2]", + "TRACE: Indentation before line 11: [2 2]", + "TRACE: Indentation after line 11: [2]", + "TRACE: Indentation before line 12: [2]", + "TRACE: Indentation after line 12: [2]", + "TRACE: Indentation before line 13: [2]", + "TRACE: Indentation after line 13: [2 4]", + "TRACE: Indentation before line 14: [2 4]", + "TRACE: Indentation after line 14: [2]", + "TRACE: Indentation before line 15: [2]", + "TRACE: Indentation after line 15: [2]", + "TRACE: Indentation before line 16: [2]", + "TRACE: Indentation after line 16: [2 4]", + "TRACE: Indentation before line 17: [2 4]", + "TRACE: Indentation after line 17: [2]", + "TRACE: Indentation before line 18: [2]", + "TRACE: Indentation after line 18: []") +} + func (s *Suite) Test_Indentation_RememberUsedVariables(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("Makefile", 123, ".if ${PKGREVISION} > 0") ind := NewIndentation() + ind.TrackBefore(mkline) ind.RememberUsedVariables(mkline.Cond()) t.CheckOutputEmpty() @@ -1595,7 +1650,7 @@ func (s *Suite) Test_Indentation_TrackAfter__checked_files(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("file.mk", - MkRcsID, + MkCvsID, "", ".if make(other.mk)", ". include \"other.mk\"", @@ -1617,7 +1672,7 @@ func (s *Suite) Test_Indentation_TrackAfter__lonely_else(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("file.mk", - MkRcsID, + MkCvsID, "", ".else") @@ -1659,7 +1714,7 @@ func (s *Suite) Test_MkLine_ForEachUsed(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "VAR=\t${VALUE} # ${varassign.comment}", ".if ${OPSYS:M${endianness}} == ${Hello:L} # ${if.comment}", ".for var in one ${two} three # ${for.comment}", @@ -1676,20 +1731,20 @@ func (s *Suite) Test_MkLine_ForEachUsed(c *check.C) { var varnames []string for _, mkline := range mklines.mklines { - mkline.ForEachUsed(func(varUse *MkVarUse, time vucTime) { + mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { varnames = append(varnames, time.String()+" "+varUse.varname) }) } c.Check(varnames, deepEquals, []string{ "run VALUE", - "parse OPSYS", - "parse endianness", + "load OPSYS", + "load endianness", // "Hello" is not a variable name, the :L modifier makes it an expression. - "parse two", - "parse TARGETS", - "parse SOURCES", - "parse OTHER_FILE", + "load two", + "load TARGETS", + "load SOURCES", + "load OTHER_FILE", "run VAR.${param}", "run param", @@ -1711,7 +1766,7 @@ func (s *Suite) Test_MkLine_UnquoteShell(c *check.C) { t := s.Init(c) test := func(input, output string) { - unquoted := (*MkLineImpl).UnquoteShell(nil, input) + unquoted := (*MkLine).UnquoteShell(nil, input) t.Check(unquoted, equals, output) } diff --git a/pkgtools/pkglint/files/mklinechecker.go b/pkgtools/pkglint/files/mklinechecker.go index 479e37fb215..6383c26410b 100644 --- a/pkgtools/pkglint/files/mklinechecker.go +++ b/pkgtools/pkglint/files/mklinechecker.go @@ -11,8 +11,8 @@ import ( // MkLineChecker provides checks for a single line from a Makefile fragment. type MkLineChecker struct { - MkLines MkLines - MkLine MkLine + MkLines *MkLines + MkLine *MkLine } func (ck MkLineChecker) Check() { @@ -181,19 +181,26 @@ func (ck MkLineChecker) checkDirectiveEnd(ind *Indentation) { directive := mkline.Directive() comment := mkline.DirectiveComment() - if directive == "endif" && comment != "" { + if ind.Empty() { + mkline.Errorf("Unmatched .%s.", directive) + return + } + + if comment == "" { + return + } + + if directive == "endif" { if args := ind.Args(); !contains(args, comment) { mkline.Warnf("Comment %q does not match condition %q.", comment, args) } } - if directive == "endfor" && comment != "" { + + if directive == "endfor" { if args := ind.Args(); !contains(args, comment) { mkline.Warnf("Comment %q does not match loop %q.", comment, args) } } - if ind.Len() <= 1 { - mkline.Errorf("Unmatched .%s.", directive) - } } func (ck MkLineChecker) checkDirectiveFor(forVars map[string]bool, indentation *Indentation) { @@ -225,8 +232,8 @@ func (ck MkLineChecker) checkDirectiveFor(forVars map[string]bool, indentation * // running pkglint over the whole pkgsrc tree did not produce any different result // whether guessed was true or false. forLoopType := NewVartype(btForLoop, List, NewACLEntry("*", aclpAllRead)) - forLoopContext := VarUseContext{forLoopType, vucTimeLoad, VucQuotPlain, false} - mkline.ForEachUsed(func(varUse *MkVarUse, time vucTime) { + forLoopContext := VarUseContext{forLoopType, VucLoadTime, VucQuotPlain, false} + mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { ck.CheckVaruse(varUse, &forLoopContext) }) } @@ -383,7 +390,7 @@ func (ck MkLineChecker) explainPermissions(varname string, vartype *Vartype, int for _, rule := range vartype.aclEntries { perms := rule.permissions.HumanString() - files := rule.glob + files := rule.matcher.originalPattern if files == "*" { files = "any file" } @@ -407,13 +414,13 @@ func (ck MkLineChecker) checkVarassignLeftRationale() { return } - isRationale := func(mkline MkLine) bool { + isRationale := func(mkline *MkLine) bool { return mkline.IsComment() && !hasPrefix(mkline.Text, "# $") && !mkline.IsCommentedVarassign() } - needsRationale := func(mkline MkLine) bool { + needsRationale := func(mkline *MkLine) bool { if !mkline.IsVarassign() && !mkline.IsCommentedVarassign() { return false } @@ -506,7 +513,7 @@ func (ck MkLineChecker) checkVarUseBuildDefs(varname string) { return } - if !(!ck.MkLines.buildDefs[varname] && ck.MkLines.FirstTimeSlice("BUILD_DEFS", varname)) { + if !(!ck.MkLines.buildDefs[varname] && ck.MkLines.once.FirstTimeSlice("BUILD_DEFS", varname)) { return } @@ -548,7 +555,7 @@ func (ck MkLineChecker) checkVaruseUndefined(vartype *Vartype, varname string) { case G.Pkgsrc.vartypes.DefinedCanon(varname): return - case !ck.MkLines.FirstTimeSlice("used but not defined: ", varname): + case !ck.MkLines.once.FirstTimeSlice("used but not defined: ", varname): return } @@ -671,7 +678,7 @@ func (ck MkLineChecker) checkVarusePermissions(varname string, vartype *Vartype, // be used at load time somewhere in the future because it is // assigned to another variable, and that variable is allowed // to be used at load time. - directly := vuc.time == vucTimeLoad + directly := vuc.time == VucLoadTime indirectly := !directly && vuc.vartype != nil && vuc.vartype.Union().Contains(aclpUseLoadtime) @@ -700,13 +707,13 @@ func (ck MkLineChecker) checkVarusePermissions(varname string, vartype *Vartype, } } - if ck.MkLines.FirstTimeSlice("checkVarusePermissions", varname) { - ck.warnVarusePermissions(varname, vartype, directly, indirectly) + if ck.MkLines.once.FirstTimeSlice("checkVarusePermissions", varname) { + ck.warnVarusePermissions(vuc.vartype, varname, vartype, directly, indirectly) } } func (ck MkLineChecker) warnVarusePermissions( - varname string, vartype *Vartype, directly, indirectly bool) { + vucVartype *Vartype, varname string, vartype *Vartype, directly, indirectly bool) { mkline := ck.MkLine @@ -718,6 +725,13 @@ func (ck MkLineChecker) warnVarusePermissions( } if indirectly { + // Some of the guessed variables may be used at load time. But since the + // variable type and these permissions are guessed, pkglint should not + // issue the following warning, since it is often wrong. + if vucVartype.Guessed() { + return + } + mkline.Warnf("%s should not be used indirectly at load time (via %s).", varname, mkline.Varname()) ck.explainPermissions(varname, vartype, @@ -841,14 +855,35 @@ func (ck MkLineChecker) checkVarUseQuoting(varUse *MkVarUse, vartype *Vartype, v mkline := ck.MkLine if mod == ":M*:Q" && !needMstar { - mkline.Notef("The :M* modifier is not needed here.") + if !vartype.Guessed() { + mkline.Notef("The :M* modifier is not needed here.") + } } else if needsQuoting == yes { modNoQ := strings.TrimSuffix(mod, ":Q") modNoM := strings.TrimSuffix(modNoQ, ":M*") correctMod := modNoM + ifelseStr(needMstar, ":M*:Q", ":Q") if correctMod == mod+":Q" && vuc.IsWordPart && !vartype.IsShell() { - if vartype.List() { + + isSingleWordConstant := func() bool { + if G.Pkg == nil { + return false + } + + varinfo := G.Pkg.redundant.vars[varname] + if varinfo == nil || !varinfo.vari.Constant() { + return false + } + + value := varinfo.vari.ConstantValue() + return len(mkline.ValueFields(value)) == 1 + } + + if vartype.List() && isSingleWordConstant() { + // Do not warn in this special case, which typically occurs + // for BUILD_DIRS or similar package-settable variables. + + } else if vartype.List() { mkline.Warnf("The list variable %s should not be embedded in a word.", varname) mkline.Explain( "When a list variable has multiple elements, this expression expands", @@ -982,7 +1017,7 @@ func (ck MkLineChecker) checkVarassign() { // checkVarassignLeft checks everything to the left of the assignment operator. func (ck MkLineChecker) checkVarassignLeft() { varname := ck.MkLine.Varname() - if hasPrefix(varname, "_") && !G.Infrastructure { + if hasPrefix(varname, "_") && !G.Infrastructure && G.Pkgsrc.vartypes.Canon(varname) == nil { ck.MkLine.Warnf("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", varname) } @@ -997,7 +1032,7 @@ func (ck MkLineChecker) checkVarassignLeft() { ck.checkTextVarUse( ck.MkLine.Varname(), NewVartype(BtVariableName, NoVartypeOptions, NewACLEntry("*", aclpAll)), - vucTimeLoad) + VucLoadTime) } func (ck MkLineChecker) checkVarassignOp() { @@ -1112,7 +1147,7 @@ func (ck MkLineChecker) checkVarassignLeftNotUsed() { return } - if !ck.MkLines.FirstTimeSlice("defined but not used: ", varname) { + if !ck.MkLines.once.FirstTimeSlice("defined but not used: ", varname) { return } @@ -1138,9 +1173,9 @@ func (ck MkLineChecker) checkVarassignRightVaruse() { mkline := ck.MkLine op := mkline.Op() - time := vucTimeRun + time := VucRunTime if op == opAssignEval || op == opAssignShell { - time = vucTimeLoad + time = VucLoadTime } vartype := G.Pkgsrc.VariableType(ck.MkLines, mkline.Varname()) @@ -1155,7 +1190,7 @@ func (ck MkLineChecker) checkVarassignRightVaruse() { } } -func (ck MkLineChecker) checkTextVarUse(text string, vartype *Vartype, time vucTime) { +func (ck MkLineChecker) checkTextVarUse(text string, vartype *Vartype, time VucTime) { if !contains(text, "$") { return } @@ -1178,7 +1213,7 @@ func (ck MkLineChecker) checkTextVarUse(text string, vartype *Vartype, time vucT // checkVarassignVaruseShell is very similar to checkVarassignRightVaruse, they just differ // in the way they determine isWordPart. -func (ck MkLineChecker) checkVarassignVaruseShell(vartype *Vartype, time vucTime) { +func (ck MkLineChecker) checkVarassignVaruseShell(vartype *Vartype, time VucTime) { if trace.Tracing { defer trace.Call(vartype, time)() } @@ -1229,7 +1264,8 @@ func (ck MkLineChecker) checkVarassignMisc() { "It is this meaning that should be described.") } - if varname == "DIST_SUBDIR" || varname == "WRKSRC" { + switch varname { + case "DIST_SUBDIR", "WRKSRC", "MASTER_SITES": // TODO: Replace regex with proper VarUse. if m, revVarname := match1(value, `\$\{(PKGNAME|PKGVERSION)[:\}]`); m { mkline.Warnf("%s should not be used in %s as it includes the PKGREVISION. "+ @@ -1265,7 +1301,7 @@ func (ck MkLineChecker) checkVarassignLeftBsdPrefs() { G.Infrastructure || mkline.Op() != opAssignDefault || ck.MkLines.Tools.SeenPrefs || - !ck.MkLines.FirstTime("include bsd.prefs.mk before using ?=") { + !ck.MkLines.once.FirstTime("include bsd.prefs.mk before using ?=") { return } @@ -1457,14 +1493,14 @@ func (ck MkLineChecker) checkDirectiveCond() { checkVarUse := func(varuse *MkVarUse) { var vartype *Vartype // TODO: Insert a better type guess here. - vuc := VarUseContext{vartype, vucTimeLoad, VucQuotPlain, false} + vuc := VarUseContext{vartype, VucLoadTime, VucQuotPlain, false} ck.CheckVaruse(varuse, &vuc) } // Skip subconditions that have already been handled as part of the !(...). done := make(map[interface{}]bool) - checkNotEmpty := func(not MkCond) { + checkNotEmpty := func(not *MkCond) { empty := not.Empty if empty != nil { ck.checkDirectiveCondEmpty(empty, true, true, not == cond.Not) @@ -1683,7 +1719,7 @@ func (ck MkLineChecker) CheckRelativePath(relativePath string, mustExist bool) { abs := path.Dir(mkline.Filename) + "/" + resolvedPath if _, err := os.Stat(abs); err != nil { - if mustExist && !ck.MkLines.indentation.IsCheckedFile(resolvedPath) { + if mustExist && !ck.MkLines.indentation.HasExists(resolvedPath) { mkline.Errorf("Relative path %q does not exist.", resolvedPath) } return diff --git a/pkgtools/pkglint/files/mklinechecker_test.go b/pkgtools/pkglint/files/mklinechecker_test.go index 402bcf0f629..985942097b4 100644 --- a/pkgtools/pkglint/files/mklinechecker_test.go +++ b/pkgtools/pkglint/files/mklinechecker_test.go @@ -9,10 +9,10 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeft(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("module.mk", - MkRcsID, + MkCvsID, "_VARNAME=\tvalue") // Only to prevent "defined but not used". - mklines.vars.Use("_VARNAME", mklines.mklines[1], vucTimeRun) + mklines.vars.Use("_VARNAME", mklines.mklines[1], VucRunTime) mklines.Check() @@ -26,7 +26,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftNotUsed__procedure_call(c * t.CreateFileLines("mk/pkg-build-options.mk") mklines := t.SetUpFileMkLines("category/package/filename.mk", - MkRcsID, + MkCvsID, "", "pkgbase := glib2", ".include \"../../mk/pkg-build-options.mk\"", @@ -53,7 +53,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftNotUsed__procedure_call_no_ t.DisableTracing() // Just for code coverage t.CreateFileLines("mk/pkg-build-options.mk") mklines := t.SetUpFileMkLines("category/package/filename.mk", - MkRcsID, + MkCvsID, "", "pkgbase := glib2", ".include \"../../mk/pkg-build-options.mk\"") @@ -67,7 +67,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftNotUsed__infra(c *check.C) t := s.Init(c) t.CreateFileLines("mk/infra.mk", - MkRcsID, + MkCvsID, "#", "# Package-settable variables:", "#", @@ -98,8 +98,9 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeft__infrastructure(c *check.C t.SetUpPkgsrc() t.CreateFileLines("mk/infra.mk", - MkRcsID, - "_VARNAME=\tvalue") + MkCvsID, + "_VARNAME=\t\tvalue", + "_SORTED_VARS.group=\tVARNAME") t.FinishSetUp() G.Check(t.File("mk/infra.mk")) @@ -108,6 +109,20 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeft__infrastructure(c *check.C "WARN: ~/mk/infra.mk:2: _VARNAME is defined but not used.") } +func (s *Suite) Test_MkLineChecker_checkVarassignLeft__documented_underscore(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("category/package/filename.mk", + MkCvsID, + "_SORTED_VARS.group=\tVARNAME") + t.FinishSetUp() + + G.Check(t.File("category/package/filename.mk")) + + t.CheckOutputEmpty() +} + func (s *Suite) Test_MkLineChecker_checkVarassignLeftUserSettable(c *check.C) { t := s.Init(c) @@ -125,7 +140,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftUserSettable(c *check.C) { "COMMENTED_SAME?=\tdefault", // commented default, same value as default "COMMENTED_DIFF?=\tpkg") // commented default, differs from default value t.CreateFileLines("mk/defaults/mk.conf", - MkRcsID, + MkCvsID, "ASSIGN_DIFF?=default", "ASSIGN_DIFF2?=default", "ASSIGN_SAME?=default", @@ -161,7 +176,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftUserSettable__before_prefs( "BEFORE=\tvalue", ".include \"../../mk/bsd.prefs.mk\"") t.CreateFileLines("mk/defaults/mk.conf", - MkRcsID, + MkCvsID, "BEFORE?=\tvalue") t.Chdir("category/package") t.FinishSetUp() @@ -184,7 +199,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftUserSettable__after_prefs(c ".include \"../../mk/bsd.prefs.mk\"", "AFTER=\tvalue") t.CreateFileLines("mk/defaults/mk.conf", - MkRcsID, + MkCvsID, "AFTER?=\t\tvalue") t.Chdir("category/package") t.FinishSetUp() @@ -199,7 +214,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftUserSettable__vartype_nil(c t := s.Init(c) t.CreateFileLines("category/package/vars.mk", - MkRcsID, + MkCvsID, "#", "# User-settable variables:", "#", @@ -227,7 +242,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftBsdPrefs__vartype_nil(c *ch t := s.Init(c) mklines := t.NewMkLines("builtin.mk", - MkRcsID, + MkCvsID, "VAR_SH?=\tvalue") mklines.Check() @@ -243,7 +258,7 @@ func (s *Suite) Test_MkLineChecker_Check__url2pkg(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "# url2pkg-marker") mklines.Check() @@ -260,7 +275,7 @@ func (s *Suite) Test_MkLineChecker_Check__buildlink3_include_prefs(c *check.C) { t.CreateFileLines("mk/bsd.prefs.mk") t.CreateFileLines("mk/bsd.fast.prefs.mk") mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", - MkRcsID, + MkCvsID, ".include \"../../mk/bsd.prefs.mk\"", ".include \"../../mk/bsd.fast.prefs.mk\"") @@ -287,7 +302,7 @@ func (s *Suite) Test_MkLineChecker_checkInclude(c *check.C) { t.CreateFileLines("devel/intltool/buildlink3.mk") t.CreateFileLines("devel/intltool/builtin.mk") mklines := t.SetUpFileMkLines("category/package/filename.mk", - MkRcsID, + MkCvsID, "", ".include \"../../pkgtools/x11-links/buildlink3.mk\"", ".include \"../../graphics/jpeg/buildlink3.mk\"", @@ -314,7 +329,7 @@ func (s *Suite) Test_MkLineChecker_checkInclude__Makefile(c *check.C) { t := s.Init(c) mklines := t.NewMkLines(t.File("Makefile"), - MkRcsID, + MkCvsID, ".include \"../../other/package/Makefile\"") mklines.Check() @@ -336,7 +351,7 @@ func (s *Suite) Test_MkLineChecker_checkInclude__Makefile_exists(c *check.C) { G.checkdirPackage(t.File("category/package")) t.CheckOutputLines( - "ERROR: ~/category/package/Makefile:20: Cannot read \"../../other/existing/Makefile\".") + "ERROR: ~/category/package/Makefile:21: Cannot read \"../../other/not-found/Makefile\".") } func (s *Suite) Test_MkLineChecker_checkInclude__hacks(c *check.C) { @@ -344,11 +359,11 @@ func (s *Suite) Test_MkLineChecker_checkInclude__hacks(c *check.C) { t.SetUpPackage("category/package") t.CreateFileLines("category/package/hacks.mk", - MkRcsID, + MkCvsID, ".include \"../../category/package/nonexistent.mk\"", ".include \"../../category/package/builtin.mk\"") t.CreateFileLines("category/package/builtin.mk", - MkRcsID) + MkCvsID) t.FinishSetUp() G.checkdirPackage(t.File("category/package")) @@ -366,7 +381,7 @@ func (s *Suite) Test_MkLineChecker__permissions_in_hacks_mk(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("hacks.mk", - MkRcsID, + MkCvsID, "OPSYS=\t${PKGREVISION}") mklines.Check() @@ -383,7 +398,7 @@ func (s *Suite) Test_MkLineChecker_checkDirective(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("category/package/filename.mk", - MkRcsID, + MkCvsID, "", ".for", ".endfor", @@ -422,7 +437,7 @@ func (s *Suite) Test_MkLineChecker_checkDirective__for_loop_varname(c *check.C) t.SetUpVartypes() mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "", ".for VAR in a b c", // Should be lowercase. ".endfor", @@ -450,7 +465,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveEnd__ending_comments(c *check.C t.SetUpVartypes() mklines := t.NewMkLines("opsys.mk", - MkRcsID, + MkCvsID, "", ".for i in 1 2 3 4 5", ". if ${OPSYS} == NetBSD", @@ -488,12 +503,32 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveEnd__ending_comments(c *check.C "WARN: opsys.mk:24: Comment \"ii\" does not match loop \"jj in 1 2\".") } +// After removing the dummy indentation in commit d5a926af, +// there was a panic: runtime error: index out of range, +// in wip/jacorb-lib/buildlink3.mk. +func (s *Suite) Test_MkLineChecker_checkDirectiveEnd__unbalanced(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + ".endfor # comment", + ".endif # comment") + + mklines.Check() + + t.CheckOutputLines( + "ERROR: filename.mk:3: Unmatched .endfor.", + "ERROR: filename.mk:4: Unmatched .endif.") +} + func (s *Suite) Test_MkLineChecker_checkDirectiveFor(c *check.C) { t := s.Init(c) t.SetUpVartypes() mklines := t.NewMkLines("for.mk", - MkRcsID, + MkCvsID, ".for dir in ${PATH:C,:, ,g}", ".endfor", "", @@ -523,7 +558,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveFor__infrastructure(c *check.C) t.SetUpPkgsrc() t.CreateFileLines("mk/file.mk", - MkRcsID, + MkCvsID, ".for i = 1 2 3", // The "=" should rather be "in". ".endfor", "", @@ -544,7 +579,7 @@ func (s *Suite) Test_MkLineChecker_checkDependencyRule(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("category/package/filename.mk", - MkRcsID, + MkCvsID, "", ".PHONY: target-1", "target-2: .PHONY", @@ -573,7 +608,7 @@ func (s *Suite) Test_MkLineChecker_checkVartype__simple_type(c *check.C) { c.Check(vartype.List(), equals, false) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "COMMENT=\tA nice package") mklines.Check() @@ -586,7 +621,7 @@ func (s *Suite) Test_MkLineChecker_checkVartype(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "DISTNAME=\tgcc-${GCC_VERSION}") mklines.vars.Define("GCC_VERSION", mklines.mklines[1]) @@ -600,7 +635,7 @@ func (s *Suite) Test_MkLineChecker_checkVartype__append_to_non_list(c *check.C) t.SetUpVartypes() mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "DISTNAME+=\tsuffix", "COMMENT=\tComment for", "COMMENT+=\tthe package") @@ -618,7 +653,7 @@ func (s *Suite) Test_MkLineChecker_checkVartype__no_tracing(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "UNKNOWN=\tvalue", "CUR_DIR!=\tpwd") t.DisableTracing() @@ -638,7 +673,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassign__URL_with_shell_special_charac G.Pkg = NewPackage(t.File("graphics/gimp-fix-ca")) t.SetUpVartypes() mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "MASTER_SITES=\thttp://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=") mklines.Check() @@ -653,7 +688,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassign__list(c *check.C) { t.SetUpVartypes() t.SetUpCommandLine("-Wall", "--explain") mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "SITES.distfile=\t-${MASTER_SITE_GITHUB:=project/}") mklines.Check() @@ -688,7 +723,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { test := func(cond string, output ...string) { mklines := t.NewMkLines("filename.mk", cond) - mklines.ForEach(func(mkline MkLine) { + mklines.ForEach(func(mkline *MkLine) { MkLineChecker{mklines, mkline}.checkDirectiveCond() }) t.CheckOutput(output) @@ -759,17 +794,17 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { "TRACE: 1 + (*MkParser).mkCondAtom(\"${VAR:Mpattern1:Mpattern2} == comparison\")", "TRACE: 1 - (*MkParser).mkCondAtom(\"${VAR:Mpattern1:Mpattern2} == comparison\")", "TRACE: 1 checkCompareVarStr ${VAR:Mpattern1:Mpattern2} == comparison", - "TRACE: 1 + MkLineChecker.CheckVaruse(filename.mk:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:parse quoting:plain wordpart:false))", + "TRACE: 1 + MkLineChecker.CheckVaruse(filename.mk:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:load quoting:plain wordpart:false))", "TRACE: 1 2 + (*Pkgsrc).VariableType(\"VAR\")", "TRACE: 1 2 3 No type definition found for \"VAR\".", "TRACE: 1 2 - (*Pkgsrc).VariableType(\"VAR\", \"=>\", (*pkglint.Vartype)(nil))", "WARN: filename.mk:1: VAR is used but not defined.", - "TRACE: 1 2 + MkLineChecker.checkVarusePermissions(\"VAR\", (no-type time:parse quoting:plain wordpart:false))", + "TRACE: 1 2 + MkLineChecker.checkVarusePermissions(\"VAR\", (no-type time:load quoting:plain wordpart:false))", "TRACE: 1 2 3 No type definition found for \"VAR\".", - "TRACE: 1 2 - MkLineChecker.checkVarusePermissions(\"VAR\", (no-type time:parse quoting:plain wordpart:false))", - "TRACE: 1 2 + (*MkLineImpl).VariableNeedsQuoting(${VAR:Mpattern1:Mpattern2}, (*pkglint.Vartype)(nil), (no-type time:parse quoting:plain wordpart:false))", - "TRACE: 1 2 - (*MkLineImpl).VariableNeedsQuoting(${VAR:Mpattern1:Mpattern2}, (*pkglint.Vartype)(nil), (no-type time:parse quoting:plain wordpart:false), \"=>\", unknown)", - "TRACE: 1 - MkLineChecker.CheckVaruse(filename.mk:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:parse quoting:plain wordpart:false))", + "TRACE: 1 2 - MkLineChecker.checkVarusePermissions(\"VAR\", (no-type time:load quoting:plain wordpart:false))", + "TRACE: 1 2 + (*MkLine).VariableNeedsQuoting(${VAR:Mpattern1:Mpattern2}, (*pkglint.Vartype)(nil), (no-type time:load quoting:plain wordpart:false))", + "TRACE: 1 2 - (*MkLine).VariableNeedsQuoting(${VAR:Mpattern1:Mpattern2}, (*pkglint.Vartype)(nil), (no-type time:load quoting:plain wordpart:false), \"=>\", unknown)", + "TRACE: 1 - MkLineChecker.CheckVaruse(filename.mk:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:load quoting:plain wordpart:false))", "TRACE: - MkLineChecker.checkDirectiveCond(\"${VAR:Mpattern1:Mpattern2} == comparison\")", "TRACE: + (*MkParser).mkCondAtom(\"${VAR:Mpattern1:Mpattern2} == comparison\")", "TRACE: - (*MkParser).mkCondAtom(\"${VAR:Mpattern1:Mpattern2} == comparison\")", @@ -783,7 +818,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassign(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "ac_cv_libpari_libs+=\t-L${BUILDLINK_PREFIX.pari}/lib") // From math/clisp-pari/Makefile, rev. 1.8 mklines.Check() @@ -803,7 +838,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions(c *check.C) { "options.mk: set", "*.mk: default, set") mklines := t.NewMkLines("options.mk", - MkRcsID, + MkCvsID, "PKG_DEVELOPER?=\tyes", "BUILD_DEFS?=\tVARBASE", "USE_TOOLS:=\t${USE_TOOLS:Nunwanted-tool}", @@ -842,7 +877,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__no_tracing(c * t.SetUpVartypes() t.DisableTracing() // Just to reach branch coverage for unknown permissions. mklines := t.NewMkLines("options.mk", - MkRcsID, + MkCvsID, "COMMENT=\tShort package description") mklines.Check() @@ -858,7 +893,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__license_defaul t.SetUpPkgsrc() mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "LICENSE?=\tgnu-gpl-v2") t.FinishSetUp() @@ -877,7 +912,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__infrastructure t.SetUpVartypes() t.CreateFileLines("mk/infra.mk", - MkRcsID, + MkCvsID, "", "PKG_DEVELOPER?=\tyes") t.CreateFileLines("mk/bsd.pkg.mk") @@ -901,13 +936,13 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftRationale(c *check.C) { t.CheckOutput(diagnostics) } test := func(lines []string, diagnostics ...string) { - testLines(append([]string{MkRcsID, ""}, lines...), diagnostics...) + testLines(append([]string{MkCvsID, ""}, lines...), diagnostics...) } lines := func(lines ...string) []string { return lines } test( lines( - MkRcsID, + MkCvsID, "ONLY_FOR_PLATFORM=\t*-*-*", // The CVS Id above is not a rationale. "NOT_FOR_PLATFORM=\t*-*-*", // Neither does this line have a rationale. ), @@ -962,7 +997,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftRationale(c *check.C) { lines( "NOT_FOR_PLATFORM=\t*-*-*", "NOT_FOR_PLATFORM=\t*-*-*"), - sprintf("ERROR: filename.mk:1: Expected %q.", MkRcsID), + sprintf("ERROR: filename.mk:1: Expected %q.", MkCvsID), "WARN: filename.mk:1: Setting variable NOT_FOR_PLATFORM should have a rationale.", "WARN: filename.mk:2: Setting variable NOT_FOR_PLATFORM should have a rationale.") @@ -971,7 +1006,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftRationale(c *check.C) { test( lines( - MkRcsID, + MkCvsID, "ONLY_FOR_PLATFORM=\t*-*-*", // The CVS Id above is not a rationale. "NOT_FOR_PLATFORM=\t*-*-*", // Neither does this line have a rationale. ), @@ -987,7 +1022,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignOpShell(c *check.C) { t.SetUpPackage("category/package", ".include \"standalone.mk\"") t.CreateFileLines("category/package/standalone.mk", - MkRcsID, + MkCvsID, "", ".include \"../../mk/bsd.prefs.mk\"", "", @@ -1060,7 +1095,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignRightVaruse(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("module.mk", - MkRcsID, + MkCvsID, "PLIST_SUBST+=\tLOCALBASE=${LOCALBASE:Q}") mklines.Check() @@ -1075,7 +1110,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("options.mk", - MkRcsID, + MkCvsID, "COMMENT=\t${GAMES_USER}", "COMMENT:=\t${PKGBASE}", "PYPKGPREFIX=\t${PKGBASE}") @@ -1096,7 +1131,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__explain(c *check.C) { t.SetUpCommandLine("-Wall", "--explain") t.SetUpVartypes() mklines := t.NewMkLines("options.mk", - MkRcsID, + MkCvsID, "COMMENT=\t${GAMES_USER}", "COMMENT:=\t${PKGBASE}", "PYPKGPREFIX=\t${PKGBASE}") @@ -1149,7 +1184,7 @@ func (s *Suite) Test_MkLineChecker_explainPermissions(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "AUTO_MKDIRS=\tyes") mklines.Check() @@ -1181,7 +1216,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time(c *check.C) t.SetUpVartypes() mklines := t.NewMkLines("options.mk", - MkRcsID, + MkCvsID, "WRKSRC:=${.CURDIR}", ".if ${PKG_SYSCONFDIR.gdm} != \"etc\"", ".endif") @@ -1205,7 +1240,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time_in_conditio "special:filename.mk: use") mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, ".if ${LOAD_TIME} && ${RUN_TIME}", ".endif") @@ -1224,7 +1259,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time_in_for_loop "special:filename.mk: use") mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, ".for pattern in ${LOAD_TIME} ${RUN_TIME}", ".endfor") @@ -1240,7 +1275,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time_guessed(c * t.SetUpVartypes() t.SetUpTool("install", "", AtRunTime) mklines := t.NewMkLines("install-docfiles.mk", - MkRcsID, + MkCvsID, "DOCFILES=\ta b c", "do-install:", ".for f in ${DOCFILES}", @@ -1280,7 +1315,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time_run_time(c "*.mk: set") mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, ".if ${LOAD_TIME} && ${RUN_TIME} && ${WRITE_ONLY}", ".elif ${LOAD_TIME_ELSEWHERE} && ${RUN_TIME_ELSEWHERE}", ".endif") @@ -1304,7 +1339,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__PKGREVISION(c *check. t.SetUpVartypes() mklines := t.NewMkLines("any.mk", - MkRcsID, + MkCvsID, ".if defined(PKGREVISION)", ".endif") @@ -1321,7 +1356,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__indirectly(c *check.C t.SetUpVartypes() mklines := t.NewMkLines("file.mk", - MkRcsID, + MkCvsID, "IGNORE_PKG.package=\t${ONLY_FOR_UNPRIVILEGED}") mklines.Check() @@ -1338,7 +1373,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__indirectly_tool(c *ch t.SetUpVartypes() mklines := t.NewMkLines("file.mk", - MkRcsID, + MkCvsID, "USE_TOOLS+=\t${PKGREVISION}") mklines.Check() @@ -1352,7 +1387,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__write_only_usable_in_ t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "VAR=\t${VAR} ${AUTO_MKDIRS}") mklines.Check() @@ -1370,7 +1405,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__usable_only_at_loadti G.Pkgsrc.vartypes.DefineParse("VAR", BtFileName, NoVartypeOptions, "*: set, use-loadtime") mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "VAR=\t${VAR}") mklines.Check() @@ -1392,7 +1427,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__assigned_to_infrastru "buildlink3.mk: none", "*: use") mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "INFRA=\t${VAR}") mklines.Check() @@ -1429,7 +1464,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__assigned_to_load_time "buildlink3.mk: none", "*.mk: use") mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "LOAD_TIME=\t${VAR}") mklines.Check() @@ -1444,7 +1479,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__multiple_times_per_fi t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", - MkRcsID, + MkCvsID, "VAR=\t${VAR} ${AUTO_MKDIRS} ${AUTO_MKDIRS} ${PKGREVISION} ${PKGREVISION}", "VAR=\t${VAR} ${AUTO_MKDIRS} ${AUTO_MKDIRS} ${PKGREVISION} ${PKGREVISION}") @@ -1466,9 +1501,14 @@ func (s *Suite) Test_MkLineChecker_warnVarusePermissions__not_directly_and_no_al t.SetUpVartypes() mklines := t.NewMkLines("mk-c.mk", - MkRcsID, + MkCvsID, "", - "TOOL_DEPENDS+=\t${BUILDLINK_API_DEPENDS.mk-c}:${BUILDLINK_PKGSRCDIR.mk-c}") + "# GUESSED_FLAGS", + "#\tDocumented here to suppress the \"defined but not used\"", + "#\twarning.", + "", + "TOOL_DEPENDS+=\t${BUILDLINK_API_DEPENDS.mk-c}:${BUILDLINK_PKGSRCDIR.mk-c}", + "GUESSED_FLAGS+=\t${BUILDLINK_CPPFLAGS}") mklines.Check() @@ -1484,9 +1524,9 @@ func (s *Suite) Test_MkLineChecker_warnVarusePermissions__not_directly_and_no_al t.Check(apiDependsType.AlternativeFiles(aclpUseLoadtime), equals, "buildlink3.mk or builtin.mk only") t.CheckOutputLines( - "WARN: mk-c.mk:3: BUILDLINK_API_DEPENDS.mk-c should not be used in any file.", - "WARN: mk-c.mk:3: The list variable BUILDLINK_API_DEPENDS.mk-c should not be embedded in a word.", - "WARN: mk-c.mk:3: BUILDLINK_PKGSRCDIR.mk-c should not be used in any file.") + "WARN: mk-c.mk:7: BUILDLINK_API_DEPENDS.mk-c should not be used in any file.", + "WARN: mk-c.mk:7: The list variable BUILDLINK_API_DEPENDS.mk-c should not be embedded in a word.", + "WARN: mk-c.mk:7: BUILDLINK_PKGSRCDIR.mk-c should not be used in any file.") } func (s *Suite) Test_MkLineChecker_checkVarassignDecreasingVersions(c *check.C) { @@ -1494,7 +1534,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignDecreasingVersions(c *check.C) t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "PYTHON_VERSIONS_ACCEPTED=\t36 __future__ # rationale", "PYTHON_VERSIONS_ACCEPTED=\t36 -13 # rationale", "PYTHON_VERSIONS_ACCEPTED=\t36 ${PKGVERSION_NOREV} # rationale", @@ -1534,7 +1574,7 @@ func (s *Suite) Test_MkLineChecker_warnVaruseToolLoadTime(c *check.C) { t.SetUpTool("after-prefs", "AFTER_PREFS", AfterPrefsMk) t.SetUpTool("at-runtime", "AT_RUNTIME", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, ".if ${NOWHERE} && ${AFTER_PREFS} && ${AT_RUNTIME} && ${MK_TOOL}", ".endif", "", @@ -1566,7 +1606,7 @@ func (s *Suite) Test_MkLineChecker_warnVaruseToolLoadTime__local_tool(c *check.C t.SetUpVartypes() t.CreateFileLines("mk/bsd.prefs.mk") mklines := t.SetUpFileMkLines("category/package/Makefile", - MkRcsID, + MkCvsID, ".include \"../../mk/bsd.prefs.mk\"", "", "TOOLS_CREATE+=\t\tmk-tool", @@ -1588,7 +1628,7 @@ func (s *Suite) Test_MkLineChecker_Check__warn_varuse_LOCALBASE(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("options.mk", - MkRcsID, + MkCvsID, "PKGNAME=\t${LOCALBASE}") mklines.Check() @@ -1607,7 +1647,7 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePkgdir(c *check.C) { mklines := t.SetUpFileMkLines("category/package/Makefile", "# dummy") - checkRelativePkgdir := func(mkline MkLine) { + checkRelativePkgdir := func(mkline *MkLine) { MkLineChecker{mklines, mkline}.CheckRelativePkgdir(relativePkgdir) } @@ -1632,7 +1672,7 @@ func (s *Suite) Test_MkLineChecker__unclosed_varuse(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "EGDIRS=\t${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d") mklines.Check() @@ -1678,7 +1718,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparison_with_shell_com t.SetUpVartypes() mklines := t.NewMkLines("security/openssl/Makefile", - MkRcsID, + MkCvsID, ".if ${PKGSRC_COMPILER} == \"gcc\" && ${CC} == \"cc\"", ".endif") @@ -1697,7 +1737,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond__compare_pattern_with_empt t.SetUpVartypes() mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, ".if ${X11BASE:Npattern} == \"\"", ".endif", "", @@ -1725,7 +1765,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCondEmpty(c *check.C) { test := func(before string, diagnosticsAndAfter ...string) { mklines := t.SetUpFileMkLines("module.mk", - MkRcsID, + MkCvsID, before, ".endif") ck := MkLineChecker{mklines, mklines.mklines[1]} @@ -1942,7 +1982,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparing_PKGSRC_COMPILER t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, ".if ${PKGSRC_COMPILER} == \"clang\"", ".elif ${PKGSRC_COMPILER} != \"gcc\"", ".endif") @@ -1974,7 +2014,7 @@ func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS_with_backticks(c *check. t.SetUpVartypes() mklines := t.NewMkLines("chat/pidgin-icb/Makefile", - MkRcsID, + MkCvsID, "CFLAGS+=\t`pkg-config pidgin --cflags`") mkline := mklines.mklines[1] @@ -1997,14 +2037,14 @@ func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "CPPFLAGS.SunOS+=\t-DPIPECOMMAND=\\\"/usr/sbin/sendmail -bs %s\\\"") mklines.Check() t.CheckOutputLines( - "WARN: Makefile:2: Unknown compiler flag \"-bs\".", - "WARN: Makefile:2: Compiler flag \"%s\\\\\\\"\" should start with a hyphen.") + "WARN: Makefile:2: Compiler flag \"-DPIPECOMMAND=\\\\\\\"/usr/sbin/sendmail\" has unbalanced double quotes.", + "WARN: Makefile:2: Compiler flag \"%s\\\\\\\"\" has unbalanced double quotes.") } func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix(c *check.C) { @@ -2012,7 +2052,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix(c *check.C t.SetUpCommandLine("--autofix", "-Wspace") lines := t.SetUpFileLines("filename.mk", - MkRcsID, + MkCvsID, ".if defined(A)", ".for a in ${A}", ".if defined(C)", @@ -2046,7 +2086,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix_multiline( t.SetUpCommandLine("-Wall", "--autofix") t.SetUpVartypes() mklines := t.SetUpFileMkLines("options.mk", - MkRcsID, + MkCvsID, ".if ${PKGNAME} == pkgname", ".if \\", " ${PLATFORM:MNetBSD-4.*}", @@ -2060,7 +2100,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix_multiline( "AUTOFIX: ~/options.mk:5: Replacing \".\" with \". \".") t.CheckFileLines("options.mk", - MkRcsID, + MkCvsID, ".if ${PKGNAME} == pkgname", ". if \\", " ${PLATFORM:MNetBSD-4.*}", @@ -2073,7 +2113,7 @@ func (s *Suite) Test_MkLineChecker_checkVarUseQuoting(c *check.C) { t.SetUpVartypes() mklines := t.SetUpFileMkLines("options.mk", - MkRcsID, + MkCvsID, "GOPATH=\t${WRKDIR}", "do-build:", "\tcd ${WRKSRC} && GOPATH=${GOPATH} PATH=${PATH} :") @@ -2097,7 +2137,7 @@ func (s *Suite) Test_MkLineChecker_checkVarUseQuoting__mstar(c *check.C) { t.SetUpCommandLine("-Wall,no-space") t.SetUpVartypes() mklines := t.SetUpFileMkLines("options.mk", - MkRcsID, + MkCvsID, "CONFIGURE_ARGS+= CFLAGS=${CFLAGS:Q}", "CONFIGURE_ARGS+= CFLAGS=${CFLAGS:M*:Q}", "CONFIGURE_ARGS+= ADA_FLAGS=${ADA_FLAGS:Q}", @@ -2146,6 +2186,79 @@ func (s *Suite) Test_MkLineChecker_checkVarUseQuoting__q_not_needed(c *check.C) "NOTE: ~/category/package/Makefile:6: The :Q operator isn't necessary for ${HOMEPAGE} here.") } +func (s *Suite) Test_MkLineChecker_checkVarUseQuoting__undefined_list_in_word_in_shell_command(c *check.C) { + t := s.Init(c) + + pkg := t.SetUpPackage("category/package", + "\t${ECHO} ./${DISTFILES}") + t.FinishSetUp() + + G.Check(pkg) + + // The variable DISTFILES is declared by the infrastructure. + // It is not defined by this package, therefore it doesn't + // appear in the RedundantScope. + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:20: The list variable DISTFILES should not be embedded in a word.") +} + +func (s *Suite) Test_MkLineChecker_checkVarUseQuoting__list_variable_with_single_constant_value(c *check.C) { + t := s.Init(c) + + pkg := t.SetUpPackage("category/package", + "BUILD_DIRS=\tonly-dir", + "", + "do-install:", + "\t${INSTALL_PROGRAM} ${WRKSRC}/${BUILD_DIRS}/program ${DESTDIR}${PREFIX}/bin/") + t.FinishSetUp() + + G.Check(pkg) + + // Don't warn here since BUILD_DIRS, although being a list + // variable, contains only a single value. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLineChecker_checkVarUseQuoting__list_variable_with_single_conditional_value(c *check.C) { + t := s.Init(c) + + pkg := t.SetUpPackage("category/package", + "BUILD_DIRS=\tonly-dir", + ".if 0", + "BUILD_DIRS=\tother-dir", + ".endif", + "", + "do-install:", + "\t${INSTALL_PROGRAM} ${WRKSRC}/${BUILD_DIRS}/program ${DESTDIR}${PREFIX}/bin/") + t.FinishSetUp() + + G.Check(pkg) + + // TODO: Don't warn here since BUILD_DIRS, although being a list + // variable, contains only a single value. + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:26: " + + "The list variable BUILD_DIRS should not be embedded in a word.") +} + +func (s *Suite) Test_MkLineChecker_checkVarUseQuoting__list_variable_with_two_constant_words(c *check.C) { + t := s.Init(c) + + pkg := t.SetUpPackage("category/package", + "BUILD_DIRS=\tfirst-dir second-dir", + "", + "do-install:", + "\t${INSTALL_PROGRAM} ${WRKSRC}/${BUILD_DIRS}/program ${DESTDIR}${PREFIX}/bin/") + t.FinishSetUp() + + G.Check(pkg) + + // Since BUILD_DIRS consists of two words, it would destroy the installation command. + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:23: " + + "The list variable BUILD_DIRS should not be embedded in a word.") +} + // The ${VARNAME:=suffix} expression should only be used with lists. // It typically appears in MASTER_SITE definitions. func (s *Suite) Test_MkLineChecker_CheckVaruse__eq_nonlist(c *check.C) { @@ -2154,7 +2267,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__eq_nonlist(c *check.C) { t.SetUpVartypes() t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") mklines := t.SetUpFileMkLines("options.mk", - MkRcsID, + MkCvsID, "WRKSRC=\t\t${WRKDIR:=/subdir}", "MASTER_SITES=\t${MASTER_SITE_GITHUB:=organization/}") @@ -2170,7 +2283,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__for(c *check.C) { t.SetUpVartypes() t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") mklines := t.SetUpFileMkLines("options.mk", - MkRcsID, + MkCvsID, ".for var in a b c", "\t: ${var}", ".endfor") @@ -2190,12 +2303,12 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__varcanon(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("mk/sys-vars.mk", - MkRcsID, + MkCvsID, "CPPPATH.Linux=\t/usr/bin/cpp") t.FinishSetUp() mklines := t.NewMkLines("module.mk", - MkRcsID, + MkCvsID, "COMMENT=\t${CPPPATH.SunOS}") ck := MkLineChecker{mklines, mklines.mklines[1]} @@ -2206,7 +2319,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__varcanon(c *check.C) { options: Guessed, aclEntries: nil, }, - time: vucTimeRun, + time: VucRunTime, quoting: VucQuotPlain, IsWordPart: false, }) @@ -2223,11 +2336,11 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__defined_in_infrastructure(c *che t.SetUpPkgsrc() t.CreateFileLines("mk/deeply/nested/infra.mk", - MkRcsID, + MkCvsID, "INFRA_VAR?=\tvalue") t.FinishSetUp() mklines := t.SetUpFileMkLines("category/package/module.mk", - MkRcsID, + MkCvsID, "do-fetch:", "\t: ${INFRA_VAR} ${UNDEFINED}") @@ -2249,7 +2362,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__build_defs(c *check.C) { t.FinishSetUp() mklines := t.SetUpFileMkLines("options.mk", - MkRcsID, + MkCvsID, "COMMENT= ${VARBASE} ${X11_TYPE}", "PKG_FAIL_REASON+= ${VARBASE} ${X11_TYPE}", "BUILD_DEFS+= X11_TYPE") @@ -2267,7 +2380,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__LOCALBASE_in_infrastructure(c *c t.SetUpPkgsrc() t.CreateFileLines("mk/infra.mk", - MkRcsID, + MkCvsID, "LOCALBASE?=\t${PREFIX}", "DEFAULT_PREFIX=\t${LOCALBASE}") t.FinishSetUp() @@ -2291,7 +2404,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__user_defined_variable_and_BUILD_ "VARBASE?=\t${PREFIX}/var", "PYTHON_VER?=\t36") mklines := t.NewMkLines("file.mk", - MkRcsID, + MkCvsID, "BUILD_DEFS+=\tPYTHON_VER", "\t: ${VARBASE}", "\t: ${VARBASE}", @@ -2309,7 +2422,7 @@ func (s *Suite) Test_MkLineChecker_checkVaruseModifiersSuffix(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("file.mk", - MkRcsID, + MkCvsID, "\t: ${HOMEPAGE:=subdir/:Q}", // wrong "\t: ${BUILD_DIRS:=subdir/}", // correct "\t: ${BIN_PROGRAMS:=.exe}") // unknown since BIN_PROGRAMS doesn't have a type @@ -2327,7 +2440,7 @@ func (s *Suite) Test_MkLineChecker_checkVaruseModifiersRange(c *check.C) { t.SetUpCommandLine("--show-autofix", "--source") t.SetUpVartypes() mklines := t.NewMkLines("mk/compiler/gcc.mk", - MkRcsID, + MkCvsID, "CC:=\t${CC:C/^/_asdf_/1:M_asdf_*:S/^_asdf_//}") mklines.Check() @@ -2342,7 +2455,7 @@ func (s *Suite) Test_MkLineChecker_checkVaruseModifiersRange(c *check.C) { // Now go through all the "almost" cases, to reach full branch coverage. mklines = t.NewMkLines("gcc.mk", - MkRcsID, + MkCvsID, "\t: ${CC:M1:M2:M3}", "\t: ${CC:C/^begin//:M2:M3}", // M1 pattern not exactly ^ "\t: ${CC:C/^/_asdf_/g:M2:M3}", // M1 options != "1" @@ -2366,7 +2479,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__deprecated_PKG_DEBUG(c *check.C) G.Pkgsrc.initDeprecatedVars() mklines := t.NewMkLines("module.mk", - MkRcsID, + MkCvsID, "\t${_PKG_SILENT}${_PKG_DEBUG} :") mklines.Check() @@ -2380,7 +2493,7 @@ func (s *Suite) Test_MkLineChecker_checkVaruseUndefined(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("mk/infra.mk", - MkRcsID, + MkCvsID, "#", "# User-settable variables:", "#", @@ -2391,7 +2504,7 @@ func (s *Suite) Test_MkLineChecker_checkVaruseUndefined(c *check.C) { t.FinishSetUp() mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "", "do-build:", "\t: ${ASSIGNED} ${COMMENTED} ${DOCUMENTED} ${UNKNOWN}") @@ -2408,7 +2521,7 @@ func (s *Suite) Test_MkLineChecker_checkVaruseUndefined__indirect_variables(c *c t.SetUpTool("echo", "ECHO", AfterPrefsMk) mklines := t.NewMkLines("net/uucp/Makefile", - MkRcsID, + MkCvsID, "\techo ${UUCP_${var}}") mklines.Check() @@ -2433,7 +2546,7 @@ func (s *Suite) Test_MkLineChecker_checkVaruseUndefined__documented(c *check.C) t.SetUpVartypes() mklines := t.NewMkLines("interpreter.mk", - MkRcsID, + MkCvsID, "#", "# Package-settable variables:", "#", @@ -2457,13 +2570,15 @@ func (s *Suite) Test_MkLineChecker_checkVarassignMisc(c *check.C) { t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://download.github.com/") t.SetUpCommandLine("-Wall,no-space") mklines := t.SetUpFileMkLines("module.mk", - MkRcsID, + MkCvsID, "EGDIR= ${PREFIX}/etc/rc.d", "RPMIGNOREPATH+= ${PREFIX}/etc/rc.d", "_TOOLS_VARNAME.sed= SED", "DIST_SUBDIR= ${PKGNAME}", "WRKSRC= ${PKGNAME}", - "SITES_distfile.tar.gz= ${MASTER_SITE_GITHUB:=user/}") + "SITES_distfile.tar.gz= ${MASTER_SITE_GITHUB:=user/}", + "MASTER_SITES= https://cdn.example.org/${PKGNAME}/", + "MASTER_SITES= https://cdn.example.org/distname-${PKGVERSION}/") t.FinishSetUp() mklines.Check() @@ -2476,7 +2591,9 @@ func (s *Suite) Test_MkLineChecker_checkVarassignMisc(c *check.C) { "WARN: ~/module.mk:5: PKGNAME should not be used in DIST_SUBDIR as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", "WARN: ~/module.mk:6: PKGNAME should not be used in WRKSRC as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", "WARN: ~/module.mk:7: SITES_distfile.tar.gz is defined but not used.", - "WARN: ~/module.mk:7: SITES_* is deprecated. Please use SITES.* instead.") + "WARN: ~/module.mk:7: SITES_* is deprecated. Please use SITES.* instead.", + "WARN: ~/module.mk:8: PKGNAME should not be used in MASTER_SITES as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", + "WARN: ~/module.mk:9: PKGVERSION should not be used in MASTER_SITES as it includes the PKGREVISION. Please use PKGVERSION_NOREV instead.") } func (s *Suite) Test_MkLineChecker_checkVarassignMisc__multiple_inclusion_guards(c *check.C) { @@ -2484,18 +2601,18 @@ func (s *Suite) Test_MkLineChecker_checkVarassignMisc__multiple_inclusion_guards t.SetUpPkgsrc() t.CreateFileLines("filename.mk", - MkRcsID, + MkCvsID, ".if !defined(FILENAME_MK)", "FILENAME_MK=\t# defined", ".endif") t.CreateFileLines("Makefile.common", - MkRcsID, + MkCvsID, ".if !defined(MAKEFILE_COMMON)", "MAKEFILE_COMMON=\t# defined", "", ".endif") t.CreateFileLines("other.mk", - MkRcsID, + MkCvsID, "COMMENT=\t# defined") t.FinishSetUp() @@ -2517,7 +2634,7 @@ func (s *Suite) Test_MkLineChecker_checkText(c *check.C) { t.SetUpCommandLine("-Wall,no-space") mklines := t.SetUpFileMkLines("module.mk", - MkRcsID, + MkCvsID, "CFLAGS+= -Wl,--rpath,${PREFIX}/lib", "PKG_FAIL_REASON+= \"Group ${GAMEGRP} doesn't exist.\"") t.FinishSetUp() @@ -2534,7 +2651,7 @@ func (s *Suite) Test_MkLineChecker_checkText__WRKSRC(c *check.C) { t.SetUpCommandLine("-Wall", "--explain") mklines := t.SetUpFileMkLines("module.mk", - MkRcsID, + MkCvsID, "pre-configure:", "\tcd ${WRKSRC}/..") @@ -2566,7 +2683,7 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePath(c *check.C) { t.CreateFileLines("wip/package/Makefile") t.CreateFileLines("wip/package/module.mk") mklines := t.SetUpFileMkLines("category/package/module.mk", - MkRcsID, + MkCvsID, "DEPENDS+= wip-package-[0-9]*:../../wip/package", ".include \"../../wip/package/module.mk\"", "", @@ -2603,7 +2720,7 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePath__absolute_path(c *check.C) t.SetUpPkgsrc() mklines := t.SetUpFileMkLines("category/package/module.mk", - MkRcsID, + MkCvsID, "DISTINFO_FILE=\t"+absPath) t.FinishSetUp() @@ -2617,7 +2734,7 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePath__include_if_exists(c *check t := s.Init(c) mklines := t.SetUpFileMkLines("filename.mk", - MkRcsID, + MkCvsID, ".include \"included.mk\"", ".sinclude \"included.mk\"") @@ -2632,9 +2749,9 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePath__wip_mk(c *check.C) { t := s.Init(c) t.CreateFileLines("wip/mk/git-package.mk", - MkRcsID) + MkCvsID) t.CreateFileLines("wip/other/version.mk", - MkRcsID) + MkCvsID) t.SetUpPackage("wip/package", ".include \"../mk/git-package.mk\"", ".include \"../other/version.mk\"") diff --git a/pkgtools/pkglint/files/mklines.go b/pkgtools/pkglint/files/mklines.go index bb578adcdf6..b140b536ded 100644 --- a/pkgtools/pkglint/files/mklines.go +++ b/pkgtools/pkglint/files/mklines.go @@ -5,28 +5,26 @@ import ( ) // MkLines contains data for the Makefile (or *.mk) that is currently checked. -type MkLines = *MkLinesImpl - -type MkLinesImpl struct { - mklines []MkLine - lines Lines - target string // Current make(1) target; only available during checkAll - vars Scope // - buildDefs map[string]bool // Variables that are registered in BUILD_DEFS, to ensure that all user-defined variables are added to it. - plistVarAdded map[string]MkLine // Identifiers that are added to PLIST_VARS. - plistVarSet map[string]MkLine // Identifiers for which PLIST.${id} is defined. - plistVarSkip bool // True if any of the PLIST_VARS identifiers refers to a variable. - Tools *Tools // Tools defined in file scope. - indentation *Indentation // Indentation depth of preprocessing directives; only available during MkLines.ForEach. - forVars map[string]bool // The variables currently used in .for loops; only available during MkLines.checkAll. - Once +type MkLines struct { + mklines []*MkLine + lines *Lines + target string // Current make(1) target; only available during checkAll + vars Scope // + buildDefs map[string]bool // Variables that are registered in BUILD_DEFS, to ensure that all user-defined variables are added to it. + plistVarAdded map[string]*MkLine // Identifiers that are added to PLIST_VARS. + plistVarSet map[string]*MkLine // Identifiers for which PLIST.${id} is defined. + plistVarSkip bool // True if any of the PLIST_VARS identifiers refers to a variable. + Tools *Tools // Tools defined in file scope. + indentation *Indentation // Indentation depth of preprocessing directives; only available during MkLines.ForEach. + forVars map[string]bool // The variables currently used in .for loops; only available during MkLines.checkAll. + once Once // TODO: Consider extracting plistVarAdded, plistVarSet, plistVarSkip into an own type. // TODO: Describe where each of the above fields is valid. } -func NewMkLines(lines Lines) MkLines { - mklines := make([]MkLine, lines.Len()) +func NewMkLines(lines *Lines) *MkLines { + mklines := make([]*MkLine, lines.Len()) for i, line := range lines.Lines { mklines[i] = MkLineParser{}.Parse(line) } @@ -34,14 +32,14 @@ func NewMkLines(lines Lines) MkLines { tools := NewTools() tools.Fallback(G.Pkgsrc.Tools) - return &MkLinesImpl{ + return &MkLines{ mklines, lines, "", NewScope(), make(map[string]bool), - make(map[string]MkLine), - make(map[string]MkLine), + make(map[string]*MkLine), + make(map[string]*MkLine), false, tools, nil, @@ -74,16 +72,16 @@ func NewMkLines(lines Lines) MkLines { // UseVar remembers that the given variable is used in the given line. // This controls the "defined but not used" warning. -func (mklines *MkLinesImpl) UseVar(mkline MkLine, varname string, time vucTime) { +func (mklines *MkLines) UseVar(mkline *MkLine, varname string, time VucTime) { mklines.vars.Use(varname, mkline, time) if G.Pkg != nil { G.Pkg.vars.Use(varname, mkline, time) } } -func (mklines *MkLinesImpl) Check() { +func (mklines *MkLines) Check() { if trace.Tracing { - defer trace.Call1(mklines.lines.FileName)() + defer trace.Call1(mklines.lines.Filename)() } // In the first pass, all additions to BUILD_DEFS and USE_TOOLS @@ -99,7 +97,7 @@ func (mklines *MkLinesImpl) Check() { SaveAutofixChanges(mklines.lines) } -func (mklines *MkLinesImpl) checkAll() { +func (mklines *MkLines) checkAll() { allowedTargets := map[string]bool{ "pre-fetch": true, "do-fetch": true, "post-fetch": true, "pre-extract": true, "do-extract": true, "post-extract": true, @@ -113,13 +111,13 @@ func (mklines *MkLinesImpl) checkAll() { "pre-package": true, "do-package": true, "post-package": true, "pre-clean": true, "do-clean": true, "post-clean": true} - mklines.lines.CheckRcsID(0, `#[\t ]+`, "# ") + mklines.lines.CheckCvsID(0, `#[\t ]+`, "# ") substContext := NewSubstContext() var varalign VaralignBlock isHacksMk := mklines.lines.BaseName == "hacks.mk" - lineAction := func(mkline MkLine) bool { + lineAction := func(mkline *MkLine) bool { if isHacksMk { // Needs to be set here because it is reset in MkLines.ForEach. mklines.Tools.SeenPrefs = true @@ -160,8 +158,8 @@ func (mklines *MkLinesImpl) checkAll() { return true } - atEnd := func(mkline MkLine) { - mklines.indentation.CheckFinish(mklines.lines.FileName) + atEnd := func(mkline *MkLine) { + mklines.indentation.CheckFinish(mklines.lines.Filename) } if trace.Tracing { @@ -175,7 +173,7 @@ func (mklines *MkLinesImpl) checkAll() { CheckLinesTrailingEmptyLines(mklines.lines) } -func (mklines *MkLinesImpl) checkVarassignPlist(mkline MkLine) { +func (mklines *MkLines) checkVarassignPlist(mkline *MkLine) { switch mkline.Varcanon() { case "PLIST_VARS": for _, id := range mkline.ValueFields(resolveVariableRefs(mklines, mkline.Value())) { @@ -192,22 +190,57 @@ func (mklines *MkLinesImpl) checkVarassignPlist(mkline MkLine) { } } +func (mklines *MkLines) SplitToParagraphs() []*Paragraph { + var paras []*Paragraph + + lines := mklines.mklines + isEmpty := func(i int) bool { + if lines[i].IsEmpty() { + return true + } + return lines[i].IsComment() && + lines[i].Text == "#" && + (i == 0 || lines[i-1].IsComment()) && + (i == len(lines)-1 || lines[i+1].IsComment()) + } + + i := 0 + for i < len(lines) { + from := i + for from < len(lines) && isEmpty(from) { + from++ + } + + to := from + for to < len(lines) && !isEmpty(to) { + to++ + } + + if from != to { + paras = append(paras, NewParagraph(mklines, from, to)) + } + i = to + } + + return paras +} + // ForEach calls the action for each line, until the action returns false. // It keeps track of the indentation (see MkLines.indentation) // and all conditional variables (see Indentation.IsConditional). -func (mklines *MkLinesImpl) ForEach(action func(mkline MkLine)) { +func (mklines *MkLines) ForEach(action func(mkline *MkLine)) { mklines.ForEachEnd( - func(mkline MkLine) bool { action(mkline); return true }, - func(mkline MkLine) {}) + func(mkline *MkLine) bool { action(mkline); return true }, + func(mkline *MkLine) {}) } // ForEachEnd calls the action for each line, until the action returns false. // It keeps track of the indentation and all conditional variables. // At the end, atEnd is called with the last line as its argument. -func (mklines *MkLinesImpl) ForEachEnd(action func(mkline MkLine) bool, atEnd func(lastMkline MkLine)) { +func (mklines *MkLines) ForEachEnd(action func(mkline *MkLine) bool, atEnd func(lastMkline *MkLine)) { // XXX: To avoid looping over the lines multiple times, it would - // be nice to have an interface LinesChecker that checks a single thing. + // be nice to have an interface LinesChecker that checks a single topic. // Multiple of these line checkers could be run in parallel, so that // the diagnostics appear in the correct order, from top to bottom. @@ -222,7 +255,9 @@ func (mklines *MkLinesImpl) ForEachEnd(action func(mkline MkLine) bool, atEnd fu mklines.indentation.TrackAfter(mkline) } - atEnd(mklines.mklines[len(mklines.mklines)-1]) + if len(mklines.mklines) > 0 { + atEnd(mklines.mklines[len(mklines.mklines)-1]) + } mklines.indentation = nil } @@ -230,8 +265,8 @@ func (mklines *MkLinesImpl) ForEachEnd(action func(mkline MkLine) bool, atEnd fu // variable and returns a slice containing all its values, fully // expanded. // -// It can only be used during a active ForEach call. -func (mklines *MkLinesImpl) ExpandLoopVar(varname string) []string { +// It can only be used during an active ForEach call. +func (mklines *MkLines) ExpandLoopVar(varname string) []string { // From the inner loop to the outer loop, just in case // that two loops should ever use the same variable. @@ -239,14 +274,14 @@ func (mklines *MkLinesImpl) ExpandLoopVar(varname string) []string { ind := mklines.indentation.levels[i] mkline := ind.mkline - if mkline == nil || !mkline.IsDirective() || mkline.Directive() != "for" { + if mkline.Directive() != "for" { continue } // TODO: If needed, add support for multi-variable .for loops. resolved := resolveVariableRefs(mklines, mkline.Args()) words := mkline.ValueFields(resolved) - if 1 < len(words) && words[0] == varname && words[1] == "in" { + if len(words) >= 3 && words[0] == varname && words[1] == "in" { return words[2:] } } @@ -254,7 +289,7 @@ func (mklines *MkLinesImpl) ExpandLoopVar(varname string) []string { return nil } -func (mklines *MkLinesImpl) collectDefinedVariables() { +func (mklines *MkLines) collectDefinedVariables() { // FIXME: This method has a wrong name. It collects not only the defined // variables but also the used ones. @@ -262,7 +297,7 @@ func (mklines *MkLinesImpl) collectDefinedVariables() { defer trace.Call0()() } - mklines.ForEach(func(mkline MkLine) { + mklines.ForEach(func(mkline *MkLine) { mklines.Tools.ParseToolLine(mklines, mkline, false, true) if !mkline.IsVarassign() && !mkline.IsCommentedVarassign() { @@ -323,14 +358,14 @@ func (mklines *MkLinesImpl) collectDefinedVariables() { } // defineVar marks a variable as defined in both the current package and the current file. -func (mklines *MkLinesImpl) defineVar(pkg *Package, mkline MkLine, varname string) { +func (mklines *MkLines) defineVar(pkg *Package, mkline *MkLine, varname string) { mklines.vars.Define(varname, mkline) if pkg != nil { pkg.vars.Define(varname, mkline) } } -func (mklines *MkLinesImpl) collectPlistVars() { +func (mklines *MkLines) collectPlistVars() { // TODO: The PLIST_VARS code above looks very similar. for _, mkline := range mklines.mklines { if mkline.IsVarassign() { @@ -355,15 +390,15 @@ func (mklines *MkLinesImpl) collectPlistVars() { } } -func (mklines *MkLinesImpl) collectElse() { +func (mklines *MkLines) collectElse() { // Make a dry-run over the lines, which sets data.elseLine (in mkline.go) as a side-effect. - mklines.ForEach(func(mkline MkLine) {}) + mklines.ForEach(func(mkline *MkLine) {}) // TODO: Check whether this ForEach is redundant because it is already run somewhere else. } -func (mklines *MkLinesImpl) collectUsedVariables() { +func (mklines *MkLines) collectUsedVariables() { for _, mkline := range mklines.mklines { - mkline.ForEachUsed(func(varUse *MkVarUse, time vucTime) { + mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { mklines.UseVar(mkline, varUse.varname, time) }) } @@ -375,7 +410,7 @@ func (mklines *MkLinesImpl) collectUsedVariables() { // documentation of the Makefile fragments from the pkgsrc infrastructure. // // Loosely based on mk/help/help.awk, revision 1.28, but much simpler. -func (mklines *MkLinesImpl) collectDocumentedVariables() { +func (mklines *MkLines) collectDocumentedVariables() { scope := NewScope() commentLines := 0 relevant := true @@ -390,7 +425,7 @@ func (mklines *MkLinesImpl) collectDocumentedVariables() { if commentLines >= 3 && relevant { for varname, mkline := range scope.used { mklines.vars.Define(varname, mkline) - mklines.vars.Use(varname, mkline, vucTimeRun) + mklines.vars.Use(varname, mkline, VucRunTime) } } @@ -426,7 +461,7 @@ func (mklines *MkLinesImpl) collectDocumentedVariables() { varcanon := varnameCanon(varname) if varcanon == strings.ToUpper(varcanon) && matches(varcanon, `[A-Z]`) && parser.EOF() { scope.Define(varcanon, mkline) - scope.Use(varcanon, mkline, vucTimeRun) + scope.Use(varcanon, mkline, VucRunTime) } if words[1] == "Copyright" { @@ -441,52 +476,101 @@ func (mklines *MkLinesImpl) collectDocumentedVariables() { finish() } -// CheckForUsedComment checks that this file (a Makefile.common) has the given +// CheckUsedBy checks that this file (a Makefile.common) has the given // relativeName in one of the "# used by" comments at the beginning of the file. -func (mklines *MkLinesImpl) CheckForUsedComment(relativeName string) { +func (mklines *MkLines) CheckUsedBy(relativeName string) { lines := mklines.lines if lines.Len() < 3 { return } + paras := mklines.SplitToParagraphs() + expected := "# used by " + relativeName - for _, line := range lines.Lines { - if line.Text == expected { - return + found := false + var usedParas []*Paragraph + + determineUsedParas := func() { + for _, para := range paras { + var hasUsedBy bool + var hasOther bool + var conflict *MkLine + + para.ForEach(func(mkline *MkLine) { + if ok, _ := mkline.IsCvsID(`#[\t ]+`); ok { + return + } + if hasPrefix(mkline.Text, "# used by ") && len(strings.Fields(mkline.Text)) == 4 { + if mkline.Text == expected { + found = true + } + hasUsedBy = true + if hasOther && conflict == nil { + conflict = mkline + } + } else { + hasOther = true + if hasUsedBy && conflict == nil { + conflict = mkline + } + } + }) + + if conflict != nil { + conflict.Warnf("The \"used by\" lines should be in a separate paragraph.") + } else if hasUsedBy { + usedParas = append(usedParas, para) + } } } + determineUsedParas() - i := 0 - for i < 2 && hasPrefix(lines.Lines[i].Text, "#") { - i++ + if len(usedParas) > 1 { + usedParas[1].FirstLine().Warnf("There should only be a single \"used by\" paragraph per file.") + } + + var prevLine *MkLine + if len(usedParas) > 0 { + prevLine = usedParas[0].LastLine() + } else { + prevLine = paras[0].LastLine() + if paras[0].to > 1 { + fix := prevLine.Autofix() + fix.Notef(SilentAutofixFormat) + fix.InsertAfter("") + fix.Apply() + } } // TODO: Sort the comments. // TODO: Discuss whether these comments are actually helpful. + // TODO: Remove lines that don't apply anymore. - fix := lines.Lines[i].Autofix() - fix.Warnf("Please add a line %q here.", expected) - fix.Explain( - "Since Makefile.common files usually don't have any comments and", - "therefore not a clearly defined purpose, they should at least", - "contain references to all files that include them, so that it is", - "easier to see what effects future changes may have.", - "", - "If there are more than five packages that use a Makefile.common,", - "that file should have a clearly defined and documented purpose,", - "and the filename should reflect that purpose.", - "Typical names are module.mk, plugin.mk or version.mk.") - fix.InsertBefore(expected) - fix.Apply() + if !found { + fix := prevLine.Autofix() + fix.Warnf("Please add a line %q here.", expected) + fix.Explain( + "Since Makefile.common files usually don't have any comments and", + "therefore not a clearly defined purpose, they should at least", + "contain references to all files that include them, so that it is", + "easier to see what effects future changes may have.", + "", + "If there are more than five packages that use a Makefile.common,", + "that file should have a clearly defined and documented purpose,", + "and the filename should reflect that purpose.", + "Typical names are module.mk, plugin.mk or version.mk.") + fix.InsertAfter(expected) + fix.Apply() + } SaveAutofixChanges(lines) } -func (mklines *MkLinesImpl) SaveAutofixChanges() { +func (mklines *MkLines) SaveAutofixChanges() { mklines.lines.SaveAutofixChanges() } -func (mklines *MkLinesImpl) EOFLine() MkLine { +func (mklines *MkLines) EOFLine() *MkLine { return MkLineParser{}.Parse(mklines.lines.EOFLine()) } @@ -504,7 +588,7 @@ type VaralignBlock struct { } type varalignBlockInfo struct { - mkline MkLine + mkline *MkLine varnameOp string // Variable name + assignment operator varnameOpWidth int // Screen width of varnameOp space string // Whitespace between varnameOp and the variable value @@ -512,7 +596,7 @@ type varalignBlockInfo struct { continuation bool // A continuation line with no value in the first line. } -func (va *VaralignBlock) Process(mkline MkLine) { +func (va *VaralignBlock) Process(mkline *MkLine) { switch { case !G.Opts.WarnSpace: return @@ -534,7 +618,7 @@ func (va *VaralignBlock) Process(mkline MkLine) { } } -func (va *VaralignBlock) processVarassign(mkline MkLine) { +func (va *VaralignBlock) processVarassign(mkline *MkLine) { switch { case mkline.Op() == opAssignEval && matches(mkline.Varname(), `^[a-z]`): // Arguments to procedures do not take part in block alignment. @@ -561,7 +645,8 @@ func (va *VaralignBlock) processVarassign(mkline MkLine) { text := strings.TrimSuffix(mkline.raw[0].orignl, "\n") data := MkLineParser{}.split(nil, text) m, a := MkLineParser{}.MatchVarassign(mkline.Line, text, data) - continuation = m && a.value == "\\" + assert(m) + continuation = a.value == "\\" } valueAlign := mkline.ValueAlign() @@ -663,7 +748,7 @@ func (va *VaralignBlock) optimalWidth(infos []*varalignBlockInfo) int { return (minVarnameOpWidth & -8) + 8 } -func (va *VaralignBlock) realign(mkline MkLine, varnameOp, oldSpace string, continuation bool, newWidth int) { +func (va *VaralignBlock) realign(mkline *MkLine, varnameOp, oldSpace string, continuation bool, newWidth int) { hasSpace := contains(oldSpace, " ") newSpace := "" @@ -688,7 +773,7 @@ func (va *VaralignBlock) realign(mkline MkLine, varnameOp, oldSpace string, cont } } -func (va *VaralignBlock) realignInitialLine(mkline MkLine, varnameOp string, oldSpace string, newSpace string, hasSpace bool, newWidth int) { +func (va *VaralignBlock) realignInitialLine(mkline *MkLine, varnameOp string, oldSpace string, newSpace string, hasSpace bool, newWidth int) { wrongColumn := tabWidth(varnameOp+oldSpace) != tabWidth(varnameOp+newSpace) fix := mkline.Autofix() @@ -726,7 +811,7 @@ func (va *VaralignBlock) realignInitialLine(mkline MkLine, varnameOp string, old fix.Apply() } -func (va *VaralignBlock) realignContinuationLines(mkline MkLine, newWidth int) { +func (va *VaralignBlock) realignContinuationLines(mkline *MkLine, newWidth int) { indentation := strings.Repeat("\t", newWidth/8) + strings.Repeat(" ", newWidth%8) fix := mkline.Autofix() fix.Notef("This line should be aligned with %q.", indentation) diff --git a/pkgtools/pkglint/files/mklines_test.go b/pkgtools/pkglint/files/mklines_test.go index f4102e1adec..283c6af18c8 100644 --- a/pkgtools/pkglint/files/mklines_test.go +++ b/pkgtools/pkglint/files/mklines_test.go @@ -3,6 +3,7 @@ package pkglint import ( "gopkg.in/check.v1" "sort" + "strings" ) func (s *Suite) Test_MkLines_Check__unusual_target(c *check.C) { @@ -11,7 +12,7 @@ func (s *Suite) Test_MkLines_Check__unusual_target(c *check.C) { t.SetUpVartypes() t.SetUpTool("cc", "CC", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "echo: echo.c", "\tcc -o ${.TARGET} ${.IMPSRC}") @@ -28,7 +29,7 @@ func (s *Suite) Test_MkLines__quoting_LDFLAGS_for_GNU_configure(c *check.C) { t.SetUpVartypes() G.Pkg = NewPackage(t.File("category/pkgbase")) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "GNU_CONFIGURE=\tyes", "CONFIGURE_ENV+=\tX_LIBS=${X11_LDFLAGS:Q}") @@ -46,7 +47,7 @@ func (s *Suite) Test_MkLines__for_loop_multiple_variables(c *check.C) { t.SetUpTool("find", "FIND", AtRunTime) t.SetUpTool("pax", "PAX", AtRunTime) mklines := t.NewMkLines("Makefile", // From audio/squeezeboxserver - MkRcsID, + MkCvsID, "", "SBS_COPY=\tsource target", "", @@ -70,7 +71,7 @@ func (s *Suite) Test_MkLines__comparing_YesNo_variable_to_string(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("databases/gdbm_compat/builtin.mk", - MkRcsID, + MkCvsID, ".if ${USE_BUILTIN.gdbm} == \"no\"", ".endif", ".if ${USE_BUILTIN.gdbm:tu} == \"no\"", // Can never be true, since "no" is not uppercase. @@ -90,19 +91,19 @@ func (s *Suite) Test_MkLines__varuse_sh_modifier(c *check.C) { t.SetUpVartypes() t.SetUpTool("sed", "SED", AfterPrefsMk) mklines := t.NewMkLines("lang/qore/module.mk", - MkRcsID, + MkCvsID, "qore-version=\tqore --short-version | ${SED} -e s/-.*//", "PLIST_SUBST+=\tQORE_VERSION=\"${qore-version:sh}\"") var vars2 []string - mklines.mklines[1].ForEachUsed(func(varUse *MkVarUse, time vucTime) { + mklines.mklines[1].ForEachUsed(func(varUse *MkVarUse, time VucTime) { vars2 = append(vars2, varUse.varname) }) c.Check(vars2, deepEquals, []string{"SED"}) var vars3 []string - mklines.mklines[2].ForEachUsed(func(varUse *MkVarUse, time vucTime) { + mklines.mklines[2].ForEachUsed(func(varUse *MkVarUse, time VucTime) { vars3 = append(vars3, varUse.varname) }) @@ -125,7 +126,7 @@ func (s *Suite) Test_MkLines__varuse_parameterized(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("converters/wv2/Makefile", - MkRcsID, + MkCvsID, "CONFIGURE_ARGS+=\t\t${CONFIGURE_ARGS.${ICONV_TYPE}-iconv}", "CONFIGURE_ARGS.gnu-iconv=\t--with-libiconv=${BUILDLINK_PREFIX.iconv}") @@ -162,7 +163,7 @@ func (s *Suite) Test_MkLines__loop_modifier(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("chat/xchat/Makefile", - MkRcsID, + MkCvsID, "GCONF_SCHEMAS=\tapps_xchat_url_handler.schemas", "post-install:", "\t${GCONF_SCHEMAS:@s@"+ @@ -179,7 +180,7 @@ func (s *Suite) Test_MkLines__PKG_SKIP_REASON_depending_on_OPSYS(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "PKG_SKIP_REASON+=\t\"Fails everywhere\"", ".if ${OPSYS} == \"Cygwin\"", "PKG_SKIP_REASON+=\t\"Fails on Cygwin\"", @@ -197,7 +198,7 @@ func (s *Suite) Test_MkLines_Check__use_list_variable_as_part_of_word(c *check.C t.SetUpVartypes() t.SetUpTool("tr", "", AtRunTime) mklines := t.NewMkLines("converters/chef/Makefile", - MkRcsID, + MkCvsID, "\tcd ${WRKSRC} && tr '\\r' '\\n' < ${DISTDIR}/${DIST_SUBDIR}/${DISTFILES} > chef.l") mklines.Check() @@ -211,7 +212,7 @@ func (s *Suite) Test_MkLines_Check__absolute_pathname_depending_on_OPSYS(c *chec t.SetUpVartypes() mklines := t.NewMkLines("games/heretic2-demo/Makefile", - MkRcsID, + MkCvsID, ".if ${OPSYS} == \"DragonFly\"", "TAR_CMD=\t/usr/bin/bsdtar", ".endif", @@ -229,7 +230,7 @@ func (s *Suite) Test_MkLines_Check__absolute_pathname_depending_on_OPSYS(c *chec "WARN: games/heretic2-demo/Makefile:5: Unknown shell command \"/usr/bin/bsdtar\".") } -func (s *Suite) Test_MkLines_CheckForUsedComment(c *check.C) { +func (s *Suite) Test_MkLines_CheckUsedBy__show_autofix(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--show-autofix") @@ -237,7 +238,7 @@ func (s *Suite) Test_MkLines_CheckForUsedComment(c *check.C) { test := func(pkgpath string, lines []string, diagnostics []string) { mklines := t.NewMkLines("Makefile.common", lines...) - mklines.CheckForUsedComment(pkgpath) + mklines.CheckUsedBy(pkgpath) t.CheckOutput(diagnostics) } @@ -255,14 +256,14 @@ func (s *Suite) Test_MkLines_CheckForUsedComment(c *check.C) { test( "category/package", lines( - MkRcsID), + MkCvsID), diagnostics()) // Still too short. test( "category/package", lines( - MkRcsID, + MkCvsID, ""), diagnostics()) @@ -270,40 +271,210 @@ func (s *Suite) Test_MkLines_CheckForUsedComment(c *check.C) { test( "sysutils/mc", lines( - MkRcsID, + MkCvsID, "", "# used by sysutils/mc"), diagnostics()) // This file is not correctly mentioned, therefore the line is inserted. - // TODO: Since the following line is of a different type, an additional empty line should be inserted. test( "category/package", lines( - MkRcsID, + MkCvsID, "", "VARNAME=\tvalue"), diagnostics( - "WARN: Makefile.common:2: Please add a line \"# used by category/package\" here.", - "AUTOFIX: Makefile.common:2: Inserting a line \"# used by category/package\" before this line.")) + "WARN: Makefile.common:1: Please add a line \"# used by category/package\" here.", + "AUTOFIX: Makefile.common:1: Inserting a line \"# used by category/package\" after this line.")) // The "used by" comments may either start in line 2 or in line 3. test( "category/package", lines( - MkRcsID, + MkCvsID, "#", "#"), diagnostics( - "WARN: Makefile.common:3: Please add a line \"# used by category/package\" here.", - "AUTOFIX: Makefile.common:3: Inserting a line \"# used by category/package\" before this line.")) + "WARN: Makefile.common:1: Please add a line \"# used by category/package\" here.", + "AUTOFIX: Makefile.common:1: Inserting a line \"# used by category/package\" after this line.")) // TODO: What if there is an introductory comment first? That should stay at the top of the file. // TODO: What if the "used by" comments appear in the second paragraph, preceded by only comments and empty lines? + // Since the first paragraph already has some comments, the "used by" + // comments need their separate paragraph, which is inserted after + // the first paragraph. + test("category/package", + lines( + MkCvsID, + "# A normal comment", + "# that spans", + "# several lines"), + diagnostics( + "AUTOFIX: Makefile.common:4: Inserting a line \"\" after this line.", + "WARN: Makefile.common:4: Please add a line \"# used by category/package\" here.", + "AUTOFIX: Makefile.common:4: Inserting a line \"# used by category/package\" after this line.")) + c.Check(G.Logger.autofixAvailable, equals, true) } +func (s *Suite) Test_MkLines_CheckUsedBy(c *check.C) { + t := s.Init(c) + + test := func(pkgpath string, lines []string, diagnostics []string) { + mklines := t.NewMkLines("Makefile.common", lines...) + + mklines.CheckUsedBy(pkgpath) + + t.CheckOutput(diagnostics) + } + + lines := func(lines ...string) []string { return lines } + diagnostics := func(diagnostics ...string) []string { return diagnostics } + + // The including package is already mentioned in the single "used by" + // paragraph. Nothing needs to be changed. + test("category/package2/Makefile", + lines( + MkCvsID, + "# This Makefile fragment is", + "# used by category/package1/Makefile, as well as", // looks similar to the formal "used by". + "# some others.", + "", + "# used by category/package2/Makefile"), + diagnostics()) + + // The including file is not yet mentioned. There is a single "used by" + // paragraph, and the including file needs to be added to that paragraph. + // It is added in the correct sorting order. The entries are simply + // sorted alphabetically. + test("category/package/Makefile", + lines( + MkCvsID, + "# This Makefile fragment is", + "# used by category/package1/Makefile, as well as", // looks similar to the formal "used by". + "# some others.", + "", + "# used by category/package2/Makefile"), + diagnostics( + "WARN: Makefile.common:6: Please add a line \"# used by category/package/Makefile\" here.")) + + // There are two separate paragraphs with "used by" lines. The first of + // them is the interesting one. The new line is added to the first paragraph. + test("category/package", + lines( + MkCvsID, + "# used by category/package1", + "", + "# used by category/package2"), + diagnostics( + "WARN: Makefile.common:4: There should only be a single \"used by\" paragraph per file.", + "WARN: Makefile.common:2: Please add a line \"# used by category/package\" here.")) + + // The empty comment also separates the two paragraphs, like in the + // previous test case. + test("category/package", + lines( + MkCvsID, + "# used by category/package1", + "#", + "# used by category/package2"), + diagnostics( + "WARN: Makefile.common:4: There should only be a single \"used by\" paragraph per file.", + "WARN: Makefile.common:2: Please add a line \"# used by category/package\" here.")) + + c.Check(G.Logger.autofixAvailable, equals, true) +} + +func (s *Suite) Test_MkLines_CheckUsedBy__separate_paragraph(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile.common", + MkCvsID, + "# a comment", + "# used by category/package", + "# a comment") + + mklines.CheckUsedBy("category/package") + + t.CheckOutputLines( + "WARN: Makefile.common:3: The \"used by\" lines should be in a separate paragraph.") +} + +func (s *Suite) Test_MkLines_ExpandLoopVar(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + ".for file in a b c d e f g h", + ". for rank in 1 2 3 4 5 6 7 8", + "CHESS_BOARD+=\t${file}${rank}", + ". endfor", + ".endfor") + + var files []string + var ranks []string + var diagonals []string + mklines.ForEach(func(mkline *MkLine) { + if mkline.IsVarassign() { + ranks = mklines.ExpandLoopVar("rank") + files = mklines.ExpandLoopVar("file") + diagonals = mklines.ExpandLoopVar("diagonals") + } + }) + + t.Check(files, deepEquals, strings.Split("abcdefgh", "")) + t.Check(ranks, deepEquals, strings.Split("12345678", "")) + t.Check(diagonals, check.HasLen, 0) +} + +func (s *Suite) Test_MkLines_ExpandLoopVar__multi(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + ".if 1", + ". for key value in 1 one 2 two 3 three", + "VAR.${key}=\t${value}", + ". endfor", + ".endif") + + var keys []string + var values []string + mklines.ForEach(func(mkline *MkLine) { + if mkline.IsVarassign() { + keys = mklines.ExpandLoopVar("key") + values = mklines.ExpandLoopVar("value") + } + }) + + // As of June 2019, multi-variable .for loops are not yet implemented. + t.Check(keys, check.HasLen, 0) + t.Check(values, check.HasLen, 0) +} + +func (s *Suite) Test_MkLines_ExpandLoopVar__malformed_for(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + ".for var in", + "VAR=\t${var}", + ".endfor") + + var values = []string{"uninitialized"} + mklines.ForEach(func(mkline *MkLine) { + if mkline.IsVarassign() { + values = mklines.ExpandLoopVar("key") + } + }) + + t.Check(values, check.HasLen, 0) +} + func (s *Suite) Test_MkLines_collectDefinedVariables(c *check.C) { t := s.Init(c) @@ -312,7 +483,7 @@ func (s *Suite) Test_MkLines_collectDefinedVariables(c *check.C) { t.CreateFileLines("mk/tools/defaults.mk", "USE_TOOLS+= autoconf autoconf213") mklines := t.NewMkLines("determine-defined-variables.mk", - MkRcsID, + MkCvsID, "", "USE_TOOLS+= autoconf213 autoconf", "", @@ -347,9 +518,9 @@ func (s *Suite) Test_MkLines_collectDefinedVariables__BUILTIN_FIND_FILES_VAR(c * t.SetUpCommandLine("-Wall,no-space") t.SetUpPackage("category/package") t.CreateFileLines("mk/buildlink3/bsd.builtin.mk", - MkRcsID) + MkCvsID) mklines := t.SetUpFileMkLines("category/package/builtin.mk", - MkRcsID, + MkCvsID, "", "BUILTIN_FIND_FILES_VAR:= H_XFT2", "BUILTIN_FIND_FILES.H_XFT2= ${X11BASE}/include/X11/Xft/Xft.h", @@ -370,7 +541,7 @@ func (s *Suite) Test_MkLines_collectDefinedVariables__no_tracing(c *check.C) { t := s.Init(c) mklines := t.SetUpFileMkLines("filename.mk", - MkRcsID, + MkCvsID, "", "BUILD_DEFS+=\tVAR1", "PLIST_VARS+=\tvar2", @@ -391,7 +562,7 @@ func (s *Suite) Test_MkLines_collectUsedVariables__simple(c *check.C) { mklines.collectUsedVariables() - c.Check(mklines.vars.used, deepEquals, map[string]MkLine{"VAR": mkline}) + c.Check(mklines.vars.used, deepEquals, map[string]*MkLine{"VAR": mkline}) c.Check(mklines.vars.FirstUse("VAR"), equals, mkline) } @@ -399,7 +570,7 @@ func (s *Suite) Test_MkLines_collectUsedVariables__nested(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "", "LHS.${lparam}=\tRHS.${rparam}", "", @@ -423,7 +594,7 @@ func (s *Suite) Test_MkLines__private_tool_undefined(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "", "\tmd5sum filename") @@ -440,7 +611,7 @@ func (s *Suite) Test_MkLines__private_tool_defined(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "TOOLS_CREATE+=\tmd5sum", "", "\tmd5sum filename") @@ -455,7 +626,7 @@ func (s *Suite) Test_MkLines_Check__indentation(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("options.mk", - MkRcsID, + MkCvsID, ". if !defined(GUARD_MK)", ". if ${OPSYS} == ${OPSYS}", ". for i in ${FILES}", @@ -507,7 +678,7 @@ func (s *Suite) Test_MkLines_Check__indentation_include(c *check.C) { t.SetUpVartypes() t.CreateFileLines("included.mk") mklines := t.SetUpFileMkLines("module.mk", - MkRcsID, + MkCvsID, "", ".if ${PKGPATH} == \"category/package\"", ".include \"included.mk\"", @@ -528,7 +699,7 @@ func (s *Suite) Test_MkLines_Check__unfinished_directives(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("opsys.mk", - MkRcsID, + MkCvsID, "", ".for i in 1 2 3 4 5", ". if ${OPSYS} == NetBSD", @@ -549,7 +720,7 @@ func (s *Suite) Test_MkLines_Check__unbalanced_directives(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("opsys.mk", - MkRcsID, + MkCvsID, "", ".for i in 1 2 3 4 5", ". if ${OPSYS} == NetBSD", @@ -571,7 +742,7 @@ func (s *Suite) Test_MkLines_Check__incomplete_subst_at_end(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("subst.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+=\tclass") @@ -594,7 +765,7 @@ func (s *Suite) Test_MkLines__wip_category_Makefile(c *check.C) { t.SetUpTool("rm", "RM", AtRunTime) t.CreateFileLines("mk/misc/category.mk") mklines := t.SetUpFileMkLines("wip/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tWIP pkgsrc packages", "", @@ -634,7 +805,7 @@ func (s *Suite) Test_MkLines_collectDocumentedVariables(c *check.C) { t.SetUpVartypes() t.SetUpTool("rm", "RM", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "#", "# Copyright 2000-2018", "#", @@ -662,7 +833,15 @@ func (s *Suite) Test_MkLines_collectDocumentedVariables(c *check.C) { "", "# VARBASE1.<param1>", "# VARBASE2.*", - "# VARBASE3.${id}") + "# VARBASE3.${id}", + "", + "# NETBSD/amd64", + "#\tThis is not a variable name.", + "#\tThe slash must not appear in a variable name.", + "", + "# _____", + "#\tThis is not a variable name.", + "#\tVariable names must have at least one letter.") // The variables that appear in the documentation are marked as // both used and defined, to prevent the "defined but not used" warnings. @@ -689,7 +868,7 @@ func (s *Suite) Test_MkLines__shell_command_indentation(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "#", "pre-configure:", "\tcd 'indented correctly'", @@ -709,7 +888,7 @@ func (s *Suite) Test_MkLines__unknown_options(c *check.C) { t.SetUpVartypes() t.SetUpOption("known", "") mklines := t.NewMkLines("options.mk", - MkRcsID, + MkCvsID, "#", "PKG_OPTIONS_VAR=\tPKG_OPTIONS.pkgbase", "PKG_SUPPORTED_OPTIONS=\tknown unknown", @@ -732,7 +911,7 @@ func (s *Suite) Test_MkLines_Check__PLIST_VARS(c *check.C) { t.CreateFileLines("mk/bsd.options.mk") mklines := t.SetUpFileMkLines("category/package/options.mk", - MkRcsID, + MkCvsID, "", "PKG_OPTIONS_VAR= PKG_OPTIONS.pkg", "PKG_SUPPORTED_OPTIONS= both only-added only-defined", @@ -766,7 +945,7 @@ func (s *Suite) Test_MkLines_Check__PLIST_VARS_indirect(c *check.C) { t.SetUpOption("option2", "") mklines := t.SetUpFileMkLines("module.mk", - MkRcsID, + MkCvsID, "", "MY_PLIST_VARS= option1 option2", "PLIST_VARS+= ${MY_PLIST_VARS}", @@ -803,7 +982,7 @@ func (s *Suite) Test_MkLines_Check__PLIST_VARS_indirect_2(c *check.C) { t.SetUpOption("c", "") mklines := t.NewMkLines("module.mk", - MkRcsID, + MkCvsID, "", "PKG_SUPPORTED_OPTIONS= a b c", "PLIST_VARS+= ${PKG_SUPPORTED_OPTIONS:S,a,,g}", @@ -827,7 +1006,7 @@ func (s *Suite) Test_MkLines_collectElse(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("module.mk", - MkRcsID, + MkCvsID, "", ".if 0", ".endif", @@ -854,7 +1033,7 @@ func (s *Suite) Test_MkLines_Check__defined_and_used_variables(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("module.mk", - MkRcsID, + MkCvsID, "", ".for lang in de fr", "PLIST_VARS+= ${lang}", @@ -880,7 +1059,7 @@ func (s *Suite) Test_MkLines_Check__hacks_mk(c *check.C) { t.SetUpCommandLine("-Wall,no-space") t.SetUpVartypes() mklines := t.NewMkLines("hacks.mk", - MkRcsID, + MkCvsID, "", "PKGNAME?= pkgbase-1.0") @@ -898,7 +1077,7 @@ func (s *Suite) Test_MkLines_Check__MASTER_SITE_in_HOMEPAGE(c *check.C) { t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") t.SetUpVartypes() mklines := t.NewMkLines("devel/catch/Makefile", - MkRcsID, + MkCvsID, "HOMEPAGE=\t${MASTER_SITE_GITHUB:=philsquared/Catch/}", "HOMEPAGE=\t${MASTER_SITE_GITHUB}", "HOMEPAGE=\t${MASTER_SITES}", @@ -915,13 +1094,70 @@ func (s *Suite) Test_MkLines_Check__MASTER_SITE_in_HOMEPAGE(c *check.C) { "WARN: devel/catch/Makefile:5: HOMEPAGE should not be defined in terms of MASTER_SITEs.") } +// Up to June 2019, pkglint wrongly replaced the HOMEPAGE +// with an empty string. +func (s *Suite) Test_MkLines_Check__autofix_MASTER_SITE_in_HOMEPAGE(c *check.C) { + t := s.Init(c) + + test := func(diagnostics ...string) { + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "MASTER_SITES= \\", + "\thttps://cdn1.example.org/ \\", + "\thttps://cdn2.example.org/", + "", + "HOMEPAGE=\t${MASTER_SITES}") + + mklines.Check() + + t.CheckOutput(diagnostics) + } + + t.SetUpVartypes() + + t.SetUpCommandLine("-Wall") + test( + "WARN: Makefile:7: HOMEPAGE should not be defined in terms of MASTER_SITEs.") + + t.SetUpCommandLine("-Wall", "--autofix") + test( + nil...) + +} + +func (s *Suite) Test_MkLines_Check__autofix_MASTER_SITE_in_HOMEPAGE_in_package(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "--autofix") + t.SetUpPackage("category/package", + "MASTER_SITES=\thttps://cdn1.example.org/ https://cdn2.example.org/", + "HOMEPAGE=\t${MASTER_SITES}") + + t.Main("-Wall", "-q", "category/package") + + // When MASTER_SITES consists of several URLs, take the first one, + // assuming that it is the most appropriate. + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:9: " + + "HOMEPAGE should not be defined in terms of MASTER_SITEs. " + + "Use https://cdn1.example.org/ directly.") + + t.Main("-Wall", "-q", "--autofix", "category/package") + + t.CheckOutputLines( + "AUTOFIX: ~/category/package/Makefile:9: " + + "Replacing \"${MASTER_SITES}\" " + + "with \"https://cdn1.example.org/\".") +} + func (s *Suite) Test_MkLines_Check__VERSION_as_word_part_in_MASTER_SITES(c *check.C) { t := s.Init(c) t.SetUpVartypes() t.SetUpMasterSite("MASTER_SITE_SOURCEFORGE", "https://download.sf.net/") mklines := t.NewMkLines("geography/viking/Makefile", - MkRcsID, + MkCvsID, "MASTER_SITES=\t${MASTER_SITE_SOURCEFORGE:=viking/}${VERSION}/") mklines.Check() @@ -937,7 +1173,7 @@ func (s *Suite) Test_MkLines_Check__shell_command_as_word_part_in_ENV_list(c *ch t.SetUpVartypes() mklines := t.NewMkLines("x11/lablgtk1/Makefile", - MkRcsID, + MkCvsID, "CONFIGURE_ENV+=\tCC=${CC}") mklines.Check() @@ -953,7 +1189,7 @@ func (s *Suite) Test_MkLines_Check__extra_warnings(c *check.C) { t.SetUpVartypes() G.Pkg = NewPackage(t.File("category/pkgbase")) mklines := t.NewMkLines("options.mk", - MkRcsID, + MkCvsID, "", ".for word in ${PKG_FAIL_REASON}", "CONFIGURE_ARGS+=\t--sharedir=${PREFIX}/share/kde", @@ -974,6 +1210,48 @@ func (s *Suite) Test_MkLines_Check__extra_warnings(c *check.C) { "NOTE: options.mk:11: You can use \"../build\" instead of \"${WRKSRC}/../build\".") } +func (s *Suite) Test_MkLines_SplitToParagraphs(c *check.C) { + t := s.Init(c) + + type lineRange struct { + from, to int + } + + test := func(mklines *MkLines, ranges ...lineRange) { + paras := mklines.SplitToParagraphs() + + var exp []*Paragraph + for _, r := range ranges { + exp = append(exp, NewParagraph(mklines, r.from, r.to)) + } + + t.Check(paras, deepEquals, exp) + } + + para := func(from, to int) lineRange { return lineRange{from, to} } + + test( + t.NewMkLines("filename.mk", + MkCvsID, + "", + "# paragraph 2", + "#", + "VAR=\tstill paragraph 2", + "", + "# paragraph 3", + "#", + "# paragraph 4"), + para(0, 1), + para(2, 5), + para(6, 7), + para(8, 9)) + + test( + t.NewMkLines("filename.mk", + ""), + nil...) +} + // Ensures that during MkLines.ForEach, the conditional variables in // MkLines.Indentation are correctly updated for each line. func (s *Suite) Test_MkLines_ForEach__conditional_variables(c *check.C) { @@ -982,7 +1260,7 @@ func (s *Suite) Test_MkLines_ForEach__conditional_variables(c *check.C) { t.SetUpCommandLine("-Wall,no-space") t.SetUpVartypes() mklines := t.NewMkLines("conditional.mk", - MkRcsID, + MkCvsID, "", ".if defined(PKG_DEVELOPER)", "DEVELOPER=\tyes", @@ -995,7 +1273,7 @@ func (s *Suite) Test_MkLines_ForEach__conditional_variables(c *check.C) { seenDeveloper := false seenUsesGettext := false - mklines.ForEach(func(mkline MkLine) { + mklines.ForEach(func(mkline *MkLine) { if mkline.IsVarassign() { switch mkline.Varname() { case "DEVELOPER": @@ -1020,7 +1298,7 @@ func (s *Suite) Test_MkLines_checkVarassignPlist__indirect(c *check.C) { t.SetUpVartypes() mklines := t.SetUpFileMkLines("plist.mk", - MkRcsID, + MkCvsID, "", "MY_PLIST_VARS=\tvar1 var2", "PLIST_VARS+=\t${MY_PLIST_VARS}", diff --git a/pkgtools/pkglint/files/mkparser.go b/pkgtools/pkglint/files/mkparser.go index b2ae6e598d8..daa9919b1aa 100644 --- a/pkgtools/pkglint/files/mkparser.go +++ b/pkgtools/pkglint/files/mkparser.go @@ -9,7 +9,7 @@ import ( // MkParser wraps a Parser and provides methods for parsing // things related to Makefiles. type MkParser struct { - Line Line + Line *Line lexer *textproc.Lexer EmitWarnings bool } @@ -28,8 +28,8 @@ func (p *MkParser) Rest() string { // The text argument is assumed to be after unescaping the # character, // which means the # is a normal character and does not introduce a Makefile comment. // For VarUse, this distinction is irrelevant. -func NewMkParser(line Line, text string, emitWarnings bool) *MkParser { - assertf((line != nil) == emitWarnings, "line must be given iff emitWarnings is set") +func NewMkParser(line *Line, text string, emitWarnings bool) *MkParser { + assert((line != nil) == emitWarnings) // line must be given iff emitWarnings is set return &MkParser{line, textproc.NewLexer(text), emitWarnings} } @@ -183,118 +183,121 @@ func (p *MkParser) varUseAlnum() *MkVarUse { func (p *MkParser) VarUseModifiers(varname string, closing byte) []MkVarUseModifier { lexer := p.lexer - // TODO: Split into VarUseModifier for parsing a single modifier. - var modifiers []MkVarUseModifier - appendModifier := func(s string) { modifiers = append(modifiers, MkVarUseModifier{s}) } - // The :S and :C modifiers may be chained without using the : as separator. mayOmitColon := false for lexer.SkipByte(':') || mayOmitColon { - mayOmitColon = false - modifierMark := lexer.Mark() - - switch lexer.PeekByte() { - case 'E', 'H', 'L', 'O', 'Q', 'R', 'T', 's', 't', 'u': - mod := lexer.NextBytesSet(textproc.Alnum) - switch mod { - - case - "E", // Extension, e.g. path/file.suffix => suffix - "H", // Head, e.g. dir/subdir/file.suffix => dir/subdir - "L", // XXX: Shouldn't this be handled specially? - "O", // Order alphabetically - "Ox", // Shuffle - "Q", // Quote shell meta-characters - "R", // Strip the file suffix, e.g. path/file.suffix => file - "T", // Basename, e.g. path/file.suffix => file.suffix - "sh", // Evaluate the variable value as shell command - "tA", // Try to convert to absolute path - "tW", // Causes the value to be treated as a single word - "tl", // To lowercase - "tu", // To uppercase - "tw", // Causes the value to be treated as list of words - "u": // Remove adjacent duplicate words (like uniq(1)) - appendModifier(mod) - continue - - case "ts": - // See devel/bmake/files/var.c:/case 't' - sep := p.varUseText(closing) - switch { - case sep == "": - lexer.SkipString(":") - case len(sep) == 1: - break - case matches(sep, `^\\\d+`): - break - default: - if p.EmitWarnings { - p.Line.Warnf("Invalid separator %q for :ts modifier of %q.", sep, varname) - } - } - appendModifier(lexer.Since(modifierMark)) - continue - } - - case '=', 'D', 'M', 'N', 'U': - lexer.Skip(1) - re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^$:\\}]|\$\$|\\.)+`, `^([^$:\\)]|\$\$|\\.)+`))) - for p.VarUse() != nil || lexer.SkipRegexp(re) { - } - arg := lexer.Since(modifierMark) - appendModifier(strings.Replace(arg, "\\:", ":", -1)) - continue - - case 'C', 'S': - if ok, _, _, _, _ := p.varUseModifierSubst(closing); ok { - appendModifier(lexer.Since(modifierMark)) - mayOmitColon = true - continue - } + modifier := p.varUseModifier(varname, closing) + if modifier != "" { + modifiers = append(modifiers, MkVarUseModifier{modifier}) + } + mayOmitColon = modifier != "" && (modifier[0] == 'S' || modifier[0] == 'C') + } + return modifiers +} - case '@': - if p.varUseModifierAt(lexer, varname) { - appendModifier(lexer.Since(modifierMark)) - continue - } +// varUseModifier parses a single variable modifier such as :Q or :S,from,to,. +// The actual parsing starts after the leading colon. +func (p *MkParser) varUseModifier(varname string, closing byte) string { + lexer := p.lexer + mark := lexer.Mark() - case '[': - if lexer.SkipRegexp(G.res.Compile(`^\[(?:[-.\d]+|#)\]`)) { - appendModifier(lexer.Since(modifierMark)) - continue + switch lexer.PeekByte() { + case 'E', 'H', 'L', 'O', 'Q', 'R', 'T', 's', 't', 'u': + mod := lexer.NextBytesSet(textproc.Alnum) + + switch mod { + case + "E", // Extension, e.g. path/file.suffix => suffix + "H", // Head, e.g. dir/subdir/file.suffix => dir/subdir + "L", // XXX: Shouldn't this be handled specially? + "O", // Order alphabetically + "Ox", // Shuffle + "Q", // Quote shell meta-characters + "R", // Strip the file suffix, e.g. path/file.suffix => file + "T", // Basename, e.g. path/file.suffix => file.suffix + "sh", // Evaluate the variable value as shell command + "tA", // Try to convert to absolute path + "tW", // Causes the value to be treated as a single word + "tl", // To lowercase + "tu", // To uppercase + "tw", // Causes the value to be treated as list of words + "u": // Remove adjacent duplicate words (like uniq(1)) + return mod + } + + if hasPrefix(mod, "ts") { + // See devel/bmake/files/var.c:/case 't' + sep := mod[2:] + p.varUseText(closing) + switch { + case sep == "": + lexer.SkipString(":") + case len(sep) == 1: + break + case matches(sep, `^\\\d+`): + break + default: + if p.EmitWarnings { + p.Line.Warnf("Invalid separator %q for :ts modifier of %q.", sep, varname) + p.Line.Explain( + "The separator for the :ts modifier must be either a single character", + "or an escape sequence like \\t or \\n or an octal or decimal escape", + "sequence; see the bmake man page for further details.") + } } + return lexer.Since(mark) + } - case '?': - lexer.Skip(1) - p.varUseText(closing) - if lexer.SkipByte(':') { - p.varUseText(closing) - appendModifier(lexer.Since(modifierMark)) - continue - } + case '=', 'D', 'M', 'N', 'U': + lexer.Skip(1) + re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^$:\\}]|\$\$|\\.)+`, `^([^$:\\)]|\$\$|\\.)+`))) + for p.VarUse() != nil || lexer.SkipRegexp(re) { } + arg := lexer.Since(mark) + return strings.Replace(arg, "\\:", ":", -1) - lexer.Reset(modifierMark) + case 'C', 'S': + if ok, _, _, _, _ := p.varUseModifierSubst(closing); ok { + return lexer.Since(mark) + } - re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^:$}]|\$\$)+`, `^([^:$)]|\$\$)+`))) - for p.VarUse() != nil || lexer.SkipRegexp(re) { + case '@': + if p.varUseModifierAt(lexer, varname) { + return lexer.Since(mark) } - modifier := lexer.Since(modifierMark) - // ${SOURCES:%.c=%.o} or ${:!uname -a:[2]} - if contains(modifier, "=") || (hasPrefix(modifier, "!") && hasSuffix(modifier, "!")) { - appendModifier(modifier) - continue + case '[': + if lexer.SkipRegexp(G.res.Compile(`^\[(?:[-.\d]+|#)\]`)) { + return lexer.Since(mark) } - if p.EmitWarnings && modifier != "" { - p.Line.Warnf("Invalid variable modifier %q for %q.", modifier, varname) + case '?': + lexer.Skip(1) + p.varUseText(closing) + if lexer.SkipByte(':') { + p.varUseText(closing) + return lexer.Since(mark) } + } + lexer.Reset(mark) + + re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^:$}]|\$\$)+`, `^([^:$)]|\$\$)+`))) + for p.VarUse() != nil || lexer.SkipRegexp(re) { } - return modifiers + modifier := lexer.Since(mark) + + // ${SOURCES:%.c=%.o} or ${:!uname -a:[2]} + if contains(modifier, "=") || (hasPrefix(modifier, "!") && hasSuffix(modifier, "!")) { + return modifier + } + + if p.EmitWarnings && modifier != "" { + p.Line.Warnf("Invalid variable modifier %q for %q.", modifier, varname) + } + + return "" } // varUseText parses any text up to the next colon or closing mark. @@ -331,10 +334,24 @@ func (p *MkParser) varUseModifierSubst(closing byte) (ok bool, regex bool, from } skipOther := func() { - for p.VarUse() != nil || - lexer.SkipString("$$") || - (len(lexer.Rest()) >= 2 && lexer.PeekByte() == '\\' && separator != '\\' && lexer.Skip(2)) || - lexer.NextBytesFunc(isOther) != "" { + for { + switch { + + case p.VarUse() != nil: + break + + case lexer.SkipString("$$"): + break + + case len(lexer.Rest()) >= 2 && lexer.PeekByte() == '\\' && separator != '\\': + _ = lexer.Skip(2) + + case lexer.NextBytesFunc(isOther) != "": + break + + default: + return + } } } @@ -388,13 +405,13 @@ func (p *MkParser) varUseModifierAt(lexer *textproc.Lexer, varname string) bool // MkCond parses a condition like ${OPSYS} == "NetBSD". // // See devel/bmake/files/cond.c. -func (p *MkParser) MkCond() MkCond { +func (p *MkParser) MkCond() *MkCond { and := p.mkCondAnd() if and == nil { return nil } - ands := []MkCond{and} + ands := []*MkCond{and} for { mark := p.lexer.Mark() p.lexer.SkipHspace() @@ -411,16 +428,16 @@ func (p *MkParser) MkCond() MkCond { if len(ands) == 1 { return and } - return &mkCond{Or: ands} + return &MkCond{Or: ands} } -func (p *MkParser) mkCondAnd() MkCond { +func (p *MkParser) mkCondAnd() *MkCond { atom := p.mkCondAtom() if atom == nil { return nil } - atoms := []MkCond{atom} + atoms := []*MkCond{atom} for { mark := p.lexer.Mark() p.lexer.SkipHspace() @@ -437,10 +454,10 @@ func (p *MkParser) mkCondAnd() MkCond { if len(atoms) == 1 { return atom } - return &mkCond{And: atoms} + return &MkCond{And: atoms} } -func (p *MkParser) mkCondAtom() MkCond { +func (p *MkParser) mkCondAtom() *MkCond { if trace.Tracing { defer trace.Call1(p.Rest())() } @@ -452,7 +469,7 @@ func (p *MkParser) mkCondAtom() MkCond { case lexer.SkipByte('!'): cond := p.mkCondAtom() if cond != nil { - return &mkCond{Not: cond} + return &MkCond{Not: cond} } case lexer.SkipByte('('): @@ -482,28 +499,28 @@ func (p *MkParser) mkCondAtom() MkCond { lexer.SkipHspace() if m := lexer.NextRegexp(G.res.Compile(`^(<|<=|==|!=|>=|>)[\t ]*(0x[0-9A-Fa-f]+|\d+(?:\.\d+)?)`)); m != nil { - return &mkCond{CompareVarNum: &MkCondCompareVarNum{lhs, m[1], m[2]}} + return &MkCond{CompareVarNum: &MkCondCompareVarNum{lhs, m[1], m[2]}} } m := lexer.NextRegexp(G.res.Compile(`^(?:<|<=|==|!=|>=|>)`)) if m == nil { - return &mkCond{Var: lhs} // See devel/bmake/files/cond.c:/\* For \.if \$/ + return &MkCond{Var: lhs} // See devel/bmake/files/cond.c:/\* For \.if \$/ } lexer.SkipHspace() op := m[0] if op == "==" || op == "!=" { if mrhs := lexer.NextRegexp(G.res.Compile(`^"([^"\$\\]*)"`)); mrhs != nil { - return &mkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, mrhs[1]}} + return &MkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, mrhs[1]}} } } if str := lexer.NextBytesSet(textproc.AlnumU); str != "" { - return &mkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, str}} + return &MkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, str}} } if rhs := p.VarUse(); rhs != nil { - return &mkCond{CompareVarVar: &MkCondCompareVarVar{lhs, op, rhs}} + return &MkCond{CompareVarVar: &MkCondCompareVarVar{lhs, op, rhs}} } if lexer.PeekByte() == '"' { @@ -511,7 +528,7 @@ func (p *MkParser) mkCondAtom() MkCond { lexer.Skip(1) if quotedRHS := p.VarUse(); quotedRHS != nil { if lexer.SkipByte('"') { - return &mkCond{CompareVarVar: &MkCondCompareVarVar{lhs, op, quotedRHS}} + return &MkCond{CompareVarVar: &MkCondCompareVarVar{lhs, op, quotedRHS}} } } lexer.Reset(mark) @@ -532,7 +549,7 @@ func (p *MkParser) mkCondAtom() MkCond { rhsText.WriteByte(lexer.Since(m)[1]) case lexer.SkipByte('"'): - return &mkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, rhsText.String()}} + return &MkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, rhsText.String()}} default: break loop } @@ -543,14 +560,14 @@ func (p *MkParser) mkCondAtom() MkCond { // See devel/bmake/files/cond.c:/^CondCvtArg if m := lexer.NextRegexp(G.res.Compile(`^(?:0x[0-9A-Fa-f]+|\d+(?:\.\d+)?)`)); m != nil { - return &mkCond{Num: m[0]} + return &MkCond{Num: m[0]} } } lexer.Reset(mark) return nil } -func (p *MkParser) mkCondFunc() *mkCond { +func (p *MkParser) mkCondFunc() *MkCond { lexer := p.lexer mark := lexer.Mark() @@ -564,14 +581,14 @@ func (p *MkParser) mkCondFunc() *mkCond { case "defined": varname := p.Varname() if varname != "" && lexer.SkipByte(')') { - return &mkCond{Defined: varname} + return &MkCond{Defined: varname} } case "empty": if varname := p.Varname(); varname != "" { modifiers := p.VarUseModifiers(varname, ')') if lexer.SkipByte(')') { - return &mkCond{Empty: &MkVarUse{varname, modifiers}} + return &MkCond{Empty: &MkVarUse{varname, modifiers}} } } @@ -585,7 +602,7 @@ func (p *MkParser) mkCondFunc() *mkCond { } arg := lexer.Since(argMark) if lexer.SkipByte(')') { - return &mkCond{Call: &MkCondCall{funcName, arg}} + return &MkCond{Call: &MkCondCall{funcName, arg}} } } @@ -608,28 +625,27 @@ func (p *MkParser) Varname() string { return lexer.Since(mark) } -func (p *MkParser) PkgbasePattern() string { +// isPkgbasePart returns whether str, when following a hyphen, +// continues the package base (as in "mysql-client"), or whether it +// starts the version (as in "mysql-1.0"). +func (*MkParser) isPkgbasePart(str string) bool { + lexer := textproc.NewLexer(str) - // isVersion returns true for "1.2", "[0-9]*", "${PKGVERSION}", "${PKGNAME:C/^.*-//}", - // but not for "client", "${PKGNAME}", "[a-z]". - isVersion := func(s string) bool { - lexer := textproc.NewLexer(s) + lexer.SkipByte('{') + lexer.SkipByte('[') + if lexer.NextByteSet(textproc.Alpha) != -1 { + return true + } - lexer.SkipByte('[') - if lexer.NextByteSet(textproc.Digit) != -1 { - return true - } + varUse := NewMkParser(nil, lexer.Rest(), false).VarUse() + if varUse != nil { + return !contains(varUse.varname, "VER") && len(varUse.modifiers) == 0 + } - lookaheadParser := NewMkParser(nil, lexer.Rest(), false) - varUse := lookaheadParser.VarUse() - if varUse != nil { - if contains(varUse.varname, "VER") || len(varUse.modifiers) > 0 { - return true - } - } + return false +} - return false - } +func (p *MkParser) PkgbasePattern() string { lexer := p.lexer start := lexer.Mark() @@ -641,11 +657,11 @@ func (p *MkParser) PkgbasePattern() string { continue } - if lexer.PeekByte() != '-' || isVersion(lexer.Rest()[1:]) { + if lexer.PeekByte() == '-' && p.isPkgbasePart(lexer.Rest()[1:]) { + lexer.Skip(1) + } else { break } - - lexer.Skip(1 /* the hyphen */) } pkgbase := lexer.Since(start) @@ -748,12 +764,6 @@ func (p *MkParser) Dependency() *DependencyPattern { return &dp } - if hasSuffix(dp.Pkgbase, "-*") { - dp.Pkgbase = strings.TrimSuffix(dp.Pkgbase, "-*") - dp.Wildcard = "*" - return &dp - } - lexer.Reset(mark) return nil } @@ -776,12 +786,10 @@ func ToVarUse(str string) *MkVarUse { // Unnecessary parentheses are omitted in this representation, // but !empty(VARNAME) is represented differently from ${VARNAME} != "". // For higher level analysis, a unified representation might be better. -type MkCond = *mkCond - -type mkCond struct { - Or []*mkCond - And []*mkCond - Not *mkCond +type MkCond struct { + Or []*MkCond + And []*MkCond + Not *MkCond Defined string Empty *MkVarUse @@ -813,7 +821,7 @@ type MkCondCall struct { } type MkCondCallback struct { - Not func(cond MkCond) + Not func(cond *MkCond) Defined func(varname string) Empty func(empty *MkVarUse) CompareVarNum func(varuse *MkVarUse, op string, num string) @@ -824,13 +832,13 @@ type MkCondCallback struct { VarUse func(varuse *MkVarUse) } -func (cond *mkCond) Walk(callback *MkCondCallback) { +func (cond *MkCond) Walk(callback *MkCondCallback) { (&MkCondWalker{}).Walk(cond, callback) } type MkCondWalker struct{} -func (w *MkCondWalker) Walk(cond MkCond, callback *MkCondCallback) { +func (w *MkCondWalker) Walk(cond *MkCond, callback *MkCondCallback) { switch { case cond.Or != nil: for _, or := range cond.Or { diff --git a/pkgtools/pkglint/files/mkparser_test.go b/pkgtools/pkglint/files/mkparser_test.go index f2428cc28b3..92ff27ab65a 100644 --- a/pkgtools/pkglint/files/mkparser_test.go +++ b/pkgtools/pkglint/files/mkparser_test.go @@ -5,6 +5,15 @@ import ( "strings" ) +func (s *Suite) Test_NewMkParser__invalid_arguments(c *check.C) { + t := s.Init(c) + + line := t.NewLine("filename.mk", 123, "") + + t.ExpectAssert(func() { NewMkParser(line, "", false) }) + t.ExpectAssert(func() { NewMkParser(nil, "", true) }) +} + func (s *Suite) Test_MkParser_MkTokens(c *check.C) { t := s.Init(c) @@ -89,7 +98,9 @@ func (s *Suite) Test_MkParser_VarUse(c *check.C) { testRest := func(input string, expectedTokens []*MkToken, expectedRest string, diagnostics ...string) { line := t.NewLines("Test_MkParser_VarUse.mk", input).Lines[0] p := NewMkParser(line, input, true) + actualTokens := p.MkTokens() + c.Check(actualTokens, deepEquals, expectedTokens) for i, expectedToken := range expectedTokens { if i < len(actualTokens) { @@ -243,6 +254,10 @@ func (s *Suite) Test_MkParser_VarUse(c *check.C) { test("${RUBY_RAILS_SUPPORTED:[#]}", varuse("RUBY_RAILS_SUPPORTED", "[#]")) + test("${GZIP_CMD:[asdf]:Q}", + varuseText("${GZIP_CMD:[asdf]:Q}", "GZIP_CMD", "Q"), + "WARN: Test_MkParser_VarUse.mk:1: Invalid variable modifier \"[asdf]\" for \"GZIP_CMD\".") + test("${DISTNAME:C/-[0-9]+$$//:C/_/-/}", varuse("DISTNAME", "C/-[0-9]+$$//", "C/_/-/")) @@ -367,6 +382,142 @@ func (s *Suite) Test_MkParser_VarUse(c *check.C) { "WARN: Test_MkParser_VarUse.mk:1: Missing closing \"}\" for \"${\".") } +func (s *Suite) Test_MkParser_varUseModifier__invalid_ts_modifier_with_warning(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "--explain") + line := t.NewLine("filename.mk", 123, "${VAR:tsabc}") + p := NewMkParser(line, "tsabc}", true) + + modifier := p.varUseModifier("VAR", '}') + + t.Check(modifier, equals, "tsabc") + t.Check(p.Rest(), equals, "}") + t.CheckOutputLines( + "WARN: filename.mk:123: Invalid separator \"abc\" for :ts modifier of \"VAR\".", + "", + "\tThe separator for the :ts modifier must be either a single character", + "\tor an escape sequence like \\t or \\n or an octal or decimal escape", + "\tsequence; see the bmake man page for further details.", + "") +} + +func (s *Suite) Test_MkParser_varUseModifier__invalid_ts_modifier_without_warning(c *check.C) { + t := s.Init(c) + + p := NewMkParser(nil, "tsabc}", false) + + modifier := p.varUseModifier("VAR", '}') + + t.Check(modifier, equals, "tsabc") + t.Check(p.Rest(), equals, "}") +} + +func (s *Suite) Test_MkParser_varUseModifier__square_bracket(c *check.C) { + t := s.Init(c) + + line := t.NewLine("filename.mk", 123, "\t${VAR:[asdf]}") + p := NewMkParser(line, "[asdf]", true) + + modifier := p.varUseModifier("VAR", '}') + + t.Check(modifier, equals, "") + t.Check(p.Rest(), equals, "") + + t.CheckOutputLines( + "WARN: filename.mk:123: Invalid variable modifier \"[asdf]\" for \"VAR\".") +} + +func (s *Suite) Test_MkParser_varUseModifier__condition_without_colon(c *check.C) { + t := s.Init(c) + + line := t.NewLine("filename.mk", 123, "${${VAR}:?yes:no}${${VAR}:?yes}") + p := NewMkParser(line, line.Text, true) + + varUse1 := p.VarUse() + varUse2 := p.VarUse() + + t.Check(varUse1, deepEquals, NewMkVarUse("${VAR}", "?yes:no")) + t.Check(varUse2, deepEquals, NewMkVarUse("${VAR}")) + t.Check(p.Rest(), equals, "") + + t.CheckOutputLines( + "WARN: filename.mk:123: Invalid variable modifier \"?yes\" for \"${VAR}\".") +} + +func (s *Suite) Test_MkParser_varUseModifier__malformed_in_parentheses(c *check.C) { + t := s.Init(c) + + line := t.NewLine("filename.mk", 123, "$(${VAR}:?yes)") + p := NewMkParser(line, line.Text, true) + + varUse := p.VarUse() + + t.Check(varUse, deepEquals, NewMkVarUse("${VAR}")) + t.Check(p.Rest(), equals, "") + + t.CheckOutputLines( + "WARN: filename.mk:123: Invalid variable modifier \"?yes\" for \"${VAR}\".", + "WARN: filename.mk:123: Please use curly braces {} instead of round parentheses () for ${VAR}.") +} + +func (s *Suite) Test_MkParser_varUseModifier__varuse_in_malformed_modifier(c *check.C) { + t := s.Init(c) + + line := t.NewLine("filename.mk", 123, "${${VAR}:?yes${INNER}}") + p := NewMkParser(line, line.Text, true) + + varUse := p.VarUse() + + t.Check(varUse, deepEquals, NewMkVarUse("${VAR}")) + t.Check(p.Rest(), equals, "") + + t.CheckOutputLines( + "WARN: filename.mk:123: Invalid variable modifier \"?yes${INNER}\" for \"${VAR}\".") +} + +func (s *Suite) Test_MkParser_varUseModifierAt__missing_at_after_variable_name(c *check.C) { + t := s.Init(c) + + line := t.NewLine("filename.mk", 123, "${VAR:@varname}") + p := NewMkParser(line, line.Text, true) + + varUse := p.VarUse() + + t.Check(varUse, deepEquals, NewMkVarUse("VAR")) + t.Check(p.Rest(), equals, "") + t.CheckOutputLines( + "WARN: filename.mk:123: Invalid variable modifier \"@varname\" for \"VAR\".") +} + +func (s *Suite) Test_MkParser_varUseModifierAt__dollar(c *check.C) { + t := s.Init(c) + + line := t.NewLine("filename.mk", 123, "${VAR:@var@$$var@}") + p := NewMkParser(line, line.Text, true) + + varUse := p.VarUse() + + t.Check(varUse, deepEquals, NewMkVarUse("VAR", "@var@$$var@")) + t.Check(p.Rest(), equals, "") + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkParser_varUseModifierAt__incomplete_without_warning(c *check.C) { + t := s.Init(c) + + p := NewMkParser(nil, "${VAR:@var@$$var}rest", false) + + varUse := p.VarUse() + + // TODO: It's inconsistent that this syntax error still produces a + // variable modifier, while most other syntax errors don't. + // FIXME: The } must not be part of the variable modifier. + t.Check(varUse, deepEquals, NewMkVarUse("VAR", "@var@$$var}rest")) + t.Check(p.Rest(), equals, "") + t.CheckOutputEmpty() +} + func (s *Suite) Test_MkParser_VarUse__ambiguous(c *check.C) { t := s.Init(c) @@ -400,13 +551,13 @@ func (s *Suite) Test_MkParser_VarUse__ambiguous(c *check.C) { func (s *Suite) Test_MkParser_MkCond(c *check.C) { t := s.Init(c) - testRest := func(input string, expectedTree MkCond, expectedRest string) { + testRest := func(input string, expectedTree *MkCond, expectedRest string) { p := NewMkParser(nil, input, false) actualTree := p.MkCond() c.Check(actualTree, deepEquals, expectedTree) c.Check(p.Rest(), equals, expectedRest) } - test := func(input string, expectedTree MkCond) { + test := func(input string, expectedTree *MkCond) { testRest(input, expectedTree, "") } varuse := NewMkVarUse @@ -414,119 +565,122 @@ func (s *Suite) Test_MkParser_MkCond(c *check.C) { t.Use(testRest, test, varuse) test("${OPSYS:MNetBSD}", - &mkCond{Var: varuse("OPSYS", "MNetBSD")}) + &MkCond{Var: varuse("OPSYS", "MNetBSD")}) test("defined(VARNAME)", - &mkCond{Defined: "VARNAME"}) + &MkCond{Defined: "VARNAME"}) test("empty(VARNAME)", - &mkCond{Empty: varuse("VARNAME")}) + &MkCond{Empty: varuse("VARNAME")}) test("!empty(VARNAME)", - &mkCond{Not: &mkCond{Empty: varuse("VARNAME")}}) + &MkCond{Not: &MkCond{Empty: varuse("VARNAME")}}) test("!empty(VARNAME:M[yY][eE][sS])", - &mkCond{Not: &mkCond{Empty: varuse("VARNAME", "M[yY][eE][sS]")}}) + &MkCond{Not: &MkCond{Empty: varuse("VARNAME", "M[yY][eE][sS]")}}) // Colons are unescaped at this point because they cannot be mistaken for separators anymore. test("!empty(USE_TOOLS:Mautoconf\\:run)", - &mkCond{Not: &mkCond{Empty: varuse("USE_TOOLS", "Mautoconf:run")}}) + &MkCond{Not: &MkCond{Empty: varuse("USE_TOOLS", "Mautoconf:run")}}) test("${VARNAME} != \"Value\"", - &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME"), "!=", "Value"}}) + &MkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME"), "!=", "Value"}}) test("${VARNAME:Mi386} != \"Value\"", - &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME", "Mi386"), "!=", "Value"}}) + &MkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME", "Mi386"), "!=", "Value"}}) test("${VARNAME} != Value", - &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME"), "!=", "Value"}}) + &MkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME"), "!=", "Value"}}) test("\"${VARNAME}\" != Value", - &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME"), "!=", "Value"}}) + &MkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME"), "!=", "Value"}}) test("${pkg} == \"${name}\"", - &mkCond{CompareVarVar: &MkCondCompareVarVar{varuse("pkg"), "==", varuse("name")}}) + &MkCond{CompareVarVar: &MkCondCompareVarVar{varuse("pkg"), "==", varuse("name")}}) test("\"${pkg}\" == \"${name}\"", - &mkCond{CompareVarVar: &MkCondCompareVarVar{varuse("pkg"), "==", varuse("name")}}) + &MkCond{CompareVarVar: &MkCondCompareVarVar{varuse("pkg"), "==", varuse("name")}}) // The right-hand side is not analyzed further to keep the data types simple. test("${ABC} == \"${A}B${C}\"", - &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("ABC"), "==", "${A}B${C}"}}) + &MkCond{CompareVarStr: &MkCondCompareVarStr{varuse("ABC"), "==", "${A}B${C}"}}) test("${ABC} == \"${A}\\\"${B}\\\\${C}$${shellvar}${D}\"", - &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("ABC"), "==", "${A}\"${B}\\${C}$${shellvar}${D}"}}) + &MkCond{CompareVarStr: &MkCondCompareVarStr{varuse("ABC"), "==", "${A}\"${B}\\${C}$${shellvar}${D}"}}) test("exists(/etc/hosts)", - &mkCond{Call: &MkCondCall{"exists", "/etc/hosts"}}) + &MkCond{Call: &MkCondCall{"exists", "/etc/hosts"}}) test("exists(${PREFIX}/var)", - &mkCond{Call: &MkCondCall{"exists", "${PREFIX}/var"}}) + &MkCond{Call: &MkCondCall{"exists", "${PREFIX}/var"}}) test("${OPSYS} == \"NetBSD\" || ${OPSYS} == \"OpenBSD\"", - &mkCond{Or: []*mkCond{ + &MkCond{Or: []*MkCond{ {CompareVarStr: &MkCondCompareVarStr{varuse("OPSYS"), "==", "NetBSD"}}, {CompareVarStr: &MkCondCompareVarStr{varuse("OPSYS"), "==", "OpenBSD"}}}}) test("${OPSYS} == \"NetBSD\" && ${MACHINE_ARCH} == \"i386\"", - &mkCond{And: []*mkCond{ + &MkCond{And: []*MkCond{ {CompareVarStr: &MkCondCompareVarStr{varuse("OPSYS"), "==", "NetBSD"}}, {CompareVarStr: &MkCondCompareVarStr{varuse("MACHINE_ARCH"), "==", "i386"}}}}) test("defined(A) && defined(B) || defined(C) && defined(D)", - &mkCond{Or: []*mkCond{ - {And: []*mkCond{ + &MkCond{Or: []*MkCond{ + {And: []*MkCond{ {Defined: "A"}, {Defined: "B"}}}, - {And: []*mkCond{ + {And: []*MkCond{ {Defined: "C"}, {Defined: "D"}}}}}) test("${MACHINE_ARCH:Mi386} || ${MACHINE_OPSYS:MNetBSD}", - &mkCond{Or: []*mkCond{ + &MkCond{Or: []*MkCond{ {Var: varuse("MACHINE_ARCH", "Mi386")}, {Var: varuse("MACHINE_OPSYS", "MNetBSD")}}}) + test("${VAR} == \"${VAR}suffix\"", + &MkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VAR"), "==", "${VAR}suffix"}}) + // Exotic cases // ".if 0" can be used to skip over a block of code. test("0", - &mkCond{Num: "0"}) + &MkCond{Num: "0"}) test("0xCAFEBABE", - &mkCond{Num: "0xCAFEBABE"}) + &MkCond{Num: "0xCAFEBABE"}) test("${VAR} == 0xCAFEBABE", - &mkCond{ + &MkCond{ CompareVarNum: &MkCondCompareVarNum{ Var: varuse("VAR"), Op: "==", Num: "0xCAFEBABE"}}) test("! ( defined(A) && empty(VARNAME) )", - &mkCond{Not: &mkCond{ - And: []*mkCond{ + &MkCond{Not: &MkCond{ + And: []*MkCond{ {Defined: "A"}, {Empty: varuse("VARNAME")}}}}) test("${REQD_MAJOR} > ${MAJOR}", - &mkCond{CompareVarVar: &MkCondCompareVarVar{varuse("REQD_MAJOR"), ">", varuse("MAJOR")}}) + &MkCond{CompareVarVar: &MkCondCompareVarVar{varuse("REQD_MAJOR"), ">", varuse("MAJOR")}}) test("${OS_VERSION} >= 6.5", - &mkCond{CompareVarNum: &MkCondCompareVarNum{varuse("OS_VERSION"), ">=", "6.5"}}) + &MkCond{CompareVarNum: &MkCondCompareVarNum{varuse("OS_VERSION"), ">=", "6.5"}}) test("${OS_VERSION} == 5.3", - &mkCond{CompareVarNum: &MkCondCompareVarNum{varuse("OS_VERSION"), "==", "5.3"}}) + &MkCond{CompareVarNum: &MkCondCompareVarNum{varuse("OS_VERSION"), "==", "5.3"}}) test("!empty(${OS_VARIANT:MIllumos})", // Probably not intended - &mkCond{Not: &mkCond{Empty: varuse("${OS_VARIANT:MIllumos}")}}) + &MkCond{Not: &MkCond{Empty: varuse("${OS_VARIANT:MIllumos}")}}) // There may be whitespace before the parenthesis; see devel/bmake/files/cond.c:^compare_function. test("defined (VARNAME)", - &mkCond{Defined: "VARNAME"}) + &MkCond{Defined: "VARNAME"}) test("${\"${PKG_OPTIONS:Moption}\":?--enable-option:--disable-option}", - &mkCond{Var: varuse("\"${PKG_OPTIONS:Moption}\"", "?--enable-option:--disable-option")}) + &MkCond{Var: varuse("\"${PKG_OPTIONS:Moption}\"", "?--enable-option:--disable-option")}) // Contrary to most other programming languages, the == operator binds // more tightly that the ! operator. @@ -534,16 +688,40 @@ func (s *Suite) Test_MkParser_MkCond(c *check.C) { // TODO: Since this operator precedence is surprising there should be a warning, // suggesting to replace "!${VAR} == value" with "${VAR} != value". test("!${VAR} == value", - &mkCond{Not: &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VAR"), "==", "value"}}}) + &MkCond{Not: &MkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VAR"), "==", "value"}}}) // Errors + testRest("defined()", + nil, + "defined()") + + testRest("empty()", + nil, + "empty()") + + testRest("empty(UNFINISHED", + nil, + "empty(UNFINISHED") + + testRest("empty(UNFINISHED:Mpattern", + nil, + "empty(UNFINISHED:Mpattern") + + testRest("exists(/$$sys)", + nil, + "exists(/$$sys)") + + testRest("exists(/unfinished", + nil, + "exists(/unfinished") + testRest("!empty(PKG_OPTIONS:Msndfile) || defined(PKG_OPTIONS:Msamplerate)", - &mkCond{Not: &mkCond{Empty: varuse("PKG_OPTIONS", "Msndfile")}}, + &MkCond{Not: &MkCond{Empty: varuse("PKG_OPTIONS", "Msndfile")}}, "|| defined(PKG_OPTIONS:Msamplerate)") testRest("${LEFT} &&", - &mkCond{Var: varuse("LEFT")}, + &MkCond{Var: varuse("LEFT")}, "&&") testRest("\"unfinished string literal", @@ -562,6 +740,61 @@ func (s *Suite) Test_MkParser_MkCond(c *check.C) { testRest("${VAR} == \"unfinished string literal", nil, "${VAR} == \"unfinished string literal") + + // A logical not must always be followed by an expression. + testRest("!<", + nil, + "!<") + + // Empty parentheses are a syntax error. + testRest("()", + nil, + "()") + + // Unfinished conditions are a syntax error. + testRest("(${VAR}", + nil, + "(${VAR}") + + // The left-hand side of the comparison can only be a variable. + // FIXME: bmake accepts this, and so should pkglint. + testRest("\"${VAR}suffix\" == value", + nil, + "\"${VAR}suffix\" == value") +} + +func (s *Suite) Test_MkParser_Varname(c *check.C) { + t := s.Init(c) + + test := func(text string) { + line := t.NewLine("filename.mk", 1, text) + p := NewMkParser(line, text, true) + + varname := p.Varname() + + t.Check(varname, equals, text) + t.Check(p.Rest(), equals, "") + } + + testRest := func(text string, expectedVarname string, expectedRest string) { + line := t.NewLine("filename.mk", 1, text) + p := NewMkParser(line, text, true) + + varname := p.Varname() + + t.Check(varname, equals, expectedVarname) + t.Check(p.Rest(), equals, expectedRest) + } + + test("VARNAME") + test("VARNAME.param") + test("VARNAME.${param}") + test("SITES_${param}") + test("SITES_distfile-1.0.tar.gz") + test("SITES.gtk+-2.0") + test("PKGPATH.category/package") + + testRest("VARNAME/rest", "VARNAME", "/rest") } // Pkglint can replace $(VAR) with ${VAR}. It doesn't look at all components @@ -574,7 +807,7 @@ func (s *Suite) Test_MkParser_VarUse__parentheses_autofix(c *check.C) { t.SetUpCommandLine("--autofix") t.SetUpVartypes() lines := t.SetUpFileLines("Makefile", - MkRcsID, + MkCvsID, "COMMENT=$(P1) $(P2)) $(P3:Q) ${BRACES} $(A.$(B.$(C)))") mklines := NewMkLines(lines) @@ -586,7 +819,7 @@ func (s *Suite) Test_MkParser_VarUse__parentheses_autofix(c *check.C) { "AUTOFIX: ~/Makefile:2: Replacing \"$(P3:Q)\" with \"${P3:Q}\".", "AUTOFIX: ~/Makefile:2: Replacing \"$(C)\" with \"${C}\".") t.CheckFileLines("Makefile", - MkRcsID, + MkCvsID, "COMMENT=${P1} ${P2}) ${P3:Q} ${BRACES} $(A.$(B.${C}))") } @@ -718,28 +951,54 @@ func (s *Suite) Test_MkParser_varUseModifierAt(c *check.C) { "") } +func (s *Suite) Test_MkParser_isPkgbasePart(c *check.C) { + + test := func(str string, expected bool) { + actual := (*MkParser)(nil).isPkgbasePart(str) + + c.Check(actual, equals, expected) + } + + test("X11", true) + test("client", true) + test("${PKGNAME}", true) + test("[a-z]", true) + test("{client,server}", true) + + test("1.2", false) + test("[0-9]*", false) + test("{5.[1-7].*,6.[0-9]*}", false) + test("${PKGVERSION}", false) + test("${PKGNAME:C/^.*-//}", false) + test(">=1.0", false) + test("_client", false) // The combination foo-_client looks strange. +} + func (s *Suite) Test_MkParser_PkgbasePattern(c *check.C) { - testRest := func(pattern, expected, rest string) { + test := func(pattern, expected, rest string) { parser := NewMkParser(nil, pattern, false) + actual := parser.PkgbasePattern() + c.Check(actual, equals, expected) c.Check(parser.Rest(), equals, rest) } - testRest("fltk", "fltk", "") - testRest("fltk|", "fltk", "|") - testRest("boost-build-1.59.*", "boost-build", "-1.59.*") - testRest("${PHP_PKG_PREFIX}-pdo-5.*", "${PHP_PKG_PREFIX}-pdo", "-5.*") - testRest("${PYPKGPREFIX}-metakit-[0-9]*", "${PYPKGPREFIX}-metakit", "-[0-9]*") + test("fltk", "fltk", "") + test("fltk-", "fltk", "-") + test("fltk|", "fltk", "|") + test("boost-build-1.59.*", "boost-build", "-1.59.*") + test("${PHP_PKG_PREFIX}-pdo-5.*", "${PHP_PKG_PREFIX}-pdo", "-5.*") + test("${PYPKGPREFIX}-metakit-[0-9]*", "${PYPKGPREFIX}-metakit", "-[0-9]*") - testRest("pkgbase-[0-9]*", "pkgbase", "-[0-9]*") + test("pkgbase-[0-9]*", "pkgbase", "-[0-9]*") - testRest("pkgbase-client-[0-9]*", "pkgbase-client", "-[0-9]*") + test("pkgbase-client-[0-9]*", "pkgbase-client", "-[0-9]*") - testRest("pkgbase-${VARIANT}-[0-9]*", "pkgbase-${VARIANT}", "-[0-9]*") + test("pkgbase-${VARIANT}-[0-9]*", "pkgbase-${VARIANT}", "-[0-9]*") - testRest("pkgbase-${VERSION}-[0-9]*", "pkgbase", "-${VERSION}-[0-9]*") + test("pkgbase-${VERSION}-[0-9]*", "pkgbase", "-${VERSION}-[0-9]*") // This PKGNAME pattern is the one from bsd.pkg.mk. // The pattern assumes that the version number does not contain a hyphen, @@ -748,16 +1007,16 @@ func (s *Suite) Test_MkParser_PkgbasePattern(c *check.C) { // Since variable substitutions are more common for version numbers // than for parts of the package name, pkglint treats the PKGNAME // as a version number. - testRest("pkgbase-${PKGNAME:C/^.*-//}-[0-9]*", "pkgbase", "-${PKGNAME:C/^.*-//}-[0-9]*") + test("pkgbase-${PKGNAME:C/^.*-//}-[0-9]*", "pkgbase", "-${PKGNAME:C/^.*-//}-[0-9]*") // Using the [a-z] pattern in the package base is only rarely seen in the wild. - testRest("pkgbase-[a-z]*-1.0", "pkgbase-[a-z]*", "-1.0") + test("pkgbase-[a-z]*-1.0", "pkgbase-[a-z]*", "-1.0") // This is a valid dependency pattern, but it's more complicated // than the patterns pkglint can handle as of January 2019. // // This pattern doesn't have a single package base, which means it cannot be parsed at all. - testRest("{ssh{,6}-[0-9]*,openssh-[0-9]*}", "", "{ssh{,6}-[0-9]*,openssh-[0-9]*}") + test("{ssh{,6}-[0-9]*,openssh-[0-9]*}", "", "{ssh{,6}-[0-9]*,openssh-[0-9]*}") } func (s *Suite) Test_MkParser_Dependency(c *check.C) { @@ -783,6 +1042,18 @@ func (s *Suite) Test_MkParser_Dependency(c *check.C) { testRest(pattern, expected, "") } + test("pkgbase>=1.0", + DependencyPattern{"pkgbase", ">=", "1.0", "", "", ""}) + + test("pkgbase>1.0", + DependencyPattern{"pkgbase", ">", "1.0", "", "", ""}) + + test("pkgbase<=1.0", + DependencyPattern{"pkgbase", "", "", "<=", "1.0", ""}) + + test("pkgbase<1.0", + DependencyPattern{"pkgbase", "", "", "<", "1.0", ""}) + test("fltk>=1.1.5rc1<1.3", DependencyPattern{"fltk", ">=", "1.1.5rc1", "<", "1.3", ""}) @@ -828,6 +1099,12 @@ func (s *Suite) Test_MkParser_Dependency(c *check.C) { testRest("gnome-control-center>=2.20.1{,nb*}", DependencyPattern{"gnome-control-center", ">=", "2.20.1", "", "", ""}, "{,nb*}") + testNil("pkgbase") + + testNil("pkgbase-") + + testNil("pkgbase-client") + testNil(">=2.20.1{,nb*}") testNil("pkgbase<=") @@ -861,7 +1138,7 @@ func (s *Suite) Test_MkCondWalker_Walk(c *check.C) { var events []string varuseStr := func(varuse *MkVarUse) string { - strs := make([]string, 1+len(varuse.modifiers), 1+len(varuse.modifiers)) + strs := make([]string, 1+len(varuse.modifiers)) strs[0] = varuse.varname for i, mod := range varuse.modifiers { strs[1+i] = mod.Text @@ -922,3 +1199,23 @@ func (s *Suite) Test_MkCondWalker_Walk(c *check.C) { " var NONEMPTY", " varUse NONEMPTY"}) } + +// Ensure that the code works even if none of the callbacks are set. +// This is only for code coverage. +func (s *Suite) Test_MkCondWalker_Walk__empty_callbacks(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("Makefile", 4, ""+ + ".if ${VAR:Mmatch} == ${OTHER} || "+ + "${STR} == Str || "+ + "${VAR} == \"${PRE}text${POST}\" || "+ + "${NUM} == 3 && "+ + "defined(VAR) && "+ + "!exists(file.mk) && "+ + "exists(${FILE}) && "+ + "(((${NONEMPTY})))") + + mkline.Cond().Walk(&MkCondCallback{}) + + t.CheckOutputEmpty() +} diff --git a/pkgtools/pkglint/files/mkshparser.go b/pkgtools/pkglint/files/mkshparser.go index 726d272ca3f..fd0297f2faf 100644 --- a/pkgtools/pkglint/files/mkshparser.go +++ b/pkgtools/pkglint/files/mkshparser.go @@ -2,7 +2,7 @@ package pkglint import "fmt" -func parseShellProgram(line Line, program string) (*MkShList, error) { +func parseShellProgram(line *Line, program string) (*MkShList, error) { if trace.Tracing { defer trace.Call(program)() } @@ -16,7 +16,7 @@ func parseShellProgram(line Line, program string) (*MkShList, error) { switch { case succeeded == 0 && lexer.error == "": return lexer.result, nil - case succeeded == 0 && rest != "": + case succeeded == 0: return nil, fmt.Errorf("splitIntoShellTokens couldn't parse %q", rest) default: return nil, &ParseError{append([]string{lexer.current}, lexer.remaining...)} diff --git a/pkgtools/pkglint/files/mkshparser_test.go b/pkgtools/pkglint/files/mkshparser_test.go index 452ca81c087..8a45a37f9cc 100644 --- a/pkgtools/pkglint/files/mkshparser_test.go +++ b/pkgtools/pkglint/files/mkshparser_test.go @@ -12,7 +12,7 @@ func (s *Suite) Test_parseShellProgram__parse_error_for_dollar(c *check.C) { test := func(text string, expProgram *MkShList, expError error, expDiagnostics ...string) { mklines := t.NewMkLines("module.mk", "\t"+text) - mklines.ForEach(func(mkline MkLine) { + mklines.ForEach(func(mkline *MkLine) { program, err := parseShellProgram(mkline.Line, text) if err == nil { diff --git a/pkgtools/pkglint/files/mkshtypes.go b/pkgtools/pkglint/files/mkshtypes.go index 4046c82b701..ea778fe3986 100644 --- a/pkgtools/pkglint/files/mkshtypes.go +++ b/pkgtools/pkglint/files/mkshtypes.go @@ -225,16 +225,11 @@ func (c *StrCommand) AnyArgMatches(pattern regex.Pattern) bool { } func (c *StrCommand) String() string { - var strs []string - for _, assignment := range c.Assignments { - strs = append(strs, assignment) - } + strs := append([]string(nil), c.Assignments...) if c.Name != "" { strs = append(strs, c.Name) } - for _, arg := range c.Args { - strs = append(strs, arg) - } + strs = append(strs, c.Args...) return strings.Join(strs, " ") } diff --git a/pkgtools/pkglint/files/mkshwalker.go b/pkgtools/pkglint/files/mkshwalker.go index 5782626ea98..012b88dcbd6 100644 --- a/pkgtools/pkglint/files/mkshwalker.go +++ b/pkgtools/pkglint/files/mkshwalker.go @@ -71,9 +71,10 @@ func NewMkShWalker() *MkShWalker { func (w *MkShWalker) Path() string { var path []string for _, level := range w.Context { - typeName := reflect.TypeOf(level.Element).Elem().Name() - if typeName == "" && reflect.TypeOf(level.Element).Kind() == reflect.Slice { - typeName = "[]" + reflect.TypeOf(level.Element).Elem().Elem().Name() + elementType := reflect.TypeOf(level.Element) + typeName := elementType.Elem().Name() + if typeName == "" { + typeName = "[]" + elementType.Elem().Elem().Name() } abbreviated := strings.TrimPrefix(typeName, "MkSh") if level.Index == -1 { @@ -91,8 +92,8 @@ func (w *MkShWalker) Path() string { func (w *MkShWalker) Walk(list *MkShList) { w.walkList(-1, list) - // If this fails, the calls to w.push and w.pop are unbalanced. - assertf(len(w.Context) == 0, "MkShWalker.Walk %v", w.Context) + // The calls to w.push and w.pop must be balanced. + assert(len(w.Context) == 0) } func (w *MkShWalker) walkList(index int, list *MkShList) { diff --git a/pkgtools/pkglint/files/mkshwalker_test.go b/pkgtools/pkglint/files/mkshwalker_test.go index a638dd1e347..4052f9cb825 100644 --- a/pkgtools/pkglint/files/mkshwalker_test.go +++ b/pkgtools/pkglint/files/mkshwalker_test.go @@ -7,9 +7,7 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { pathFor := map[string]bool{} outputPathFor := func(kinds ...string) { - for key := range pathFor { - pathFor[key] = false - } + pathFor = make(map[string]bool) for _, kind := range kinds { pathFor[kind] = true } @@ -212,7 +210,7 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " Word 2") outputPathFor("Redirects", "Redirect", "Word") - test(""+ + test( "echo 'hello world' 1>/dev/null 2>&1 0</dev/random", " List with 1 andOrs", @@ -240,3 +238,55 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " Word /dev/random", " Path List.AndOr[0].Pipeline[0].Command[0].SimpleCommand.[]MkShRedirection.Redirection[2].ShToken[2]") } + +func (s *Suite) Test_MkShWalker_Walk__empty_callback(c *check.C) { + + test := func(program string) { + list, err := parseShellProgram(dummyLine, program) + assertNil(err, "") + + walker := NewMkShWalker() + walker.Walk(list) + + c.Check(walker.Parent(0), equals, nil) + } + + test("" + + "if condition; then action; else case selector in pattern) case-item-action ;; esac; fi; " + + "set -e; " + + "cd ${WRKSRC}/locale; " + + "for lang in *.po; do " + + " [ \"$${lang}\" = \"wxstd.po\" ] && continue; " + + " ${TOOLS_PATH.msgfmt} -c -o \"$${lang%.po}.mo\" \"$${lang}\"; " + + "done; " + + "while :; do fun() { :; } 1>&2; done") + + test( + "echo 'hello world' 1>/dev/null 2>&1 0</dev/random") +} + +func (s *Suite) Test_MkShWalker_Walk__assertion(c *check.C) { + t := s.Init(c) + + list, err := parseShellProgram(dummyLine, "echo \"hello, world\"") + assertNil(err, "") + + walker := NewMkShWalker() + + // This callback intentionally breaks the assertion. + walker.Callback.Word = func(word *ShToken) { walker.push(0, "extra word") } + + t.ExpectAssert(func() { walker.Walk(list) }) +} + +// Just for code coverage, to keep the main code symmetrical. +func (s *Suite) Test_MkShWalker_walkCommand__empty(c *check.C) { + walker := NewMkShWalker() + walker.walkCommand(0, &MkShCommand{}) +} + +// Just for code coverage, to keep the main code symmetrical. +func (s *Suite) Test_MkShWalker_walkCompoundCommand__empty(c *check.C) { + walker := NewMkShWalker() + walker.walkCompoundCommand(0, &MkShCompoundCommand{}) +} diff --git a/pkgtools/pkglint/files/mktokenslexer_test.go b/pkgtools/pkglint/files/mktokenslexer_test.go index 00926c1e369..b52156463b5 100644 --- a/pkgtools/pkglint/files/mktokenslexer_test.go +++ b/pkgtools/pkglint/files/mktokenslexer_test.go @@ -225,7 +225,6 @@ func (s *Suite) Test_MkTokensLexer__constructor_uses_shared_array(c *check.C) { tokens[0].Text = "modified text" tokens[0].Varuse = NewMkVarUse("MODIFIED", "Mpattern") - tokens = tokens[0:0] t.Check(lexer.Rest(), equals, "modified text") } diff --git a/pkgtools/pkglint/files/mktypes_test.go b/pkgtools/pkglint/files/mktypes_test.go index 0fc7ebb6d9b..70fb3cae63b 100644 --- a/pkgtools/pkglint/files/mktypes_test.go +++ b/pkgtools/pkglint/files/mktypes_test.go @@ -64,7 +64,7 @@ func (s *Suite) Test_MkVarUseModifier_ChangesWords__empty(c *check.C) { mkline := t.NewMkLine("filename.mk", 123, "\t${VAR:}") n := 0 - mkline.ForEachUsed(func(varUse *MkVarUse, time vucTime) { + mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { n += 100 for _, mod := range varUse.modifiers { mod.ChangesWords() @@ -151,3 +151,16 @@ func (s *Suite) Test_MkVarUseModifier_Subst__no_tracing(c *check.C) { c.Check(ok, equals, true) c.Check(result, equals, "to a to b") } + +// Since the replacement text is not a simple string, the :C modifier +// cannot be treated like the :S modifier. The variable could contain +// one of the special characters that would need to be escaped in the +// replacement text. +func (s *Suite) Test_MkVarUseModifier_Subst__C_with_complex_replacement(c *check.C) { + mod := MkVarUseModifier{"C,from,${VAR},"} + + result, ok := mod.Subst("from a to b") + + c.Check(ok, equals, false) + c.Check(result, equals, "") +} diff --git a/pkgtools/pkglint/files/options.go b/pkgtools/pkglint/files/options.go index e25d4357888..62e044ae9c3 100755 --- a/pkgtools/pkglint/files/options.go +++ b/pkgtools/pkglint/files/options.go @@ -1,10 +1,10 @@ package pkglint -func CheckLinesOptionsMk(mklines MkLines) { +func CheckLinesOptionsMk(mklines *MkLines) { ck := OptionsLinesChecker{ mklines, - make(map[string]MkLine), - make(map[string]MkLine), + make(map[string]*MkLine), + make(map[string]*MkLine), nil} ck.Check() @@ -14,10 +14,10 @@ func CheckLinesOptionsMk(mklines MkLines) { // // See mk/bsd.options.mk for a detailed description. type OptionsLinesChecker struct { - mklines MkLines + mklines *MkLines - declaredOptions map[string]MkLine - handledOptions map[string]MkLine + declaredOptions map[string]*MkLine + handledOptions map[string]*MkLine optionsInDeclarationOrder []string } @@ -27,7 +27,7 @@ func (ck *OptionsLinesChecker) Check() { mklines.Check() mlex := NewMkLinesLexer(mklines) - mlex.SkipWhile(func(mkline MkLine) bool { return mkline.IsComment() || mkline.IsEmpty() }) + mlex.SkipWhile(func(mkline *MkLine) bool { return mkline.IsComment() || mkline.IsEmpty() }) if !ck.lookingAtPkgOptionsVar(mlex) { return @@ -70,7 +70,7 @@ func (ck *OptionsLinesChecker) lookingAtPkgOptionsVar(mlex *MkLinesLexer) bool { // checkLineUpper checks a line from the upper part of an options.mk file, // before bsd.options.mk is included. -func (ck *OptionsLinesChecker) handleUpperLine(mkline MkLine) bool { +func (ck *OptionsLinesChecker) handleUpperLine(mkline *MkLine) bool { switch { case mkline.IsComment(): break @@ -110,7 +110,7 @@ func (ck *OptionsLinesChecker) handleUpperLine(mkline MkLine) bool { return true } -func (ck *OptionsLinesChecker) handleLowerLine(mkline MkLine) { +func (ck *OptionsLinesChecker) handleLowerLine(mkline *MkLine) { if mkline.IsDirective() { directive := mkline.Directive() if directive == "if" || directive == "elif" { @@ -122,7 +122,7 @@ func (ck *OptionsLinesChecker) handleLowerLine(mkline MkLine) { } } -func (ck *OptionsLinesChecker) handleLowerCondition(mkline MkLine, cond MkCond) { +func (ck *OptionsLinesChecker) handleLowerCondition(mkline *MkLine, cond *MkCond) { recordUsedOption := func(varuse *MkVarUse) { if varuse.varname == "PKG_OPTIONS" && len(varuse.modifiers) == 1 { diff --git a/pkgtools/pkglint/files/options_test.go b/pkgtools/pkglint/files/options_test.go index 47b5374b20f..d57114b0559 100755 --- a/pkgtools/pkglint/files/options_test.go +++ b/pkgtools/pkglint/files/options_test.go @@ -16,10 +16,10 @@ func (s *Suite) Test_CheckLinesOptionsMk(c *check.C) { t.SetUpOption("x11", "") t.CreateFileLines("mk/bsd.options.mk", - MkRcsID) + MkCvsID) mklines := t.SetUpFileMkLines("category/package/options.mk", - MkRcsID, + MkCvsID, "", "PKG_OPTIONS_VAR= PKG_OPTIONS.mc", "PKG_OPTIONS_REQUIRED_GROUPS= screen", @@ -76,13 +76,13 @@ func (s *Suite) Test_CheckLinesOptionsMk__edge_cases(c *check.C) { t.SetUpVartypes() t.SetUpOption("option1", "Description for option1") t.CreateFileLines("mk/compiler.mk", - MkRcsID) + MkCvsID) t.CreateFileLines("mk/bsd.options.mk", - MkRcsID) + MkCvsID) t.DisableTracing() mklines := t.SetUpFileMkLines("category/package/options.mk", - MkRcsID) + MkCvsID) CheckLinesOptionsMk(mklines) @@ -90,7 +90,7 @@ func (s *Suite) Test_CheckLinesOptionsMk__edge_cases(c *check.C) { "WARN: ~/category/package/options.mk:EOF: Expected definition of PKG_OPTIONS_VAR.") mklines = t.SetUpFileMkLines("category/package/options.mk", - MkRcsID, + MkCvsID, "PKG_SUPPORTED_OPTIONS=\toption1") CheckLinesOptionsMk(mklines) @@ -99,7 +99,7 @@ func (s *Suite) Test_CheckLinesOptionsMk__edge_cases(c *check.C) { "WARN: ~/category/package/options.mk:2: Expected definition of PKG_OPTIONS_VAR.") mklines = t.SetUpFileMkLines("category/package/options.mk", - MkRcsID, + MkCvsID, "PKG_OPTIONS_VAR=\tPKG_OPTIONS.pkgbase", "PKG_SUPPORTED_OPTIONS=\toption1", ".include \"../../mk/compiler.mk\"") @@ -111,7 +111,7 @@ func (s *Suite) Test_CheckLinesOptionsMk__edge_cases(c *check.C) { "Option \"option1\" should be handled below in an .if block.") mklines = t.SetUpFileMkLines("category/package/options.mk", - MkRcsID, + MkCvsID, "PKG_OPTIONS_VAR=\tPKG_OPTIONS.pkgbase", "PKG_SUPPORTED_OPTIONS=\toption1", ".include \"../../mk/bsd.options.mk\"", @@ -142,10 +142,10 @@ func (s *Suite) Test_CheckLinesOptionsMk__unexpected_line(c *check.C) { t.SetUpVartypes() t.CreateFileLines("mk/bsd.options.mk", - MkRcsID) + MkCvsID) mklines := t.SetUpFileMkLines("category/package/options.mk", - MkRcsID, + MkCvsID, "", "PKG_OPTIONS_VAR= PKG_OPTIONS.mc", "", @@ -169,10 +169,10 @@ func (s *Suite) Test_CheckLinesOptionsMk__malformed_condition(c *check.C) { t.SetUpOption("x11", "") t.CreateFileLines("mk/bsd.options.mk", - MkRcsID) + MkCvsID) mklines := t.SetUpFileMkLines("category/package/options.mk", - MkRcsID, + MkCvsID, "", "PKG_OPTIONS_VAR= PKG_OPTIONS.mc", "PKG_SUPPORTED_OPTIONS= # none", @@ -202,7 +202,7 @@ func (s *Suite) Test_CheckLinesOptionsMk__PLIST_VARS_based_on_PKG_SUPPORTED_OPTI t.SetUpPackage("category/package") t.CreateFileLines("mk/bsd.options.mk") t.SetUpFileMkLines("category/package/options.mk", - MkRcsID, + MkCvsID, "", "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", "PKG_SUPPORTED_OPTIONS+=\tone", @@ -246,7 +246,7 @@ func (s *Suite) Test_OptionsLinesChecker_handleLowerCondition__foreign_variable( t.SetUpPackage("category/package", ".include \"options.mk\"") t.CreateFileLines("category/package/options.mk", - MkRcsID, + MkCvsID, "", "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", "PKG_SUPPORTED_OPTIONS=\topt", @@ -273,7 +273,7 @@ func (s *Suite) Test_CheckLinesOptionsMk__autofix(c *check.C) { t.SetUpPackage("category/package", ".include \"options.mk\"") t.CreateFileLines("category/package/options.mk", - MkRcsID, + MkCvsID, "", "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", "PKG_SUPPORTED_OPTIONS=\t# none", @@ -312,7 +312,7 @@ func (s *Suite) Test_CheckLinesOptionsMk__autofix(c *check.C) { "AUTOFIX: options.mk:10: Replacing \".\" with \". \".") t.CheckFileLinesDetab("options.mk", - MkRcsID, + MkCvsID, "", "PKG_OPTIONS_VAR= PKG_OPTIONS.package", "PKG_SUPPORTED_OPTIONS= # none", diff --git a/pkgtools/pkglint/files/package.go b/pkgtools/pkglint/files/package.go index 9c9fa93cdda..cae544b7d44 100644 --- a/pkgtools/pkglint/files/package.go +++ b/pkgtools/pkglint/files/package.go @@ -26,11 +26,13 @@ type Package struct { EffectivePkgname string // PKGNAME or DISTNAME from the package Makefile, including nb13 EffectivePkgbase string // EffectivePkgname without the version EffectivePkgversion string // The version part of the effective PKGNAME, excluding nb13 - EffectivePkgnameLine MkLine // The origin of the three Effective* values + EffectivePkgnameLine *MkLine // The origin of the three Effective* values Plist PlistContent // Files and directories mentioned in the PLIST files - vars Scope - bl3 map[string]MkLine // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. + vars Scope + redundant *RedundantScope + + bl3 map[string]*MkLine // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. // Remembers the Makefile fragments that have already been included. // The key to the map is the filename relative to the package directory. @@ -43,25 +45,24 @@ type Package struct { // TODO: Set an upper limit, to prevent denial of service. included Once - seenMakefileCommon bool // Does the package have any .includes? + // Does the package have any .includes? + seenInclude bool // Files from .include lines that are nested inside .if. // They often depend on OPSYS or on the existence of files in the build environment. - conditionalIncludes map[string]MkLine + conditionalIncludes map[string]*MkLine // Files from .include lines that are not nested. // These are cross-checked with buildlink3.mk whether they are unconditional there, too. - unconditionalIncludes map[string]MkLine + unconditionalIncludes map[string]*MkLine - once Once IgnoreMissingPatches bool // In distinfo, don't warn about patches that cannot be found. } func NewPackage(dir string) *Package { pkgpath := G.Pkgsrc.ToRel(dir) - if strings.Count(pkgpath, "/") != 1 { - assertf(false, "Package directory %q must be two subdirectories below the pkgsrc root %q.", - dir, G.Pkgsrc.File(".")) - } + + // Package directory must be two subdirectories below the pkgsrc root. + assert(strings.Count(pkgpath, "/") == 1) pkg := Package{ dir: dir, @@ -72,10 +73,10 @@ func NewPackage(dir string) *Package { DistinfoFile: "${PKGDIR}/distinfo", // TODO: Redundant, see the vars.Fallback below. Plist: NewPlistContent(), vars: NewScope(), - bl3: make(map[string]MkLine), + bl3: make(map[string]*MkLine), included: Once{}, - conditionalIncludes: make(map[string]MkLine), - unconditionalIncludes: make(map[string]MkLine), + conditionalIncludes: make(map[string]*MkLine), + unconditionalIncludes: make(map[string]*MkLine), } pkg.vars.DefineAll(G.Pkgsrc.UserDefinedVars) @@ -101,6 +102,16 @@ func (pkg *Package) File(relativeFileName string) string { return cleanpath(resolveVariableRefs(nil, pkg.dir+"/"+relativeFileName)) } +// Rel returns the path by which the given filename (as seen from the +// current working directory) can be reached as a relative path from +// the package directory. +// +// Example: +// NewPackage("category/package").Rel("other/package") == "../../other/package" +func (pkg *Package) Rel(filename string) string { + return relpath(pkg.dir, filename) +} + func (pkg *Package) checkPossibleDowngrade() { if trace.Tracing { defer trace.Call0()() @@ -121,14 +132,14 @@ func (pkg *Package) checkPossibleDowngrade() { return } - if change.Action == "Updated" { + if change.Action == Updated { pkgversionNorev := replaceAll(pkgversion, `nb\d+$`, "") - changeNorev := replaceAll(change.Version, `nb\d+$`, "") + changeNorev := replaceAll(change.Version(), `nb\d+$`, "") cmp := pkgver.Compare(pkgversionNorev, changeNorev) switch { case cmp < 0: mkline.Warnf("The package is being downgraded from %s (see %s) to %s.", - change.Version, mkline.Line.RefToLocation(change.Location), pkgversion) + change.Version(), mkline.Line.RefToLocation(change.Location), pkgversion) mkline.Explain( "The files in doc/CHANGES-*, in which all version changes are", "recorded, have a higher version number than what the package says.", @@ -137,9 +148,11 @@ func (pkg *Package) checkPossibleDowngrade() { case cmp > 0 && !isLocallyModified(mkline.Filename): mkline.Notef("Package version %q is greater than the latest %q from %s.", - pkgversion, change.Version, mkline.Line.RefToLocation(change.Location)) + pkgversion, change.Version(), mkline.Line.RefToLocation(change.Location)) mkline.Explain( "Each update to a package should be mentioned in the doc/CHANGES file.", + "That file is used for the quarterly statistics of updated packages.", + "", "To do this after updating a package, run", sprintf("%q,", bmake("cce")), "which is the abbreviation for commit-changes-entry.") @@ -150,13 +163,13 @@ func (pkg *Package) checkPossibleDowngrade() { // checkLinesBuildlink3Inclusion checks whether the package Makefile and // the corresponding buildlink3.mk agree for all included buildlink3.mk // files whether they are included conditionally or unconditionally. -func (pkg *Package) checkLinesBuildlink3Inclusion(mklines MkLines) { +func (pkg *Package) checkLinesBuildlink3Inclusion(mklines *MkLines) { if trace.Tracing { defer trace.Call0()() } // Collect all the included buildlink3.mk files from the file. - includedFiles := make(map[string]MkLine) + includedFiles := make(map[string]*MkLine) for _, mkline := range mklines.mklines { if mkline.IsInclude() { includedFile := mkline.IncludedFile() @@ -178,7 +191,7 @@ func (pkg *Package) checkLinesBuildlink3Inclusion(mklines MkLines) { } } -func (pkg *Package) load() ([]string, MkLines, MkLines) { +func (pkg *Package) load() ([]string, *MkLines, *MkLines) { // Load the package Makefile and all included files, // to collect all used and defined variables and similar data. mklines, allLines := pkg.loadPackageMakefile() @@ -190,9 +203,6 @@ func (pkg *Package) load() ([]string, MkLines, MkLines) { if pkg.Pkgdir != "." { files = append(files, dirglob(pkg.File(pkg.Pkgdir))...) } - if G.Opts.CheckExtra { - files = append(files, dirglob(pkg.File(pkg.Filesdir))...) - } files = append(files, dirglob(pkg.File(pkg.Patchdir))...) if pkg.DistinfoFile != pkg.vars.fallback["DISTINFO_FILE"] { files = append(files, pkg.File(pkg.DistinfoFile)) @@ -206,9 +216,8 @@ func (pkg *Package) load() ([]string, MkLines, MkLines) { !matches(filename, `patch-`) && !contains(filename, pkg.Pkgdir+"/") && !contains(filename, pkg.Filesdir+"/") { - if fragmentMklines := LoadMk(filename, MustSucceed); fragmentMklines != nil { - fragmentMklines.collectUsedVariables() - } + fragmentMklines := LoadMk(filename, MustSucceed) + fragmentMklines.collectUsedVariables() } if hasPrefix(basename, "PLIST") { pkg.loadPlistDirs(filename) @@ -218,7 +227,7 @@ func (pkg *Package) load() ([]string, MkLines, MkLines) { return files, mklines, allLines } -func (pkg *Package) check(filenames []string, mklines, allLines MkLines) { +func (pkg *Package) check(filenames []string, mklines, allLines *MkLines) { haveDistinfo := false havePatches := false @@ -233,19 +242,19 @@ func (pkg *Package) check(filenames []string, mklines, allLines MkLines) { st, err := os.Lstat(filename) switch { case err != nil: - // For missing custom distinfo file, an error message is already generated + // For a missing custom distinfo file, an error message is already generated // for the line where DISTINFO_FILE is defined. // // For all other cases it is next to impossible to reach this branch // since all those files come from calls to dirglob. break - case path.Base(filename) == "Makefile": + case path.Base(filename) == "Makefile" && strings.Count(G.Pkgsrc.ToRel(filename), "/") == 2: G.checkExecutable(filename, st.Mode()) pkg.checkfilePackageMakefile(filename, mklines, allLines) default: - G.checkDirent(filename, st.Mode()) + pkg.checkDirent(filename, st.Mode()) } if contains(filename, "/patches/patch-") { @@ -253,7 +262,8 @@ func (pkg *Package) check(filenames []string, mklines, allLines MkLines) { } else if hasSuffix(filename, "/distinfo") { haveDistinfo = true } - pkg.checkLocallyModified(filename) + pkg.checkOwnerMaintainer(filename) + pkg.checkFreeze(filename) } if pkg.Pkgdir == "." { @@ -267,16 +277,59 @@ func (pkg *Package) check(filenames []string, mklines, allLines MkLines) { } } -func (pkg *Package) loadPackageMakefile() (MkLines, MkLines) { +// checkDirent checks a directory entry based on its filename and its mode +// (regular file, directory, symlink). +func (pkg *Package) checkDirent(dirent string, mode os.FileMode) { + // TODO: merge duplicate code in Pkglint.checkMode + + basename := path.Base(dirent) + + switch { + + case mode.IsRegular(): + pkgsrcRel := G.Pkgsrc.ToRel(dirent) + depth := strings.Count(pkgsrcRel, "/") + G.checkReg(dirent, basename, depth) + + case hasPrefix(basename, "work"): + if G.Opts.Import { + NewLineWhole(dirent).Errorf("Must be cleaned up before committing the package.") + } + return + + case mode.IsDir(): + switch { + case basename == "files", + basename == "patches", + matches(dirent, `(?:^|/)files/[^/]*$`), + isEmptyDir(dirent): + break + + default: + NewLineWhole(dirent).Warnf("Unknown directory name.") + } + + case mode&os.ModeSymlink != 0: + NewLineWhole(dirent).Warnf("Invalid symlink name.") + + default: + NewLineWhole(dirent).Errorf("Only files and directories are allowed in pkgsrc.") + } +} + +func (pkg *Package) loadPackageMakefile() (*MkLines, *MkLines) { filename := pkg.File("Makefile") if trace.Tracing { defer trace.Call1(filename)() } - mainLines := NewMkLines(NewLines(filename, nil)) + mainLines := LoadMk(filename, NotEmpty|LogErrors) + if mainLines == nil { + return nil, nil + } + allLines := NewMkLines(NewLines("", nil)) - if _, result := pkg.readMakefile(filename, mainLines, allLines, ""); !result { - LoadMk(filename, NotEmpty|LogErrors) // Just for the LogErrors. + if !pkg.parse(mainLines, allLines, "") { return nil, nil } @@ -330,162 +383,182 @@ func (pkg *Package) loadPackageMakefile() (MkLines, MkLines) { } // TODO: What is allLines used for, is it still necessary? Would it be better as a field in Package? -func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines MkLines, includingFileForUsedCheck string) (exists bool, result bool) { +func (pkg *Package) parse(mklines *MkLines, allLines *MkLines, includingFileForUsedCheck string) bool { if trace.Tracing { - defer trace.Call1(filename)() + defer trace.Call1(mklines.lines.Filename)() } - fileMklines := LoadMk(filename, NotEmpty) // TODO: Document why omitting LogErrors is correct here. - if fileMklines == nil { - return false, false - } + result := true - exists = true - result = true + lineAction := func(mkline *MkLine) bool { + result = pkg.parseLine(mklines, mkline, allLines) + return result + } - isMainMakefile := len(mainLines.mklines) == 0 + atEnd := func(mkline *MkLine) {} + mklines.ForEachEnd(lineAction, atEnd) - handleIncludeLine := func(mkline MkLine) YesNoUnknown { - includedFile, incDir, incBase := pkg.findIncludedFile(mkline, filename) + if includingFileForUsedCheck != "" { + mklines.CheckUsedBy(G.Pkgsrc.ToRel(includingFileForUsedCheck)) + } - if includedFile == "" { - return unknown + // For every included buildlink3.mk, include the corresponding builtin.mk + // automatically since the pkgsrc infrastructure does the same. + filename := mklines.lines.Filename + if path.Base(filename) == "buildlink3.mk" { + builtin := cleanpath(path.Dir(filename) + "/builtin.mk") + builtinRel := relpath(pkg.dir, builtin) + if pkg.included.FirstTime(builtinRel) && fileExists(builtin) { + builtinMkLines := LoadMk(builtin, MustSucceed|LogErrors) + pkg.parse(builtinMkLines, allLines, "") } + } - dirname, _ := path.Split(filename) - dirname = cleanpath(dirname) - fullIncluded := dirname + "/" + includedFile - relIncludedFile := relpath(pkg.dir, fullIncluded) - - if !pkg.diveInto(filename, includedFile) { - return unknown - } + return result +} - if !pkg.included.FirstTime(relIncludedFile) { - return unknown - } +func (pkg *Package) parseLine(mklines *MkLines, mkline *MkLine, allLines *MkLines) bool { + allLines.mklines = append(allLines.mklines, mkline) + allLines.lines.Lines = append(allLines.lines.Lines, mkline.Line) - pkg.collectUsedBy(mkline, incDir, incBase, includedFile) + if mkline.IsInclude() { + includingFile := mkline.Filename + includedFile := mkline.IncludedFile() + includedMkLines, skip := pkg.loadIncluded(mkline, includingFile) - if trace.Tracing { - trace.Step1("Including %q.", fullIncluded) + if includedMkLines == nil { + if skip || mklines.indentation.HasExists(includedFile) { + return true // See https://github.com/rillig/pkglint/issues/1 + } + mkline.Errorf("Cannot read %q.", includedFile) + return false } - fullIncluding := ifelseStr(incBase == "Makefile.common" && incDir != "", filename, "") - innerExists, innerResult := pkg.readMakefile(fullIncluded, mainLines, allLines, fullIncluding) - if !innerExists { - if fileMklines.indentation.IsCheckedFile(includedFile) { - return yes // See https://github.com/rillig/pkglint/issues/1 - } + filenameForUsedCheck := "" + dir, base := path.Split(includedFile) + if dir != "" && base == "Makefile.common" && dir != "../../"+pkg.Pkgpath+"/" { + filenameForUsedCheck = includingFile + } + if !pkg.parse(includedMkLines, allLines, filenameForUsedCheck) { + return false + } + } - // Only look in the directory relative to the - // current file and in the package directory. - // Make(1) has a list of include directories, but pkgsrc - // doesn't make use of that, so pkglint also doesn't - // need this extra complexity. - pkgBasedir := pkg.File(".") - if dirname != pkgBasedir { // Prevent unnecessary syscalls - dirname = pkgBasedir - - fullIncludedFallback := dirname + "/" + includedFile - innerExists, innerResult = pkg.readMakefile(fullIncludedFallback, mainLines, allLines, fullIncluding) - } + if mkline.IsVarassign() { + varname, op, value := mkline.Varname(), mkline.Op(), mkline.Value() - if !innerExists { - mkline.Errorf("Cannot read %q.", includedFile) + if op != opAssignDefault || !pkg.vars.Defined(varname) { + if trace.Tracing { + trace.Stepf("varassign(%q, %q, %q)", varname, op, value) } + pkg.vars.Define(varname, mkline) } + } + return true +} - if !innerResult { - result = false - return no - } +// loadIncluded loads the lines from the file given by the .include directive +// in mkline. +// +// The returned lines may be nil in two different cases: if skip is true, +// the included file is not processed further for whatever reason. But if +// skip is false, the file could not be read and an appropriate error message +// has already been logged. +func (pkg *Package) loadIncluded(mkline *MkLine, includingFile string) (includedMklines *MkLines, skip bool) { + includedFile := pkg.resolveIncludedFile(mkline, includingFile) - return unknown + if includedFile == "" { + return nil, true } - lineAction := func(mkline MkLine) bool { - if isMainMakefile { - mainLines.mklines = append(mainLines.mklines, mkline) - mainLines.lines.Lines = append(mainLines.lines.Lines, mkline.Line) - } - allLines.mklines = append(allLines.mklines, mkline) - allLines.lines.Lines = append(allLines.lines.Lines, mkline.Line) + dirname, _ := path.Split(includingFile) + dirname = cleanpath(dirname) + fullIncluded := dirname + "/" + includedFile + relIncludedFile := relpath(pkg.dir, fullIncluded) - if mkline.IsInclude() { - includeResult := handleIncludeLine(mkline) - if includeResult != unknown { - return includeResult == yes - } - } + if !pkg.diveInto(includingFile, includedFile) { + return nil, true + } - if mkline.IsVarassign() { - varname, op, value := mkline.Varname(), mkline.Op(), mkline.Value() + if !pkg.included.FirstTime(relIncludedFile) { + return nil, true + } - if op != opAssignDefault || !pkg.vars.Defined(varname) { - if trace.Tracing { - trace.Stepf("varassign(%q, %q, %q)", varname, op, value) - } - pkg.vars.Define(varname, mkline) - } - } - return true + pkg.collectSeenInclude(mkline, includedFile) + + if trace.Tracing { + trace.Step1("Including %q.", fullIncluded) + } + includedMklines = LoadMk(fullIncluded, 0) + if includedMklines != nil { + return includedMklines, false } - atEnd := func(mkline MkLine) {} - fileMklines.ForEachEnd(lineAction, atEnd) + // Only look in the directory relative to the current file + // and in the package directory; see + // devel/bmake/files/parse.c, function Parse_include_file. + // + // Bmake has a list of include directories that can be specified + // on the command line using the -I option, but pkgsrc doesn't + // make use of that, so pkglint also doesn't need this extra + // complexity. + pkgBasedir := pkg.File(".") - if includingFileForUsedCheck != "" { - fileMklines.CheckForUsedComment(G.Pkgsrc.ToRel(includingFileForUsedCheck)) + // Prevent unnecessary syscalls + if dirname == pkgBasedir { + return nil, false } - // For every included buildlink3.mk, include the corresponding builtin.mk - // automatically since the pkgsrc infrastructure does the same. - if path.Base(filename) == "buildlink3.mk" { - builtin := cleanpath(path.Dir(filename) + "/builtin.mk") - builtinRel := relpath(pkg.dir, builtin) - if pkg.included.FirstTime(builtinRel) && fileExists(builtin) { - pkg.readMakefile(builtin, mainLines, allLines, "") - } + dirname = pkgBasedir + + fullIncludedFallback := dirname + "/" + includedFile + includedMklines = LoadMk(fullIncludedFallback, 0) + if includedMklines == nil { + return nil, false } - return + mkline.Notef("The path to the included file should be %q.", + relpath(path.Dir(mkline.Filename), fullIncludedFallback)) + mkline.Explain( + "The .include directive first searches the file relative to the including file.", + "And if that doesn't exist, falls back to the current directory, which in the", + "case of a pkgsrc package is the package directory.", + "", + "This fallback mechanism is not necessary for pkgsrc, therefore it should not", + "be used. One less thing to learn for package developers.") + + return includedMklines, false } -func (pkg *Package) diveInto(includingFile string, includedFile string) bool { +// diveInto decides whether to load the includedFile. +// +// The includingFile is relative to the current working directory, +// the includedFile is taken directly from the .include directive. +func (*Package) diveInto(includingFile string, includedFile string) bool { - // The variables that appear in these files are largely modeled by - // pkglint in the file vardefs.go. Therefore parsing these files again - // doesn't make much sense. if hasSuffix(includedFile, "/bsd.pkg.mk") || IsPrefs(includedFile) { return false } - // All files that are included from outside of the pkgsrc infrastructure - // are relevant. This is typically mk/compiler.mk or the various - // mk/*.buildlink3.mk files. if !contains(includingFile, "/mk/") { return true } - // The mk/*.buildlink3.mk files often come with a companion file called - // mk/*.builtin.mk, which also defines variables that are visible from - // the package. - // - // This case is needed for getting the redundancy check right. Without it - // there will be warnings about redundant assignments to the - // BUILTIN_CHECK.pthread variable. - if contains(includingFile, "buildlink3.mk") && contains(includedFile, "builtin.mk") { + if hasSuffix(includingFile, "buildlink3.mk") && hasSuffix(includedFile, "builtin.mk") { return true } return false } -func (pkg *Package) collectUsedBy(mkline MkLine, incDir string, incBase string, includedFile string) { +func (pkg *Package) collectSeenInclude(mkline *MkLine, includedFile string) { + if mkline.Basename != "Makefile" { + return + } + + incDir, incBase := path.Split(includedFile) switch { case - mkline.Basename != "Makefile", hasPrefix(incDir, "../../mk/"), incBase == "buildlink3.mk", incBase == "builtin.mk", @@ -494,53 +567,45 @@ func (pkg *Package) collectUsedBy(mkline MkLine, incDir string, incBase string, } if trace.Tracing { - trace.Step1("Including %q sets seenMakefileCommon.", includedFile) + trace.Step1("Including %q sets seenInclude.", includedFile) } - pkg.seenMakefileCommon = true + pkg.seenInclude = true } -func (pkg *Package) findIncludedFile(mkline MkLine, includingFilename string) (includedFile, incDir, incBase string) { +// resolveIncludedFile resolves Makefile variables such as ${PKGPATH} to +// their actual values. +func (pkg *Package) resolveIncludedFile(mkline *MkLine, includingFilename string) string { // TODO: resolveVariableRefs uses G.Pkg implicitly. It should be made explicit. // TODO: Try to combine resolveVariableRefs and ResolveVarsInRelativePath. - includedFile = resolveVariableRefs(nil, mkline.ResolveVarsInRelativePath(mkline.IncludedFile())) + includedFile := resolveVariableRefs(nil, mkline.ResolveVarsInRelativePath(mkline.IncludedFile())) if containsVarRef(includedFile) { if trace.Tracing && !contains(includingFilename, "/mk/") { - trace.Stepf("%s:%s: Skipping include file %q. This may result in false warnings.", + trace.Stepf("%s:%s: Skipping unresolvable include file %q.", mkline.Filename, mkline.Linenos(), includedFile) } - includedFile = "" + return "" } - incDir, incBase = path.Split(includedFile) - if includedFile != "" { - if mkline.Basename != "buildlink3.mk" { - if matches(includedFile, `^\.\./\.\./(.*)/buildlink3\.mk$`) { - pkg.bl3[includedFile] = mkline - if trace.Tracing { - trace.Step1("Buildlink3 file in package: %q", includedFile) - } + if mkline.Basename != "buildlink3.mk" { + if matches(includedFile, `^\.\./\.\./(.*)/buildlink3\.mk$`) { + pkg.bl3[includedFile] = mkline + if trace.Tracing { + trace.Step1("Buildlink3 file in package: %q", includedFile) } } } - return + return includedFile } -func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines, allLines MkLines) { +func (pkg *Package) checkfilePackageMakefile(filename string, mklines *MkLines, allLines *MkLines) { if trace.Tracing { defer trace.Call1(filename)() } vars := pkg.vars - if !vars.Defined("PLIST_SRC") && - !vars.Defined("GENERATE_PLIST") && - !vars.Defined("META_PACKAGE") && - !fileExists(pkg.File(pkg.Pkgdir+"/PLIST")) && - !fileExists(pkg.File(pkg.Pkgdir+"/PLIST.common")) { - // TODO: Move these technical details into the explanation, making space for an understandable warning. - NewLineWhole(filename).Warnf("Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset.") - } + pkg.checkPlist() if (vars.Defined("NO_CHECKSUM") || vars.Defined("META_PACKAGE")) && isEmptyDir(pkg.File(pkg.Patchdir)) { @@ -570,7 +635,7 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines, a } } - if !vars.Defined("LICENSE") && !vars.Defined("META_PACKAGE") && pkg.once.FirstTime("LICENSE") { + if !vars.Defined("LICENSE") && !vars.Defined("META_PACKAGE") { line := NewLineWhole(filename) line.Errorf("Each package must define its LICENSE.") // TODO: Explain why the LICENSE is necessary. @@ -579,9 +644,9 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines, a sprintf("run %q.", bmake("guess-license"))) } - scope := NewRedundantScope() - scope.Check(allLines) // Updates the variables in the scope - pkg.checkGnuConfigureUseLanguages(scope) + pkg.redundant = NewRedundantScope() + pkg.redundant.Check(allLines) // Updates the variables in the scope + pkg.checkGnuConfigureUseLanguages() pkg.checkUseLanguagesCompilerMk(allLines) pkg.determineEffectivePkgVars() @@ -600,14 +665,68 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines, a } pkg.checkUpdate() + allLines.collectDefinedVariables() // To get the tool definitions mklines.Tools = allLines.Tools // TODO: also copy the other collected data mklines.Check() + pkg.CheckVarorder(mklines) + SaveAutofixChanges(mklines.lines) } -func (pkg *Package) checkGnuConfigureUseLanguages(s *RedundantScope) { +// checkPlist checks whether the package needs a PLIST file, +// or whether that file should be omitted since it is autogenerated. +func (pkg *Package) checkPlist() { + vars := pkg.vars + if vars.Defined("PLIST_SRC") || vars.Defined("GENERATE_PLIST") { + return + } + + needsPlist, line := pkg.needsPlist() + hasPlist := fileExists(pkg.File(pkg.Pkgdir+"/PLIST")) || + fileExists(pkg.File(pkg.Pkgdir+"/PLIST.common")) + + if needsPlist && !hasPlist { + line.Warnf("This package should have a PLIST file.") + line.Explain( + "The PLIST file provides the list of files that will be", + "installed by the package. Having this list ensures that", + "a package update doesn't accidentally modify the list", + "of installed files.", + "", + seeGuide("PLIST issues", "plist")) + } + + if hasPlist && !needsPlist { + line.Warnf("This package should not have a PLIST file.") + } +} + +func (pkg *Package) needsPlist() (bool, *Line) { + vars := pkg.vars + + // TODO: In the below code, it shouldn't be necessary to mention + // each variable name twice. + + if vars.Defined("PERL5_PACKLIST") { + return false, vars.LastDefinition("PERL5_PACKLIST").Line + } + + if vars.Defined("PERL5_USE_PACKLIST") { + needed := strings.ToLower(vars.LastValue("PERL5_USE_PACKLIST")) == "no" + return needed, vars.LastDefinition("PERL5_USE_PACKLIST").Line + } + + if vars.Defined("META_PACKAGE") { + return false, vars.LastDefinition("META_PACKAGE").Line + } + + return true, NewLineWhole(pkg.File("Makefile")) +} + +func (pkg *Package) checkGnuConfigureUseLanguages() { + s := pkg.redundant gnuConfigure := s.vars["GNU_CONFIGURE"] if gnuConfigure == nil || !gnuConfigure.vari.Constant() { @@ -619,7 +738,7 @@ func (pkg *Package) checkGnuConfigureUseLanguages(s *RedundantScope) { return } - var wrongLines []MkLine + var wrongLines []*MkLine for _, mkline := range useLanguages.vari.WriteLocations() { if G.Pkgsrc.IsInfra(mkline.Line.Filename) { @@ -684,7 +803,7 @@ func (pkg *Package) determineEffectivePkgVars() { } } - if pkgname != "" && (pkgname == distname || pkgname == "${DISTNAME}") { + if pkgnameLine != nil && (pkgname == distname || pkgname == "${DISTNAME}") { if pkgnameLine.VarassignComment() == "" { pkgnameLine.Notef("This assignment is probably redundant " + "since PKGNAME is ${DISTNAME} by default.") @@ -693,7 +812,7 @@ func (pkg *Package) determineEffectivePkgVars() { } } - if pkgname == "" && distname != "" && !containsVarRef(distname) && !matches(distname, rePkgname) { + if pkgname == "" && distnameLine != nil && !containsVarRef(distname) && !matches(distname, rePkgname) { distnameLine.Warnf("As DISTNAME is not a valid package name, please define the PKGNAME explicitly.") } @@ -798,12 +917,12 @@ func (pkg *Package) checkUpdate() { // the most common variables appear in a fixed order. // The order itself is a little arbitrary but provides // at least a bit of consistency. -func (pkg *Package) CheckVarorder(mklines MkLines) { +func (pkg *Package) CheckVarorder(mklines *MkLines) { if trace.Tracing { defer trace.Call0()() } - if pkg.seenMakefileCommon { + if pkg.seenInclude { return } @@ -818,77 +937,70 @@ func (pkg *Package) CheckVarorder(mklines MkLines) { ) type Variable struct { - varname string - repetition Repetition + Name string + Repetition Repetition } - type Section struct { - repetition Repetition - vars []Variable - } - - variable := func(name string, repetition Repetition) Variable { return Variable{name, repetition} } - section := func(repetition Repetition, vars ...Variable) Section { return Section{repetition, vars} } + emptyLine := Variable{"", once} // See doc/Makefile-example. // See https://netbsd.org/docs/pkgsrc/pkgsrc.html#components.Makefile. - var sections = []Section{ - section(once, - variable("GITHUB_PROJECT", optional), // either here or below MASTER_SITES - variable("GITHUB_TAG", optional), - variable("DISTNAME", optional), - variable("PKGNAME", optional), - variable("PKGREVISION", optional), - variable("CATEGORIES", once), - variable("MASTER_SITES", many), - variable("GITHUB_PROJECT", optional), // either here or at the very top - variable("GITHUB_TAG", optional), - variable("DIST_SUBDIR", optional), - variable("EXTRACT_SUFX", optional), - variable("DISTFILES", many), - variable("SITES.*", many)), - section(optional, - variable("PATCH_SITES", optional), // or once? - variable("PATCH_SITE_SUBDIR", optional), - variable("PATCHFILES", optional), // or once? - variable("PATCH_DIST_ARGS", optional), - variable("PATCH_DIST_STRIP", optional), - variable("PATCH_DIST_CAT", optional)), - section(once, - variable("MAINTAINER", optional), - variable("OWNER", optional), - variable("HOMEPAGE", optional), - variable("COMMENT", once), - variable("LICENSE", once)), - section(optional, - variable("LICENSE_FILE", optional), - variable("RESTRICTED", optional), - variable("NO_BIN_ON_CDROM", optional), - variable("NO_BIN_ON_FTP", optional), - variable("NO_SRC_ON_CDROM", optional), - variable("NO_SRC_ON_FTP", optional)), - section(optional, - variable("BROKEN_EXCEPT_ON_PLATFORM", many), - variable("BROKEN_ON_PLATFORM", many), - variable("NOT_FOR_PLATFORM", many), - variable("ONLY_FOR_PLATFORM", many), - variable("NOT_FOR_COMPILER", many), - variable("ONLY_FOR_COMPILER", many), - variable("NOT_FOR_UNPRIVILEGED", optional), - variable("ONLY_FOR_UNPRIVILEGED", optional)), - section(optional, - variable("BUILD_DEPENDS", many), - variable("TOOL_DEPENDS", many), - variable("DEPENDS", many))} - - relevantLines := (func() []MkLine { + var variables = []Variable{ + {"GITHUB_PROJECT", optional}, // either here or below MASTER_SITES + {"GITHUB_TAG", optional}, + {"DISTNAME", optional}, + {"PKGNAME", optional}, + {"PKGREVISION", optional}, + {"CATEGORIES", once}, + {"MASTER_SITES", many}, + {"GITHUB_PROJECT", optional}, // either here or at the very top + {"GITHUB_TAG", optional}, + {"DIST_SUBDIR", optional}, + {"EXTRACT_SUFX", optional}, + {"DISTFILES", many}, + {"SITES.*", many}, + emptyLine, + {"PATCH_SITES", optional}, // or once? + {"PATCH_SITE_SUBDIR", optional}, + {"PATCHFILES", optional}, // or once? + {"PATCH_DIST_ARGS", optional}, + {"PATCH_DIST_STRIP", optional}, + {"PATCH_DIST_CAT", optional}, + emptyLine, + {"MAINTAINER", optional}, + {"OWNER", optional}, + {"HOMEPAGE", optional}, + {"COMMENT", once}, + {"LICENSE", once}, + emptyLine, + {"LICENSE_FILE", optional}, + {"RESTRICTED", optional}, + {"NO_BIN_ON_CDROM", optional}, + {"NO_BIN_ON_FTP", optional}, + {"NO_SRC_ON_CDROM", optional}, + {"NO_SRC_ON_FTP", optional}, + emptyLine, + {"BROKEN_EXCEPT_ON_PLATFORM", many}, + {"BROKEN_ON_PLATFORM", many}, + {"NOT_FOR_PLATFORM", many}, + {"ONLY_FOR_PLATFORM", many}, + {"NOT_FOR_COMPILER", many}, + {"ONLY_FOR_COMPILER", many}, + {"NOT_FOR_UNPRIVILEGED", optional}, + {"ONLY_FOR_UNPRIVILEGED", optional}, + emptyLine, + {"BUILD_DEPENDS", many}, + {"TOOL_DEPENDS", many}, + {"DEPENDS", many}} + + relevantLines := (func() []*MkLine { firstRelevant := -1 lastRelevant := -1 relevantVars := make(map[string]bool) - for _, section := range sections { - for _, variable := range section.vars { - relevantVars[variable.varname] = true + for _, variable := range variables { + if variable != emptyLine { + relevantVars[variable.Name] = true } } @@ -931,6 +1043,8 @@ func (pkg *Package) CheckVarorder(mklines MkLines) { return mklines.mklines[firstRelevant : lastRelevant+1] })() + // If there are foreign variables, skip the whole check. + // The check is only intended for the most simple packages. skip := func() bool { interesting := relevantLines @@ -938,78 +1052,88 @@ func (pkg *Package) CheckVarorder(mklines MkLines) { for len(interesting) > 0 && interesting[0].IsComment() { interesting = interesting[1:] } - if len(interesting) > 0 && (interesting[0].IsVarassign() || interesting[0].IsCommentedVarassign()) { + + if len(interesting) > 0 && interesting[0].IsVarassign() { return interesting[0].Varcanon() } return "" } - for _, section := range sections { - for _, variable := range section.vars { - switch variable.repetition { - case optional: - if varcanon() == variable.varname { - interesting = interesting[1:] - } - case once: - if varcanon() == variable.varname { - interesting = interesting[1:] - } else if section.repetition == once { - if variable.varname != "LICENSE" { - if trace.Tracing { - trace.Stepf("Wrong varorder because %s is missing.", variable.varname) - } - return false - } - } - case many: - for varcanon() == variable.varname { - interesting = interesting[1:] - } + for _, variable := range variables { + if variable == emptyLine { + for len(interesting) > 0 && (interesting[0].IsEmpty() || interesting[0].IsComment()) { + interesting = interesting[1:] } + continue } - for len(interesting) > 0 && (interesting[0].IsEmpty() || interesting[0].IsComment()) { - interesting = interesting[1:] + switch variable.Repetition { + case optional: + if varcanon() == variable.Name { + interesting = interesting[1:] + } + case once: + if varcanon() == variable.Name { + interesting = interesting[1:] + } else if variable.Name != "LICENSE" { + if trace.Tracing { + trace.Stepf("Wrong varorder because %s is missing.", variable.Name) + } + return false + } + case many: + for varcanon() == variable.Name { + interesting = interesting[1:] + } } } return len(interesting) == 0 } - if len(relevantLines) == 0 || skip() { - return - } + // canonical returns the canonical ordering of the variables. It mentions all the + // variables that occur in the relevant section, as well as the "once" variables. + canonical := func() string { + var canonical []string + for _, variable := range variables { + if variable == emptyLine { + if canonical[len(canonical)-1] != "empty line" { + canonical = append(canonical, "empty line") + } + continue + } - var canonical []string - for _, section := range sections { - for _, variable := range section.vars { found := false for _, mkline := range relevantLines { - if mkline.IsVarassign() || mkline.IsCommentedVarassign() { - if mkline.Varcanon() == variable.varname { - canonical = append(canonical, mkline.Varname()) - found = true - } + if (mkline.IsVarassign() || mkline.IsCommentedVarassign()) && + mkline.Varcanon() == variable.Name { + + canonical = append(canonical, mkline.Varname()) + found = true + break } } - if !found && section.repetition == once && variable.repetition == once { - canonical = append(canonical, variable.varname) + + if !found && variable.Repetition == once { + canonical = append(canonical, variable.Name) } } - if len(canonical) > 0 && canonical[len(canonical)-1] != "empty line" { - canonical = append(canonical, "empty line") + + if canonical[len(canonical)-1] == "empty line" { + canonical = canonical[:len(canonical)-1] } + return strings.Join(canonical, ", ") } - if len(canonical) > 0 && canonical[len(canonical)-1] == "empty line" { - canonical = canonical[:len(canonical)-1] + + if len(relevantLines) == 0 || skip() { + return } // TODO: This leads to very long and complicated warnings. // Those parts that are correct should not be mentioned, // except if they are helpful for locating the mistakes. mkline := relevantLines[0] - mkline.Warnf("The canonical order of the variables is %s.", strings.Join(canonical, ", ")) + mkline.Warnf("The canonical order of the variables is %s.", canonical()) mkline.Explain( "In simple package Makefiles, some common variables should be", "arranged in a specific order.", @@ -1040,13 +1164,13 @@ func (pkg *Package) checkFileMakefileExt(filename string) { sprintf("content can be queried using %q.", makeHelp("help"))) } -// checkLocallyModified checks files that are about to be committed. +// checkOwnerMaintainer checks files that are about to be committed. // Depending on whether the package has a MAINTAINER or an OWNER, // the wording differs. // // Pkglint assumes that the local username is the same as the NetBSD // username, which fits most scenarios. -func (pkg *Package) checkLocallyModified(filename string) { +func (pkg *Package) checkOwnerMaintainer(filename string) { if trace.Tracing { defer trace.Call(filename)() } @@ -1069,7 +1193,7 @@ func (pkg *Package) checkLocallyModified(filename string) { return } - if !isLocallyModified(filename) || !fileExists(filename) { + if !isLocallyModified(filename) { return } @@ -1089,40 +1213,52 @@ func (pkg *Package) checkLocallyModified(filename string) { } } -func (pkg *Package) checkIncludeConditionally(mkline MkLine, indentation *Indentation) { - conditionalVars := mkline.ConditionalVars() - if len(conditionalVars) == 0 { - conditionalVars = indentation.Varnames() - mkline.SetConditionalVars(conditionalVars) +func (pkg *Package) checkFreeze(filename string) { + freezeStart := G.Pkgsrc.FreezeStart + if freezeStart == "" { + return } - if path.Dir(abspath(mkline.Filename)) == abspath(pkg.File(".")) { - includedFile := mkline.IncludedFile() + if !isLocallyModified(filename) { + return + } - if indentation.IsConditional() { - pkg.conditionalIncludes[includedFile] = mkline - if other := pkg.unconditionalIncludes[includedFile]; other != nil { - mkline.Warnf( - "%q is included conditionally here (depending on %s) "+ - "and unconditionally in %s.", - cleanpath(includedFile), strings.Join(mkline.ConditionalVars(), ", "), mkline.RefTo(other)) - } + line := NewLineWhole(filename) + line.Notef("Pkgsrc is frozen since %s.", freezeStart) + line.Explain( + "During a pkgsrc freeze, changes to pkgsrc should only be made very carefully.", + "See https://www.netbsd.org/developers/pkgsrc/ for the exact rules.") +} - } else { - pkg.unconditionalIncludes[includedFile] = mkline - if other := pkg.conditionalIncludes[includedFile]; other != nil { - mkline.Warnf( - "%q is included unconditionally here "+ - "and conditionally in %s (depending on %s).", - cleanpath(includedFile), mkline.RefTo(other), strings.Join(other.ConditionalVars(), ", ")) - } +func (pkg *Package) checkIncludeConditionally(mkline *MkLine, indentation *Indentation) { + mkline.SetConditionalVars(indentation.Varnames()) + + includedFile := mkline.IncludedFile() + key := pkg.Rel(mkline.IncludedFile()) + + if indentation.IsConditional() { + pkg.conditionalIncludes[key] = mkline + if other := pkg.unconditionalIncludes[key]; other != nil { + mkline.Warnf( + "%q is included conditionally here (depending on %s) "+ + "and unconditionally in %s.", + cleanpath(includedFile), strings.Join(mkline.ConditionalVars(), ", "), mkline.RefTo(other)) } - // TODO: Check whether the conditional variables are the same on both places. - // Ideally they should match, but there may be some differences in internal - // variables, which need to be filtered out before comparing them, like it is - // already done with *_MK variables. + } else { + pkg.unconditionalIncludes[key] = mkline + if other := pkg.conditionalIncludes[key]; other != nil { + mkline.Warnf( + "%q is included unconditionally here "+ + "and conditionally in %s (depending on %s).", + cleanpath(includedFile), mkline.RefTo(other), strings.Join(other.ConditionalVars(), ", ")) + } } + + // TODO: Check whether the conditional variables are the same on both places. + // Ideally they should match, but there may be some differences in internal + // variables, which need to be filtered out before comparing them, like it is + // already done with *_MK variables. } func (pkg *Package) loadPlistDirs(plistFilename string) { @@ -1155,11 +1291,11 @@ func (pkg *Package) AutofixDistinfo(oldSha1, newSha1 string) { // checkUseLanguagesCompilerMk checks that after including mk/compiler.mk // or mk/endian.mk for the first time, there are no more changes to // USE_LANGUAGES, as these would be ignored by the pkgsrc infrastructure. -func (pkg *Package) checkUseLanguagesCompilerMk(mklines MkLines) { +func (pkg *Package) checkUseLanguagesCompilerMk(mklines *MkLines) { var seen Once - handleVarassign := func(mkline MkLine) { + handleVarassign := func(mkline *MkLine) { if mkline.Varname() != "USE_LANGUAGES" { return } @@ -1179,7 +1315,7 @@ func (pkg *Package) checkUseLanguagesCompilerMk(mklines MkLines) { "The file compiler.mk guards itself against multiple inclusion.") } - handleInclude := func(mkline MkLine) { + handleInclude := func(mkline *MkLine) { dirname, _ := path.Split(mkline.Filename) dirname = cleanpath(dirname) fullIncluded := dirname + "/" + mkline.IncludedFile() @@ -1188,7 +1324,7 @@ func (pkg *Package) checkUseLanguagesCompilerMk(mklines MkLines) { seen.FirstTime(relIncludedFile) } - mklines.ForEach(func(mkline MkLine) { + mklines.ForEach(func(mkline *MkLine) { switch { case mkline.IsVarassign(): handleVarassign(mkline) diff --git a/pkgtools/pkglint/files/package_test.go b/pkgtools/pkglint/files/package_test.go index 4e0db10e589..34778304f43 100644 --- a/pkgtools/pkglint/files/package_test.go +++ b/pkgtools/pkglint/files/package_test.go @@ -2,6 +2,7 @@ package pkglint import ( "gopkg.in/check.v1" + "os" "strings" ) @@ -9,11 +10,13 @@ func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__file_but_not_package t := s.Init(c) t.CreateFileLines("category/dependency/buildlink3.mk") + t.CreateFileLines("category/dependency/module.mk") G.Pkg = NewPackage(t.File("category/package")) mklines := t.NewMkLines("category/package/buildlink3.mk", - MkRcsID, + MkCvsID, "", - ".include \"../../category/dependency/buildlink3.mk\"") + ".include \"../../category/dependency/buildlink3.mk\"", + ".include \"../../category/dependency/module.mk\"") G.Pkg.checkLinesBuildlink3Inclusion(mklines) @@ -31,7 +34,7 @@ func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__package_but_not_file G.Pkg.bl3["../../category/dependency/buildlink3.mk"] = t.NewMkLine("../../category/dependency/buildlink3.mk", 1, "") mklines := t.NewMkLines("category/package/buildlink3.mk", - MkRcsID) + MkCvsID) t.EnableTracingToLog() G.Pkg.checkLinesBuildlink3Inclusion(mklines) @@ -50,8 +53,7 @@ func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__package_but_not_file func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__no_tracing(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "PKGNAME=\tpackage-1.0") + t.SetUpPackage("category/package") t.CreateFileDummyBuildlink3("category/package/buildlink3.mk") t.FinishSetUp() @@ -118,7 +120,7 @@ func (s *Suite) Test_Package_CheckVarorder__only_required_variables(c *check.C) pkg := NewPackage(t.File("x11/9term")) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "DISTNAME=9term", "CATEGORIES=x11", @@ -137,7 +139,7 @@ func (s *Suite) Test_Package_CheckVarorder__with_optional_variables(c *check.C) pkg := NewPackage(t.File("x11/9term")) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "GITHUB_PROJECT=project", "DISTNAME=9term", @@ -158,7 +160,7 @@ func (s *Suite) Test_Package_CheckVarorder__no_tracing(c *check.C) { pkg := NewPackage(t.File("x11/9term")) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "DISTNAME=9term", "CATEGORIES=x11", @@ -179,9 +181,8 @@ func (s *Suite) Test_Package_CheckVarorder__comments_do_not_crash(c *check.C) { t := s.Init(c) pkg := NewPackage(t.File("x11/9term")) - - pkg.CheckVarorder(t.NewMkLines("Makefile", - MkRcsID, + mklines := t.NewMkLines("Makefile", + MkCvsID, "", "GITHUB_PROJECT=project", "", @@ -189,7 +190,9 @@ func (s *Suite) Test_Package_CheckVarorder__comments_do_not_crash(c *check.C) { "", "DISTNAME=9term", "# comment", - "CATEGORIES=x11")) + "CATEGORIES=x11") + + pkg.CheckVarorder(mklines) t.CheckOutputLines( "WARN: Makefile:3: The canonical order of the variables is " + @@ -201,9 +204,8 @@ func (s *Suite) Test_Package_CheckVarorder__comments_are_ignored(c *check.C) { t := s.Init(c) pkg := NewPackage(t.File("x11/9term")) - - pkg.CheckVarorder(t.NewMkLines("Makefile", - MkRcsID, + mklines := t.NewMkLines("Makefile", + MkCvsID, "", "DISTNAME=\tdistname-1.0", "CATEGORIES=\tsysutils", @@ -211,17 +213,62 @@ func (s *Suite) Test_Package_CheckVarorder__comments_are_ignored(c *check.C) { "MAINTAINER=\tpkgsrc-users@NetBSD.org", "# comment", "COMMENT=\tComment", - "LICENSE=\tgnu-gpl-v2")) + "LICENSE=\tgnu-gpl-v2") + + pkg.CheckVarorder(mklines) t.CheckOutputEmpty() } +func (s *Suite) Test_Package_CheckVarorder__commented_variable_assignment(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("x11/9term")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "DISTNAME=\tdistname-1.0", + "CATEGORIES=\tsysutils", + "", + "MAINTAINER=\tpkgsrc-users@NetBSD.org", + "#HOMEPAGE=\thttps://example.org/", + "COMMENT=\tComment", + "LICENSE=\tgnu-gpl-v2") + + pkg.CheckVarorder(mklines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_CheckVarorder__skip_because_of_foreign_variable(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("x11/9term")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "DISTNAME=\tdistname-1.0", + "USE_TOOLS+=gmake", + "CATEGORIES=\tsysutils", + "", + "MAINTAINER=\tpkgsrc-users@NetBSD.org", + "#HOMEPAGE=\thttps://example.org/", + "COMMENT=\tComment", + "LICENSE=\tgnu-gpl-v2") + + t.EnableTracingToLog() + pkg.CheckVarorder(mklines) + + t.CheckOutputLinesMatching(`.*varorder.*`, + "TRACE: 1 Skipping varorder because of line 4.") +} + func (s *Suite) Test_Package_CheckVarorder__skip_if_there_are_directives(c *check.C) { t := s.Init(c) pkg := NewPackage(t.File("category/package")) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "DISTNAME=\tdistname-1.0", "CATEGORIES=\tsysutils", @@ -249,9 +296,8 @@ func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_top(c *check.C t := s.Init(c) pkg := NewPackage(t.File("x11/9term")) - - pkg.CheckVarorder(t.NewMkLines("Makefile", - MkRcsID, + mklines := t.NewMkLines("Makefile", + MkCvsID, "", "GITHUB_PROJECT=\t\tautocutsel", "DISTNAME=\t\tautocutsel-0.10.0", @@ -260,7 +306,9 @@ func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_top(c *check.C "GITHUB_TAG=\t\t${PKGVERSION_NOREV}", "", "COMMENT=\tComment", - "LICENSE=\tgnu-gpl-v2")) + "LICENSE=\tgnu-gpl-v2") + + pkg.CheckVarorder(mklines) t.CheckOutputEmpty() } @@ -269,9 +317,8 @@ func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_bottom(c *chec t := s.Init(c) pkg := NewPackage(t.File("x11/9term")) - - pkg.CheckVarorder(t.NewMkLines("Makefile", - MkRcsID, + mklines := t.NewMkLines("Makefile", + MkCvsID, "", "DISTNAME=\t\tautocutsel-0.10.0", "CATEGORIES=\t\tx11", @@ -280,7 +327,9 @@ func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_bottom(c *chec "GITHUB_TAG=\t\t${PKGVERSION_NOREV}", "", "COMMENT=\tComment", - "LICENSE=\tgnu-gpl-v2")) + "LICENSE=\tgnu-gpl-v2") + + pkg.CheckVarorder(mklines) t.CheckOutputEmpty() } @@ -289,10 +338,10 @@ func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { t := s.Init(c) t.CreateFileLines("mk/bsd.pkg.mk", "# dummy") - t.CreateFileLines("x11/Makefile", MkRcsID) - t.CreateFileLines("x11/9term/PLIST", PlistRcsID, "bin/9term") + t.CreateFileLines("x11/Makefile", MkCvsID) + t.CreateFileLines("x11/9term/PLIST", PlistCvsID, "bin/9term") t.CreateFileLines("x11/9term/Makefile", - MkRcsID, + MkCvsID, "", "DISTNAME=\t9term-1.0", "CATEGORIES=\tx11", @@ -318,9 +367,8 @@ func (s *Suite) Test_Package_CheckVarorder__MASTER_SITES(c *check.C) { t := s.Init(c) pkg := NewPackage(t.File("category/package")) - - pkg.CheckVarorder(t.NewMkLines("Makefile", - MkRcsID, + mklines := t.NewMkLines("Makefile", + MkCvsID, "", "PKGNAME=\tpackage-1.0", "CATEGORIES=\tcategory", @@ -328,7 +376,9 @@ func (s *Suite) Test_Package_CheckVarorder__MASTER_SITES(c *check.C) { "MASTER_SITES+=\thttp://mirror.example.org/", "", "COMMENT=\tComment", - "LICENSE=\tgnu-gpl-v2")) + "LICENSE=\tgnu-gpl-v2") + + pkg.CheckVarorder(mklines) // No warning that "MASTER_SITES appears too late" t.CheckOutputEmpty() @@ -339,9 +389,8 @@ func (s *Suite) Test_Package_CheckVarorder__diagnostics(c *check.C) { t.SetUpVartypes() pkg := NewPackage(t.File("category/package")) - - pkg.CheckVarorder(t.NewMkLines("Makefile", - MkRcsID, + mklines := t.NewMkLines("Makefile", + MkCvsID, "", "CATEGORIES= net", "", @@ -357,7 +406,9 @@ func (s *Suite) Test_Package_CheckVarorder__diagnostics(c *check.C) { "MAINTAINER= maintainer@example.org", "HOMEPAGE= https://github.com/project/pkgbase/", "", - ".include \"../../mk/bsd.pkg.mk\"")) + ".include \"../../mk/bsd.pkg.mk\"") + + pkg.CheckVarorder(mklines) t.CheckOutputLines( "WARN: Makefile:3: The canonical order of the variables is " + @@ -366,8 +417,8 @@ func (s *Suite) Test_Package_CheckVarorder__diagnostics(c *check.C) { "MAINTAINER, HOMEPAGE, COMMENT, LICENSE.") // After moving the variables according to the warning: - pkg.CheckVarorder(t.NewMkLines("Makefile", - MkRcsID, + mklines = t.NewMkLines("Makefile", + MkCvsID, "", "GITHUB_PROJECT= pkgbase", "DISTNAME= v1.0", @@ -381,11 +432,131 @@ func (s *Suite) Test_Package_CheckVarorder__diagnostics(c *check.C) { "COMMENT= Comment", "LICENSE= gnu-gpl-v3", "", - ".include \"../../mk/bsd.pkg.mk\"")) + ".include \"../../mk/bsd.pkg.mk\"") + + pkg.CheckVarorder(mklines) t.CheckOutputEmpty() } +func (s *Suite) Test_Package_CheckVarorder__comment_at_end_of_section(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + pkg := NewPackage(t.File("category/package")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "CATEGORIES= net", + "SITES.*= # none", + "# comment after the last variable of a section", + "", + "MAINTAINER= maintainer@example.org", + "HOMEPAGE= https://github.com/project/pkgbase/", + "COMMENT= Comment", + "LICENSE= gnu-gpl-v3", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + t.EnableTracingToLog() + pkg.CheckVarorder(mklines) + + // The varorder code is not skipped, not even because of the comment + // after SITES.*. + t.CheckOutputLinesMatching(`.*varorder.*`, + nil...) +} + +func (s *Suite) Test_Package_CheckVarorder__comments_between_sections(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + pkg := NewPackage(t.File("category/package")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "CATEGORIES= net", + "", + "# comment 1", + "", + "# comment 2", + "", + "MAINTAINER= maintainer@example.org", + "HOMEPAGE= https://github.com/project/pkgbase/", + "COMMENT= Comment", + "LICENSE= gnu-gpl-v3", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + pkg.CheckVarorder(mklines) + + // The empty line between the comments is not treated as a section separator. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_CheckVarorder__commented_varassign(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + pkg := NewPackage(t.File("category/package")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "CATEGORIES= net", + "#MASTER_SITES= # none", + "", + "HOMEPAGE= https://github.com/project/pkgbase/", + "#HOMEPAGE= https://github.com/project/pkgbase/", + "#HOMEPAGE= https://github.com/project/pkgbase/", + "#HOMEPAGE= https://github.com/project/pkgbase/", + "#HOMEPAGE= https://github.com/project/pkgbase/", + "LICENSE= gnu-gpl-v3", + "COMMENT= Comment", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + pkg.CheckVarorder(mklines) + + // The order of the variables LICENSE and COMMENT is intentionally + // wrong to force the warning. + // + // Up to June 2019 (308099138a62) pkglint mentioned in the warning + // each commented variable assignment, even repeatedly for the same + // variable name. + // + // These variable assignments should be in the correct order, even + // if they are commented out. It's not necessary though to list a + // variable more than once. + t.CheckOutputLines( + "WARN: Makefile:3: The canonical order of the variables is " + + "CATEGORIES, MASTER_SITES, empty line, HOMEPAGE, COMMENT, LICENSE.") +} + +func (s *Suite) Test_Package_CheckVarorder__DEPENDS(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + pkg := NewPackage(t.File("category/package")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "CATEGORIES= net", + "", + "COMMENT= Comment", + "LICENSE= license", + "MAINTAINER= maintainer@example.org", // In wrong order + "", + "DEPENDS+= dependency>=1.0:../../category/dependency", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + pkg.CheckVarorder(mklines) + + t.CheckOutputLines( + "WARN: Makefile:3: The canonical order of the variables is " + + "CATEGORIES, empty line, MAINTAINER, COMMENT, LICENSE, empty line, DEPENDS.") +} + func (s *Suite) Test_Package_nbPart(c *check.C) { t := s.Init(c) @@ -450,6 +621,19 @@ func (s *Suite) Test_Package_determineEffectivePkgVars__simple_reference(c *chec "This assignment is probably redundant since PKGNAME is ${DISTNAME} by default.") } +func (s *Suite) Test_Package_determineEffectivePkgVars__commented(c *check.C) { + t := s.Init(c) + + pkg := t.SetUpPackage("category/package", + "DISTNAME=\tdistname-1.0", + "PKGNAME=\t${DISTNAME} # intentionally") + t.FinishSetUp() + + G.Check(pkg) + + t.CheckOutputEmpty() +} + func (s *Suite) Test_Package_determineEffectivePkgVars__invalid_DISTNAME(c *check.C) { t := s.Init(c) @@ -464,6 +648,20 @@ func (s *Suite) Test_Package_determineEffectivePkgVars__invalid_DISTNAME(c *chec "As DISTNAME is not a valid package name, please define the PKGNAME explicitly.") } +func (s *Suite) Test_Package_determineEffectivePkgVars__indirect_DISTNAME(c *check.C) { + t := s.Init(c) + + pkg := t.SetUpPackage("category/package", + "DISTNAME=\t${DISTFILES:[1]:C,\\..*,,}") + t.FinishSetUp() + + G.Check(pkg) + + // No warning since the case of DISTNAME being dependent on another + // variable is too difficult to analyze. + t.CheckOutputEmpty() +} + func (s *Suite) Test_Package_determineEffectivePkgVars__C_modifier(c *check.C) { t := s.Init(c) @@ -516,13 +714,62 @@ func (s *Suite) Test_Package_checkPossibleDowngrade(c *check.C) { t.CheckOutputLines( "WARN: Makefile:5: The package is being downgraded from 1.8 (see ../../doc/CHANGES-2018:1) to 1.0nb15.") - G.Pkgsrc.LastChange["category/pkgbase"].Version = "1.0nb22" + G.Pkgsrc.LastChange["category/pkgbase"].target = "1.0nb22" G.Pkg.checkPossibleDowngrade() t.CheckOutputEmpty() } +func (s *Suite) Test_Package_checkPossibleDowngrade__moved(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/pkgbase", + "PKGNAME=\tpackage-1.0") + t.CreateFileLines("doc/CHANGES-2018", + "\tUpdated category/old-package to 1.8 [committer 2018-01-05]", + "\tMoved category/old-package to category/pkgbase [committer 2018-01-05]") + t.FinishSetUp() + + pkg := NewPackage(t.File("category/pkgbase")) + pkg.load() + pkg.determineEffectivePkgVars() + pkg.checkPossibleDowngrade() + + t.Check(G.Pkgsrc.LastChange["category/pkgbase"].Action, equals, Moved) + // No warning because the latest action is not Updated. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkPossibleDowngrade__locally_modified_update(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "PKGNAME=\tpackage-1.8") + t.CreateFileLines("doc/CHANGES-2018", + "\tUpdated category/package to 1.0 [committer 2018-01-05]") + t.CreateFileLines("category/package/CVS/Entries", + "/Makefile//modified//") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + // Since the Makefile is locally modified, pkglint doesn't issue + // any warning since it assumes the package is being upgraded. + t.CheckOutputEmpty() + + // When the Makefile is no longer locally modified, the warning + // is activated again. + t.Remove("category/package/CVS/Entries") + G.cvsEntriesDir = "" + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "NOTE: ~/category/package/Makefile:4: Package version \"1.8\" " + + "is greater than the latest \"1.0\" from ../../doc/CHANGES-2018:1.") +} + func (s *Suite) Test_Package_loadPackageMakefile__dump(c *check.C) { t := s.Init(c) @@ -530,17 +777,17 @@ func (s *Suite) Test_Package_loadPackageMakefile__dump(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("category/Makefile") t.CreateFileLines("category/package/PLIST", - PlistRcsID, + PlistCvsID, "bin/program") t.CreateFileLines("category/package/distinfo", - RcsID, + CvsID, "", "SHA1 (distfile-1.0.tar.gz) = 12341234...", "RMD160 (distfile-1.0.tar.gz) = 12341234...", "SHA512 (distfile-1.0.tar.gz) = 12341234...", "Size (distfile-1.0.tar.gz) = 12341234...") t.CreateFileLines("category/package/Makefile", - MkRcsID, + MkCvsID, "", "CATEGORIES=category", "", @@ -554,7 +801,7 @@ func (s *Suite) Test_Package_loadPackageMakefile__dump(c *check.C) { t.CheckOutputLines( "Whole Makefile (with all included files) follows:", - "~/category/package/Makefile:1: "+MkRcsID, + "~/category/package/Makefile:1: "+MkCvsID, "~/category/package/Makefile:2: ", "~/category/package/Makefile:3: CATEGORIES=category", "~/category/package/Makefile:4: ", @@ -577,7 +824,7 @@ func (s *Suite) Test_Package__varuse_at_load_time(c *check.C) { "_TOOLS_VARNAME.nice=NICE") t.CreateFileLines("category/pkgbase/Makefile", - MkRcsID, + MkCvsID, "", "PKGNAME= loadtime-vartest-1.0", "CATEGORIES= misc", @@ -652,11 +899,102 @@ func (s *Suite) Test_Package__varuse_at_load_time(c *check.C) { "NOTE: ~/category/pkgbase/Makefile:26: Consider the :sh modifier instead of != for \"echo true=${TRUE:Q}\".") } +// Demonstrates that Makefile fragments are handled differently, +// depending on the directory they are in. +func (s *Suite) Test_Package_load__extra_files(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "PKGDIR=\t../../category/other") + t.SetUpPackage("category/other") + t.Chdir("category/package") + t.CreateFileLines("gnu-style.mk", + "ifeq ($(CC),gcc)", + "IS_GCC=\tyes", + "else", + "IS_GCC=\tno", + "endif") + t.CreateFileLines("patches/patch-Makefile.mk", + CvsID, + "", + "Documentation", + "", + "--- Makefile.mk.orig", + "--- Makefile.mk", + "@@ -1,1 +1,1 @@", + "- old", + "+ new") + t.CreateFileLines("patches/readme.mk", + "This is not a BSD-style Makefile.") + t.Copy("gnu-style.mk", "files/gnu-style.mk") + t.Copy("gnu-style.mk", "../../category/other/gnu-style.mk") + + t.FinishSetUp() + + G.Check(".") + + t.CheckOutputLines( + // All *.mk files in the package directory are assumed + // to be BSD-style Makefiles, therefore the many warnings. + "WARN: gnu-style.mk:1: Please use curly braces {} instead of round parentheses () for CC.", + "ERROR: gnu-style.mk:1: Unknown Makefile line format: \"ifeq ($(CC),gcc)\".", + "ERROR: gnu-style.mk:3: Unknown Makefile line format: \"else\".", + "ERROR: gnu-style.mk:5: Unknown Makefile line format: \"endif\".", + + // Since the patches directory should contain only patches, + // each other file is treated as a file belonging to pkgsrc, + // therefore *.mk is interpreted as a Makefile fragment. + "ERROR: patches/readme.mk:1: Unknown Makefile line format: \"This is not a BSD-style Makefile.\".", + "ERROR: distinfo: Patch \"patches/patch-Makefile.mk\" is not recorded. Run \""+confMake+" makepatchsum\".", + + // The following diagnostics are duplicated because the files from + // the package directory are loaded once during Package.load, just + // for collecting the used variables. And then a second time in + // Package.check to perform the actual checks. + // + // The above diagnostics are only those from parsing the file, to + // correctly classify the lines. This is because the main purpose + // of Package.load above is to load the files and collect some + // data, not to perform the actual checks. + // + // Therefore, the below lines contain two more diagnostics. + "WARN: gnu-style.mk:1: Please use curly braces {} instead of round parentheses () for CC.", + "ERROR: gnu-style.mk:1: Unknown Makefile line format: \"ifeq ($(CC),gcc)\".", + "ERROR: gnu-style.mk:3: Unknown Makefile line format: \"else\".", + "ERROR: gnu-style.mk:5: Unknown Makefile line format: \"endif\".", + "ERROR: gnu-style.mk:1: Expected \"# $NetBSD: package_test.go,v 1.48 2019/06/30 20:56:19 rillig Exp $\".", + "WARN: gnu-style.mk:2: IS_GCC is defined but not used.", + + // There is no warning about files/gnu-style.mk since pkglint + // doesn't even attempt at guessing the file type. Files placed + // in this directory can have an arbitrary format. + + "ERROR: ../../category/other/distinfo: Patch \"../../category/package/patches/"+ + "patch-Makefile.mk\" is not recorded. Run \""+confMake+" makepatchsum\".", + + // All *.mk files from PKGDIR are loaded to see which variables + // they define, in order to make the check for unused variables + // more reliable. + // + // All files that belong to the package itself, and not to pkgsrc + // should therefore be placed in the files/ directory. + "WARN: ../../category/other/gnu-style.mk:1: "+ + "Please use curly braces {} instead of round parentheses () for CC.", + "ERROR: ../../category/other/gnu-style.mk:1: Unknown Makefile line format: \"ifeq ($(CC),gcc)\".", + "ERROR: ../../category/other/gnu-style.mk:3: Unknown Makefile line format: \"else\".", + "ERROR: ../../category/other/gnu-style.mk:5: Unknown Makefile line format: \"endif\".", + "ERROR: ../../category/other/gnu-style.mk:1: Expected \"# $NetBSD: package_test.go,v 1.48 2019/06/30 20:56:19 rillig Exp $\".", + "WARN: ../../category/other/gnu-style.mk:2: IS_GCC is defined but not used.", + + "ERROR: patches/patch-Makefile.mk: Contains no patch.", + "WARN: patches/readme.mk: Patch files should be named \"patch-\", followed by letters, '-', '_', '.', and digits only.") +} + func (s *Suite) Test_Package_loadPackageMakefile(c *check.C) { t := s.Init(c) t.CreateFileLines("category/package/Makefile", - MkRcsID, + MkCvsID, "", "PKGNAME=pkgname-1.67", "DISTNAME=distfile_1_67", @@ -680,7 +1018,7 @@ func (s *Suite) Test_Package__relative_included_filenames_in_same_directory(c *c "DISTNAME=\tdistfile_1_67", ".include \"../../category/package/other.mk\"") t.CreateFileLines("category/package/other.mk", - MkRcsID, + MkCvsID, "PKGNAME=\tpkgname-1.67", "DISTNAME=\tdistfile_1_67", ".include \"../../category/package/other.mk\"") @@ -706,7 +1044,7 @@ func (s *Suite) Test_Package_loadPackageMakefile__PECL_VERSION(c *check.C) { t := s.Init(c) t.CreateFileLines("lang/php/ext.mk", - MkRcsID, + MkCvsID, "", "PHPEXT_MK= # defined", "PHPPKGSRCDIR= ../../lang/php72", @@ -728,6 +1066,109 @@ func (s *Suite) Test_Package_loadPackageMakefile__PECL_VERSION(c *check.C) { G.Check(pkg) } +func (s *Suite) Test_Package_check__files_Makefile(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/files/Makefile", + "This file may contain anything.") + + t.Main("category/package/files/Makefile") + + // Since there is nothing to check in files/*, pkglint could + // as well report this as a usage error. + // + // Until June 2019, checking individual files in FILESDIR had + // been enabled by the -Call command line option. + t.CheckOutputLines( + "Looks fine.") + + t.Main("category/package") + + t.CheckOutputLines( + "Looks fine.") +} + +func (s *Suite) Test_Package_check__patches_Makefile(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/patches/Makefile", + "This file may contain anything.") + + t.Main("category/package") + + t.CheckOutputLines( + "WARN: ~/category/package/patches/Makefile: Patch files should be "+ + "named \"patch-\", followed by letters, '-', '_', '.', and digits only.", + "0 errors and 1 warning found.") +} + +func (s *Suite) Test_Package_checkDirent__errors(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Call", "-Wall,no-space") + t.SetUpPkgsrc() + t.CreateFileLines("category/package/files/subdir/file") + t.CreateFileLines("category/package/files/subdir/subsub/file") + t.FinishSetUp() + + pkg := NewPackage(t.File("category/package")) + pkg.checkDirent(t.File("category/package/options.mk"), 0444) + pkg.checkDirent(t.File("category/package/files/subdir"), 0555|os.ModeDir) + pkg.checkDirent(t.File("category/package/files/subdir/subsub"), 0555|os.ModeDir) + pkg.checkDirent(t.File("category/package/files"), 0555|os.ModeDir) + + t.CheckOutputLines( + "ERROR: ~/category/package/options.mk: Cannot be read.", + "WARN: ~/category/package/files/subdir/subsub: Unknown directory name.") +} + +func (s *Suite) Test_Package_checkDirent__file_selection(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Call", "-Wall,no-space") + t.SetUpPkgsrc() + t.CreateFileLines("doc/CHANGES-2018", + CvsID) + t.CreateFileLines("category/package/buildlink3.mk", + MkCvsID) + t.CreateFileLines("category/package/unexpected.txt", + CvsID) + t.FinishSetUp() + + pkg := NewPackage(t.File("category/package")) + pkg.checkDirent(t.File("doc/CHANGES-2018"), 0444) + pkg.checkDirent(t.File("category/package/buildlink3.mk"), 0444) + pkg.checkDirent(t.File("category/package/unexpected.txt"), 0444) + + t.CheckOutputLines( + "WARN: ~/category/package/buildlink3.mk:EOF: Expected a BUILDLINK_TREE line.", + "WARN: ~/category/package/unexpected.txt: Unexpected file found.") +} + +// Since all required information is passed to G.checkDirent via parameters, +// this test produces the expected results even though none of these files actually exists. +func (s *Suite) Test_Package_checkDirent__skipped(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.FinishSetUp() + t.Chdir("category/package") + pkg := NewPackage(".") + + pkg.checkDirent("work", os.ModeSymlink) + pkg.checkDirent("work.i386", os.ModeSymlink) + pkg.checkDirent("work.hostname", os.ModeSymlink) + pkg.checkDirent("other", os.ModeSymlink) + + pkg.checkDirent("device", os.ModeDevice) + + t.CheckOutputLines( + "WARN: other: Invalid symlink name.", + "ERROR: device: Only files and directories are allowed in pkgsrc.") +} + func (s *Suite) Test_Package_checkIncludeConditionally__conditional_and_unconditional_include(c *check.C) { t := s.Init(c) @@ -741,7 +1182,7 @@ func (s *Suite) Test_Package_checkIncludeConditionally__conditional_and_uncondit t.CreateFileLines("sysutils/coreutils/buildlink3.mk", "") t.CreateFileLines("category/package/options.mk", - MkRcsID, + MkCvsID, "", ".if !empty(PKG_OPTIONS:Mzlib)", ". include \"../../devel/zlib/buildlink3.mk\"", @@ -762,6 +1203,66 @@ func (s *Suite) Test_Package_checkIncludeConditionally__conditional_and_uncondit "WARN: options.mk:3: Expected definition of PKG_OPTIONS_VAR.") } +func (s *Suite) Test_Package_checkIncludeConditionally__mixed(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.CreateFileLines("including.mk", + MkCvsID, + "", + ".include \"included.mk\"", + ".if ${OPSYS} == \"Linux\"", + ".include \"included.mk\"", + ".endif", + "", + ".include \"included.mk\"", + ".if ${OPSYS} == \"Linux\"", + ".include \"included.mk\"", + ".endif") + t.CreateFileLines("included.mk", + MkCvsID) + t.FinishSetUp() + + G.Check(".") + + t.CheckOutputLines( + "WARN: including.mk:5: \"included.mk\" is included "+ + "conditionally here (depending on OPSYS) and unconditionally in line 3.", + "WARN: including.mk:8: \"included.mk\" is included "+ + "unconditionally here and conditionally in line 5 (depending on OPSYS).", + "WARN: including.mk:10: \"included.mk\" is included "+ + "conditionally here (depending on OPSYS) and unconditionally in line 8.") +} + +func (s *Suite) Test_Package_checkIncludeConditionally__other_directory(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../category/package-base/including.mk\"") + t.CreateFileLines("category/package-base/including.mk", + MkCvsID, + "", + ".include \"included.mk\"", + ".if ${OPSYS} == \"Linux\"", + ".include \"included.mk\"", + ".endif", + "", + ".include \"included.mk\"", + ".if ${OPSYS} == \"Linux\"", + ".include \"included.mk\"", + ".endif") + t.CreateFileLines("category/package-base/included.mk", + MkCvsID) + + t.Main("-Wall", "-Call", "category/package") + + // TODO: Understand why ../../category/package-base/including.mk is + // not checked for (un)conditional includes. + t.CheckOutputLines( + "Looks fine.") +} + // See https://github.com/rillig/pkglint/issues/1 func (s *Suite) Test_Package__include_without_exists(c *check.C) { t := s.Init(c) @@ -793,7 +1294,7 @@ func (s *Suite) Test_Package__include_after_exists(c *check.C) { } // See https://github.com/rillig/pkglint/issues/1 -func (s *Suite) Test_Package_readMakefile__include_other_after_exists(c *check.C) { +func (s *Suite) Test_Package_parse__include_other_after_exists(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package", @@ -815,7 +1316,7 @@ func (s *Suite) Test_Package__redundant_master_sites(c *check.C) { t.SetUpPkgsrc() t.SetUpMasterSite("MASTER_SITE_R_CRAN", "http://cran.r-project.org/src/") t.CreateFileLines("math/R/Makefile.extension", - MkRcsID, + MkCvsID, "", "PKGNAME?=\tR-${R_PKGNAME}-${R_PKGVER}", "MASTER_SITES?=\t${MASTER_SITE_R_CRAN:=contrib/}", @@ -823,7 +1324,7 @@ func (s *Suite) Test_Package__redundant_master_sites(c *check.C) { "NO_CHECKSUM=\tyes", "LICENSE?=\tgnu-gpl-v2") t.CreateFileLines("math/R-date/Makefile", - MkRcsID, + MkCvsID, "", "R_PKGNAME=\tdate", "R_PKGVER=\t1.2.3", @@ -888,13 +1389,10 @@ func (s *Suite) Test_NewPackage(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("category/Makefile", - MkRcsID) + MkCvsID) t.FinishSetUp() - c.Check( - func() { NewPackage("category") }, - check.PanicMatches, - `Pkglint internal error: Package directory "category" must be two subdirectories below the pkgsrc root ".*".`) + t.ExpectAssert(func() { NewPackage("category") }) } // Before 2018-09-09, the .CURDIR variable did not have a fallback value. @@ -909,17 +1407,17 @@ func (s *Suite) Test__distinfo_from_other_package(c *check.C) { t.SetUpPkgsrc() t.Chdir(".") t.CreateFileLines("x11/gst-x11/Makefile", - MkRcsID, + MkCvsID, ".include \"../../multimedia/gst-base/Makefile.common\"", ".include \"../../mk/bsd.pkg.mk\"") t.CreateFileLines("multimedia/gst-base/Makefile.common", - MkRcsID, + MkCvsID, ".include \"plugins.mk\"") t.CreateFileLines("multimedia/gst-base/plugins.mk", - MkRcsID, + MkCvsID, "DISTINFO_FILE=\t${.CURDIR}/../../multimedia/gst-base/distinfo") t.CreateFileLines("multimedia/gst-base/distinfo", - RcsID, + CvsID, "", "SHA1 (patch-aa) = 1234") t.FinishSetUp() @@ -927,7 +1425,7 @@ func (s *Suite) Test__distinfo_from_other_package(c *check.C) { G.Check("x11/gst-x11") t.CheckOutputLines( - "WARN: x11/gst-x11/Makefile: Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset.", + "WARN: x11/gst-x11/Makefile: This package should have a PLIST file.", "ERROR: x11/gst-x11/Makefile: Each package must define its LICENSE.", "WARN: x11/gst-x11/Makefile: Each package should define a COMMENT.", "WARN: x11/gst-x11/../../multimedia/gst-base/distinfo:3: Patch file \"patch-aa\" does not exist in directory \"../../x11/gst-x11/patches\".") @@ -989,10 +1487,33 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__META_PACKAGE_with_distinf G.Check(pkg) t.CheckOutputLines( - "WARN: ~/category/package/distinfo: " + + "WARN: ~/category/package/Makefile:20: This package should not have a PLIST file.", + "WARN: ~/category/package/distinfo: "+ "This file should not exist since NO_CHECKSUM or META_PACKAGE is set.") } +func (s *Suite) Test_Package_checkfilePackageMakefile__META_PACKAGE_with_patch(c *check.C) { + t := s.Init(c) + + pkg := t.SetUpPackage("category/package", + "META_PACKAGE=\tyes") + t.Remove("category/package/PLIST") + t.CreateFileDummyPatch("category/package/patches/patch-aa") + t.CreateFileLines("category/package/distinfo", + CvsID, + "", + "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") + + t.FinishSetUp() + + G.Check(pkg) + + // At first it may sound strange to have a META_PACKAGE with patches. + // As of June 2019, there are 21 meta packages having a patches + // directory, being referred to by PATCHDIR. + t.CheckOutputEmpty() +} + func (s *Suite) Test_Package_checkfilePackageMakefile__USE_IMAKE_and_USE_X11(c *check.C) { t := s.Init(c) @@ -1007,6 +1528,64 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__USE_IMAKE_and_USE_X11(c * "NOTE: ~/category/package/Makefile:21: USE_IMAKE makes USE_X11 in line 20 redundant.") } +func (s *Suite) Test_Package_checkfilePackageMakefile__USE_IMAKE_without_USE_X11(c *check.C) { + t := s.Init(c) + + pkg := t.SetUpPackage("category/package", + "USE_IMAKE=\tyes") + t.FinishSetUp() + + G.Check(pkg) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkfilePackageMakefile__USE_IMAKE_and_USE_X11_in_infra(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("mk/x11.buildlink3.mk", + MkCvsID, + "USE_X11=\tyes") + pkg := t.SetUpPackage("category/package", + ".include \"../../mk/x11.buildlink3.mk\"", + "USE_IMAKE=\tyes") + t.FinishSetUp() + + G.Check(pkg) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkfilePackageMakefile__PLIST_common(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.Copy("category/package/PLIST", "category/package/PLIST.common") + t.Remove("category/package/PLIST") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + // No warning about missing PLIST file. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkfilePackageMakefile__files_Makefile(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "-Call") + t.SetUpPackage("category/package", + "#LICENSE=\t# none") + t.CreateFileLines("category/package/files/Makefile", + "#") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/Makefile: Each package must define its LICENSE.") +} + func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__no_C(c *check.C) { t := s.Init(c) @@ -1059,7 +1638,7 @@ func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__realistic_compiler_m "", ".include \"../../mk/compiler.mk\"") t.CreateFileLines("mk/compiler.mk", - MkRcsID, + MkCvsID, ".include \"bsd.prefs.mk\"", "", "USE_LANGUAGES?=\tc", @@ -1116,6 +1695,55 @@ func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__ok(c *check.C) { t.CheckOutputEmpty() } +func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__not_constant_1(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".if 0", + "GNU_CONFIGURE=\tyes", + ".endif", + "USE_LANGUAGES=\tc++ objc") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__not_constant_2(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "GNU_CONFIGURE=\tyes", + ".if 0", + "USE_LANGUAGES=\tc++ objc", + ".endif") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_loadPlistDirs(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/PLIST.common", + PlistCvsID, + "@exec echo hello", + "${PLIST.condition}dir/subdir/file", + "@unexec echo bye") + t.FinishSetUp() + + pkg := NewPackage(t.File("category/package")) + pkg.load() + + // FIXME: dir/subdir should also be included in pkg.Plist.Dirs. + t.Check(pkg.Plist.Dirs, deepEquals, map[string]bool{ + "bin": true}) +} + func (s *Suite) Test_Package_checkUseLanguagesCompilerMk__too_late(c *check.C) { t := s.Init(c) @@ -1123,7 +1751,7 @@ func (s *Suite) Test_Package_checkUseLanguagesCompilerMk__too_late(c *check.C) { ".include \"../../mk/compiler.mk\"", "USE_LANGUAGES=\tc c99 fortran ada c++14") t.CreateFileLines("mk/compiler.mk", - MkRcsID) + MkCvsID) t.FinishSetUp() G.Check(t.File("category/package")) @@ -1137,14 +1765,40 @@ func (s *Suite) Test_Package_checkUseLanguagesCompilerMk__compiler_mk(c *check.C t := s.Init(c) t.SetUpPackage("category/package", - ".include \"compiler.mk\"", "USE_LANGUAGES=\tc c99 fortran ada c++14", ".include \"../../mk/compiler.mk\"", + ".include \"compiler.mk\"", "USE_LANGUAGES=\tc c99 fortran ada c++14") t.CreateFileLines("category/package/compiler.mk", - MkRcsID) + MkCvsID, + "USE_LANGUAGES=\tc++") t.CreateFileLines("mk/compiler.mk", - MkRcsID) + MkCvsID) + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:20: "+ + "Variable USE_LANGUAGES is overwritten in compiler.mk:2.", + "WARN: ~/category/package/compiler.mk:2: "+ + "Modifying USE_LANGUAGES after including ../../mk/compiler.mk has no effect.", + "WARN: ~/category/package/Makefile:23: "+ + "Modifying USE_LANGUAGES after including ../../mk/compiler.mk has no effect.") +} + +func (s *Suite) Test_Package_checkUseLanguagesCompilerMk__endian_mk(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"endian.mk\"", + "USE_LANGUAGES=\tc c99 fortran ada c++14", + ".include \"../../mk/endian.mk\"", + "USE_LANGUAGES=\tc c99 fortran ada c++14") + t.CreateFileLines("category/package/endian.mk", + MkCvsID) + t.CreateFileLines("mk/endian.mk", + MkCvsID) t.FinishSetUp() G.Check(t.File("category/package")) @@ -1156,7 +1810,164 @@ func (s *Suite) Test_Package_checkUseLanguagesCompilerMk__compiler_mk(c *check.C "Modifying USE_LANGUAGES after including ../../mk/compiler.mk has no effect.") } -func (s *Suite) Test_Package_readMakefile__skipping(c *check.C) { +func (s *Suite) Test_Package_parse__simple(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.FinishSetUp() + + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() + + t.CheckOutputLines( + "FirstTime: suppress-varorder.mk") +} + +func (s *Suite) Test_Package_parse__nonexistent_Makefile(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.Remove("Makefile") + t.FinishSetUp() + + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() + + t.CheckOutputLines( + "ERROR: Makefile: Cannot be read.") +} + +func (s *Suite) Test_Package_parse__include_in_same_directory(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"version.mk\"") + t.Chdir("category/package") + t.CreateFileLines("version.mk", + MkCvsID) + t.FinishSetUp() + + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() + + t.CheckOutputLines( + "FirstTime: suppress-varorder.mk", + "FirstTime: version.mk") +} + +func (s *Suite) Test_Package_parse__nonexistent_include(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"version.mk\"") + t.Chdir("category/package") + t.FinishSetUp() + + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() + + t.CheckOutputLines( + "FirstTime: suppress-varorder.mk", + "FirstTime: version.mk", + "ERROR: Makefile:20: Cannot read \"version.mk\".") +} + +// When reading the package Makefile, pkglint loads and interprets each +// file only once. This is especially important for packages with a large +// dependency graph containing many common subdependencies. +func (s *Suite) Test_Package_parse__include_twice(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"version.mk\"", + ".include \"version.mk\"") + t.Chdir("category/package") + t.CreateFileLines("version.mk", + MkCvsID) + t.FinishSetUp() + + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() + + t.CheckOutputLines( + "FirstTime: suppress-varorder.mk", + "FirstTime: version.mk") +} + +func (s *Suite) Test_Package_parse__include_in_other_directory(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../category/other/version.mk\"") + t.Chdir("category/package") + t.CreateFileLines("../../category/other/version.mk", + MkCvsID) + t.FinishSetUp() + + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() + + t.CheckOutputLines( + "FirstTime: suppress-varorder.mk", + "FirstTime: ../../category/other/version.mk") +} + +// Demonstrates that Package.included contains the file paths of the +// included files, relative to the package directory. +func (s *Suite) Test_Package_parse__includes_in_other_directory(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../category/other/module.mk\"") + t.Chdir("category/package") + t.CreateFileLines("../../category/other/module.mk", + MkCvsID, + ".include \"version.mk\"") + t.CreateFileLines("../../category/other/version.mk", + MkCvsID) + t.FinishSetUp() + + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() + + t.CheckOutputLines( + "FirstTime: suppress-varorder.mk", + "FirstTime: ../../category/other/module.mk", + "FirstTime: ../../category/other/version.mk") +} + +func (s *Suite) Test_Package_parse__nonexistent_in_other_directory(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../category/other/module.mk\"") + t.Chdir("category/package") + t.CreateFileLines("../../category/other/module.mk", + MkCvsID, + ".include \"version.mk\"") + t.FinishSetUp() + + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() + + t.CheckOutputLines( + "FirstTime: suppress-varorder.mk", + "FirstTime: ../../category/other/module.mk", + "FirstTime: ../../category/other/version.mk", + "ERROR: ../../category/other/module.mk:2: Cannot read \"version.mk\".") +} + +func (s *Suite) Test_Package_parse__skipping(c *check.C) { t := s.Init(c) t.SetUpCommandLine("-Wall,no-space") @@ -1184,11 +1995,10 @@ func (s *Suite) Test_Package_readMakefile__skipping(c *check.C) { c.Check(relevant, deepEquals, []string{ "TRACE: 1 2 3 4 ~/category/package/Makefile:20: " + - "Skipping include file \"${MYSQL_PKGSRCDIR:S/-client$/-server/}/buildlink3.mk\". " + - "This may result in false warnings."}) + "Skipping unresolvable include file \"${MYSQL_PKGSRCDIR:S/-client$/-server/}/buildlink3.mk\"."}) } -func (s *Suite) Test_Package_readMakefile__not_found(c *check.C) { +func (s *Suite) Test_Package_parse__not_found(c *check.C) { t := s.Init(c) pkg := t.SetUpPackage("category/package", @@ -1203,11 +2013,11 @@ func (s *Suite) Test_Package_readMakefile__not_found(c *check.C) { "ERROR: ~/devel/zlib/buildlink3.mk:1: Cannot read \"../../enoent/enoent/buildlink3.mk\".") } -func (s *Suite) Test_Package_readMakefile__relative(c *check.C) { +func (s *Suite) Test_Package_parse__relative(c *check.C) { t := s.Init(c) t.CreateFileLines("category/package/extra.mk", - MkRcsID) + MkCvsID) pkg := t.SetUpPackage("category/package", ".include \"../package/extra.mk\"") t.FinishSetUp() @@ -1223,7 +2033,7 @@ func (s *Suite) Test_Package_readMakefile__relative(c *check.C) { // When a buildlink3.mk file is included, the corresponding builtin.mk // file is included by the pkgsrc infrastructure. Therefore all variables // declared in the builtin.mk file become known in the package. -func (s *Suite) Test_Package_readMakefile__builtin_mk(c *check.C) { +func (s *Suite) Test_Package_parse__builtin_mk(c *check.C) { t := s.Init(c) t.SetUpTool("echo", "ECHO", AtRunTime) @@ -1234,7 +2044,7 @@ func (s *Suite) Test_Package_readMakefile__builtin_mk(c *check.C) { "\techo ${VAR_FROM_BUILTIN} ${OTHER_VAR}") t.CreateFileDummyBuildlink3("category/lib1/buildlink3.mk") t.CreateFileLines("category/lib1/builtin.mk", - MkRcsID, + MkCvsID, "VAR_FROM_BUILTIN=\t# defined") t.FinishSetUp() @@ -1247,7 +2057,7 @@ func (s *Suite) Test_Package_readMakefile__builtin_mk(c *check.C) { // Ensures that the paths in Package.included are indeed relative to the // package directory. This hadn't been the case until March 2019. -func (s *Suite) Test_Package_readMakefile__included(c *check.C) { +func (s *Suite) Test_Package_parse__included(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package", @@ -1256,127 +2066,332 @@ func (s *Suite) Test_Package_readMakefile__included(c *check.C) { t.SetUpPackage("devel/library") t.CreateFileDummyBuildlink3("devel/library/buildlink3.mk") t.CreateFileLines("devel/library/builtin.mk", - MkRcsID) + MkCvsID) t.CreateFileLines("lang/language/module.mk", - MkRcsID, + MkCvsID, ".include \"version.mk\"") t.CreateFileLines("lang/language/version.mk", - MkRcsID) + MkCvsID) t.FinishSetUp() - pkg := NewPackage(t.File("category/package")) + t.Chdir("category/package") + pkg := NewPackage(".") + pkg.included.Trace = true pkg.loadPackageMakefile() - expected := []string{ - "../../devel/library/buildlink3.mk", - "../../devel/library/builtin.mk", - "../../lang/language/module.mk", - "../../lang/language/version.mk", - "suppress-varorder.mk"} - - seen := pkg.included - for _, filename := range expected { - if !seen.Seen(filename) { - c.Errorf("File %q is not seen.", filename) - } + t.CheckOutputLines( + "FirstTime: suppress-varorder.mk", + "FirstTime: ../../devel/library/buildlink3.mk", + "FirstTime: ../../devel/library/builtin.mk", + "FirstTime: ../../lang/language/module.mk", + "FirstTime: ../../lang/language/version.mk") +} + +func (s *Suite) Test_Package_parse__include_Makefile_common_same_directory(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/dependency") + t.CreateFileLines("category/dependency/Makefile.common", + MkCvsID, + "#", + "#") + t.SetUpPackage("category/package", + ".include \"../../category/dependency/Makefile.common\"", + ".include \"Makefile.common\"") + t.CreateFileLines("category/package/Makefile.common", + MkCvsID, + "#", + "#") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/dependency/Makefile.common:1: " + + "Please add a line \"# used by category/package/Makefile\" here.") +} + +func (s *Suite) Test_Package_parse__include_Makefile_common_explicit(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/dependency") + t.CreateFileLines("category/dependency/Makefile.common", + MkCvsID, + "#", + "#") + t.SetUpPackage("category/package", + ".include \"../../category/dependency/Makefile.common\"", + ".include \"../../category/package/Makefile.common\"") + t.CreateFileLines("category/package/Makefile.common", + MkCvsID, + "#", + "#") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/dependency/Makefile.common:1: " + + "Please add a line \"# used by category/package/Makefile\" here.") +} + +func (s *Suite) Test_Package_parse__fallback_lookup_in_package_directory(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("mk/pthread.buildlink3.mk", + MkCvsID, + ".include \"../../mk/pthread.builtin.mk\"") + t.CreateFileLines("mk/pthread.builtin.mk", + MkCvsID) + t.SetUpPackage("category/package", + ".include \"../../mk/pthread.buildlink3.mk\"") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "NOTE: ~/mk/pthread.buildlink3.mk:2: " + + "The path to the included file should be \"pthread.builtin.mk\".") +} + +func (s *Suite) Test_Package_collectSeenInclude__builtin_mk(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"builtin.mk\"") + t.CreateFileLines("category/package/builtin.mk", + MkCvsID) + t.FinishSetUp() + + pkg := NewPackage(t.File("category/package")) + pkg.load() + + t.Check(pkg.seenInclude, equals, true) +} + +func (s *Suite) Test_Package_diveInto(c *check.C) { + t := s.Init(c) + + test := func(including, included string, expected bool) { + actual := (*Package)(nil).diveInto(including, included) + t.Check(actual, equals, expected) } - t.Check(seen.seen, check.HasLen, 5) + + // The variables that appear in these files are largely modeled by + // pkglint in the file vardefs.go. Therefore parsing these files again + // doesn't make much sense. + test("Makefile", "../../mk/bsd.pkg.mk", false) + test("Makefile", "../../mk/bsd.prefs.mk", false) + test("Makefile", "../../mk/bsd.fast.prefs.mk", false) + + // All files that are included from outside of the pkgsrc infrastructure + // are relevant. This is typically mk/compiler.mk or the various + // mk/*.buildlink3.mk files. + test("Makefile", "Makefile.common", true) + test("Makefile", "../../mk/compiler.mk", true) + + // The mk/*.buildlink3.mk files often come with a companion file called + // mk/*.builtin.mk, which also defines variables that are visible from + // the package. + // + // This case is needed for getting the redundancy check right. Without it + // there will be warnings about redundant assignments to the + // BUILTIN_CHECK.pthread variable. + test("pthread.buildlink3.mk", "pthread.builtin.mk", true) + test("../../mk/pthread.buildlink3.mk", "pthread.builtin.mk", true) + test("../../mk/pthread.buildlink3.mk", "../../mk/pthread.builtin.mk", true) + + // Files other than the companion builtin.mk are ignored. + test("../../mk/pthread.buildlink3.mk", "pthread.internals.mk", false) + + // Files that are included from within the pkgsrc infrastructure are not + // interesting since their content is largely modeled by pkglint in the + // file vardefs.go. + test("../../mk/one.mk", "two.mk", false) + test("../../mk/one.mk", "../../mk/two.mk", false) + test("../../mk/one.mk", "../lang/go/version.mk", false) +} + +func (s *Suite) Test_Package_collectSeenInclude__multiple(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"001.mk\"", + ".include \"002.mk\"") + t.CreateFileLines("category/package/001.mk", + MkCvsID) + t.CreateFileLines("category/package/002.mk", + MkCvsID) + t.FinishSetUp() + + t.EnableTracingToLog() + G.Check(t.File("category/package")) + t.EnableSilentTracing() + + // TODO: It's not necessary to trace this message three times. + t.CheckOutputLinesMatching(`^TRACE: .*seenInclude`, + "TRACE: 1 2 3 4 Including \"suppress-varorder.mk\" sets seenInclude.", + "TRACE: 1 2 3 4 Including \"001.mk\" sets seenInclude.", + "TRACE: 1 2 3 4 Including \"002.mk\" sets seenInclude.") } // Just for code coverage. -func (s *Suite) Test_Package_findIncludedFile__no_tracing(c *check.C) { +func (s *Suite) Test_Package_resolveIncludedFile__no_tracing(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package", + ".include \"../../mk/${UNKNOWN_PKGPATH}.mk\"", ".include \"../../${UNKNOWN_PKGPATH}/buildlink3.mk\"", ".include \"../../lang/language/buildlink3.mk\"") t.CreateFileLines("lang/language/buildlink3.mk", - MkRcsID) + MkCvsID) t.FinishSetUp() pkg := NewPackage(t.File("category/package")) t.DisableTracing() + pkg.included.Trace = true pkg.loadPackageMakefile() - expected := []string{ - "../../lang/language/buildlink3.mk", - "../../lang/language/builtin.mk", - "suppress-varorder.mk"} + t.CheckOutputLines( + "FirstTime: suppress-varorder.mk", + "FirstTime: ../../lang/language/buildlink3.mk", + "FirstTime: ../../lang/language/builtin.mk") +} - seen := pkg.included - for _, filename := range expected { - if !seen.Seen(filename) { - c.Errorf("File %q is not seen.", filename) - } - } - t.Check(seen.seen, check.HasLen, 3) +func (s *Suite) Test_Package_resolveIncludedFile__skipping(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../mk/known.mk\"", + ".include \"../../${UNKNOWN_PKGPATH}/buildlink3.mk\"", + ".include \"../../lang/language/buildlink3.mk\"") + t.CreateFileLines("mk/known.mk", + MkCvsID, + ".include \"${UNKNOWN}.mk\"") + t.CreateFileLines("lang/language/buildlink3.mk", + MkCvsID) + t.FinishSetUp() + pkg := NewPackage(t.File("category/package")) + + t.EnableTracingToLog() + pkg.loadPackageMakefile() + + // The trace log does not contain the message that mk/known.mk includes + // a file that is skipped. This is because most package authors are not + // involved in the pkgsrc infrastructure, therefore there's no point in + // logging anything about these files. + t.CheckOutputLinesMatching(`.*Skipping.*`, + "TRACE: 1 2 ~/category/package/Makefile:21: "+ + "Skipping unresolvable include file \"../../${UNKNOWN_PKGPATH}/buildlink3.mk\".") } -func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { +// In packages without specific MAINTAINER, everyone may commit changes. +func (s *Suite) Test_Package_checkOwnerMaintainer__no_maintainer(c *check.C) { t := s.Init(c) G.Username = "example-user" t.CreateFileLines("category/package/CVS/Entries", "/Makefile//modified//") + t.SetUpPackage("category/package", + "MAINTAINER=\tpkgsrc-users@NetBSD.org") + t.FinishSetUp() + + G.Check(t.File("category/package")) - // In packages without specific MAINTAINER, everyone may commit changes. + t.CheckOutputEmpty() +} - pkg := t.SetUpPackage("category/package", - "MAINTAINER=\tpkgsrc-users@NetBSD.org") +// A package with a MAINTAINER may be edited by the maintainer itself. +func (s *Suite) Test_Package_checkOwnerMaintainer__maintainer_equal(c *check.C) { + t := s.Init(c) + + G.Username = "maintainer" + t.CreateFileLines("category/package/CVS/Entries", + "/Makefile//modified//") + t.SetUpPackage("category/package", + "MAINTAINER=\tmaintainer@example.org") t.FinishSetUp() - G.Check(pkg) + G.Check(t.File("category/package")) t.CheckOutputEmpty() +} - // A package with a MAINTAINER may be edited with care. +// A package with a MAINTAINER may be edited by everyone, with care. +func (s *Suite) Test_Package_checkOwnerMaintainer__maintainer_unequal(c *check.C) { + t := s.Init(c) + G.Username = "example-user" + t.CreateFileLines("category/package/CVS/Entries", + "/Makefile//modified//") t.SetUpPackage("category/package", "MAINTAINER=\tmaintainer@example.org") + t.FinishSetUp() - G.Check(pkg) + G.Check(t.File("category/package")) t.CheckOutputLines( "NOTE: ~/category/package/Makefile: " + "Please only commit changes that maintainer@example.org would approve.") +} - // A package with an OWNER may NOT be edited by others. +// A package with an OWNER may be edited by the owner itself. +func (s *Suite) Test_Package_checkOwnerMaintainer__owner_equal(c *check.C) { + t := s.Init(c) - pkg = t.SetUpPackage("category/package", - "#MAINTAINER=\t# undefined", + G.Username = "owner" + t.CreateFileLines("category/package/CVS/Entries", + "/Makefile//modified//") + t.SetUpPackage("category/package", "OWNER=\towner@example.org") + t.FinishSetUp() - G.Check(pkg) + G.Check(t.File("category/package")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkOwnerMaintainer__owner_unequal(c *check.C) { + t := s.Init(c) + + G.Username = "example-user" + t.CreateFileLines("category/package/CVS/Entries", + "/Makefile//modified//") + t.SetUpPackage("category/package", + "OWNER=\towner@example.org") + t.FinishSetUp() + + G.Check(t.File("category/package")) t.CheckOutputLines( "WARN: ~/category/package/Makefile: " + "Don't commit changes to this file without asking the OWNER, owner@example.org.") +} - // In a package with both OWNER and MAINTAINER, OWNER wins. +// In a package with both OWNER and MAINTAINER, OWNER wins. +func (s *Suite) Test_Package_checkOwnerMaintainer__both(c *check.C) { + t := s.Init(c) - pkg = t.SetUpPackage("category/package", + G.Username = "example-user" + t.CreateFileLines("category/package/CVS/Entries", + "/Makefile//modified//") + t.SetUpPackage("category/package", "MAINTAINER=\tmaintainer@example.org", "OWNER=\towner@example.org") + t.FinishSetUp() - G.Check(pkg) + G.Check(t.File("category/package")) t.CheckOutputLines( "WARN: ~/category/package/Makefile: "+ "Don't commit changes to this file without asking the OWNER, owner@example.org.", + // FIXME: OWNER is stronger than MAINTAINER, therefore this note should disappear. "NOTE: ~/category/package/Makefile: "+ "Please only commit changes that maintainer@example.org would approve.") - - // ... unless you are the owner, of course. - - G.Username = "owner" - - G.Check(pkg) - - t.CheckOutputEmpty() } // Just for code coverage. -func (s *Suite) Test_Package_checkLocallyModified__no_tracing(c *check.C) { +func (s *Suite) Test_Package_checkOwnerMaintainer__no_tracing(c *check.C) { t := s.Init(c) G.Username = "example-user" @@ -1395,7 +2410,7 @@ func (s *Suite) Test_Package_checkLocallyModified__no_tracing(c *check.C) { "that maintainer@example.org would approve.") } -func (s *Suite) Test_Package_checkLocallyModified__directory(c *check.C) { +func (s *Suite) Test_Package_checkOwnerMaintainer__directory(c *check.C) { t := s.Init(c) G.Username = "example-user" @@ -1407,7 +2422,7 @@ func (s *Suite) Test_Package_checkLocallyModified__directory(c *check.C) { pkg := t.SetUpPackage("category/package", "MAINTAINER=\tmaintainer@example.org") t.CreateFileLines("category/package/distinfo", - RcsID, + CvsID, "", "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") t.FinishSetUp() @@ -1420,6 +2435,28 @@ func (s *Suite) Test_Package_checkLocallyModified__directory(c *check.C) { "maintainer@example.org would approve.") } +func (s *Suite) Test_Package_checkFreeze(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "--explain") + pkg := t.SetUpPackage("category/package") + t.CreateFileLines("category/package/CVS/Entries", + "/Makefile//modified//") + t.CreateFileLines("doc/CHANGES-2018", + "\tmk/bsd.pkg.mk: started freeze for 2018Q2 [freezer 2018-03-20]") + t.FinishSetUp() + + G.Check(pkg) + + t.CheckOutputLines( + "NOTE: ~/category/package/Makefile: Pkgsrc is frozen since 2018-03-20.", + "", + "\tDuring a pkgsrc freeze, changes to pkgsrc should only be made very", + "\tcarefully. See https://www.netbsd.org/developers/pkgsrc/ for the", + "\texact rules.", + "") +} + // In practice the distinfo file can always be autofixed since it has // just been read successfully and the corresponding patch file could // also be autofixed right before. @@ -1441,14 +2478,14 @@ func (s *Suite) Test_Package__using_common_Makefile_overriding_DISTINFO_FILE(c * t.SetUpPackage("security/pinentry") t.CreateFileLines("security/pinentry/Makefile.common", - MkRcsID, + MkCvsID, "DISTINFO_FILE=\t${.CURDIR}/../../security/pinentry/distinfo") t.SetUpPackage("security/pinentry-fltk", ".include \"../../security/pinentry/Makefile.common\"", "DISTINFO_FILE=\t${.CURDIR}/distinfo") t.CreateFileDummyPatch("security/pinentry-fltk/patches/patch-aa") t.CreateFileLines("security/pinentry-fltk/distinfo", - RcsID, + CvsID, "", "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") t.FinishSetUp() @@ -1472,10 +2509,10 @@ func (s *Suite) Test_Package__redundant_variable_in_unrelated_files(c *check.C) ".include \"../../devel/py-trytond/Makefile.common\"", ".include \"../../lang/python/egg.mk\"") t.CreateFileLines("devel/py-trytond/Makefile.common", - MkRcsID, + MkCvsID, "PY_PATCHPLIST=\tyes") t.CreateFileLines("lang/python/egg.mk", - MkRcsID, + MkCvsID, "PY_PATCHPLIST=\tyes") t.FinishSetUp() @@ -1494,7 +2531,7 @@ func (s *Suite) Test_Package__redundant_variable_in_unrelated_files(c *check.C) // This is necessary to load the correct variable assignments for the // redundancy check, in particular variable assignments that serve as // arguments to "procedure calls", such as mk/find-files.mk. -func (s *Suite) Test_Package_readMakefile__include_infrastructure(c *check.C) { +func (s *Suite) Test_Package_parse__include_infrastructure(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--dumpmakefile") @@ -1515,9 +2552,9 @@ func (s *Suite) Test_Package_readMakefile__include_infrastructure(c *check.C) { t.CheckOutputLines( "Whole Makefile (with all included files) follows:", - "~/category/package/Makefile:1: "+MkRcsID, + "~/category/package/Makefile:1: "+MkCvsID, "~/category/package/Makefile:2: ", - "~/category/package/Makefile:3: DISTNAME=\tdistname-1.0", + "~/category/package/Makefile:3: DISTNAME=\tpackage-1.0", "~/category/package/Makefile:4: #PKGNAME=\tpackage-1.0", "~/category/package/Makefile:5: CATEGORIES=\tcategory", "~/category/package/Makefile:6: MASTER_SITES=\t# none", @@ -1528,7 +2565,7 @@ func (s *Suite) Test_Package_readMakefile__include_infrastructure(c *check.C) { "~/category/package/Makefile:11: LICENSE=\t2-clause-bsd", "~/category/package/Makefile:12: ", "~/category/package/Makefile:13: .include \"suppress-varorder.mk\"", - "~/category/package/suppress-varorder.mk:1: "+MkRcsID, + "~/category/package/suppress-varorder.mk:1: "+MkCvsID, "~/category/package/Makefile:14: # empty", "~/category/package/Makefile:15: # empty", "~/category/package/Makefile:16: # empty", @@ -1556,13 +2593,13 @@ func (s *Suite) Test_Package__Makefile_files(c *check.C) { t.SetUpPackage("category/package") t.CreateFileLines("category/package/Makefile.common", - MkRcsID) + MkCvsID) t.CreateFileLines("category/package/Makefile.orig", - MkRcsID) + MkCvsID) t.CreateFileLines("category/package/Makefile.php", - MkRcsID) + MkCvsID) t.CreateFileLines("category/package/ext.mk", - MkRcsID) + MkCvsID) t.FinishSetUp() G.Check(t.File("category/package")) @@ -1573,3 +2610,88 @@ func (s *Suite) Test_Package__Makefile_files(c *check.C) { "NOTE: ~/category/package/Makefile.php: " + "Consider renaming \"Makefile.php\" to \"php.mk\".") } + +func (s *Suite) Test_Package__patch_in_FILESDIR(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "-Call") + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/files/patch-aa", + "This file can contain anything, no matter what the filename says.") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + // No warnings. The files in FILESDIR are independent of pkgsrc + // and may contain anything. There are no naming conventions or + // anything else. + t.CheckOutputEmpty() +} + +// When a package defines PLIST_SRC, it may or may not use the +// PLIST file from the package directory. Therefore the check +// is skipped completely. +func (s *Suite) Test_Package_checkPlist__PLIST_SRC(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "PLIST_SRC=\t${WRKDIR}/PLIST") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkPlist__META_PACKAGE(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "META_PACKAGE=\tyes") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:20: This package should not have a PLIST file.", + "WARN: ~/category/package/distinfo: This file should not exist "+ + "since NO_CHECKSUM or META_PACKAGE is set.") +} + +func (s *Suite) Test_Package_checkPlist__Perl5_packlist(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/p5-Packlist", + "PERL5_PACKLIST=\tauto/Packlist/.packlist") + t.FinishSetUp() + + G.Check(t.File("category/p5-Packlist")) + + t.CheckOutputLines( + "WARN: ~/category/p5-Packlist/Makefile:20: This package should not have a PLIST file.") +} + +func (s *Suite) Test_Package_checkPlist__PERL5_USE_PACKLIST_no(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/p5-NoPacklist", + "PERL5_USE_PACKLIST=\tno") + t.FinishSetUp() + + G.Check(t.File("category/p5-NoPacklist")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkPlist__PERL5_USE_PACKLIST_yes(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/p5-Packlist", + "PERL5_USE_PACKLIST=\tyes") + t.FinishSetUp() + + G.Check(t.File("category/p5-Packlist")) + + t.CheckOutputLines( + "WARN: ~/category/p5-Packlist/Makefile:20: This package should not have a PLIST file.") +} diff --git a/pkgtools/pkglint/files/paragraph.go b/pkgtools/pkglint/files/paragraph.go index 26c701e3fd8..2b457202093 100644 --- a/pkgtools/pkglint/files/paragraph.go +++ b/pkgtools/pkglint/files/paragraph.go @@ -9,24 +9,25 @@ import "strings" // If the paragraph adds an identifier to SUBST_CLASSES, the rest of the SUBST // block should be defined in the same paragraph. type Paragraph struct { - mklines []MkLine + mklines *MkLines + from int + to int } -func NewParagraph(mklines []MkLine) *Paragraph { - return &Paragraph{mklines} +func NewParagraph(mklines *MkLines, from, to int) *Paragraph { + for i := from; i < to; i++ { + assert(!mklines.mklines[i].IsEmpty()) + } + return &Paragraph{mklines, from, to} } -func (p *Paragraph) Clear() { - p.mklines = nil -} +func (p *Paragraph) FirstLine() *MkLine { return p.mklines.mklines[p.from] } +func (p *Paragraph) LastLine() *MkLine { return p.mklines.mklines[p.to-1] } -func (p *Paragraph) Add(mkline MkLine) { - assertf(!mkline.IsEmpty(), "A paragraph must not contain empty lines.") - p.mklines = append(p.mklines, mkline) -} +func (p *Paragraph) MkLines() []*MkLine { return p.mklines.mklines[p.from:p.to] } -func (p *Paragraph) ForEach(action func(mkline MkLine)) { - for _, mkline := range p.mklines { +func (p *Paragraph) ForEach(action func(mkline *MkLine)) { + for _, mkline := range p.MkLines() { action(mkline) } } @@ -44,21 +45,21 @@ func (p *Paragraph) Align() { // No warning or note is logged. Therefore this method must only be used to // realign the whole paragraph after one of its lines has been modified. func (p *Paragraph) AlignTo(column int) { - for _, mkline := range p.mklines { + p.ForEach(func(mkline *MkLine) { if !mkline.IsVarassign() { - continue + return } align := mkline.ValueAlign() oldWidth := tabWidth(align) if tabWidth(rtrimHspace(align)) > column { - continue + return } if oldWidth == column && !hasSuffix(strings.TrimRight(align, "\t"), " ") { - continue + return } if mkline.IsMultiline() && !mkline.FirstLineContainsValue() { - continue + return } trimmed := strings.TrimRightFunc(align, isHspaceRune) @@ -68,5 +69,5 @@ func (p *Paragraph) AlignTo(column int) { fix.Notef(SilentAutofixFormat) fix.ReplaceAfter(trimmed, align[len(trimmed):], newSpace) fix.Apply() - } + }) } diff --git a/pkgtools/pkglint/files/paragraph_test.go b/pkgtools/pkglint/files/paragraph_test.go index 0873945eac8..97bae968dc4 100644 --- a/pkgtools/pkglint/files/paragraph_test.go +++ b/pkgtools/pkglint/files/paragraph_test.go @@ -2,34 +2,13 @@ package pkglint import "gopkg.in/check.v1" -func (s *Suite) Test_Paragraph_Clear(c *check.C) { +func (s *Suite) Test_Paragraph__empty_line(c *check.C) { t := s.Init(c) - para := NewParagraph(nil) + mklines := t.NewMkLines("filename.mk", + "") - para.Clear() - - t.Check(para.mklines, check.IsNil) - - para.Add(t.NewMkLine("filename.mk", 123, "#")) - - para.Clear() - - t.Check(para.mklines, check.IsNil) -} - -func (s *Suite) Test_Paragraph_Add__empty_line(c *check.C) { - t := s.Init(c) - - para := NewParagraph(nil) - - para.Clear() - - t.Check(para.mklines, check.IsNil) - - t.ExpectPanic( - func() { para.Add(t.NewMkLine("filename.mk", 123, "")) }, - "Pkglint internal error: A paragraph must not contain empty lines.") + t.ExpectAssert(func() { _ = NewParagraph(mklines, 0, 1) }) } func (s *Suite) Test_Paragraph_Align(c *check.C) { @@ -37,15 +16,10 @@ func (s *Suite) Test_Paragraph_Align(c *check.C) { t.SetUpCommandLine("-Wall", "--autofix") mklines := t.SetUpFileMkLines("filename.mk", - MkRcsID, + MkCvsID, "VAR=value", "VAR=\t\t\tvalue") - para := NewParagraph(nil) - for _, mkline := range mklines.mklines { - // Strictly speaking, lines 1 and 2 don't belong to the paragraph, - // but aligning the lines works nevertheless. - para.Add(mkline) - } + para := NewParagraph(mklines, 1, 3) para.Align() mklines.SaveAutofixChanges() @@ -55,7 +29,7 @@ func (s *Suite) Test_Paragraph_Align(c *check.C) { "AUTOFIX: ~/filename.mk:3: Replacing \"\\t\\t\\t\" with \"\\t\".") t.CheckFileLinesDetab("filename.mk", - MkRcsID, + MkCvsID, "VAR= value", "VAR= value") } @@ -65,30 +39,27 @@ func (s *Suite) Test_Paragraph_AlignTo(c *check.C) { t.SetUpCommandLine("-Wall", "--autofix") mklines := t.SetUpFileMkLines("filename.mk", - MkRcsID, + MkCvsID, "VAR=value", "VAR=\t\tvalue", + "# comment between the variable assignments", "VAR=\t \tvalue", "VAR=\t\t\tvalue") - para := NewParagraph(nil) - for _, mkline := range mklines.mklines { - // Strictly speaking, lines 1 and 2 don't belong to the paragraph, - // but aligning the lines works nevertheless. - para.Add(mkline) - } + para := NewParagraph(mklines, 1, 6) para.AlignTo(16) mklines.SaveAutofixChanges() t.CheckOutputLines( "AUTOFIX: ~/filename.mk:2: Replacing \"\" with \"\\t\\t\".", - "AUTOFIX: ~/filename.mk:4: Replacing \"\\t \\t\" with \"\\t\\t\".", - "AUTOFIX: ~/filename.mk:5: Replacing \"\\t\\t\\t\" with \"\\t\\t\".") + "AUTOFIX: ~/filename.mk:5: Replacing \"\\t \\t\" with \"\\t\\t\".", + "AUTOFIX: ~/filename.mk:6: Replacing \"\\t\\t\\t\" with \"\\t\\t\".") t.CheckFileLinesDetab("filename.mk", - MkRcsID, + MkCvsID, "VAR= value", "VAR= value", + "# comment between the variable assignments", "VAR= value", "VAR= value") } @@ -98,16 +69,13 @@ func (s *Suite) Test_Paragraph_AlignTo__continued_lines(c *check.C) { t.SetUpCommandLine("-Wall", "--autofix") mklines := t.SetUpFileMkLines("filename.mk", - MkRcsID, + MkCvsID, "VAR= \\", " value", "VAR= value1 \\", "value2 \\", "\t\tvalue3") - para := NewParagraph(nil) - for _, mkline := range mklines.mklines { - para.Add(mkline) - } + para := NewParagraph(mklines, 1, 3) para.AlignTo(16) mklines.SaveAutofixChanges() @@ -116,7 +84,7 @@ func (s *Suite) Test_Paragraph_AlignTo__continued_lines(c *check.C) { "AUTOFIX: ~/filename.mk:4: Replacing \" \" with \"\\t\\t\".") t.CheckFileLinesDetab("filename.mk", - MkRcsID, + MkCvsID, // Since this line does not contain the actual value, it doesn't need to be aligned. "VAR= \\", " value", @@ -131,13 +99,10 @@ func (s *Suite) Test_Paragraph_AlignTo__outlier(c *check.C) { t.SetUpCommandLine("-Wall", "--autofix") mklines := t.SetUpFileMkLines("filename.mk", - MkRcsID, + MkCvsID, "VAR= value", "VERY_LONG_VARIABLE_NAME= value1") - para := NewParagraph(nil) - for _, mkline := range mklines.mklines { - para.Add(mkline) - } + para := NewParagraph(mklines, 1, 3) para.AlignTo(8) mklines.SaveAutofixChanges() @@ -146,7 +111,7 @@ func (s *Suite) Test_Paragraph_AlignTo__outlier(c *check.C) { "AUTOFIX: ~/filename.mk:2: Replacing \" \" with \"\\t\".") t.CheckFileLinesDetab("filename.mk", - MkRcsID, + MkCvsID, "VAR= value", // The space is preserved since this line is an outlier. "VERY_LONG_VARIABLE_NAME= value1") diff --git a/pkgtools/pkglint/files/patches.go b/pkgtools/pkglint/files/patches.go index db9b760c5ff..ad2532e4f3f 100644 --- a/pkgtools/pkglint/files/patches.go +++ b/pkgtools/pkglint/files/patches.go @@ -7,16 +7,12 @@ import ( "strings" ) -func CheckLinesPatch(lines Lines) { - if trace.Tracing { - defer trace.Call1(lines.FileName)() - } - +func CheckLinesPatch(lines *Lines) { (&PatchChecker{lines, NewLinesLexer(lines), false, false}).Check() } type PatchChecker struct { - lines Lines + lines *Lines llex *LinesLexer seenDocumentation bool previousLineEmpty bool @@ -29,11 +25,7 @@ const ( ) func (ck *PatchChecker) Check() { - if trace.Tracing { - defer trace.Call0()() - } - - if ck.lines.CheckRcsID(0, ``, "") { + if ck.lines.CheckCvsID(0, ``, "") { ck.llex.Skip() } if ck.llex.EOF() { @@ -94,25 +86,17 @@ func (ck *PatchChecker) Check() { } CheckLinesTrailingEmptyLines(ck.lines) - sha1Before, err := computePatchSha1Hex(ck.lines.FileName) - if SaveAutofixChanges(ck.lines) && G.Pkg != nil && err == nil { - sha1After, err := computePatchSha1Hex(ck.lines.FileName) - if err == nil { - G.Pkg.AutofixDistinfo(sha1Before, sha1After) - } + sha1Before := computePatchSha1Hex(ck.lines) + if SaveAutofixChanges(ck.lines) && G.Pkg != nil { + linesAfter := Load(ck.lines.Filename, 0) + sha1After := computePatchSha1Hex(linesAfter) + G.Pkg.AutofixDistinfo(sha1Before, sha1After) } } // See https://www.gnu.org/software/diffutils/manual/html_node/Detailed-Unified.html func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { - if trace.Tracing { - defer trace.Call0()() - } - - patchedFileType := guessFileType(patchedFile) - if trace.Tracing { - trace.Stepf("guessFileType(%q) = %s", patchedFile, patchedFileType) - } + isConfigure := ck.isConfigure(patchedFile) hasHunks := false for { @@ -125,12 +109,9 @@ func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { hasHunks = true linesToDel := toInt(m[2], 1) linesToAdd := toInt(m[4], 1) - if trace.Tracing { - trace.Stepf("hunk -%d +%d", linesToDel, linesToAdd) - } ck.checktextUniHunkCr() - ck.checktextRcsid(text) + ck.checktextCvsID(text) for !ck.llex.EOF() && (linesToDel > 0 || linesToAdd > 0 || hasPrefix(ck.llex.CurrentLine().Text, "\\")) { line := ck.llex.CurrentLine() @@ -149,15 +130,15 @@ func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { case hasPrefix(text, " "), hasPrefix(text, "\t"): linesToDel-- linesToAdd-- - ck.checklineContext(text[1:], patchedFileType) + ck.checktextCvsID(text) case hasPrefix(text, "-"): linesToDel-- case hasPrefix(text, "+"): linesToAdd-- - ck.checktextRcsid(text) - ck.checklineAdded(text[1:], patchedFileType) + ck.checktextCvsID(text) + ck.checkConfigure(text[1:], isConfigure) case hasPrefix(text, "\\"): // \ No newline at end of file (or a translation of that message) @@ -196,11 +177,7 @@ func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { } } -func (ck *PatchChecker) checkBeginDiff(line Line, patchedFiles int) { - if trace.Tracing { - defer trace.Call0()() - } - +func (ck *PatchChecker) checkBeginDiff(line *Line, patchedFiles int) { if !ck.seenDocumentation && patchedFiles == 0 { line.Errorf("Each patch must be documented.") line.Explain( @@ -222,6 +199,7 @@ func (ck *PatchChecker) checkBeginDiff(line Line, patchedFiles int) { "After submitting a patch upstream, the corresponding bug report should", "be mentioned in this file, to prevent duplicate work.") } + if G.Opts.WarnSpace && !ck.previousLineEmpty { fix := line.Autofix() fix.Notef("Empty line expected.") @@ -230,64 +208,48 @@ func (ck *PatchChecker) checkBeginDiff(line Line, patchedFiles int) { } } -func (ck *PatchChecker) checklineContext(text string, patchedFileType FileType) { - if trace.Tracing { - defer trace.Call2(text, patchedFileType.String())() - } - - ck.checktextRcsid(text) - - if G.Opts.WarnExtra { - ck.checklineAdded(text, patchedFileType) +func (ck *PatchChecker) checkConfigure(addedText string, isConfigure bool) { + if !isConfigure { + return } -} - -func (ck *PatchChecker) checklineAdded(addedText string, patchedFileType FileType) { - if trace.Tracing { - defer trace.Call2(addedText, patchedFileType.String())() + if !hasSuffix(addedText, ": Avoid regenerating within pkgsrc") { + return } line := ck.llex.PreviousLine() - switch patchedFileType { - case ftConfigure: - if hasSuffix(addedText, ": Avoid regenerating within pkgsrc") { - line.Errorf("This code must not be included in patches.") - line.Explain( - "It is generated automatically by pkgsrc after the patch phase.", - "", - "For more details, look for \"configure-scripts-override\" in", - "mk/configure/gnu-configure.mk.") - } - } + line.Errorf("This code must not be included in patches.") + line.Explain( + "It is generated automatically by pkgsrc after the patch phase.", + "", + "For more details, look for \"configure-scripts-override\" in", + "mk/configure/gnu-configure.mk.") } func (ck *PatchChecker) checktextUniHunkCr() { - if trace.Tracing { - defer trace.Call0()() - } - line := ck.llex.PreviousLine() - if hasSuffix(line.Text, "\r") { - // This code has been introduced around 2006. - // As of 2018, this might be fixed by now. - fix := line.Autofix() - fix.Errorf("The hunk header must not end with a CR character.") - fix.Explain( - "The MacOS X patch utility cannot handle these.") - fix.Replace("\r\n", "\n") - fix.Apply() + if !hasSuffix(line.Text, "\r") { + return } + + // This code has been introduced around 2006. + // As of 2018, this might be fixed by now. + fix := line.Autofix() + fix.Errorf("The hunk header must not end with a CR character.") + fix.Explain( + "The MacOS X patch utility cannot handle these.") + fix.Replace("\r\n", "\n") + fix.Apply() } -func (ck *PatchChecker) checktextRcsid(text string) { +func (ck *PatchChecker) checktextCvsID(text string) { if strings.IndexByte(text, '$') == -1 { return } if m, tagname := match1(text, `\$(Author|Date|Header|Id|Locker|Log|Name|RCSfile|Revision|Source|State|NetBSD)(?::[^\$]*)?\$`); m { if matches(text, rePatchUniHunk) { - ck.llex.PreviousLine().Warnf("Found RCS tag \"$%s$\". Please remove it.", tagname) + ck.llex.PreviousLine().Warnf("Found CVS tag \"$%s$\". Please remove it.", tagname) } else { - ck.llex.PreviousLine().Warnf("Found RCS tag \"$%s$\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".", tagname) + ck.llex.PreviousLine().Warnf("Found CVS tag \"$%s$\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".", tagname) } } } @@ -303,32 +265,10 @@ func (ck *PatchChecker) isEmptyLine(text string) bool { hasPrefix(text, "=============") } -type FileType uint8 - -const ( - ftConfigure FileType = iota - ftUnknown -) - -func (ft FileType) String() string { - return [...]string{ - "configure file", - "unknown", - }[ft] -} - -// This is used to select the proper subroutine for detecting absolute pathnames. -func guessFileType(filename string) (fileType FileType) { - if trace.Tracing { - defer trace.Call(filename, trace.Result(&fileType))() - } - - basename := path.Base(filename) - basename = strings.TrimSuffix(basename, ".in") // doesn't influence the content type - - switch { - case basename == "configure" || basename == "configure.ac": - return ftConfigure +func (*PatchChecker) isConfigure(filename string) bool { + switch path.Base(filename) { + case "configure", "configure.in", "configure.ac": + return true } - return ftUnknown + return false } diff --git a/pkgtools/pkglint/files/patches_test.go b/pkgtools/pkglint/files/patches_test.go index ab78bf9e194..e9713b7f246 100644 --- a/pkgtools/pkglint/files/patches_test.go +++ b/pkgtools/pkglint/files/patches_test.go @@ -7,7 +7,7 @@ func (s *Suite) Test_CheckLinesPatch__with_comment(c *check.C) { t := s.Init(c) lines := t.NewLines("patch-WithComment", - RcsID, + CvsID, "", "This part describes:", "* the purpose of the patch,", @@ -35,7 +35,7 @@ func (s *Suite) Test_CheckLinesPatch__without_empty_line__autofix(c *check.C) { t.Chdir("category/package") patchLines := t.SetUpFileLines("patch-WithoutEmptyLines", - RcsID, + CvsID, "Text", "--- file.orig", "+++ file", @@ -45,7 +45,7 @@ func (s *Suite) Test_CheckLinesPatch__without_empty_line__autofix(c *check.C) { "+new line", " context after") t.CreateFileLines("distinfo", - RcsID, + CvsID, "", // The hash is taken from a breakpoint at the beginning of AutofixDistinfo, oldSha1 "SHA1 (some patch) = 49abd735b7e706ea9ed6671628bb54e91f7f5ffb") @@ -62,7 +62,7 @@ func (s *Suite) Test_CheckLinesPatch__without_empty_line__autofix(c *check.C) { "with \"4938fc8c0b483dc2e33e741b0da883d199e78164\".") t.CheckFileLines("patch-WithoutEmptyLines", - RcsID, + CvsID, "", "Text", "", @@ -74,7 +74,7 @@ func (s *Suite) Test_CheckLinesPatch__without_empty_line__autofix(c *check.C) { "+new line", " context after") t.CheckFileLines("distinfo", - RcsID, + CvsID, "", "SHA1 (some patch) = 4938fc8c0b483dc2e33e741b0da883d199e78164") } @@ -83,7 +83,7 @@ func (s *Suite) Test_CheckLinesPatch__no_comment_and_no_empty_lines(c *check.C) t := s.Init(c) patchLines := t.SetUpFileLines("patch-WithoutEmptyLines", - RcsID, + CvsID, "--- file.orig", "+++ file", "@@ -1,1 +1,1 @@", @@ -107,7 +107,7 @@ func (s *Suite) Test_CheckLinesPatch__without_comment(c *check.C) { t := s.Init(c) lines := t.NewLines("patch-WithoutComment", - RcsID, + CvsID, "", "--- file.orig", "+++ file", @@ -123,16 +123,19 @@ func (s *Suite) Test_CheckLinesPatch__without_comment(c *check.C) { "ERROR: patch-WithoutComment:3: Each patch must be documented.") } -// Autogenerated git "comments" don't count as real comments since they -// don't convey any intention of a human developer. -func (s *Suite) Test_CheckLinesPatch__git_without_comment(c *check.C) { +// Autogenerated "comments" from Git or other tools don't count as real +// comments since they don't convey any intention of a human developer. +func (s *Suite) Test_PatchChecker_isEmptyLine(c *check.C) { t := s.Init(c) lines := t.NewLines("patch-aa", - RcsID, + CvsID, "", "diff --git a/aa b/aa", "index 1234567..1234567 100644", + "Index: from Subversion", + "============= separator or conflict marker", + "", "--- a/aa", "+++ b/aa", "@@ -1,1 +1,1 @@", @@ -142,7 +145,7 @@ func (s *Suite) Test_CheckLinesPatch__git_without_comment(c *check.C) { CheckLinesPatch(lines) t.CheckOutputLines( - "ERROR: patch-aa:5: Each patch must be documented.") + "ERROR: patch-aa:8: Each patch must be documented.") } // The output of BSD Make typically contains "*** Error code". @@ -152,7 +155,7 @@ func (s *Suite) Test_CheckLinesPatch__error_code(c *check.C) { t := s.Init(c) lines := t.NewLines("patch-ErrorCode", - RcsID, + CvsID, "", "*** Error code 1", // Looks like a context diff but isn't. "", @@ -173,7 +176,7 @@ func (s *Suite) Test_CheckLinesPatch__wrong_header_order(c *check.C) { t := s.Init(c) lines := t.NewLines("patch-WrongOrder", - RcsID, + CvsID, "", "Text", "Text", @@ -197,7 +200,7 @@ func (s *Suite) Test_CheckLinesPatch__context_diff(c *check.C) { t := s.Init(c) lines := t.NewLines("patch-ctx", - RcsID, + CvsID, "", "diff -cr history.c.orig history.c", "*** history.c.orig", @@ -214,7 +217,7 @@ func (s *Suite) Test_CheckLinesPatch__no_patch(c *check.C) { t := s.Init(c) lines := t.NewLines("patch-aa", - RcsID, + CvsID, "", "-- oldfile", "++ newfile") @@ -229,7 +232,7 @@ func (s *Suite) Test_CheckLinesPatch__two_patched_files(c *check.C) { t := s.Init(c) lines := t.NewLines("patch-aa", - RcsID, + CvsID, "", "A single patch file can apply to more than one file at a time.", "It shouldn't though, to keep the relation between patch files", @@ -257,7 +260,7 @@ func (s *Suite) Test_CheckLinesPatch__documentation_that_looks_like_patch_lines( t := s.Init(c) lines := t.NewLines("patch-aa", - RcsID, + CvsID, "", "--- oldfile", "", @@ -275,7 +278,7 @@ func (s *Suite) Test_CheckLinesPatch__only_unified_header_but_no_content(c *chec t := s.Init(c) lines := t.NewLines("patch-unified", - RcsID, + CvsID, "", "Documentation for the patch", "", @@ -292,7 +295,7 @@ func (s *Suite) Test_CheckLinesPatch__only_context_header_but_no_content(c *chec t := s.Init(c) lines := t.NewLines("patch-context", - RcsID, + CvsID, "", "Documentation for the patch", "", @@ -311,7 +314,7 @@ func (s *Suite) Test_CheckLinesPatch__no_newline_with_text_following(c *check.C) t := s.Init(c) lines := t.NewLines("patch-aa", - RcsID, + CvsID, "", "comment", "", @@ -334,7 +337,7 @@ func (s *Suite) Test_CheckLinesPatch__no_newline(c *check.C) { t := s.Init(c) lines := t.NewLines("patch-aa", - RcsID, + CvsID, "", "comment", "", @@ -357,7 +360,7 @@ func (s *Suite) Test_CheckLinesPatch__empty_lines_left_out_at_eof(c *check.C) { t := s.Init(c) lines := t.NewLines("patch-aa", - RcsID, + CvsID, "", "comment", "", @@ -382,7 +385,7 @@ func (s *Suite) Test_CheckLinesPatch__context_lines_with_tab_instead_of_space(c t := s.Init(c) lines := t.NewLines("patch-aa", - RcsID, + CvsID, "", "comment", "", @@ -406,7 +409,7 @@ func (s *Suite) Test_CheckLinesPatch__autofix_empty_patch(c *check.C) { t.SetUpCommandLine("-Wall", "--autofix") lines := t.NewLines("patch-aa", - RcsID) + CvsID) CheckLinesPatch(lines) @@ -420,7 +423,7 @@ func (s *Suite) Test_CheckLinesPatch__autofix_long_empty_patch(c *check.C) { t.SetUpCommandLine("-Wall", "--autofix") lines := t.NewLines("patch-aa", - RcsID, + CvsID, "") CheckLinesPatch(lines) @@ -433,7 +436,7 @@ func (s *Suite) Test_CheckLinesPatch__crlf_autofix(c *check.C) { t.SetUpCommandLine("-Wall", "--autofix") lines := t.SetUpFileLines("patch-aa", - RcsID, + CvsID, "", "Documentation", "", @@ -456,7 +459,7 @@ func (s *Suite) Test_CheckLinesPatch__autogenerated(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("patch-aa", - RcsID, + CvsID, "", "Documentation", "", @@ -476,7 +479,7 @@ func (s *Suite) Test_CheckLinesPatch__empty_context_lines_in_hunk(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("patch-aa", - RcsID, + CvsID, "", "Documentation", "", @@ -501,7 +504,7 @@ func (s *Suite) Test_CheckLinesPatch__invalid_line_in_hunk(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("patch-aa", - RcsID, + CvsID, "", "Documentation", "", @@ -519,91 +522,119 @@ func (s *Suite) Test_CheckLinesPatch__invalid_line_in_hunk(c *check.C) { "ERROR: ~/patch-aa:10: Invalid line in unified patch hunk: <<<<<<<<") } -// Just for code coverage. -func (s *Suite) Test_PatchChecker_checklineContext__no_tracing(c *check.C) { +func (s *Suite) Test_PatchChecker_Check__missing_CVS_Id(c *check.C) { t := s.Init(c) - lines := t.NewLines("patch-WithComment", - RcsID, + lines := t.SetUpFileLines("patch-aa", + "This first line is missing the CVS Id", + "", + "Documentation") + + CheckLinesPatch(lines) + + t.CheckOutputLines( + sprintf("ERROR: ~/patch-aa:1: Expected %q.", CvsID), + "NOTE: ~/patch-aa:1: Empty line expected before this line.", + "ERROR: ~/patch-aa: Contains no patch.") +} + +func (s *Suite) Test_PatchChecker_checkUnifiedDiff__lines_at_end(c *check.C) { + t := s.Init(c) + + lines := t.SetUpFileLines("patch-aa", + CvsID, "", "Documentation", "", - "--- file.orig", - "+++ file", - "@@ -5,3 +5,3 @@", - " context before", - "-old line", - "+new line", - " context after") - t.DisableTracing() + "--- old", + "+++ new", + "@@ -1,1 +1,1 @@", + "- old", + "+ new", + "", + "This line is not part of the patch. Since it is separated from", + "the patch by an empty line, there is no reason for a warning.") CheckLinesPatch(lines) t.CheckOutputEmpty() } -func (s *Suite) Test_PatchChecker_checklineAdded__shell(c *check.C) { +func (s *Suite) Test_PatchChecker_checkBeginDiff__multiple_patches_without_documentation(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("patch-aa", - RcsID, + CvsID, "", - "Documentation", + "--- old", + "+++ new", + "@@ -1,1 +1,1 @@", + "- old", + "+ new", "", - "--- configure.sh.orig", - "+++ configure.sh", + "--- old", + "+++ new", "@@ -1,1 +1,1 @@", - "-old line", - "+new line") + "- old", + "+ new") CheckLinesPatch(lines) - t.CheckOutputEmpty() + // The "must be documented" error message is only given before the first + // patch since that's the only place where the documentation is expected. + // Since each pkgsrc patch should only patch a single file, this situation + // is an edge case anyway. + t.CheckOutputLines( + "ERROR: ~/patch-aa:3: Each patch must be documented.", + "WARN: ~/patch-aa: Contains patches for 2 files, should be only one.") } -func (s *Suite) Test_PatchChecker_checklineAdded__text(c *check.C) { +func (s *Suite) Test_PatchChecker_checkConfigure__no_GNU(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("patch-aa", - RcsID, + CvsID, "", "Documentation", "", - "--- configure.tex.orig", - "+++ configure.tex", + "--- configure.sh.orig", + "+++ configure.sh", "@@ -1,1 +1,1 @@", "-old line", - "+new line") + "+: Avoid regenerating within pkgsrc") CheckLinesPatch(lines) + // No warning since configure.sh is probably not a GNU-style + // configure file. t.CheckOutputEmpty() } -func (s *Suite) Test_PatchChecker_checklineAdded__unknown(c *check.C) { +func (s *Suite) Test_PatchChecker_checkConfigure__GNU(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("patch-aa", - RcsID, + CvsID, "", "Documentation", "", - "--- configure.unknown.orig", - "+++ configure.unknown", + "--- configure.orig", + "+++ configure", "@@ -1,1 +1,1 @@", "-old line", - "+new line") + "+: Avoid regenerating within pkgsrc") CheckLinesPatch(lines) - t.CheckOutputEmpty() + t.CheckOutputLines( + "ERROR: ~/patch-aa:9: This code must not be included in patches.") } -func (s *Suite) Test_PatchChecker_checktextRcsid(c *check.C) { +func (s *Suite) Test_PatchChecker_checktextCvsID(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("patch-aa", - RcsID, + CvsID, "", "Documentation", "", @@ -612,17 +643,13 @@ func (s *Suite) Test_PatchChecker_checktextRcsid(c *check.C) { "@@ -1,3 +1,3 @@ $"+"Id$", " $"+"Id$", "-old line", - "+new line", + "+new line $varname", " $"+"Author: authorship $") CheckLinesPatch(lines) t.CheckOutputLines( - "WARN: ~/patch-aa:7: Found RCS tag \"$"+"Id$\". Please remove it.", - "WARN: ~/patch-aa:8: Found RCS tag \"$"+"Id$\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".", - "WARN: ~/patch-aa:11: Found RCS tag \"$"+"Author$\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".") -} - -func (s *Suite) Test_FileType_String(c *check.C) { - c.Check(ftUnknown.String(), equals, "unknown") + "WARN: ~/patch-aa:7: Found CVS tag \"$"+"Id$\". Please remove it.", + "WARN: ~/patch-aa:8: Found CVS tag \"$"+"Id$\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".", + "WARN: ~/patch-aa:11: Found CVS tag \"$"+"Author$\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".") } diff --git a/pkgtools/pkglint/files/pkglint.1 b/pkgtools/pkglint/files/pkglint.1 index 85e02e03a0b..88839eaf136 100644 --- a/pkgtools/pkglint/files/pkglint.1 +++ b/pkgtools/pkglint/files/pkglint.1 @@ -1,4 +1,4 @@ -.\" $NetBSD: pkglint.1,v 1.55 2019/03/10 19:01:50 rillig Exp $ +.\" $NetBSD: pkglint.1,v 1.56 2019/06/30 20:56:19 rillig Exp $ .\" From FreeBSD: portlint.1,v 1.8 1997/11/25 14:53:14 itojun Exp .\" .\" Copyright (c) 1997 by Jun-ichiro Itoh <itojun@itojun.org>. @@ -8,7 +8,7 @@ .\" Thomas Klausner <wiz@NetBSD.org>, 2012. .\" Roland Illig <rillig@NetBSD.org>, 2015-2019. .\" -.Dd January 14, 2018 +.Dd June 17, 2019 .Dt PKGLINT 1 .Os .Sh NAME @@ -88,8 +88,6 @@ For a list of warnings, see below. Enable all checks. .It Cm none Disable all checks. -.It Cm [no-]extra -Check remaining files in the package directory. .It Cm [no-]global Check inter-package consistency for distfile hashes and used licenses. .El diff --git a/pkgtools/pkglint/files/pkglint.go b/pkgtools/pkglint/files/pkglint.go index f88412eb77f..122fe89de61 100644 --- a/pkgtools/pkglint/files/pkglint.go +++ b/pkgtools/pkglint/files/pkglint.go @@ -2,6 +2,7 @@ package pkglint import ( "fmt" + "io" "netbsd.org/pkglint/getopt" "netbsd.org/pkglint/histogram" "netbsd.org/pkglint/regex" @@ -25,13 +26,14 @@ type Pkglint struct { Pkgsrc Pkgsrc // Global data, mostly extracted from mk/*. Pkg *Package // The package that is currently checked, or nil. - Todo []string // The files or directories that still need to be checked. - Wip bool // Is the currently checked file or package from pkgsrc-wip? - Infrastructure bool // Is the currently checked file from the pkgsrc infrastructure? - Testing bool // Is pkglint in self-testing mode (only during development)? - Username string // For checking against OWNER and MAINTAINER - cvsEntriesDir string // Cached to avoid I/O - cvsEntriesLines Lines + Todo []string // The files or directories that still need to be checked. + Wip bool // Is the currently checked file or package from pkgsrc-wip? + Infrastructure bool // Is the currently checked file from the pkgsrc infrastructure? + Testing bool // Is pkglint in self-testing mode (only during development)? + Username string // For checking against OWNER and MAINTAINER + + cvsEntriesDir string // Cached to avoid I/O + cvsEntries map[string]CvsEntry Logger Logger @@ -63,8 +65,6 @@ func NewPkglint() Pkglint { // This is to ensure that tests are properly initialized and shut down. func unusablePkglint() Pkglint { return Pkglint{} } -func (pkglint *Pkglint) usable() bool { return pkglint != nil } - type InterPackage struct { hashes map[string]*Hash // Maps "alg:filename" => hash (inter-package check). usedLicenses map[string]struct{} // Maps "license name" => true (inter-package check). @@ -118,7 +118,6 @@ func (ip *InterPackage) Bl3(name string, loc *Location) *Location { } type CmdOpts struct { - CheckExtra, CheckGlobal bool // TODO: Are these Warn* options really all necessary? @@ -160,86 +159,104 @@ var ( trace tracePkg.Tracer ) -func Main() int { - G.Logger.out = NewSeparatorWriter(os.Stdout) - G.Logger.err = NewSeparatorWriter(os.Stderr) - trace.Out = os.Stdout - exitCode := G.Main(os.Args...) - if G.Opts.Profiling { - G = unusablePkglint() // Free all memory. - runtime.GC() // For detecting possible memory leaks; see qa-pkglint. - } - return exitCode -} - // Main runs the main program with the given arguments. -// argv[0] is the program name. +// args[0] is the program name. // // Note: during tests, calling this method disables tracing // because the getopt parser resets all options before the actual parsing. // One of these options is trace.Tracing, which is connected to --debug. // // It also discards the -Wall option that is used by default in other tests. -func (pkglint *Pkglint) Main(argv ...string) (exitCode int) { +func (pkglint *Pkglint) Main(stdout io.Writer, stderr io.Writer, args []string) (exitCode int) { + G.Logger.out = NewSeparatorWriter(stdout) + G.Logger.err = NewSeparatorWriter(stderr) + trace.Out = stdout + defer func() { if r := recover(); r != nil { - if _, ok := r.(pkglintFatal); ok { - exitCode = 1 - } else { - panic(r) - } + _ = r.(pkglintFatal) + exitCode = 1 } }() - if exitcode := pkglint.ParseCommandLine(argv); exitcode != -1 { + if exitcode := pkglint.ParseCommandLine(args); exitcode != -1 { return exitcode } if pkglint.Opts.Profiling { + defer pkglint.setUpProfiling()() + } - defer func() { - pkglint.fileCache.table = nil - pkglint.fileCache.mapping = nil - runtime.GC() - - fd, err := os.Create("pkglint.heapdump") - assertNil(err, "heapDump.create") + pkglint.prepareMainLoop() - debug.WriteHeapDump(fd.Fd()) + for len(pkglint.Todo) > 0 { + item := pkglint.Todo[0] + pkglint.Todo = pkglint.Todo[1:] + pkglint.Check(item) + } - err = fd.Close() - assertNil(err, "heapDump.close") - }() + pkglint.Pkgsrc.checkToplevelUnusedLicenses() - f, err := os.Create("pkglint.pprof") - if err != nil { - dummyLine.Fatalf("Cannot create profiling file: %s", err) - } - defer f.Close() + pkglint.Logger.ShowSummary() + if pkglint.Logger.errors != 0 { + return 1 + } + return 0 +} - err = pprof.StartCPUProfile(f) - assertNil(err, "Cannot start profiling") - defer pprof.StopCPUProfile() +func (pkglint *Pkglint) setUpProfiling() func() { - pkglint.res.Profiling() - pkglint.Logger.histo = histogram.New() - pkglint.loaded = histogram.New() - defer func() { - pkglint.Logger.out.Write("") - pkglint.Logger.histo.PrintStats(pkglint.Logger.out.out, "loghisto", -1) - pkglint.res.PrintStats(pkglint.Logger.out.out) - pkglint.loaded.PrintStats(pkglint.Logger.out.out, "loaded", 10) - pkglint.Logger.out.WriteLine(sprintf("fileCache: %d hits, %d misses", pkglint.fileCache.hits, pkglint.fileCache.misses)) - }() + var cleanups []func() + atExit := func(cleanup func()) { + cleanups = append(cleanups, cleanup) } - for _, arg := range pkglint.Opts.args { - pkglint.Todo = append(pkglint.Todo, filepath.ToSlash(arg)) - } - if len(pkglint.Todo) == 0 { - pkglint.Todo = []string{"."} + atExit(func() { + pkglint.fileCache.table = nil + pkglint.fileCache.mapping = nil + runtime.GC() + + fd, err := os.Create("pkglint.heapdump") + assertNil(err, "heapDump.create") + + debug.WriteHeapDump(fd.Fd()) + + err = fd.Close() + assertNil(err, "heapDump.close") + + G = unusablePkglint() // Free all memory. + runtime.GC() // For detecting possible memory leaks; see qa-pkglint. + }) + + f, err := os.Create("pkglint.pprof") + if err != nil { + dummyLine.Fatalf("Cannot create profiling file: %s", err) + } + atExit(func() { assertNil(f.Close(), "") }) + + err = pprof.StartCPUProfile(f) + assertNil(err, "Cannot start profiling") + atExit(pprof.StopCPUProfile) + + pkglint.res.Profiling() + pkglint.Logger.histo = histogram.New() + pkglint.loaded = histogram.New() + atExit(func() { + pkglint.Logger.out.Write("") + pkglint.Logger.histo.PrintStats(pkglint.Logger.out.out, "loghisto", -1) + pkglint.res.PrintStats(pkglint.Logger.out.out) + pkglint.loaded.PrintStats(pkglint.Logger.out.out, "loaded", 10) + pkglint.Logger.out.WriteLine(sprintf("fileCache: %d hits, %d misses", pkglint.fileCache.hits, pkglint.fileCache.misses)) + }) + + return func() { + for i := range cleanups { + cleanups[len(cleanups)-1-i]() + } } +} +func (pkglint *Pkglint) prepareMainLoop() { firstDir := pkglint.Todo[0] if fileExists(firstDir) { firstDir = path.Dir(firstDir) @@ -262,20 +279,6 @@ func (pkglint *Pkglint) Main(argv ...string) (exitCode int) { assertNil(err, "user.Current") // On Windows, this is `Computername\Username`. pkglint.Username = replaceAll(currentUser.Username, `^.*\\`, "") - - for len(pkglint.Todo) > 0 { - item := pkglint.Todo[0] - pkglint.Todo = pkglint.Todo[1:] - pkglint.Check(item) - } - - pkglint.Pkgsrc.checkToplevelUnusedLicenses() - - pkglint.Logger.ShowSummary() - if pkglint.Logger.errors != 0 { - return 1 - } - return 0 } func (pkglint *Pkglint) ParseCommandLine(args []string) int { @@ -301,7 +304,6 @@ func (pkglint *Pkglint) ParseCommandLine(args []string) int { opts.AddFlagVar('V', "version", &gopts.ShowVersion, false, "show the version number of pkglint") warn := opts.AddFlagGroup('W', "warning", "warning,...", "enable or disable groups of warnings") - check.AddFlagVar("extra", &gopts.CheckExtra, false, "check various additional files") check.AddFlagVar("global", &gopts.CheckGlobal, false, "inter-package checks") warn.AddFlagVar("extra", &gopts.WarnExtra, false, "enable some extra warnings") @@ -330,6 +332,14 @@ func (pkglint *Pkglint) ParseCommandLine(args []string) int { return 0 } + pkglint.Todo = nil + for _, arg := range pkglint.Opts.args { + pkglint.Todo = append(pkglint.Todo, filepath.ToSlash(arg)) + } + if len(pkglint.Todo) == 0 { + pkglint.Todo = []string{"."} + } + return -1 } @@ -349,12 +359,22 @@ func (pkglint *Pkglint) Check(dirent string) { } st, err := os.Lstat(dirent) - if err != nil || !st.Mode().IsDir() && !st.Mode().IsRegular() { + if err != nil { + NewLineWhole(dirent).Errorf("No such file or directory.") + return + } + + pkglint.checkMode(dirent, st.Mode()) +} + +func (pkglint *Pkglint) checkMode(dirent string, mode os.FileMode) { + // TODO: merge duplicate code in Package.checkDirent + isDir := mode.IsDir() + isReg := mode.IsRegular() + if !isDir && !isReg { NewLineWhole(dirent).Errorf("No such file or directory.") return } - isDir := st.Mode().IsDir() - isReg := st.Mode().IsRegular() dir := dirent if !isDir { @@ -374,12 +394,12 @@ func (pkglint *Pkglint) Check(dirent string) { if isReg { depth := strings.Count(pkgsrcRel, "/") - pkglint.checkExecutable(dirent, st.Mode()) + pkglint.checkExecutable(dirent, mode) pkglint.checkReg(dirent, basename, depth) return } - if isDir && isEmptyDir(dirent) { + if isEmptyDir(dirent) { return } @@ -395,42 +415,6 @@ func (pkglint *Pkglint) Check(dirent string) { } } -// checkDirent checks a directory entry based on its filename and its mode -// (regular file, directory, symlink). -func (pkglint *Pkglint) checkDirent(dirent string, mode os.FileMode) { - basename := path.Base(dirent) - - switch { - - case mode.IsRegular(): - pkgsrcRel := pkglint.Pkgsrc.ToRel(dirent) - depth := strings.Count(pkgsrcRel, "/") - pkglint.checkReg(dirent, basename, depth) - - case hasPrefix(basename, "work"): - if pkglint.Opts.Import { - NewLineWhole(dirent).Errorf("Must be cleaned up before committing the package.") - } - return - - case mode.IsDir(): - switch { - case basename == "files" || basename == "patches" || isIgnoredFilename(basename): - // Ok - case matches(dirent, `(?:^|/)files/[^/]*$`): - // Ok - case !isEmptyDir(dirent): - NewLineWhole(dirent).Warnf("Unknown directory name.") - } - - case mode&os.ModeSymlink != 0: - NewLineWhole(dirent).Warnf("Invalid symlink name.") - - default: - NewLineWhole(dirent).Errorf("Only files and directories are allowed in pkgsrc.") - } -} - // checkdirPackage checks a complete pkgsrc package, including each // of the files individually, and also when seen in combination. func (pkglint *Pkglint) checkdirPackage(dir string) { @@ -456,7 +440,7 @@ func findPkgsrcTopdir(dirname string) string { return "" } -func resolveVariableRefs(mklines MkLines, text string) (resolved string) { +func resolveVariableRefs(mklines *MkLines, text string) (resolved string) { // TODO: How does this fit into the Scope type, which is newer than this function? if !contains(text, "${") { @@ -507,9 +491,9 @@ func CheckFileOther(filename string) { } } -func CheckLinesDescr(lines Lines) { +func CheckLinesDescr(lines *Lines) { if trace.Tracing { - defer trace.Call1(lines.FileName)() + defer trace.Call1(lines.Filename)() } for _, line := range lines.Lines { @@ -541,9 +525,9 @@ func CheckLinesDescr(lines Lines) { SaveAutofixChanges(lines) } -func CheckLinesMessage(lines Lines) { +func CheckLinesMessage(lines *Lines) { if trace.Tracing { - defer trace.Call1(lines.FileName)() + defer trace.Call1(lines.Filename)() } // For now, skip all checks when the MESSAGE may be built from multiple @@ -558,7 +542,7 @@ func CheckLinesMessage(lines Lines) { explanation := func() []string { return []string{ "A MESSAGE file should consist of a header line, having 75 \"=\"", - "characters, followed by a line containing only the RCS Id, then an", + "characters, followed by a line containing only the CVS Id, then an", "empty line, your text and finally the footer line, which is the", "same as the header line."} } @@ -577,9 +561,9 @@ func CheckLinesMessage(lines Lines) { fix.Explain(explanation()...) fix.InsertBefore(hline) fix.Apply() - lines.CheckRcsID(0, ``, "") - } else if 1 < lines.Len() { - lines.CheckRcsID(1, ``, "") + lines.CheckCvsID(0, ``, "") + } else { + lines.CheckCvsID(1, ``, "") } for _, line := range lines.Lines { ck := LineChecker{line} @@ -685,8 +669,7 @@ func (pkglint *Pkglint) checkReg(filename, basename string, depth int) { NewLineWhole(filename).Warnf("Patch files should be named \"patch-\", followed by letters, '-', '_', '.', and digits only.") case (hasPrefix(basename, "Makefile") || hasSuffix(basename, ".mk")) && - !(hasPrefix(filename, "files/") || contains(filename, "/files/")) && - !(hasPrefix(filename, "patches/") || contains(filename, "/patches/")): + !pathContainsDir(filename, "files"): CheckFileMk(filename) case hasPrefix(basename, "PLIST"): @@ -699,7 +682,7 @@ func (pkglint *Pkglint) checkReg(filename, basename string, depth int) { _ = pkglint.Pkgsrc.loadDocChangesFromFile(filename) case matches(filename, `(?:^|/)files/[^/]*$`): - // Skip + // Skip files directly in the files/ directory, but not those further down. case basename == "spec": if !hasPrefix(pkglint.Pkgsrc.ToRel(filename), "regress/") { @@ -711,9 +694,6 @@ func (pkglint *Pkglint) checkReg(filename, basename string, depth int) { default: NewLineWhole(filename).Warnf("Unexpected file found.") - if pkglint.Opts.CheckExtra { - CheckFileOther(filename) - } } } @@ -759,7 +739,7 @@ func (pkglint *Pkglint) checkExecutable(filename string, mode os.FileMode) { fix.Apply() } -func CheckLinesTrailingEmptyLines(lines Lines) { +func CheckLinesTrailingEmptyLines(lines *Lines) { max := lines.Len() last := max @@ -776,30 +756,16 @@ func CheckLinesTrailingEmptyLines(lines Lines) { // The command can be "sed" or "gsed" or "${SED}". // If a tool is returned, usable tells whether that tool has been added // to USE_TOOLS in the current scope (file or package). -func (pkglint *Pkglint) Tool(mklines MkLines, command string, time ToolTime) (tool *Tool, usable bool) { - varname := "" - if varUse := ToVarUse(command); varUse != nil { - varname = varUse.varname - } - +func (pkglint *Pkglint) Tool(mklines *MkLines, command string, time ToolTime) (tool *Tool, usable bool) { tools := pkglint.tools(mklines) - if t := tools.ByName(command); t != nil { - if tools.Usable(t, time) { - return t, true - } - tool = t + if varUse := ToVarUse(command); varUse != nil { + tool = tools.ByVarname(varUse.varname) + } else { + tool = tools.ByName(command) } - if t := tools.ByVarname(varname); t != nil { - if tools.Usable(t, time) { - return t, true - } - if tool == nil { - tool = t - } - } - return + return tool, tool != nil && tools.Usable(tool, time) } // ToolByVarname looks up the tool by its variable name, e.g. "SED". @@ -808,11 +774,11 @@ func (pkglint *Pkglint) Tool(mklines MkLines, command string, time ToolTime) (to // It is not guaranteed to be usable (added to USE_TOOLS), only defined; // that must be checked by the calling code, // see Tool.UsableAtLoadTime and Tool.UsableAtRunTime. -func (pkglint *Pkglint) ToolByVarname(mklines MkLines, varname string) *Tool { +func (pkglint *Pkglint) ToolByVarname(mklines *MkLines, varname string) *Tool { return pkglint.tools(mklines).ByVarname(varname) } -func (pkglint *Pkglint) tools(mklines MkLines) *Tools { +func (pkglint *Pkglint) tools(mklines *MkLines) *Tools { if mklines != nil { return mklines.Tools } else { @@ -820,18 +786,53 @@ func (pkglint *Pkglint) tools(mklines MkLines) *Tools { } } -func (pkglint *Pkglint) loadCvsEntries(filename string) Lines { +func (pkglint *Pkglint) loadCvsEntries(filename string) map[string]CvsEntry { dir := path.Dir(filename) if dir == pkglint.cvsEntriesDir { - return pkglint.cvsEntriesLines + return pkglint.cvsEntries + } + + var entries map[string]CvsEntry + + handle := func(line *Line, add bool, text string) { + if !hasPrefix(text, "/") { + return + } + + fields := strings.Split(text, "/") + if len(fields) != 6 { + line.Errorf("Invalid line: %s", line.Text) + return + } + + if add { + entries[fields[1]] = CvsEntry{fields[1], fields[2], fields[3], fields[4], fields[5]} + } else { + delete(entries, fields[1]) + } } lines := Load(dir+"/CVS/Entries", 0) - if lines == nil { - return nil + if lines != nil { + entries = make(map[string]CvsEntry) + for _, line := range lines.Lines { + handle(line, true, line.Text) + } + + logLines := Load(dir+"/CVS/Entries.Log", 0) + if logLines != nil { + for _, line := range logLines.Lines { + text := line.Text + if hasPrefix(text, "A ") { + handle(line, true, text[2:]) + } else if hasPrefix(text, "R ") { + handle(line, false, text[2:]) + } + } + } } pkglint.cvsEntriesDir = dir - pkglint.cvsEntriesLines = lines - return lines + pkglint.cvsEntries = entries + return entries } diff --git a/pkgtools/pkglint/files/pkglint_test.go b/pkgtools/pkglint/files/pkglint_test.go index 8c8f0a4aaf8..0c0d07ac537 100644 --- a/pkgtools/pkglint/files/pkglint_test.go +++ b/pkgtools/pkglint/files/pkglint_test.go @@ -9,6 +9,8 @@ import ( "strings" ) +func (pkglint *Pkglint) usable() bool { return pkglint.fileCache != nil } + func (s *Suite) Test_Pkglint_Main__help(c *check.C) { t := s.Init(c) @@ -39,7 +41,6 @@ func (s *Suite) Test_Pkglint_Main__help(c *check.C) { " Flags for -C, --check:", " all all of the following", " none none of the following", - " extra check various additional files (disabled)", " global inter-package checks (disabled)", "", " Flags for -W, --warning:", @@ -102,19 +103,6 @@ func (s *Suite) Test_Pkglint_Main__unknown_option(c *check.C) { // See Test_Pkglint_Main__help for the complete output. } -// This test covers the code path for unexpected panics. -func (s *Suite) Test_Pkglint_Main__panic(c *check.C) { - t := s.Init(c) - - pkg := t.SetUpPackage("category/package") - - G.Logger.out = nil // Force an error that cannot happen in practice. - - c.Check( - func() { t.Main(pkg) }, - check.PanicMatches, `(?s).*\bnil pointer\b.*`) -} - // Demonstrates which infrastructure files are necessary to actually run // pkglint in a realistic scenario. // @@ -131,7 +119,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("doc/CHANGES-2018", - RcsID, + CvsID, "", "Changes to the packages collection and infrastructure in 2018:", "", @@ -139,7 +127,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { // See Pkgsrc.loadSuggestedUpdates. t.CreateFileLines("doc/TODO", - RcsID, + CvsID, "", "Suggested package updates", "", @@ -148,7 +136,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { // The MASTER_SITES in the package Makefile are searched here. // See Pkgsrc.loadMasterSites. t.CreateFileLines("mk/fetch/sites.mk", - MkRcsID, + MkCvsID, "", "MASTER_SITE_GITHUB+=\thttps://github.com/") @@ -163,7 +151,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { // so that it can be used in CATEGORIES in the package Makefile. // The category "tools" on the other hand is not valid. t.CreateFileLines("sysutils/Makefile", - MkRcsID) + MkCvsID) // The package Makefile in this test is quite simple, containing just the // standard variable definitions. The data for checking the variable @@ -171,7 +159,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { // (as defined in the previous lines), and partly in the pkglint // code directly. Many details can be found in vartypecheck.go. t.CreateFileLines("sysutils/checkperms/Makefile", - MkRcsID, + MkCvsID, "", "DISTNAME=\tcheckperms-1.11", "CATEGORIES=\tsysutils tools", @@ -186,14 +174,14 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { t.CreateFileLines("sysutils/checkperms/MESSAGE", "===========================================================================", - RcsID, + CvsID, "", "After installation, this package has to be configured in a special way.", "", "===========================================================================") t.CreateFileLines("sysutils/checkperms/PLIST", - PlistRcsID, + PlistCvsID, "bin/checkperms", "man/man1/checkperms.1") @@ -204,7 +192,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { "Make the package work on MS-DOS") t.CreateFileLines("sysutils/checkperms/patches/patch-checkperms.c", - RcsID, + CvsID, "", "A simple patch demonstrating that pkglint checks for missing", "removed lines. The hunk headers says that one line is to be", @@ -217,7 +205,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { "+// Header 2", "+// Header 3") t.CreateFileLines("sysutils/checkperms/distinfo", - RcsID, + CvsID, "", "SHA1 (checkperms-1.12.tar.gz) = 34c084b4d06bcd7a8bba922ff57677e651eeced5", "RMD160 (checkperms-1.12.tar.gz) = cd95029aa930b6201e9580b3ab7e36dd30b8f925", @@ -248,6 +236,20 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { "(Run \"pkglint -F\" to automatically fix some issues.)") } +func (s *Suite) Test_Pkglint_Main__autofix_exitcode(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("filename.mk", + "") + + exitcode := t.Main("-Wall", "--autofix", t.File("filename.mk")) + + t.CheckOutputLines( + "AUTOFIX: ~/filename.mk:1: Inserting a line \"# $NetBSD: pkglint_test.go,v 1.44 2019/06/30 20:56:19 rillig Exp $\" before this line.") + t.Check(exitcode, equals, 0) +} + // Run pkglint in a realistic environment. // // env \ @@ -277,10 +279,7 @@ func (s *Suite) Test_Pkglint__realistic(c *check.C) { cmdline := os.Getenv("PKGLINT_TESTCMDLINE") if cmdline != "" { - G.Logger.out = NewSeparatorWriter(os.Stdout) - G.Logger.err = NewSeparatorWriter(os.Stderr) - trace.Out = os.Stdout - G.Main(append([]string{"pkglint"}, strings.Fields(cmdline)...)...) + G.Main(os.Stdout, os.Stderr, append([]string{"pkglint"}, strings.Fields(cmdline)...)) } } @@ -385,7 +384,7 @@ func (s *Suite) Test_Pkglint_Check(c *check.C) { t.CreateFileLines("mk/bsd.pkg.mk") t.CreateFileLines("category/package/Makefile") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tCategory\u0007", "", @@ -393,7 +392,7 @@ func (s *Suite) Test_Pkglint_Check(c *check.C) { "", ".include \"../mk/misc/category.mk\"") t.CreateFileLines("Makefile", - MkRcsID, + MkCvsID, "COMMENT=\tToplevel\u0005") G.Check(t.File(".")) @@ -418,20 +417,31 @@ func (s *Suite) Test_Pkglint_Check(c *check.C) { "ERROR: ~/category/package/nonexistent: No such file or directory.") } +func (s *Suite) Test_Pkglint_checkMode__neither_file_nor_directory(c *check.C) { + t := s.Init(c) + + G.checkMode("/dev/null", os.ModeDevice) + + t.CheckOutputLines( + "ERROR: /dev/null: No such file or directory.") +} + // Pkglint must never be trapped in an endless loop, even when // resolving the value of a variable that refers back to itself. func (s *Suite) Test_resolveVariableRefs__circular_reference(c *check.C) { t := s.Init(c) - mkline := t.NewMkLine("filename.mk", 1, "GCC_VERSION=${GCC_VERSION}") + mkline := t.NewMkLine("filename.mk", 1, "VAR=\t1:${VAR}+ 2:${VAR}") G.Pkg = NewPackage(t.File("category/pkgbase")) - G.Pkg.vars.Define("GCC_VERSION", mkline) + G.Pkg.vars.Define("VAR", mkline) // TODO: It may be better to define MkLines.Resolve and Package.Resolve, // to clearly state the scope of the involved variables. - resolved := resolveVariableRefs(nil, "gcc-${GCC_VERSION}") + resolved := resolveVariableRefs(nil, "the a:${VAR} b:${VAR}") - c.Check(resolved, equals, "gcc-${GCC_VERSION}") + // TODO: The ${VAR} after "b:" should also be expanded since there + // is no recursion. + c.Check(resolved, equals, "the a:1:${VAR}+ 2:${VAR} b:${VAR}") } func (s *Suite) Test_resolveVariableRefs__multilevel(c *check.C) { @@ -495,7 +505,7 @@ func (s *Suite) Test_CheckLinesDescr(c *check.C) { "WARN: DESCR:25: File too long (should be no more than 24 lines).") } -func (s *Suite) Test_CheckLinesMessage__short(c *check.C) { +func (s *Suite) Test_CheckLinesMessage__one_line_of_text(c *check.C) { t := s.Init(c) lines := t.NewLines("MESSAGE", @@ -507,6 +517,18 @@ func (s *Suite) Test_CheckLinesMessage__short(c *check.C) { "WARN: MESSAGE:1: File too short.") } +func (s *Suite) Test_CheckLinesMessage__one_hline(c *check.C) { + t := s.Init(c) + + lines := t.NewLines("MESSAGE", + strings.Repeat("=", 75)) + + CheckLinesMessage(lines) + + t.CheckOutputLines( + "WARN: MESSAGE:1: File too short.") +} + func (s *Suite) Test_CheckLinesMessage__malformed(c *check.C) { t := s.Init(c) @@ -546,7 +568,7 @@ func (s *Suite) Test_CheckLinesMessage__autofix(c *check.C) { "=============================================\" after this line.") t.CheckFileLines("MESSAGE", "===========================================================================", - RcsID, + CvsID, "1", "2", "3", @@ -564,7 +586,7 @@ func (s *Suite) Test_CheckLinesMessage__common(c *check.C) { "MESSAGE_SRC+=\t${.CURDIR}/MESSAGE") t.CreateFileLines("category/package/MESSAGE.common", hline, - RcsID, + CvsID, "common line") t.CreateFileLines("category/package/MESSAGE", hline) @@ -584,7 +606,7 @@ func (s *Suite) Test_Pkglint_checkReg__alternatives(c *check.C) { lines := t.SetUpFileLines("category/package/ALTERNATIVES", "bin/tar bin/gnu-tar") - t.Main(lines.FileName) + t.Main(lines.Filename) t.CheckOutputLines( "ERROR: ~/category/package/ALTERNATIVES:1: Alternative implementation \"bin/gnu-tar\" must be an absolute path.", @@ -679,7 +701,7 @@ func (s *Suite) Test_Pkglint_Tool__prefer_mk_over_pkgsrc(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") - mklines := t.NewMkLines("Makefile", MkRcsID) + mklines := t.NewMkLines("Makefile", MkCvsID) global := G.Pkgsrc.Tools.Define("tool", "TOOL", mkline) local := mklines.Tools.Define("tool", "TOOL", mkline) @@ -698,7 +720,7 @@ func (s *Suite) Test_Pkglint_Tool__prefer_mk_over_pkgsrc(c *check.C) { func (s *Suite) Test_Pkglint_Tool__lookup_by_name_fallback(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("Makefile", MkRcsID) + mklines := t.NewMkLines("Makefile", MkCvsID) t.SetUpTool("tool", "", Nowhere) loadTimeTool, loadTimeUsable := G.Tool(mklines, "tool", LoadTime) @@ -718,7 +740,7 @@ func (s *Suite) Test_Pkglint_Tool__lookup_by_varname(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") - mklines := t.NewMkLines("Makefile", MkRcsID) + mklines := t.NewMkLines("Makefile", MkCvsID) global := G.Pkgsrc.Tools.Define("tool", "TOOL", mkline) local := mklines.Tools.Define("tool", "TOOL", mkline) @@ -738,7 +760,7 @@ func (s *Suite) Test_Pkglint_Tool__lookup_by_varname(c *check.C) { func (s *Suite) Test_Pkglint_Tool__lookup_by_varname_fallback(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("Makefile", MkRcsID) + mklines := t.NewMkLines("Makefile", MkCvsID) G.Pkgsrc.Tools.def("tool", "TOOL", false, Nowhere, nil) loadTimeTool, loadTimeUsable := G.Tool(mklines, "${TOOL}", LoadTime) @@ -754,7 +776,7 @@ func (s *Suite) Test_Pkglint_Tool__lookup_by_varname_fallback(c *check.C) { func (s *Suite) Test_Pkglint_Tool__lookup_by_varname_fallback_runtime(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("Makefile", MkRcsID) + mklines := t.NewMkLines("Makefile", MkCvsID) G.Pkgsrc.Tools.def("tool", "TOOL", false, AtRunTime, nil) loadTimeTool, loadTimeUsable := G.Tool(mklines, "${TOOL}", LoadTime) @@ -770,7 +792,7 @@ func (s *Suite) Test_Pkglint_ToolByVarname__prefer_mk_over_pkgsrc(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") - mklines := t.NewMkLines("Makefile", MkRcsID) + mklines := t.NewMkLines("Makefile", MkCvsID) global := G.Pkgsrc.Tools.Define("tool", "TOOL", mkline) local := mklines.Tools.Define("tool", "TOOL", mkline) @@ -783,7 +805,7 @@ func (s *Suite) Test_Pkglint_ToolByVarname__prefer_mk_over_pkgsrc(c *check.C) { func (s *Suite) Test_Pkglint_ToolByVarname(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("Makefile", MkRcsID) + mklines := t.NewMkLines("Makefile", MkCvsID) G.Pkgsrc.Tools.def("tool", "TOOL", false, AtRunTime, nil) c.Check(G.ToolByVarname(mklines, "TOOL").String(), equals, "tool:TOOL::AtRunTime") @@ -825,71 +847,30 @@ func (s *Suite) Test_Pkglint_Check__invalid_files_before_import(c *check.C) { "ERROR: ~/category/package/work: Must be cleaned up before committing the package.") } -func (s *Suite) Test_Pkglint_checkDirent__errors(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("-Call", "-Wall,no-space") - t.SetUpPkgsrc() - t.CreateFileLines("category/package/files/subdir/file") - t.CreateFileLines("category/package/files/subdir/subsub/file") - t.FinishSetUp() - - G.checkDirent(t.File("category/package/options.mk"), 0444) - G.checkDirent(t.File("category/package/files/subdir"), 0555|os.ModeDir) - G.checkDirent(t.File("category/package/files/subdir/subsub"), 0555|os.ModeDir) - G.checkDirent(t.File("category/package/files"), 0555|os.ModeDir) - - t.CheckOutputLines( - "ERROR: ~/category/package/options.mk: Cannot be read.", - "WARN: ~/category/package/files/subdir/subsub: Unknown directory name.") -} - -func (s *Suite) Test_Pkglint_checkDirent__file_selection(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("-Call", "-Wall,no-space") - t.SetUpPkgsrc() - t.CreateFileLines("doc/CHANGES-2018", - RcsID) - t.CreateFileLines("category/package/buildlink3.mk", - MkRcsID) - t.CreateFileLines("category/package/unexpected.txt", - RcsID) - t.FinishSetUp() - - G.checkDirent(t.File("doc/CHANGES-2018"), 0444) - G.checkDirent(t.File("category/package/buildlink3.mk"), 0444) - G.checkDirent(t.File("category/package/unexpected.txt"), 0444) - - t.CheckOutputLines( - "WARN: ~/category/package/buildlink3.mk:EOF: Expected a BUILDLINK_TREE line.", - "WARN: ~/category/package/unexpected.txt: Unexpected file found.") -} - func (s *Suite) Test_Pkglint_checkReg__readme_and_todo(c *check.C) { t := s.Init(c) t.CreateFileLines("category/Makefile", - MkRcsID) + MkCvsID) t.CreateFileLines("category/package/files/README", "Extra file that is installed later.") t.CreateFileDummyPatch("category/package/patches/patch-README") t.CreateFileLines("category/package/Makefile", - MkRcsID, + MkCvsID, "CATEGORIES=category", "", "COMMENT=Comment", "LICENSE=2-clause-bsd") t.CreateFileLines("category/package/PLIST", - PlistRcsID, + PlistCvsID, "bin/program") t.CreateFileLines("category/package/README", "This package ...") t.CreateFileLines("category/package/TODO", "Make this package work.") t.CreateFileLines("category/package/distinfo", - RcsID, + CvsID, "", "SHA1 (patch-README) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") @@ -942,6 +923,17 @@ func (s *Suite) Test_Pkglint_checkReg__unknown_file_in_patches(c *check.C) { "Patch files should be named \"patch-\", followed by letters, '-', '_', '.', and digits only.") } +func (s *Suite) Test_Pkglint_checkReg__patch_for_Makefile_fragment(c *check.C) { + t := s.Init(c) + + t.CreateFileDummyPatch("category/package/patches/patch-compiler.mk") + t.Chdir("category/package") + + G.checkReg(t.File("patches/patch-compiler.mk"), "patch-compiler.mk", 3) + + t.CheckOutputEmpty() +} + func (s *Suite) Test_Pkglint_checkReg__file_in_files(c *check.C) { t := s.Init(c) @@ -966,23 +958,6 @@ func (s *Suite) Test_Pkglint_checkReg__spec(c *check.C) { "WARN: ~/category/package/spec: Only packages in regress/ may have spec files.") } -// Since all required information is passed to G.checkDirent via parameters, -// this test produces the expected results even though none of these files actually exists. -func (s *Suite) Test_Pkglint_checkDirent__skipped(c *check.C) { - t := s.Init(c) - - G.checkDirent("work", os.ModeSymlink) - G.checkDirent("work.i386", os.ModeSymlink) - G.checkDirent("work.hostname", os.ModeSymlink) - G.checkDirent("other", os.ModeSymlink) - - G.checkDirent("device", os.ModeDevice) - - t.CheckOutputLines( - "WARN: other: Invalid symlink name.", - "ERROR: device: Only files and directories are allowed in pkgsrc.") -} - // A package that is very incomplete may produce lots of warnings. // This case is unrealistic since most packages are either generated by url2pkg // or copied from an existing working package. @@ -991,12 +966,12 @@ func (s *Suite) Test_Pkglint_checkdirPackage(c *check.C) { t.Chdir("category/package") t.CreateFileLines("Makefile", - MkRcsID) + MkCvsID) G.checkdirPackage(".") t.CheckOutputLines( - "WARN: Makefile: Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset.", + "WARN: Makefile: This package should have a PLIST file.", "WARN: distinfo: A package that downloads files should have a distinfo file.", "ERROR: Makefile: Each package must define its LICENSE.", "WARN: Makefile: Each package should define a COMMENT.") @@ -1008,19 +983,19 @@ func (s *Suite) Test_Pkglint_checkdirPackage__PKGDIR(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("category/Makefile") t.CreateFileLines("other/package/Makefile", - MkRcsID) + MkCvsID) t.CreateFileLines("other/package/PLIST", - PlistRcsID, + PlistCvsID, "bin/program") t.CreateFileLines("other/package/distinfo", - RcsID, + CvsID, "", "SHA1 (patch-aa) = da39a3ee5e6b4b0d3255bfef95601890afd80709") t.CreateFileLines("category/package/patches/patch-aa", - RcsID) + CvsID) t.Chdir("category/package") t.CreateFileLines("Makefile", - MkRcsID, + MkCvsID, "", "CATEGORIES=\tcategory", "", @@ -1057,7 +1032,7 @@ func (s *Suite) Test_Pkglint_checkdirPackage__meta_package_without_license(c *ch t.Chdir("category/package") t.CreateFileLines("Makefile", - MkRcsID, + MkCvsID, "", "META_PACKAGE=\tyes") t.SetUpVartypes() @@ -1156,6 +1131,34 @@ func (s *Suite) Test_CheckFileOther__no_tracing(c *check.C) { func (s *Suite) Test_Pkglint_checkExecutable(c *check.C) { t := s.Init(c) + filename := t.CreateFileLines("file.mk") + err := os.Chmod(filename, 0555) + assertNil(err, "") + + G.checkExecutable(filename, 0555) + + t.CheckOutputLines( + "WARN: ~/file.mk: Should not be executable.") + + t.SetUpCommandLine("--autofix") + + G.checkExecutable(filename, 0555) + + t.CheckOutputMatches( + "AUTOFIX: ~/file.mk: Clearing executable bits") + + // On Windows, this is effectively a no-op test since there is no + // execute-bit. The only relevant permissions bit is whether a + // file is readonly or not. + st, err := os.Lstat(filename) + if t.Check(err, check.IsNil) { + t.Check(st.Mode()&0111, equals, os.FileMode(0)) + } +} + +func (s *Suite) Test_Pkglint_checkExecutable__error(c *check.C) { + t := s.Init(c) + filename := t.File("file.mk") G.checkExecutable(filename, 0555) @@ -1176,7 +1179,7 @@ func (s *Suite) Test_Pkglint_checkExecutable__already_committed(c *check.C) { t := s.Init(c) t.CreateFileLines("CVS/Entries", - "/file.mk/modified////") + "/file.mk//modified//") filename := t.File("file.mk") G.checkExecutable(filename, 0555) @@ -1185,7 +1188,7 @@ func (s *Suite) Test_Pkglint_checkExecutable__already_committed(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_Main(c *check.C) { +func (s *Suite) Test_Pkglint_Main(c *check.C) { t := s.Init(c) out, err := os.Create(t.CreateFileLines("out")) @@ -1198,19 +1201,7 @@ func (s *Suite) Test_Main(c *check.C) { t.FinishSetUp() runMain := func(out *os.File, commandLine ...string) { - args := os.Args - stdout := os.Stdout - stderr := os.Stderr - defer func() { - os.Stderr = stderr - os.Stdout = stdout - os.Args = args - }() - os.Args = commandLine - os.Stdout = out - os.Stderr = out - - exitCode := Main() + exitCode := G.Main(out, out, commandLine) c.Check(exitCode, equals, 0) } @@ -1246,3 +1237,43 @@ func (s *Suite) Test_InterPackage_Bl3__same_identifier(c *check.C) { "ERROR: category/package2/buildlink3.mk:3: Duplicate package identifier " + "\"package1\" already appeared in ../../category/package1/buildlink3.mk:3.") } + +func (s *Suite) Test_Pkglint_loadCvsEntries(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("CVS/Entries", + "/invalid/", + "must be silently ignored", + "/name/revision/timestamp/options/tagdate") + + t.Check(isCommitted(t.File("name")), equals, true) + + t.CheckOutputLines( + "ERROR: ~/CVS/Entries:1: Invalid line: /invalid/") +} + +func (s *Suite) Test_Pkglint_loadCvsEntries__with_Entries_Log(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("CVS/Entries", + "/invalid/", + "must be silently ignored", + "/name//modified//", + "/removed//modified//") + + t.CreateFileLines("CVS/Entries.Log", + "A /invalid/", + "A /added//modified//", + "must be silently ignored", + "R /invalid/", + "R /removed//modified//") + + t.Check(isCommitted(t.File("name")), equals, true) + t.Check(isCommitted(t.File("added")), equals, true) + t.Check(isCommitted(t.File("removed")), equals, false) + + t.CheckOutputLines( + "ERROR: ~/CVS/Entries:1: Invalid line: /invalid/", + "ERROR: ~/CVS/Entries.Log:1: Invalid line: A /invalid/", + "ERROR: ~/CVS/Entries.Log:4: Invalid line: R /invalid/") +} diff --git a/pkgtools/pkglint/files/pkgsrc.go b/pkgtools/pkglint/files/pkgsrc.go index 680c52e9d76..d40ce28c361 100644 --- a/pkgtools/pkglint/files/pkgsrc.go +++ b/pkgtools/pkglint/files/pkgsrc.go @@ -33,10 +33,13 @@ type Pkgsrc struct { PkgOptions map[string]string // "x11" => "Provides X11 support" - suggestedUpdates []SuggestedUpdate // - suggestedWipUpdates []SuggestedUpdate // - LastChange map[string]*Change // - listVersions map[string][]string // See ListVersions + suggestedUpdates []SuggestedUpdate + suggestedWipUpdates []SuggestedUpdate + + LastChange map[string]*Change + FreezeStart string // e.g. "2018-01-01", or "" + + listVersions map[string][]string // See Pkgsrc.ListVersions // Variables that may be overridden by the pkgsrc user. // They are typically defined in mk/defaults/mk.conf. @@ -60,6 +63,7 @@ func NewPkgsrc(dir string) Pkgsrc { nil, nil, make(map[string]*Change), + "", make(map[string][]string), NewScope(), make(map[string]string), @@ -184,9 +188,9 @@ func (src *Pkgsrc) Latest(category string, re regex.Pattern, repl string) string // => {"php-53", "php-56", "php-73"} func (src *Pkgsrc) ListVersions(category string, re regex.Pattern, repl string, errorIfEmpty bool) []string { if G.Testing { - assertf( - hasPrefix(string(re), "^") && hasSuffix(string(re), "$"), - "Regular expression %q must be anchored at both ends.", re) + // Regular expression must be anchored at both ends, to avoid typos. + assert(hasPrefix(string(re), "^")) + assert(hasSuffix(string(re), "$")) } // TODO: Maybe convert cache key to a struct, to save allocations. @@ -243,7 +247,7 @@ func (src *Pkgsrc) ListVersions(category string, re regex.Pattern, repl string, return naturalLess(names[i], names[j]) }) - var repls = make([]string, len(names), len(names)) + var repls = make([]string, len(names)) for i, name := range names { repls[i] = replaceAll(name, re, repl) } @@ -262,9 +266,7 @@ func (src *Pkgsrc) checkToplevelUnusedLicenses() { licenseName := licenseFile.Name() if !G.InterPackage.LicenseUsed(licenseName) { licensePath := licensesDir + "/" + licenseName - if fileExists(licensePath) { - NewLineWhole(licensePath).Warnf("This license seems to be unused.") - } + NewLineWhole(licensePath).Warnf("This license seems to be unused.") } } } @@ -291,24 +293,15 @@ func (src *Pkgsrc) loadTools() { } // TODO: parse bsd.prefs.mk and bsd.pkg.mk instead of hardcoding this. - toolDefs := [...]struct { - Name string - Varname string - Validity Validity - }{ - {"echo", "ECHO", AfterPrefsMk}, - {"echo -n", "ECHO_N", AfterPrefsMk}, - {"false", "FALSE", AtRunTime}, // from bsd.pkg.mk - {"test", "TEST", AfterPrefsMk}, - {"true", "TRUE", AfterPrefsMk}} - - for _, toolDef := range toolDefs { - tools.def(toolDef.Name, toolDef.Varname, true, toolDef.Validity, nil) - } + tools.def("echo", "ECHO", true, AfterPrefsMk, nil) + tools.def("echo -n", "ECHO_N", true, AfterPrefsMk, nil) + tools.def("false", "FALSE", true, AtRunTime, nil) // from bsd.pkg.mk + tools.def("test", "TEST", true, AfterPrefsMk, nil) + tools.def("true", "TRUE", true, AfterPrefsMk, nil) for _, basename := range toolFiles { mklines := src.LoadMk("mk/tools/"+basename, MustSucceed|NotEmpty) - mklines.ForEach(func(mkline MkLine) { + mklines.ForEach(func(mkline *MkLine) { tools.ParseToolLine(mklines, mkline, true, !mklines.indentation.IsConditional()) }) } @@ -316,7 +309,7 @@ func (src *Pkgsrc) loadTools() { for _, relativeName := range [...]string{"mk/bsd.prefs.mk", "mk/bsd.pkg.mk"} { mklines := src.LoadMk(relativeName, MustSucceed|NotEmpty) - mklines.ForEach(func(mkline MkLine) { + mklines.ForEach(func(mkline *MkLine) { if mkline.IsVarassign() { varname := mkline.Varname() switch varname { @@ -349,7 +342,7 @@ func (src *Pkgsrc) loadUntypedVars() { // Setting guessed to false prevents the vartype.guessed case in MkLineChecker.CheckVaruse. unknownType := NewVartype(BtUnknown, NoVartypeOptions, NewACLEntry("*", aclpAll)) - define := func(varcanon string, mkline MkLine) { + define := func(varcanon string, mkline *MkLine) { switch { case src.vartypes.DefinedCanon(varcanon): // Already defined, can also be a tool. @@ -376,23 +369,21 @@ func (src *Pkgsrc) loadUntypedVars() { } handleMkFile := func(path string) { - mklines := LoadMk(path, 0) - if mklines != nil && len(mklines.mklines) > 0 { - mklines.collectDefinedVariables() - mklines.collectUsedVariables() - for varname, mkline := range mklines.vars.firstDef { - define(varnameCanon(varname), mkline) - } - for varname, mkline := range mklines.vars.used { - define(varnameCanon(varname), mkline) - } + mklines := LoadMk(path, MustSucceed) + mklines.collectDefinedVariables() + mklines.collectUsedVariables() + for varname, mkline := range mklines.vars.firstDef { + define(varnameCanon(varname), mkline) + } + for varname, mkline := range mklines.vars.used { + define(varnameCanon(varname), mkline) } } handleFile := func(pathName string, info os.FileInfo, err error) error { assertNil(err, "handleFile %q", pathName) baseName := info.Name() - if hasSuffix(baseName, ".mk") || baseName == "mk.conf" { + if info.Mode().IsRegular() && (hasSuffix(baseName, ".mk") || baseName == "mk.conf") { handleMkFile(filepath.ToSlash(pathName)) } return nil @@ -402,7 +393,7 @@ func (src *Pkgsrc) loadUntypedVars() { assertNil(err, "Walk error in pkgsrc infrastructure") } -func (src *Pkgsrc) parseSuggestedUpdates(lines Lines) []SuggestedUpdate { +func (src *Pkgsrc) parseSuggestedUpdates(lines *Lines) []SuggestedUpdate { if lines == nil { return nil } @@ -444,79 +435,77 @@ func (src *Pkgsrc) loadSuggestedUpdates() { src.suggestedWipUpdates = src.parseSuggestedUpdates(Load(src.File("wip/TODO"), NotEmpty)) } -func (src *Pkgsrc) loadDocChangesFromFile(filename string) []*Change { - - warn := !G.Wip - - parseChange := func(line Line) *Change { - lex := textproc.NewLexer(line.Text) - - space := lex.NextHspace() - if space == "" { - return nil - } - - if space != "\t" { - if warn { - line.Warnf("Package changes should be indented using a single tab, not %q.", space) - line.Explain( - "To avoid this formatting mistake in the future, just run", - sprintf("%q", bmake("cce")), - "after committing the update to the package.") - } +func (*Pkgsrc) parseDocChange(line *Line, warn bool) *Change { + lex := textproc.NewLexer(line.Text) - return nil - } + space := lex.NextHspace() + if space == "" { + return nil + } - f := strings.Fields(lex.Rest()) - n := len(f) - if n != 4 && n != 6 { - return nil + if space != "\t" { + if warn { + line.Warnf("Package changes should be indented using a single tab, not %q.", space) + line.Explain( + "To avoid this formatting mistake in the future, just run", + sprintf("%q", bmake("cce")), + "after committing the update to the package.") } - action, pkgpath, author, date := f[0], f[1], f[len(f)-2], f[len(f)-1] - if !hasPrefix(author, "[") || !hasSuffix(date, "]") { - return nil - } - author, date = author[1:], date[:len(date)-1] - - newChange := func(version string) *Change { - return &Change{ - Location: line.Location, - Action: intern(action), - Pkgpath: intern(pkgpath), - Version: intern(version), - Author: intern(author), - Date: intern(date), - } - } + return nil + } - switch { - case action == "Added" && f[2] == "version" && n == 6: - return newChange(f[3]) + f := strings.Fields(lex.Rest()) + n := len(f) + if n != 4 && n != 6 { + return nil + } - case (action == "Updated" || action == "Downgraded") && f[2] == "to" && n == 6: - return newChange(f[3]) + action := ParseChangeAction(f[0]) + pkgpath := f[1] + author := f[len(f)-2] + date := f[len(f)-1] - case action == "Removed" && (n == 6 && f[2] == "successor" || n == 4): - return newChange("") + if !hasPrefix(author, "[") || !hasSuffix(date, "]") { + return nil + } + author, date = author[1:], date[:len(date)-1] - case (action == "Renamed" || action == "Moved") && f[2] == "to" && n == 6: - return newChange("") + switch { + case + action == Added && f[2] == "version", + action == Updated && f[2] == "to", + action == Downgraded && f[2] == "to", + action == Removed && (f[2] == "successor" || n == 4), + (action == Renamed || action == Moved) && f[2] == "to": + return &Change{ + Location: line.Location, + Action: action, + Pkgpath: intern(pkgpath), + target: intern(ifelseStr(n == 6, f[3], "")), + Author: intern(author), + Date: intern(date), } + } + if warn { line.Warnf("Unknown doc/CHANGES line: %s", line.Text) line.Explain( "See mk/misc/developer.mk for the rules.") - - return nil } + return nil +} + +func (src *Pkgsrc) loadDocChangesFromFile(filename string) []*Change { + + warn := !G.Wip + // Each date in the file should be from the same year as the filename says. // This check has been added in 2018. // For years earlier than 2018 pkglint doesn't care because it's not a big issue anyway. year := "" - if m, yyyy := match1(filename, `-(\d+)$`); m && yyyy >= "2018" { + if _, yyyy := match1(filename, `-(\d\d\d\d)$`); yyyy >= "2018" { year = yyyy } @@ -527,6 +516,13 @@ func (src *Pkgsrc) loadDocChangesFromFile(filename string) []*Change { if hasPrefix(line.Text, "\tmk/") { infra = true + if hasPrefix(line.Text, "\tmk/bsd.pkg.mk: started freeze for") { + if m, freezeDate := match1(line.Text, `(\d\d\d\d-\d\d-\d\d)\]$`); m { + src.FreezeStart = freezeDate + } + } else if hasPrefix(line.Text, "\tmk/bsd.pkg.mk: freeze ended for") { + src.FreezeStart = "" + } } if infra { if hasSuffix(line.Text, "]") { @@ -535,7 +531,7 @@ func (src *Pkgsrc) loadDocChangesFromFile(filename string) []*Change { continue } - change := parseChange(line) + change := src.parseDocChange(line, warn) if change == nil { continue } @@ -546,14 +542,14 @@ func (src *Pkgsrc) loadDocChangesFromFile(filename string) []*Change { continue } - if year != "" && change.Date[0:4] != year { - line.Warnf("Year %s for %s does not match the filename %s.", + if year != "" && len(change.Date) >= 4 && change.Date[0:4] != year { + line.Warnf("Year %q for %s does not match the filename %s.", change.Date[0:4], change.Pkgpath, filename) } if len(changes) >= 2 && year != "" { if prev := changes[len(changes)-2]; change.Date < prev.Date { - line.Warnf("Date %s for %s is earlier than %s in %s.", + line.Warnf("Date %q for %s is earlier than %q in %s.", change.Date, change.Pkgpath, prev.Date, line.RefToLocation(prev.Location)) line.Explain( "The entries in doc/CHANGES should be in chronological order, and", @@ -597,12 +593,14 @@ func (src *Pkgsrc) loadDocChanges() { } } - sort.Strings(filenames) src.LastChange = make(map[string]*Change) for _, filename := range filenames { changes := src.loadDocChangesFromFile(docDir + "/" + filename) for _, change := range changes { src.LastChange[change.Pkgpath] = change + if change.Action == Renamed || change.Action == Moved { + src.LastChange[change.Target()] = change + } } } } @@ -782,17 +780,18 @@ func (src *Pkgsrc) initDeprecatedVars() { } // Load loads the file relative to the pkgsrc top directory. -func (src *Pkgsrc) Load(filename string, options LoadOptions) Lines { +func (src *Pkgsrc) Load(filename string, options LoadOptions) *Lines { return Load(src.File(filename), options) } // LoadMk loads the Makefile relative to the pkgsrc top directory. -func (src *Pkgsrc) LoadMk(filename string, options LoadOptions) MkLines { +func (src *Pkgsrc) LoadMk(filename string, options LoadOptions) *MkLines { return LoadMk(src.File(filename), options) } -// ReadDir reads the file listing from the given directory (relative to the pkgsrc root), -// filtering out any ignored files (CVS/*) and empty directories. +// ReadDir lists the files and subdirectories from the given directory +// (relative to the pkgsrc root), filtering out any ignored files (CVS/*) +// and empty directories. func (src *Pkgsrc) ReadDir(dirName string) []os.FileInfo { dir := src.File(dirName) files, err := ioutil.ReadDir(dir) @@ -901,7 +900,7 @@ func (src *Pkgsrc) loadPkgOptions() { // VariableType returns the type of the variable // (possibly guessed based on the variable name), // or nil if the type cannot even be guessed. -func (src *Pkgsrc) VariableType(mklines MkLines, varname string) (vartype *Vartype) { +func (src *Pkgsrc) VariableType(mklines *MkLines, varname string) (vartype *Vartype) { if trace.Tracing { defer trace.Call(varname, trace.Result(&vartype))() } @@ -976,6 +975,8 @@ func (src *Pkgsrc) guessVariableType(varname string) (vartype *Vartype) { return listType(BtCFlag, aclpAllRuntime) case hasSuffix(varname, "_LDFLAGS"): return listType(BtLdFlag, aclpAllRuntime) + case hasSuffix(varname, "FLAGS"): + return listType(BtShellWord, aclpAll) case hasSuffix(varbase, "_MK"): // TODO: Add BtGuard for inclusion guards, since these variables may only be checked using defined(). return plainType(BtUnknown, aclpAll) @@ -1005,13 +1006,64 @@ func (src *Pkgsrc) guessVariableType(varname string) (vartype *Vartype) { // Change describes a modification to a single package, from the doc/CHANGES-* files. type Change struct { Location Location - Action string - Pkgpath string - Version string + Action ChangeAction // Added, Updated, Downgraded, Renamed, Moved, Removed + Pkgpath string // For renamed or moved packages, the previous PKGPATH + target string Author string Date string } +// Version returns the version number for an Added, Updated or Downgraded package. +func (ch *Change) Version() string { + assert(ch.Action == Added || ch.Action == Updated || ch.Action == Downgraded) + return ch.target +} + +// Target returns the target PKGPATH for a Renamed or Moved package. +func (ch *Change) Target() string { + assert(ch.Action == Renamed || ch.Action == Moved) + return ch.target +} + +// Successor returns the successor for a Removed package. +func (ch *Change) Successor() string { + assert(ch.Action == Removed) + return ch.target +} + +type ChangeAction uint8 + +const ( + Added ChangeAction = 1 + iota + Updated + Downgraded + Renamed + Moved + Removed +) + +func ParseChangeAction(s string) ChangeAction { + switch s { + case "Added": + return Added + case "Updated": + return Updated + case "Downgraded": + return Downgraded + case "Renamed": + return Renamed + case "Moved": + return Moved + case "Removed": + return Removed + } + return 0 +} + +func (ca ChangeAction) String() string { + return [...]string{"", "Added", "Updated", "Downgraded", "Renamed", "Moved", "Removed"}[ca] +} + // SuggestedUpdate describes a desired package update, from the doc/TODO file. type SuggestedUpdate struct { Line Location diff --git a/pkgtools/pkglint/files/pkgsrc_test.go b/pkgtools/pkglint/files/pkgsrc_test.go index e61c672de77..a57fcc7d0c0 100644 --- a/pkgtools/pkglint/files/pkgsrc_test.go +++ b/pkgtools/pkglint/files/pkgsrc_test.go @@ -10,12 +10,17 @@ func (s *Suite) Test_Pkgsrc_loadMasterSites(c *check.C) { t := s.Init(c) t.CreateFileLines("mk/fetch/sites.mk", - MkRcsID, + MkCvsID, "", "MASTER_SITE_A+= https://example.org/distfiles/", "MASTER_SITE_B+= https://b.example.org/distfiles/ \\", " https://b2.example.org/distfiles/", - "MASTER_SITE_A+= https://a.example.org/distfiles/") + "MASTER_SITE_A+= https://a.example.org/distfiles/ ${other}", + "", + "MASTER_SITE_BACKUP+=\t", + "\thttps://backup.example.org/", + "", + "OTHER_VARIABLE=\tyes # only for code coverage") G.Pkgsrc.loadMasterSites() @@ -25,6 +30,11 @@ func (s *Suite) Test_Pkgsrc_loadMasterSites(c *check.C) { c.Check(G.Pkgsrc.MasterSiteURLToVar["https://a.example.org/distfiles/"], equals, "MASTER_SITE_A") c.Check(G.Pkgsrc.MasterSiteVarToURL["MASTER_SITE_A"], equals, "https://example.org/distfiles/") c.Check(G.Pkgsrc.MasterSiteVarToURL["MASTER_SITE_B"], equals, "https://b.example.org/distfiles/") + + // Ignored entries: + c.Check(G.Pkgsrc.MasterSiteURLToVar["${other}"], equals, "") + c.Check(G.Pkgsrc.MasterSiteURLToVar["https://backup.example.org/"], equals, "") + c.Check(G.Pkgsrc.MasterSiteVarToURL["MASTER_SITE_BACKUP"], equals, "") } func (s *Suite) Test_Pkgsrc_parseSuggestedUpdates(c *check.C) { @@ -57,26 +67,30 @@ func (s *Suite) Test_Pkgsrc_checkToplevelUnusedLicenses(c *check.C) { t.CreateFileLines("licenses/gnu-gpl-v3") t.CreateFileLines("Makefile", - MkRcsID, + MkCvsID, "SUBDIR+=\tcategory") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "COMMENT=\tExample category", "", "SUBDIR+=\tpackage", + "SUBDIR+=\tpackage2", "", ".include \"../mk/misc/category.mk\"") t.SetUpPackage("category/package", "LICENSE=\t2-clause-bsd") + t.SetUpPackage("category/package2", + "LICENSE=\tmissing") t.Main("-r", "-Cglobal", t.File(".")) t.CheckOutputLines( + "WARN: ~/category/package2/Makefile:11: License file ~/licenses/missing does not exist.", "WARN: ~/licenses/gnu-gpl-v2: This license seems to be unused.", // Added by Tester.SetUpPkgsrc "WARN: ~/licenses/gnu-gpl-v3: This license seems to be unused.", - "0 errors and 2 warnings found.") + "0 errors and 3 warnings found.") } func (s *Suite) Test_Pkgsrc_loadUntypedVars(c *check.C) { @@ -85,7 +99,7 @@ func (s *Suite) Test_Pkgsrc_loadUntypedVars(c *check.C) { t.SetUpPkgsrc() t.SetUpTool("echo", "ECHO", AtRunTime) t.CreateFileLines("mk/infra.mk", - MkRcsID, + MkCvsID, "#", "# System-provided variables:", "#", @@ -109,7 +123,7 @@ func (s *Suite) Test_Pkgsrc_loadUntypedVars(c *check.C) { t.FinishSetUp() mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "", "do-build:", "\t: ${INFRA_MK} ${UNTYPED.three} ${ECHO}", @@ -125,12 +139,25 @@ func (s *Suite) Test_Pkgsrc_loadUntypedVars(c *check.C) { "WARN: filename.mk:6: INDIRECT_param is used but not defined.") } +func (s *Suite) Test_Pkgsrc_loadUntypedVars__badly_named_directory(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("mk/subdir.mk/file.mk", + MkCvsID) + t.FinishSetUp() + + // Even when a directory is named *.mk, pkglint doesn't crash. + t.CheckOutputEmpty() +} + func (s *Suite) Test_Pkgsrc_loadTools(c *check.C) { t := s.Init(c) t.CreateFileLines("mk/tools/bsd.tools.mk", ".include \"flex.mk\"", ".include \"gettext.mk\"", + ".include \"../nonexistent.mk\"", // Is skipped because of the slash. ".include \"strip.mk\"", ".include \"replace.mk\"") t.CreateFileLines("mk/tools/defaults.mk", @@ -197,10 +224,10 @@ func (s *Suite) Test_Pkgsrc_loadTools__BUILD_DEFS(c *check.C) { "pre-configure:", "\t@${ECHO} ${PKG_SYSCONFDIR} ${VARBASE}") t.CreateFileLines("mk/bsd.pkg.mk", - MkRcsID, + MkCvsID, "_BUILD_DEFS+=\tPKG_SYSCONFBASEDIR PKG_SYSCONFDIR") t.CreateFileLines("mk/defaults/mk.conf", - MkRcsID, + MkCvsID, "", "VARBASE=\t\t/var/pkg", "PKG_SYSCONFBASEDIR=\t/usr/pkg/etc", @@ -217,6 +244,21 @@ func (s *Suite) Test_Pkgsrc_loadTools__BUILD_DEFS(c *check.C) { "The user-defined variable VARBASE is used but not added to BUILD_DEFS.") } +func (s *Suite) Test_Pkgsrc_loadDocChanges(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("doc/CHANGES-2018", + CvsID, + "", + "\tUpdated pkgpath to 1.0 [author 2018-01-01]", + "\tRenamed pkgpath to new-pkg [author 2018-02-01]", + "\tMoved pkgpath to category/new-pkg [author 2018-03-01]") + t.FinishSetUp() + + t.Check(G.Pkgsrc.LastChange["pkgpath"].Action, equals, Moved) +} + func (s *Suite) Test_Pkgsrc_loadDocChanges__not_found(c *check.C) { t := s.Init(c) @@ -241,6 +283,7 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile(c *check.C) { "\tRemoved category/package [author5 2018-01-09]", // Too far in the future "\tRemoved category/package successor category/package2 [author6 2018-01-06]", "\tDowngraded category/package to 1.2 [author7 2018-01-07]", + "\tReworked category/package to 1.2 [author8 2018-01-08]", "", "\ttoo few fields", "\ttoo many many many many many fields", @@ -251,24 +294,25 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile(c *check.C) { c.Assert(len(changes), equals, 7) c.Check(*changes[0], equals, Change{changes[0].Location, - "Added", "category/package", "1.0", "author1", "2015-01-01"}) + Added, "category/package", "1.0", "author1", "2015-01-01"}) c.Check(*changes[1], equals, Change{changes[1].Location, - "Updated", "category/package", "1.5", "author2", "2018-01-02"}) + Updated, "category/package", "1.5", "author2", "2018-01-02"}) c.Check(*changes[2], equals, Change{changes[2].Location, - "Renamed", "category/package", "", "author3", "2018-01-03"}) + Renamed, "category/package", "category/pkg", "author3", "2018-01-03"}) c.Check(*changes[3], equals, Change{changes[3].Location, - "Moved", "category/package", "", "author4", "2018-01-04"}) + Moved, "category/package", "other/package", "author4", "2018-01-04"}) c.Check(*changes[4], equals, Change{changes[4].Location, - "Removed", "category/package", "", "author5", "2018-01-09"}) + Removed, "category/package", "", "author5", "2018-01-09"}) c.Check(*changes[5], equals, Change{changes[5].Location, - "Removed", "category/package", "", "author6", "2018-01-06"}) + Removed, "category/package", "category/package2", "author6", "2018-01-06"}) c.Check(*changes[6], equals, Change{changes[6].Location, - "Downgraded", "category/package", "1.2", "author7", "2018-01-07"}) + Downgraded, "category/package", "1.2", "author7", "2018-01-07"}) t.CheckOutputLines( - "WARN: ~/doc/CHANGES-2018:1: Year 2015 for category/package does not match the filename ~/doc/CHANGES-2018.", - "WARN: ~/doc/CHANGES-2018:6: Date 2018-01-06 for category/package is earlier than 2018-01-09 in line 5.", - "WARN: ~/doc/CHANGES-2018:12: Unknown doc/CHANGES line: \tAdded another [new package]") + "WARN: ~/doc/CHANGES-2018:1: Year \"2015\" for category/package does not match the filename ~/doc/CHANGES-2018.", + "WARN: ~/doc/CHANGES-2018:6: Date \"2018-01-06\" for category/package is earlier than \"2018-01-09\" in line 5.", + "WARN: ~/doc/CHANGES-2018:8: Unknown doc/CHANGES line: \tReworked category/package to 1.2 [author8 2018-01-08]", + "WARN: ~/doc/CHANGES-2018:13: Unknown doc/CHANGES line: \tAdded another [new package]") } func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__not_found(c *check.C) { @@ -287,12 +331,14 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__wip_suppresses_warnings(c *c t.SetUpPackage("wip/package") t.CreateFileLines("doc/CHANGES-2018", - RcsID, + CvsID, "", "Changes to the packages collection and infrastructure in 2018:", "", "\tUpdated sysutils/checkperms to 1.10 [rillig 2018-01-05]", - "\tUpdated sysutils/checkperms to 1.11 [rillig 2018-01-01]") + "\tUpdated sysutils/checkperms to 1.11 [rillig 2018-01-01]", + "\t\tWrong indentation", + "\tInvalid pkgpath to 1.16 [rillig 2019-06-16]") t.Main(t.File("wip/package")) @@ -305,7 +351,7 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__wrong_indentation(c *check.C t.SetUpPackage("category/package") t.CreateFileLines("doc/CHANGES-2018", - RcsID, + CvsID, "", "Changes to the packages collection and infrastructure in 2018:", "", @@ -328,7 +374,7 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__infrastructure(c *check.C) { t.SetUpPackage("category/package") t.CreateFileLines("doc/CHANGES-2018", - RcsID, + CvsID, "", "Changes to the packages collection and infrastructure in 2018:", "", @@ -345,13 +391,135 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__infrastructure(c *check.C) { "Looks fine.") } +func (s *Suite) Test_Pkgsrc_parseDocChange(c *check.C) { + t := s.Init(c) + + test := func(text string, diagnostics ...string) { + line := t.NewLine("doc/CHANGES-2019", 123, text) + _ = (*Pkgsrc)(nil).parseDocChange(line, true) + t.CheckOutput(diagnostics) + } + + test(CvsID, + nil...) + test("", + nil...) + test("Changes to the packages collection and infrastructure in 2019:", + nil...) + + test("\tAdded something [author date]", + "WARN: doc/CHANGES-2019:123: Unknown doc/CHANGES line: \tAdded something [author date]") + + test("\t\tToo large indentation", + "WARN: doc/CHANGES-2019:123: Package changes should be indented using a single tab, not \"\\t\\t\".") + test("\t Too large indentation", + "WARN: doc/CHANGES-2019:123: Package changes should be indented using a single tab, not \"\\t \".") + + // TODO: Add a warning here, since it's easy to forget a bracket. + test("\t1 2 3 4", + nil...) + test("\t1 2 3 4 5", + nil...) + test("\t1 2 3 4 5 6", + nil...) + test("\t1 2 3 4 5 6 7", + nil...) + test("\t1 2 [3 4", + nil...) + test("\t1 2 [3 4]", + "WARN: doc/CHANGES-2019:123: Unknown doc/CHANGES line: \t1 2 [3 4]") + test("\tAdded 2 [3 4]", + "WARN: doc/CHANGES-2019:123: Unknown doc/CHANGES line: \tAdded 2 [3 4]") + + test("\tAdded pkgpath version 1.0 [author date]", + nil...) + // "to" is wrong + test("\tAdded pkgpath to 1.0 [author date]", + "WARN: doc/CHANGES-2019:123: Unknown doc/CHANGES line: \tAdded pkgpath to 1.0 [author date]") + + test("\tUpdated pkgpath to 1.0 [author date]", + nil...) + // "from" is wrong + test("\tUpdated pkgpath from 1.0 [author date]", + "WARN: doc/CHANGES-2019:123: Unknown doc/CHANGES line: \tUpdated pkgpath from 1.0 [author date]") + + test("\tDowngraded pkgpath to 1.0 [author date]", + nil...) + // "from" is wrong + test("\tDowngraded pkgpath from 1.0 [author date]", + "WARN: doc/CHANGES-2019:123: Unknown doc/CHANGES line: \tDowngraded pkgpath from 1.0 [author date]") + + test("\tRemoved pkgpath [author date]", + nil...) + test("\tRemoved pkgpath successor pkgpath [author date]", + nil...) + // "and" is wrong + test("\tRemoved pkgpath and pkgpath [author date]", + "WARN: doc/CHANGES-2019:123: Unknown doc/CHANGES line: \tRemoved pkgpath and pkgpath [author date]") + + test("\tRenamed pkgpath to other [author date]", + nil...) + // "from" is wrong + test("\tRenamed pkgpath from previous [author date]", + "WARN: doc/CHANGES-2019:123: Unknown doc/CHANGES line: \tRenamed pkgpath from previous [author date]") + + test("\tMoved pkgpath to other [author date]", + nil...) + // "from" is wrong + test("\tMoved pkgpath from previous [author date]", + "WARN: doc/CHANGES-2019:123: Unknown doc/CHANGES line: \tMoved pkgpath from previous [author date]") + + // "Split" is wrong + // TODO: Add a warning since this is probably a typo. + test("\tSplit pkgpath into a and b [author date]", + nil...) +} + +func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__old(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("doc/CHANGES-2010", + CvsID, + "", + "Changes to the packages collection and infrastructure in 2015:", + "", + "\tInvalid line [3 4]") + t.CreateFileLines("doc/CHANGES-2015", + CvsID, + "", + "Changes to the packages collection and infrastructure in 2015:", + "", + "\tUpdated pkgpath to 1.0 [author 2015-07-01]", + "\tInvalid line [3 4]", + // The date of the below entry is earlier than that of the above entry; + // this error is ignored because the 2015 file is too old. + "\tUpdated pkgpath to 1.2 [author 2015-02-01]") + t.CreateFileLines("doc/CHANGES-2018", + CvsID, + "", + "Changes to the packages collection and infrastructure in 2018:", + "", + "\tUpdated pkgpath to 1.0 [author date]", + "\tUpdated pkgpath to 1.0 [author d]") + t.FinishSetUp() + + // The 2010 file is so old that it is skipped completely. + // The 2015 file is so old that the date is not checked. + // Since 2018, each date in the file must match the filename. + t.CheckOutputLines( + "WARN: ~/doc/CHANGES-2015:6: Unknown doc/CHANGES line: \tInvalid line [3 4]", + "WARN: ~/doc/CHANGES-2018:5: Year \"date\" for pkgpath does not match the filename ~/doc/CHANGES-2018.", + "WARN: ~/doc/CHANGES-2018:6: Date \"d\" for pkgpath is earlier than \"date\" in line 5.") +} + func (s *Suite) Test_Pkgsrc_parseSuggestedUpdates__wip(c *check.C) { t := s.Init(c) pkg := t.SetUpPackage("wip/package", "DISTNAME=\tpackage-1.11") t.CreateFileLines("wip/TODO", - RcsID, + CvsID, "", "Suggested package updates", "", @@ -372,7 +540,7 @@ func (s *Suite) Test_Pkgsrc__deprecated(c *check.C) { t.SetUpVartypes() G.Pkgsrc.initDeprecatedVars() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "USE_PERL5=\t\tyes", "SUBST_POSTCMD.class=\t${ECHO}", "CPPFLAGS+=\t\t${BUILDLINK_CPPFLAGS.${PKG_JVM}}") @@ -576,9 +744,13 @@ func (s *Suite) Test_Pkgsrc_ListVersions__go(c *check.C) { func (s *Suite) Test_Pkgsrc_ListVersions__invalid_argument(c *check.C) { t := s.Init(c) - t.ExpectPanic( - func() { G.Pkgsrc.ListVersions("databases", `postgresql[0-9]+`, "$0", true) }, - "Pkglint internal error: Regular expression \"postgresql[0-9]+\" must be anchored at both ends.") + t.ExpectAssert(func() { G.Pkgsrc.ListVersions("databases", `postgresql[0-9]+`, "$0", true) }) + t.ExpectAssert(func() { G.Pkgsrc.ListVersions("databases", `^postgresql[0-9]+`, "$0", true) }) + + G.Testing = false + versions := G.Pkgsrc.ListVersions("databases", `^postgresql[0-9]+`, "$0", false) + + t.Check(versions, check.HasLen, 0) } func (s *Suite) Test_Pkgsrc_loadPkgOptions(c *check.C) { @@ -612,7 +784,7 @@ func (s *Suite) Test_Pkgsrc_loadTools__no_tools_found(c *check.C) { "FATAL: ~/mk/tools/bsd.tools.mk: Must not be empty.") t.CreateFileLines("mk/tools/bsd.tools.mk", - MkRcsID) + MkCvsID) t.ExpectFatal( G.Pkgsrc.loadTools, @@ -624,6 +796,7 @@ func (s *Suite) Test_Pkgsrc_VariableType(c *check.C) { t := s.Init(c) t.SetUpVartypes() + t.SetUpTool("echo", "ECHO", AtRunTime) test := func(varname string, vartype string) { actualType := G.Pkgsrc.VariableType(nil, varname) @@ -647,6 +820,8 @@ func (s *Suite) Test_Pkgsrc_VariableType(c *check.C) { test("MY_CMD_CFLAGS", "CFlag (list, guessed)") test("MY_CMD_LDFLAGS", "LdFlag (list, guessed)") test("PLIST.abcde", "Yes (package-settable)") + test("TOOLS_ECHO", "Pathname") + test("TOOLS_UNKNOWN", "") } // Guessing the variable type works for both plain and parameterized variable names. @@ -678,7 +853,7 @@ func (s *Suite) Test_Pkgsrc_VariableType__from_mk(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("mk/sys-vars.mk", - MkRcsID, + MkCvsID, "", "PKGSRC_MAKE_ENV?=\t# none", "CPPPATH?=\tcpp", @@ -717,7 +892,7 @@ func (s *Suite) Test_Pkgsrc_guessVariableType__SKIP(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "MY_CHECK_SKIP=\t*.c \"bad*pathname\"", "MY_CHECK_SKIP+=\t*.cpp", ".if ${MY_CHECK_SKIP}", @@ -741,3 +916,106 @@ func (s *Suite) Test_Pkgsrc_guessVariableType__SKIP(c *check.C) { "WARN: filename.mk:2: The pathname pattern \"\\\"bad*pathname\\\"\" " + "contains the invalid characters \"\\\"\\\"\".") } + +func (s *Suite) Test_Pkgsrc__frozen(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("doc/CHANGES-2018", + "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2018Q2 branch [freezer 2018-03-25]") + t.FinishSetUp() + + t.Check(G.Pkgsrc.FreezeStart, equals, "2018-03-25") +} + +func (s *Suite) Test_Pkgsrc__not_frozen(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("doc/CHANGES-2018", + "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2018Q2 branch [freezer 2018-03-25]", + "\tmk/bsd.pkg.mk: freeze ended for pkgsrc-2018Q2 branch [freezer 2018-03-27]") + t.FinishSetUp() + + t.Check(G.Pkgsrc.FreezeStart, equals, "") +} + +func (s *Suite) Test_Pkgsrc__frozen_with_typo(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("doc/CHANGES-2018", + // The closing bracket is missing. + "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2018Q2 branch [freezer 2018-03-25") + t.FinishSetUp() + + t.Check(G.Pkgsrc.FreezeStart, equals, "") +} + +func (s *Suite) Test_Change_Version(c *check.C) { + t := s.Init(c) + + loc := Location{"doc/CHANGES-2019", 5, 5} + added := Change{loc, Added, "category/path", "1.0", "author", "2019-01-01"} + updated := Change{loc, Updated, "category/path", "1.0", "author", "2019-01-01"} + downgraded := Change{loc, Downgraded, "category/path", "1.0", "author", "2019-01-01"} + removed := Change{loc, Removed, "category/path", "1.0", "author", "2019-01-01"} + + t.Check(added.Version(), equals, "1.0") + t.Check(updated.Version(), equals, "1.0") + t.Check(downgraded.Version(), equals, "1.0") + t.ExpectAssert(func() { removed.Version() }) +} + +func (s *Suite) Test_Change_Target(c *check.C) { + t := s.Init(c) + + loc := Location{"doc/CHANGES-2019", 5, 5} + renamed := Change{loc, Renamed, "category/path", "category/other", "author", "2019-01-01"} + moved := Change{loc, Moved, "category/path", "category/other", "author", "2019-01-01"} + downgraded := Change{loc, Downgraded, "category/path", "1.0", "author", "2019-01-01"} + + t.Check(renamed.Target(), equals, "category/other") + t.Check(moved.Target(), equals, "category/other") + t.ExpectAssert(func() { downgraded.Target() }) +} + +func (s *Suite) Test_Change_Successor(c *check.C) { + t := s.Init(c) + + loc := Location{"doc/CHANGES-2019", 5, 5} + removed := Change{loc, Removed, "category/path", "", "author", "2019-01-01"} + removedSucc := Change{loc, Removed, "category/path", "category/successor", "author", "2019-01-01"} + downgraded := Change{loc, Downgraded, "category/path", "1.0", "author", "2019-01-01"} + + t.Check(removed.Successor(), equals, "") + t.Check(removedSucc.Successor(), equals, "category/successor") + t.ExpectAssert(func() { downgraded.Successor() }) +} + +func (s *Suite) Test_ChangeAction_String(c *check.C) { + t := s.Init(c) + + t.Check(Added.String(), equals, "Added") + t.Check(Removed.String(), equals, "Removed") +} + +func (s *Suite) Test_Pkgsrc_ReadDir(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("dir/aaa-subdir/file") + t.CreateFileLines("dir/subdir/file") + t.CreateFileLines("dir/file") + t.CreateFileLines("dir/.git/file") + t.CreateFileLines("dir/CVS/Entries") + t.CreateFileLines("dir/empty/empty/empty/empty/CVS/Entries") + + infos := G.Pkgsrc.ReadDir("dir") + + var names []string + for _, info := range infos { + names = append(names, info.Name()) + } + + t.Check(names, deepEquals, []string{"aaa-subdir", "file", "subdir"}) +} diff --git a/pkgtools/pkglint/files/plist.go b/pkgtools/pkglint/files/plist.go index 09c17fc2737..da941944e0d 100644 --- a/pkgtools/pkglint/files/plist.go +++ b/pkgtools/pkglint/files/plist.go @@ -7,14 +7,14 @@ import ( "strings" ) -func CheckLinesPlist(pkg *Package, lines Lines) { +func CheckLinesPlist(pkg *Package, lines *Lines) { if trace.Tracing { - defer trace.Call1(lines.FileName)() + defer trace.Call1(lines.Filename)() } - lines.CheckRcsID(0, `@comment `, "@comment ") + idOk := lines.CheckCvsID(0, `@comment `, "@comment ") - if lines.Len() == 1 { + if idOk && lines.Len() == 1 { line := lines.Lines[0] line.Warnf("PLIST files shouldn't be empty.") line.Explain( @@ -26,6 +26,7 @@ func CheckLinesPlist(pkg *Package, lines Lines) { "", "Meta packages also don't need a PLIST file", "since their only purpose is to declare dependencies.") + return } ck := PlistChecker{ @@ -48,12 +49,12 @@ type PlistChecker struct { } type PlistLine struct { - Line + *Line conditions []string // e.g. PLIST.docs text string // Line.Text without any conditions of the form ${PLIST.cond} } -func (ck *PlistChecker) Check(plainLines Lines) { +func (ck *PlistChecker) Check(plainLines *Lines) { plines := ck.NewLines(plainLines) ck.collectFilesAndDirs(plines) @@ -77,7 +78,7 @@ func (ck *PlistChecker) Check(plainLines Lines) { } } -func (ck *PlistChecker) NewLines(lines Lines) []*PlistLine { +func (ck *PlistChecker) NewLines(lines *Lines) []*PlistLine { plines := make([]*PlistLine, lines.Len()) for i, line := range lines.Lines { var conditions []string @@ -306,28 +307,14 @@ func (ck *PlistChecker) checkPathInfo(pline *PlistLine, dirname, basename string } func (ck *PlistChecker) checkPathLib(pline *PlistLine, dirname, basename string) { - pkg := ck.pkg switch { - case pkg != nil && pkg.EffectivePkgbase != "" && hasPrefix(pline.text, "lib/"+pkg.EffectivePkgbase+"/"): - return - - case pline.text == "lib/charset.alias" && (pkg == nil || pkg.Pkgpath != "converters/libiconv"): - pline.Errorf("Only the libiconv package may install lib/charset.alias.") - return case hasPrefix(pline.text, "lib/locale/"): pline.Errorf("\"lib/locale\" must not be listed. Use ${PKGLOCALEDIR}/locale and set USE_PKGLOCALEDIR instead.") return } - switch ext := path.Ext(basename); ext { - case ".la": - if pkg != nil && !pkg.vars.Defined("USE_LIBTOOL") && ck.once.FirstTime("USE_LIBTOOL") { - pline.Warnf("Packages that install libtool libraries should define USE_LIBTOOL.") - } - } - if contains(basename, ".a") || contains(basename, ".so") { if m, noext := match1(pline.text, `^(.*)(?:\.a|\.so[0-9.]*)$`); m { if laLine := ck.allFiles[noext+".la"]; laLine != nil { @@ -335,6 +322,21 @@ func (ck *PlistChecker) checkPathLib(pline *PlistLine, dirname, basename string) } } } + + pkg := ck.pkg + if pkg == nil { + return + } + + if pline.text == "lib/charset.alias" && pkg.Pkgpath != "converters/libiconv" { + pline.Errorf("Only the libiconv package may install lib/charset.alias.") + } + + if hasSuffix(basename, ".la") && !pkg.vars.Defined("USE_LIBTOOL") { + if ck.once.FirstTime("USE_LIBTOOL") { + pline.Warnf("Packages that install libtool libraries should define USE_LIBTOOL.") + } + } } func (ck *PlistChecker) checkPathMan(pline *PlistLine) { @@ -380,43 +382,12 @@ func (ck *PlistChecker) checkPathShare(pline *PlistLine) { text := pline.text switch { - case hasPrefix(text, "share/icons/") && pkg != nil: - if hasPrefix(text, "share/icons/hicolor/") && pkg.Pkgpath != "graphics/hicolor-icon-theme" { - f := "../../graphics/hicolor-icon-theme/buildlink3.mk" - if !pkg.included.Seen(f) && ck.once.FirstTime("hicolor-icon-theme") { - pline.Errorf("Packages that install hicolor icons must include %q in the Makefile.", f) - } - } - - if text == "share/icons/hicolor/icon-theme.cache" && pkg.Pkgpath != "graphics/hicolor-icon-theme" { - pline.Errorf("The file icon-theme.cache must not appear in any PLIST file.") - pline.Explain( - "Remove this line and add the following line to the package Makefile.", - "", - ".include \"../../graphics/hicolor-icon-theme/buildlink3.mk\"") - } - - if hasPrefix(text, "share/icons/gnome") && pkg.Pkgpath != "graphics/gnome-icon-theme" { - f := "../../graphics/gnome-icon-theme/buildlink3.mk" - if !pkg.included.Seen(f) { - pline.Errorf("The package Makefile must include %q.", f) - pline.Explain( - "Packages that install GNOME icons must maintain the icon theme", - "cache.") - } - } - - if contains(text[12:], "/") && !pkg.vars.Defined("ICON_THEMES") && ck.once.FirstTime("ICON_THEMES") { - pline.Warnf("Packages that install icon theme files should set ICON_THEMES.") - } + case pkg != nil && hasPrefix(text, "share/icons/"): + ck.checkPathShareIcons(pline) case hasPrefix(text, "share/doc/html/"): pline.Warnf("Use of \"share/doc/html\" is deprecated. Use \"share/doc/${PKGBASE}\" instead.") - case pkg != nil && pkg.EffectivePkgbase != "" && (hasPrefix(text, "share/doc/"+pkg.EffectivePkgbase+"/") || - hasPrefix(text, "share/examples/"+pkg.EffectivePkgbase+"/")): - // Fine. - case hasPrefix(text, "share/info/"): pline.Warnf("Info pages should be installed into info/, not share/info/.") pline.Explain( @@ -427,6 +398,40 @@ func (ck *PlistChecker) checkPathShare(pline *PlistLine) { } } +func (ck *PlistChecker) checkPathShareIcons(pline *PlistLine) { + pkg := ck.pkg + text := pline.text + + if hasPrefix(text, "share/icons/hicolor/") && pkg.Pkgpath != "graphics/hicolor-icon-theme" { + f := "../../graphics/hicolor-icon-theme/buildlink3.mk" + if !pkg.included.Seen(f) && ck.once.FirstTime("hicolor-icon-theme") { + pline.Errorf("Packages that install hicolor icons must include %q in the Makefile.", f) + } + } + + if text == "share/icons/hicolor/icon-theme.cache" && pkg.Pkgpath != "graphics/hicolor-icon-theme" { + pline.Errorf("The file icon-theme.cache must not appear in any PLIST file.") + pline.Explain( + "Remove this line and add the following line to the package Makefile.", + "", + ".include \"../../graphics/hicolor-icon-theme/buildlink3.mk\"") + } + + if hasPrefix(text, "share/icons/gnome") && pkg.Pkgpath != "graphics/gnome-icon-theme" { + f := "../../graphics/gnome-icon-theme/buildlink3.mk" + if !pkg.included.Seen(f) { + pline.Errorf("The package Makefile must include %q.", f) + pline.Explain( + "Packages that install GNOME icons must maintain the icon theme", + "cache.") + } + } + + if contains(text[12:], "/") && !pkg.vars.Defined("ICON_THEMES") && ck.once.FirstTime("ICON_THEMES") { + pline.Warnf("Packages that install icon theme files should set ICON_THEMES.") + } +} + func (pline *PlistLine) CheckTrailingWhitespace() { if hasSuffix(pline.text, " ") || hasSuffix(pline.text, "\t") { pline.Errorf("Pkgsrc does not support filenames ending in whitespace.") @@ -499,7 +504,7 @@ type plistLineSorter struct { header []*PlistLine // Does not take part in sorting middle []*PlistLine // Only this part is sorted footer []*PlistLine // Does not take part in sorting, typically contains @exec or @pkgdir - unsortable Line // Some lines are so difficult to sort that only humans can do that + unsortable *Line // Some lines are so difficult to sort that only humans can do that changed bool // Whether the sorting actually changed something autofixed bool // Whether the newly sorted file has been written to disk } @@ -518,7 +523,7 @@ func NewPlistLineSorter(plines []*PlistLine) *plistLineSorter { header := plines[0:headerEnd] middle := plines[headerEnd:footerStart] footer := plines[footerStart:] - var unsortable Line + var unsortable *Line for _, pline := range middle { if unsortable == nil && (hasPrefix(pline.text, "@") || contains(pline.text, "$")) { @@ -564,7 +569,7 @@ func (s *plistLineSorter) Sort() { fix.Describef(int(firstLine.firstLine), "Sorting the whole file.") fix.Apply() - var lines []Line + var lines []*Line for _, pline := range s.header { lines = append(lines, pline.Line) } diff --git a/pkgtools/pkglint/files/plist_test.go b/pkgtools/pkglint/files/plist_test.go index 7340f3e038f..9e68c6b23b7 100644 --- a/pkgtools/pkglint/files/plist_test.go +++ b/pkgtools/pkglint/files/plist_test.go @@ -48,6 +48,18 @@ func (s *Suite) Test_CheckLinesPlist(c *check.C) { "ERROR: PLIST:18: Duplicate filename \"share/tzinfo\", already appeared in line 17.") } +func (s *Suite) Test_CheckLinesPlist__single_file_no_comment(c *check.C) { + t := s.Init(c) + + lines := t.NewLines("PLIST", + "bin/program") + + CheckLinesPlist(nil, lines) + + t.CheckOutputLines( + "ERROR: PLIST:1: Expected \"" + PlistCvsID + "\".") +} + // When a PLIST contains multiple libtool libraries, USE_LIBTOOL needs only // be defined once in the package Makefile. Therefore, a single warning is enough. func (s *Suite) Test_CheckLinesPlist__multiple_libtool_libraries(c *check.C) { @@ -55,7 +67,7 @@ func (s *Suite) Test_CheckLinesPlist__multiple_libtool_libraries(c *check.C) { G.Pkg = NewPackage(t.File("category/pkgbase")) lines := t.NewLines("PLIST", - PlistRcsID, + PlistCvsID, "lib/libc.la", "lib/libm.la") @@ -69,7 +81,7 @@ func (s *Suite) Test_CheckLinesPlist__empty(c *check.C) { t := s.Init(c) lines := t.NewLines("PLIST", - PlistRcsID) + PlistCvsID) CheckLinesPlist(nil, lines) @@ -81,10 +93,22 @@ func (s *Suite) Test_CheckLinesPlist__common_end(c *check.C) { t := s.Init(c) t.CreateFileLines("PLIST.common", - PlistRcsID, + PlistCvsID, "bin/common") lines := t.SetUpFileLines("PLIST.common_end", - PlistRcsID, + PlistCvsID, + "sbin/common_end") + + CheckLinesPlist(nil, lines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_CheckLinesPlist__common_end_without_common(c *check.C) { + t := s.Init(c) + + lines := t.SetUpFileLines("PLIST.common_end", + PlistCvsID, "sbin/common_end") CheckLinesPlist(nil, lines) @@ -97,7 +121,7 @@ func (s *Suite) Test_CheckLinesPlist__condition(c *check.C) { G.Pkg = NewPackage(t.File("category/pkgbase")) lines := t.NewLines("PLIST", - PlistRcsID, + PlistCvsID, "${PLIST.bincmds}bin/subdir/command") CheckLinesPlist(G.Pkg, lines) @@ -110,7 +134,7 @@ func (s *Suite) Test_CheckLinesPlist__sorting(c *check.C) { t := s.Init(c) lines := t.NewLines("PLIST", - PlistRcsID, + PlistCvsID, "@comment Do not remove", "sbin/i386/6c", "sbin/program", @@ -137,7 +161,7 @@ func (s *Suite) Test_plistLineSorter_Sort(c *check.C) { t.SetUpCommandLine("--autofix") lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "@comment Do not remove", "A", "b", @@ -163,7 +187,7 @@ func (s *Suite) Test_plistLineSorter_Sort(c *check.C) { cleanedLines := append(append(lines.Lines[0:5], lines.Lines[6:8]...), lines.Lines[9:]...) // Remove ${UNKNOWN} and @exec sorter2 := NewPlistLineSorter((&PlistChecker{nil, nil, nil, "", Once{}, false}). - NewLines(NewLines(lines.FileName, cleanedLines))) + NewLines(NewLines(lines.Filename, cleanedLines))) c.Check(sorter2.unsortable, check.IsNil) @@ -172,7 +196,7 @@ func (s *Suite) Test_plistLineSorter_Sort(c *check.C) { t.CheckOutputLines( "AUTOFIX: ~/PLIST:3: Sorting the whole file.") t.CheckFileLines("PLIST", - PlistRcsID, + PlistCvsID, "@comment Do not remove", // The header ends here "A", "C", @@ -193,7 +217,7 @@ func (s *Suite) Test_PlistChecker_checkLine(c *check.C) { t := s.Init(c) lines := t.NewLines("PLIST", - PlistRcsID, + PlistCvsID, "bin/program", "${PLIST.var}bin/conditional-program", "${PLIST.linux}${PLIST.arm}bin/arm-linux-only", @@ -227,7 +251,7 @@ func (s *Suite) Test_PlistChecker_checkPathMan__gz(c *check.C) { G.Pkg = NewPackage(t.File("category/pkgbase")) lines := t.NewLines("PLIST", - PlistRcsID, + PlistCvsID, "man/man3/strerror.3.gz") CheckLinesPlist(G.Pkg, lines) @@ -240,7 +264,7 @@ func (s *Suite) Test_PlistChecker_checkPath__PKGMANDIR(c *check.C) { t := s.Init(c) lines := t.NewLines("PLIST", - PlistRcsID, + PlistCvsID, "${PKGMANDIR}/man1/sh.1") CheckLinesPlist(nil, lines) @@ -253,7 +277,7 @@ func (s *Suite) Test_PlistChecker_checkPath__python_egg(c *check.C) { t := s.Init(c) lines := t.NewLines("PLIST", - PlistRcsID, + PlistCvsID, "${PYSITELIB}/gdspy-${PKGVERSION}-py${PYVERSSUFFIX}.egg-info/PKG-INFO") CheckLinesPlist(nil, lines) @@ -266,7 +290,7 @@ func (s *Suite) Test_PlistChecker__autofix(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "lib/libvirt/connection-driver/libvirt_driver_storage.la", "${PLIST.hal}lib/libvirt/connection-driver/libvirt_driver_nodedev.la", "${PLIST.xen}lib/libvirt/connection-driver/libvirt_driver_libxl.la", @@ -304,7 +328,7 @@ func (s *Suite) Test_PlistChecker__autofix(c *check.C) { "AUTOFIX: ~/PLIST:6: Replacing \"${PKGMANDIR}/\" with \"man/\".", "AUTOFIX: ~/PLIST:2: Sorting the whole file.") t.CheckFileLines("PLIST", - PlistRcsID, + PlistCvsID, "${PLIST.xen}lib/libvirt/connection-driver/libvirt_driver_libxl.la", "${PLIST.hal}lib/libvirt/connection-driver/libvirt_driver_nodedev.la", "lib/libvirt/connection-driver/libvirt_driver_storage.la", @@ -334,7 +358,7 @@ func (s *Suite) Test_PlistChecker__remove_same_entries(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "${PLIST.option1}bin/true", "bin/true", "${PLIST.option1}bin/true", @@ -363,7 +387,7 @@ func (s *Suite) Test_PlistChecker__remove_same_entries(c *check.C) { "AUTOFIX: ~/PLIST:8: Deleting this line.", "AUTOFIX: ~/PLIST:2: Sorting the whole file.") t.CheckFileLines("PLIST", - PlistRcsID, + PlistCvsID, "${PLIST.option2}bin/false", "${PLIST.option3}bin/false", "bin/true") @@ -378,7 +402,7 @@ func (s *Suite) Test_PlistChecker__autofix_with_only(c *check.C) { t.SetUpCommandLine("-Wall", "--autofix", "--only", "matches nothing") lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "sbin/program", "bin/program") @@ -386,7 +410,7 @@ func (s *Suite) Test_PlistChecker__autofix_with_only(c *check.C) { t.CheckOutputEmpty() t.CheckFileLines("PLIST", - PlistRcsID, + PlistCvsID, "sbin/program", "bin/program") } @@ -395,7 +419,7 @@ func (s *Suite) Test_PlistChecker__exec_MKDIR(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "bin/program", "@exec ${MKDIR} %D/share/mk/subdir") @@ -408,7 +432,7 @@ func (s *Suite) Test_PlistChecker__empty_line(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "", "bin/program") @@ -424,7 +448,7 @@ func (s *Suite) Test_PlistChecker__empty_line(c *check.C) { t.CheckOutputLines( "AUTOFIX: ~/PLIST:2: Deleting this line.") t.CheckFileLines("PLIST", - PlistRcsID, + PlistCvsID, "bin/program") } @@ -432,7 +456,7 @@ func (s *Suite) Test_PlistChecker__invalid_line_type(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "---invalid", "+++invalid", "<<<<<<<< merge conflict", @@ -454,7 +478,7 @@ func (s *Suite) Test_PlistChecker_checkPathNonAscii(c *check.C) { t.SetUpCommandLine("-Wall", "--explain") lines := t.NewLines("PLIST", - PlistRcsID, + PlistCvsID, "dir1/fr\xFCher", // German, "back then", encoded in ISO 8859-1 @@ -506,7 +530,7 @@ func (s *Suite) Test_PlistChecker__doc(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "doc/html/index.html") CheckLinesPlist(nil, lines) @@ -518,22 +542,52 @@ func (s *Suite) Test_PlistChecker__doc(c *check.C) { func (s *Suite) Test_PlistChecker__PKGLOCALEDIR(c *check.C) { t := s.Init(c) - lines := t.SetUpFileLines("PLIST", - PlistRcsID, + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/PLIST", + PlistCvsID, "${PKGLOCALEDIR}/file") - G.Pkg = NewPackage(t.File("category/package")) + t.FinishSetUp() - CheckLinesPlist(G.Pkg, lines) + G.Check(t.File("category/package")) t.CheckOutputLines( - "WARN: ~/PLIST:2: PLIST contains ${PKGLOCALEDIR}, but USE_PKGLOCALEDIR is not set in the package Makefile.") + "WARN: ~/category/package/PLIST:2: PLIST contains ${PKGLOCALEDIR}, " + + "but USE_PKGLOCALEDIR is not set in the package Makefile.") +} + +func (s *Suite) Test_PlistChecker__PKGLOCALEDIR_with_USE_PKGLOCALEDIR(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "USE_PKGLOCALEDIR=\tyes") + t.CreateFileLines("category/package/PLIST", + PlistCvsID, + "${PKGLOCALEDIR}/file") + t.FinishSetUp() + + G.Check(t.File("category/package")) +} + +func (s *Suite) Test_PlistChecker__PKGLOCALEDIR_without_package(c *check.C) { + t := s.Init(c) + + lines := t.SetUpFileLines("PLIST", + PlistCvsID, + "${PKGLOCALEDIR}/file") + + CheckLinesPlist(nil, lines) + + // When a PLIST file is checked on its own, outside of checking a + // package, there can be no warning that USE_PKGLOCALEDIR is missing + // in the package. + t.CheckOutputEmpty() } func (s *Suite) Test_PlistChecker_checkPath__unwanted_entries(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "share/perllocal.pod", "share/pkgbase/CVS/Entries", "share/pkgbase/Makefile.orig") @@ -549,23 +603,55 @@ func (s *Suite) Test_PlistChecker_checkPath__unwanted_entries(c *check.C) { func (s *Suite) Test_PlistChecker_checkPathInfo(c *check.C) { t := s.Init(c) - lines := t.SetUpFileLines("PLIST", - PlistRcsID, + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.CreateFileLines("PLIST", + PlistCvsID, "info/gmake.1.info") - G.Pkg = NewPackage(t.File("category/package")) + t.FinishSetUp() - CheckLinesPlist(G.Pkg, lines) + G.Check(".") t.CheckOutputLines( - "WARN: ~/PLIST:2: Packages that install info files should set INFO_FILES in the Makefile.") + "WARN: PLIST:2: Packages that install info files should set INFO_FILES in the Makefile.") +} + +func (s *Suite) Test_PlistChecker_checkPathInfo__with_package(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "INFO_FILES=\tyes") + t.Chdir("category/package") + t.CreateFileLines("PLIST", + PlistCvsID, + "info/gmake.1.info") + t.FinishSetUp() + + G.Check(".") + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_PlistChecker_checkPathInfo__without_package(c *check.C) { + t := s.Init(c) + + lines := t.SetUpFileLines("PLIST", + PlistCvsID, + "info/gmake.1.info") + + CheckLinesPlist(nil, lines) + + t.CheckOutputEmpty() } func (s *Suite) Test_PlistChecker_checkPathLib(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "lib/charset.alias", + "lib/liberty-1.0.a", + "lib/liberty-1.0.archive", "lib/liberty-1.0.la", "lib/locale/de_DE/liberty.mo", "lib/package/liberty-1.0.so") @@ -576,34 +662,69 @@ func (s *Suite) Test_PlistChecker_checkPathLib(c *check.C) { t.CheckOutputLines( "ERROR: ~/PLIST:2: Only the libiconv package may install lib/charset.alias.", - "WARN: ~/PLIST:3: Packages that install libtool libraries should define USE_LIBTOOL.", - "ERROR: ~/PLIST:4: \"lib/locale\" must not be listed. "+ + "WARN: ~/PLIST:3: Redundant library found. The libtool library is in line 5.", + "WARN: ~/PLIST:5: Packages that install libtool libraries should define USE_LIBTOOL.", + "ERROR: ~/PLIST:6: \"lib/locale\" must not be listed. "+ "Use ${PKGLOCALEDIR}/locale and set USE_PKGLOCALEDIR instead.") } +func (s *Suite) Test_PlistChecker_checkPathLib__libiconv(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("converters/libiconv") + t.Chdir("converters/libiconv") + t.CreateFileLines("PLIST", + PlistCvsID, + "lib/charset.alias") + t.FinishSetUp() + + G.Check(".") + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_PlistChecker_checkPathLib__libtool(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "USE_LIBTOOL=\tyes") + t.Chdir("category/package") + t.CreateFileLines("PLIST", + PlistCvsID, + "lib/libname.la") + t.FinishSetUp() + + G.Check(".") + + t.CheckOutputEmpty() +} + func (s *Suite) Test_PlistChecker_checkPathMan(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, + "man/cat1/formatted.0", + "man/man1/formatted.1", "man/man1/program.8", "man/manx/program.x") CheckLinesPlist(nil, lines) t.CheckOutputLines( - "WARN: ~/PLIST:2: Mismatch between the section (1) and extension (8) of the manual page.", - "WARN: ~/PLIST:3: Unknown section \"x\" for manual page.") + "WARN: ~/PLIST:4: Mismatch between the section (1) and extension (8) of the manual page.", + "WARN: ~/PLIST:5: Unknown section \"x\" for manual page.") } func (s *Suite) Test_PlistChecker_checkPathShare(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "share/doc/html/package/index.html", "share/doc/package/index.html", "share/icons/hicolor/icon-theme.cache", + "share/icons/open_24x24.svg", "share/info/program.1.info", "share/man/man1/program.1") G.Pkg = NewPackage(t.File("category/package")) @@ -616,11 +737,11 @@ func (s *Suite) Test_PlistChecker_checkPathShare(c *check.C) { "ERROR: ~/PLIST:4: Packages that install hicolor icons must include \"../../graphics/hicolor-icon-theme/buildlink3.mk\" in the Makefile.", "ERROR: ~/PLIST:4: The file icon-theme.cache must not appear in any PLIST file.", "WARN: ~/PLIST:4: Packages that install icon theme files should set ICON_THEMES.", - "WARN: ~/PLIST:5: Info pages should be installed into info/, not share/info/.", - "WARN: ~/PLIST:6: Man pages should be installed into man/, not share/man/.") + "WARN: ~/PLIST:6: Info pages should be installed into info/, not share/info/.", + "WARN: ~/PLIST:7: Man pages should be installed into man/, not share/man/.") } -func (s *Suite) Test_PlistChecker_checkPathShare__gnome_icon_theme(c *check.C) { +func (s *Suite) Test_PlistChecker_checkPathShareIcons__using_gnome_icon_theme(c *check.C) { t := s.Init(c) t.CreateFileDummyBuildlink3("graphics/gnome-icon-theme/buildlink3.mk") @@ -628,7 +749,7 @@ func (s *Suite) Test_PlistChecker_checkPathShare__gnome_icon_theme(c *check.C) { "ICON_THEMES=\tyes", ".include \"../../graphics/gnome-icon-theme/buildlink3.mk\"") t.CreateFileLines("graphics/gnome-icon-theme-extras/PLIST", - PlistRcsID, + PlistCvsID, "share/icons/gnome/16x16/devices/media-optical-cd-audio.png", "share/icons/gnome/16x16/devices/media-optical-dvd.png") t.FinishSetUp() @@ -648,42 +769,126 @@ func (s *Suite) Test_PlistChecker_checkPathShare__gnome_icon_theme(c *check.C) { t.CheckOutputEmpty() } +func (s *Suite) Test_PlistChecker_checkPathShareIcons__gnome_icon_theme_itself(c *check.C) { + t := s.Init(c) + + t.CreateFileDummyBuildlink3("graphics/gnome-icon-theme/buildlink3.mk", + "ICON_THEMES=\tyes") + t.SetUpPackage("graphics/gnome-icon-theme", + ".include \"../../graphics/gnome-icon-theme/buildlink3.mk\"") + t.CreateFileLines("graphics/gnome-icon-theme/PLIST", + PlistCvsID, + "share/icons/gnome/16x16/devices/media-optical-cd-audio.png", + "share/icons/gnome/16x16/devices/media-optical-dvd.png") + t.FinishSetUp() + t.Chdir(".") + + G.Check("graphics/gnome-icon-theme") + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_PlistChecker_checkPathShareIcons(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("graphics/hicolor-icon-theme") + t.CreateFileLines("graphics/hicolor-icon-theme/PLIST", + PlistCvsID, + "share/icons/hicolor/icon-theme.cache", + "share/icons/hicolor/open.svg") + t.SetUpPackage("graphics/other") + t.Copy("graphics/hicolor-icon-theme/PLIST", "graphics/other/PLIST") + t.Chdir(".") + t.FinishSetUp() + + G.Check("graphics/hicolor-icon-theme") + G.Check("graphics/other") + + t.CheckOutputLines( + "WARN: graphics/hicolor-icon-theme/PLIST:2: "+ + "Packages that install icon theme files should set ICON_THEMES.", + "ERROR: graphics/other/PLIST:2: Packages that install hicolor icons "+ + "must include \"../../graphics/hicolor-icon-theme/buildlink3.mk\" in the Makefile.", + "ERROR: graphics/other/PLIST:2: The file icon-theme.cache must not appear in any PLIST file.", + "WARN: graphics/other/PLIST:2: "+ + "Packages that install icon theme files should set ICON_THEMES.") +} + +func (s *Suite) Test_PlistChecker_checkPathShareIcons__hicolor_ok(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../graphics/hicolor-icon-theme/buildlink3.mk\"") + t.CreateFileLines("category/package/PLIST", + PlistCvsID, + "share/icons/hicolor/open.svg") + t.CreateFileLines("graphics/hicolor-icon-theme/buildlink3.mk", + MkCvsID, + "ICON_THEMES=\tyes") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputEmpty() +} + func (s *Suite) Test_PlistLine_CheckTrailingWhitespace(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("PLIST", - PlistRcsID, - "bin/program \t") + PlistCvsID, + "bin/space ", + "bin/space-tab \t", + "bin/tab\t") CheckLinesPlist(nil, lines) t.CheckOutputLines( - "WARN: ~/PLIST:2: Non-ASCII filename \"bin/program \\t\".", - "ERROR: ~/PLIST:2: Pkgsrc does not support filenames ending in whitespace.") + "ERROR: ~/PLIST:2: Pkgsrc does not support filenames ending in whitespace.", + "WARN: ~/PLIST:3: Non-ASCII filename \"bin/space-tab \\t\".", + "ERROR: ~/PLIST:3: Pkgsrc does not support filenames ending in whitespace.", + "ERROR: ~/PLIST:4: Pkgsrc does not support filenames ending in whitespace.") } func (s *Suite) Test_PlistLine_CheckDirective(c *check.C) { t := s.Init(c) lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "@unexec rmdir %D/bin", + "@unexec rmdir %D/bin || true", + "@unexec rmdir %D/bin || ${TRUE}", + "@unexec echo 'uninstalling'", "@exec ldconfig", + "@exec ldconfig || /usr/bin/true", "@comment This is a comment", "@dirrm %D/bin", "@imake-man 1 2 3 4", "@imake-man 1 2 ${IMAKE_MANNEWSUFFIX}", + "@imake-man 1 2 3", "@unknown") CheckLinesPlist(nil, lines) t.CheckOutputLines( "WARN: ~/PLIST:2: Please remove this line. It is no longer necessary.", - "ERROR: ~/PLIST:3: The ldconfig command must be used with \"||/usr/bin/true\".", - "WARN: ~/PLIST:5: @dirrm is obsolete. Please remove this line.", - "WARN: ~/PLIST:6: Invalid number of arguments for imake-man, should be 3.", - "WARN: ~/PLIST:7: IMAKE_MANNEWSUFFIX is not meant to appear in PLISTs.", - "WARN: ~/PLIST:8: Unknown PLIST directive \"@unknown\".") + "ERROR: ~/PLIST:6: The ldconfig command must be used with \"||/usr/bin/true\".", + "WARN: ~/PLIST:9: @dirrm is obsolete. Please remove this line.", + "WARN: ~/PLIST:10: Invalid number of arguments for imake-man, should be 3.", + "WARN: ~/PLIST:11: IMAKE_MANNEWSUFFIX is not meant to appear in PLISTs.", + "WARN: ~/PLIST:13: Unknown PLIST directive \"@unknown\".") +} + +func (s *Suite) Test_NewPlistLineSorter__only_comments(c *check.C) { + t := s.Init(c) + + lines := t.NewLines("PLIST", + PlistCvsID, + "@comment intentionally left empty") + + CheckLinesPlist(nil, lines) + + t.CheckOutputEmpty() } func (s *Suite) Test_plistLineSorter__unsortable(c *check.C) { @@ -691,7 +896,7 @@ func (s *Suite) Test_plistLineSorter__unsortable(c *check.C) { t.SetUpCommandLine("-Wall", "--show-autofix") lines := t.SetUpFileLines("PLIST", - PlistRcsID, + PlistCvsID, "bin/program${OPSYS}", "@exec true", "bin/program1") @@ -701,8 +906,8 @@ func (s *Suite) Test_plistLineSorter__unsortable(c *check.C) { t.CheckOutputLines( "TRACE: + CheckLinesPlist(\"~/PLIST\")", - "TRACE: 1 + (*LinesImpl).CheckRcsID(\"@comment \", \"@comment \")", - "TRACE: 1 - (*LinesImpl).CheckRcsID(\"@comment \", \"@comment \")", + "TRACE: 1 + (*Lines).CheckCvsID(\"@comment \", \"@comment \")", + "TRACE: 1 - (*Lines).CheckCvsID(\"@comment \", \"@comment \")", "TRACE: 1 ~/PLIST:2: bin/program${OPSYS}: This line prevents pkglint from sorting the PLIST automatically.", "TRACE: 1 + SaveAutofixChanges()", "TRACE: 1 - SaveAutofixChanges()", diff --git a/pkgtools/pkglint/files/redundantscope.go b/pkgtools/pkglint/files/redundantscope.go index f40d923dd63..104ad1a9560 100644 --- a/pkgtools/pkglint/files/redundantscope.go +++ b/pkgtools/pkglint/files/redundantscope.go @@ -27,13 +27,13 @@ func NewRedundantScope() *RedundantScope { return &RedundantScope{vars: make(map[string]*redundantScopeVarinfo)} } -func (s *RedundantScope) Check(mklines MkLines) { - mklines.ForEach(func(mkline MkLine) { +func (s *RedundantScope) Check(mklines *MkLines) { + mklines.ForEach(func(mkline *MkLine) { s.checkLine(mklines, mkline) }) } -func (s *RedundantScope) checkLine(mklines MkLines, mkline MkLine) { +func (s *RedundantScope) checkLine(mklines *MkLines, mkline *MkLine) { s.updateIncludePath(mkline) switch { @@ -44,7 +44,7 @@ func (s *RedundantScope) checkLine(mklines MkLines, mkline MkLine) { s.handleVarUse(mkline) } -func (s *RedundantScope) updateIncludePath(mkline MkLine) { +func (s *RedundantScope) updateIncludePath(mkline *MkLine) { if mkline.firstLine == 1 { s.includePath.push(mkline.Location.Filename) } else { @@ -52,7 +52,7 @@ func (s *RedundantScope) updateIncludePath(mkline MkLine) { } } -func (s *RedundantScope) handleVarassign(mkline MkLine, ind *Indentation) { +func (s *RedundantScope) handleVarassign(mkline *MkLine, ind *Indentation) { varname := mkline.Varname() info := s.get(varname) @@ -154,10 +154,10 @@ func (s *RedundantScope) handleVarassign(mkline MkLine, ind *Indentation) { } } -func (s *RedundantScope) handleVarUse(mkline MkLine) { +func (s *RedundantScope) handleVarUse(mkline *MkLine) { switch { case mkline.IsVarassign(): - mkline.ForEachUsed(func(varUse *MkVarUse, time vucTime) { + mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { varname := varUse.varname info := s.get(varname) info.vari.Read(mkline) @@ -195,7 +195,7 @@ func (s *RedundantScope) access(varname string) { info.includePaths = append(info.includePaths, s.includePath.copy()) } -func (s *RedundantScope) onRedundant(redundant MkLine, because MkLine) { +func (s *RedundantScope) onRedundant(redundant *MkLine, because *MkLine) { if redundant.Op() == opAssignDefault { redundant.Notef("Default assignment of %s has no effect because of %s.", because.Varname(), redundant.RefTo(because)) @@ -205,7 +205,7 @@ func (s *RedundantScope) onRedundant(redundant MkLine, because MkLine) { } } -func (s *RedundantScope) onOverwrite(overwritten MkLine, by MkLine) { +func (s *RedundantScope) onOverwrite(overwritten *MkLine, by *MkLine) { overwritten.Warnf("Variable %s is overwritten in %s.", overwritten.Varname(), overwritten.RefTo(by)) overwritten.Explain( diff --git a/pkgtools/pkglint/files/redundantscope_test.go b/pkgtools/pkglint/files/redundantscope_test.go index 868f8c2eec4..de1eef2b74a 100644 --- a/pkgtools/pkglint/files/redundantscope_test.go +++ b/pkgtools/pkglint/files/redundantscope_test.go @@ -1118,7 +1118,7 @@ func (s *Suite) Test_RedundantScope__procedure_call_implemented_package(c *check t.SetUpPackage("x11/Xaos", ".include \"../../devel/gettext-lib/buildlink3.mk\"") t.CreateFileLines("devel/gettext-lib/builtin.mk", - MkRcsID, + MkCvsID, "", ".include \"../../mk/bsd.fast.prefs.mk\"", "", @@ -1126,7 +1126,7 @@ func (s *Suite) Test_RedundantScope__procedure_call_implemented_package(c *check ".if !empty(CHECK_BUILTIN.gettext:M[nN][oO])", ".endif") t.CreateFileLines("devel/gettext-lib/buildlink3.mk", - MkRcsID, + MkCvsID, "CHECK_BUILTIN.gettext:= yes", ".include \"builtin.mk\"", "CHECK_BUILTIN.gettext:= no") @@ -1148,12 +1148,12 @@ func (s *Suite) Test_RedundantScope__procedure_call_infrastructure(c *check.C) { t.SetUpPackage("x11/alacarte", ".include \"../../mk/pthread.buildlink3.mk\"") t.CreateFileLines("mk/pthread.buildlink3.mk", - MkRcsID, + MkCvsID, "CHECK_BUILTIN.gettext:= yes", ".include \"pthread.builtin.mk\"", "CHECK_BUILTIN.gettext:= no") t.CreateFileLines("mk/pthread.builtin.mk", - MkRcsID, + MkCvsID, "CHECK_BUILTIN.gettext?= no", ".if !empty(CHECK_BUILTIN.gettext:M[nN][oO])", ".endif") @@ -1248,7 +1248,7 @@ func (s *Suite) Test_RedundantScope__included_OPSYS_variable(c *check.C) { t.SetUpPackage("category/dependency") t.CreateFileDummyBuildlink3("category/dependency/buildlink3.mk") t.CreateFileLines("category/dependency/builtin.mk", - MkRcsID, + MkCvsID, "CONFIGURE_ARGS.Darwin+= darwin") t.FinishSetUp() @@ -1501,7 +1501,7 @@ func (s *Suite) Test_RedundantScope_handleVarassign__conditional(c *check.C) { t.Check( writeLocations, deepEquals, - []MkLine{mklines.mklines[0], mklines.mklines[2]}) + []*MkLine{mklines.mklines[0], mklines.mklines[2]}) } // Ensures that commented variables do not influence the redundancy check. diff --git a/pkgtools/pkglint/files/shell.go b/pkgtools/pkglint/files/shell.go index 905546ec992..8d044b1d49d 100644 --- a/pkgtools/pkglint/files/shell.go +++ b/pkgtools/pkglint/files/shell.go @@ -16,15 +16,16 @@ import ( // Or it is a variable assignment line from a Makefile with a left-hand // side variable that is of some shell-like type; see Vartype.IsShell. type ShellLineChecker struct { - MkLines MkLines - mkline MkLine + MkLines *MkLines + mkline *MkLine // checkVarUse is set to false when checking a single shell word // in order to skip duplicate warnings in variable assignments. checkVarUse bool } -func NewShellLineChecker(mklines MkLines, mkline MkLine) *ShellLineChecker { +func NewShellLineChecker(mklines *MkLines, mkline *MkLine) *ShellLineChecker { + assertNotNil(mklines) return &ShellLineChecker{mklines, mkline, true} } @@ -36,7 +37,7 @@ func (ck *ShellLineChecker) Explain(explanation ...string) { } var shellCommandsType = NewVartype(BtShellCommands, NoVartypeOptions, NewACLEntry("*", aclpAllRuntime)) -var shellWordVuc = &VarUseContext{shellCommandsType, vucTimeUnknown, VucQuotPlain, false} +var shellWordVuc = &VarUseContext{shellCommandsType, VucUnknownTime, VucQuotPlain, false} func (ck *ShellLineChecker) CheckWord(token string, checkQuoting bool, time ToolTime) { if trace.Tracing { @@ -216,7 +217,7 @@ func (ck *ShellLineChecker) checkVaruseToken(atoms *[]*ShAtom, quoting ShQuoting } if ck.checkVarUse { - vuc := VarUseContext{shellCommandsType, vucTimeUnknown, quoting.ToVarUseContext(), true} + vuc := VarUseContext{shellCommandsType, VucUnknownTime, quoting.ToVarUseContext(), true} MkLineChecker{ck.MkLines, ck.mkline}.CheckVaruse(varuse, &vuc) } @@ -291,10 +292,7 @@ func (ck *ShellLineChecker) variableNeedsQuoting(shVarname string) bool { case "d", "f", "i", "id", "file", "src", "dst", "prefix": return false // Probably ok } - if hasSuffix(shVarname, "dir") { - return false // Probably ok - } - return true + return !hasSuffix(shVarname, "dir") // Probably ok } func (ck *ShellLineChecker) CheckShellCommandLine(shelltext string) { @@ -485,9 +483,7 @@ func (scc *SimpleCommandChecker) checkCommandStart() { scc.checkInstallCommand(shellword) switch { - case shellword == "${RUN}" || shellword == "": - // FIXME: ${RUN} must never appear as a simple command. - // It should always be trimmed before passing the shell program to the SimpleCommandChecker. + case shellword == "": break case scc.handleForbiddenCommand(): break @@ -504,7 +500,7 @@ func (scc *SimpleCommandChecker) checkCommandStart() { case scc.handleComment(): break default: - if G.Opts.WarnExtra && !(scc.MkLines != nil && scc.MkLines.indentation.DependsOn("OPSYS")) { + if G.Opts.WarnExtra && !scc.MkLines.indentation.DependsOn("OPSYS") { scc.mkline.Warnf("Unknown shell command %q.", shellword) scc.mkline.Explain( "To make the package portable to all platforms that pkgsrc supports,", @@ -560,24 +556,25 @@ func (scc *SimpleCommandChecker) handleCommandVariable() bool { } shellword := scc.strcmd.Name - if varuse := ToVarUse(shellword); varuse != nil { - varname := varuse.varname + varuse := NewMkParser(nil, shellword, false).VarUse() + if varuse == nil { + return false + } - if vartype := G.Pkgsrc.VariableType(scc.MkLines, varname); vartype != nil && vartype.basicType.name == "ShellCommand" { - scc.checkInstallCommand(shellword) - return true - } + varname := varuse.varname - // When the package author has explicitly defined a command - // variable, assume it to be valid. - if scc.MkLines != nil && scc.MkLines.vars.DefinedSimilar(varname) { - return true - } - if G.Pkg != nil && G.Pkg.vars.DefinedSimilar(varname) { - return true - } + if vartype := G.Pkgsrc.VariableType(scc.MkLines, varname); vartype != nil && vartype.basicType.name == "ShellCommand" { + scc.checkInstallCommand(shellword) + return true } - return false + + // When the package author has explicitly defined a command + // variable, assume it to be valid. + if scc.MkLines.vars.DefinedSimilar(varname) { + return true + } + + return G.Pkg != nil && G.Pkg.vars.DefinedSimilar(varname) } func (scc *SimpleCommandChecker) handleShellBuiltin() bool { @@ -919,8 +916,9 @@ func (spc *ShellProgramChecker) canFail(cmd *MkShCommand) bool { if simple.Name == nil { for _, assignment := range simple.Assignments { - if contains(assignment.MkText, "`") || contains(assignment.MkText, "$(") { - if !contains(assignment.MkText, "|| ${TRUE}") { + text := assignment.MkText + if contains(text, "`") || contains(text, "$(") { + if !contains(text, "|| ${TRUE}") && !contains(text, "|| true") { return true } } @@ -929,7 +927,7 @@ func (spc *ShellProgramChecker) canFail(cmd *MkShCommand) bool { } for _, redirect := range simple.Redirections { - if redirect.Target != nil && !hasSuffix(redirect.Op, "&") { + if !hasSuffix(redirect.Op, "&") { return true } } @@ -955,7 +953,7 @@ func (spc *ShellProgramChecker) canFail(cmd *MkShCommand) bool { args := simple.Args argc := len(args) switch toolName { - case "echo", "env", "printf", "tr": + case "basename", "dirname", "echo", "env", "printf", "tr": return false case "sed", "gsed": if argc == 2 && args[0].MkText == "-e" { @@ -1062,7 +1060,7 @@ func (ck *ShellLineChecker) checkInstallCommand(shellcmd string) { // Example: "word1 word2;;;" => "word1", "word2", ";;", ";" // // TODO: Document what this function should be used for. -func splitIntoShellTokens(line Line, text string) (tokens []string, rest string) { +func splitIntoShellTokens(line *Line, text string) (tokens []string, rest string) { if trace.Tracing { defer trace.Call(line, text)() } diff --git a/pkgtools/pkglint/files/shell_test.go b/pkgtools/pkglint/files/shell_test.go index dee5358868d..3a127740a61 100644 --- a/pkgtools/pkglint/files/shell_test.go +++ b/pkgtools/pkglint/files/shell_test.go @@ -144,7 +144,7 @@ func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine(c *check.C) { "\t"+shellCommand) ck := NewShellLineChecker(mklines, mklines.mklines[0]) - mklines.ForEach(func(mkline MkLine) { + mklines.ForEach(func(mkline *MkLine) { ck.CheckShellCommandLine(ck.mkline.ShellCommand()) }) @@ -262,7 +262,7 @@ func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__strip(c *check.C) { mklines := t.NewMkLines("filename.mk", "\t"+shellCommand) - mklines.ForEach(func(mkline MkLine) { + mklines.ForEach(func(mkline *MkLine) { ck := NewShellLineChecker(mklines, mkline) ck.CheckShellCommandLine(mkline.ShellCommand()) }) @@ -352,7 +352,8 @@ func (s *Suite) Test_ShellProgramChecker_checkPipeExitcode(c *check.C) { "\t sed s,s,s < input | right-side", "\t ./unknown | right-side", "\t var=value | right-side", - "\t if :; then :; fi | right-side") + "\t if :; then :; fi | right-side", + "\t var=`cat file` | right-side") for _, mkline := range mklines.mklines { ck := NewShellLineChecker(mklines, mkline) @@ -366,7 +367,8 @@ func (s *Suite) Test_ShellProgramChecker_checkPipeExitcode(c *check.C) { "WARN: Makefile:7: The exitcode of \"sed\" at the left of the | operator is ignored.", "WARN: Makefile:8: The exitcode of \"sed\" at the left of the | operator is ignored.", "WARN: Makefile:9: The exitcode of \"./unknown\" at the left of the | operator is ignored.", - "WARN: Makefile:11: The exitcode of the command at the left of the | operator is ignored.") + "WARN: Makefile:11: The exitcode of the command at the left of the | operator is ignored.", + "WARN: Makefile:12: The exitcode of the command at the left of the | operator is ignored.") } // TODO: Document the exact purpose of this test, or split it into useful tests. @@ -386,12 +388,12 @@ func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__implementation(c *c c.Check(tokens, deepEquals, []string{text}) c.Check(rest, equals, "") - mklines.ForEach(func(mkline MkLine) { ck.CheckWord(text, false, RunTime) }) + mklines.ForEach(func(mkline *MkLine) { ck.CheckWord(text, false, RunTime) }) t.CheckOutputLines( "WARN: filename.mk:1: Unknown shell command \"echo\".") - mklines.ForEach(func(mkline MkLine) { ck.CheckShellCommandLine(text) }) + mklines.ForEach(func(mkline *MkLine) { ck.CheckShellCommandLine(text) }) // No parse errors t.CheckOutputLines( @@ -530,7 +532,7 @@ func (s *Suite) Test_ShellLineChecker_CheckWord__PKGMANDIR(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("chat/ircII/Makefile", - MkRcsID, + MkCvsID, "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/man", "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/${PKGMANDIR}") @@ -542,11 +544,26 @@ func (s *Suite) Test_ShellLineChecker_CheckWord__PKGMANDIR(c *check.C) { "NOTE: chat/ircII/Makefile:3: This variable value should be aligned to column 25.") } +func (s *Suite) Test_ShellLineChecker_CheckWord__empty(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "JAVA_CLASSPATH=\t# empty") + + mklines.Check() + + t.CheckOutputEmpty() +} + func (s *Suite) Test_ShellLineChecker_unescapeBackticks__unfinished(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "", "pre-configure:", "\t`${VAR}", // Error in first shell word @@ -565,17 +582,19 @@ func (s *Suite) Test_ShellLineChecker_unescapeBackticks__unfinished(c *check.C) func (s *Suite) Test_ShellLineChecker_unescapeBackticks__unfinished_direct(c *check.C) { t := s.Init(c) - mkline := t.NewMkLine("dummy.mk", 123, "\t# shell command") + mklines := t.NewMkLines("dummy.mk", + MkCvsID, + "\t# shell command") // This call is unrealistic. It doesn't happen in practice, and this // direct, forcing test is only to reach the code coverage. atoms := []*ShAtom{ NewShAtom(shtText, "`", shqBackt)} - NewShellLineChecker(nil, mkline). + NewShellLineChecker(mklines, mklines.mklines[1]). unescapeBackticks(&atoms, shqBackt) t.CheckOutputLines( - "ERROR: dummy.mk:123: Unfinished backticks after \"\".") + "ERROR: dummy.mk:2: Unfinished backticks after \"\".") } func (s *Suite) Test_ShellLineChecker_variableNeedsQuoting(c *check.C) { @@ -616,7 +635,7 @@ func (s *Suite) Test_ShellLineChecker_variableNeedsQuoting__integration(c *check t.SetUpVartypes() t.SetUpTool("cp", "", AtRunTime) mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "", // It's a bit silly to use shell variables in CONFIGURE_ARGS, // but as of January 2019 that's the only way to run ShellLine.variableNeedsQuoting. @@ -661,7 +680,7 @@ func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__shell_variables(c * t.SetUpTool("sed", "SED", AtRunTime) text := "for f in *.pl; do ${SED} s,@PREFIX@,${PREFIX}, < $f > $f.tmp && ${MV} $f.tmp $f; done" mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "\t"+text) @@ -776,7 +795,7 @@ func (s *Suite) Test_ShellLineChecker__shell_comment_with_line_continuation(c *c t := s.Init(c) mklines := t.SetUpFileMkLines("Makefile", - MkRcsID, + MkCvsID, "pre-install:", "\t"+"# comment\\", "\t"+"echo \"hello\"") @@ -839,6 +858,57 @@ func (s *Suite) Test_ShellLineChecker_checkWordQuoting(c *check.C) { test( "$$$$", nil...) + + test( + "``", + nil...) +} + +func (s *Suite) Test_ShellLineChecker_checkShVarUsePlain__default_warning_level(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine( /* none */ ) + t.SetUpVartypes() + t.SetUpTool("echo", "", AtRunTime) + + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "CONFIGURE_ARGS+=\techo $$@ $$var", + "", + "pre-configure:", + "\techo $$@ $$var") + + mklines.Check() + + // Using $@ outside of double quotes is so obviously wrong that + // the warning is issued by default. + t.CheckOutputLines( + "WARN: filename.mk:2: The $@ shell variable should only be used in double quotes.", + "WARN: filename.mk:5: The $@ shell variable should only be used in double quotes.") +} + +func (s *Suite) Test_ShellLineChecker_checkShVarUsePlain__Wall(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpTool("echo", "", AtRunTime) + + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "CONFIGURE_ARGS+=\techo $$@ $$var", + "", + "pre-configure:", + "\techo $$@ $$var") + + mklines.Check() + + // FIXME: It is inconsistent that the check for unquoted shell + // variables is enabled for CONFIGURE_ARGS (where shell variables + // don't make sense at all) but not for real shell commands. + t.CheckOutputLines( + "WARN: filename.mk:2: The $@ shell variable should only be used in double quotes.", + "WARN: filename.mk:2: Unquoted shell variable \"var\".", + "WARN: filename.mk:5: The $@ shell variable should only be used in double quotes.") } func (s *Suite) Test_ShellLineChecker_unescapeBackticks(c *check.C) { @@ -907,7 +977,7 @@ func (s *Suite) Test_ShellLineChecker_unescapeBackticks__dquotBacktDquot(c *chec t.SetUpTool("echo", "", AtRunTime) mklines := t.NewMkLines("dummy.mk", - MkRcsID, + MkCvsID, "\t var=\"`echo \"\"`\"") mklines.Check() @@ -921,7 +991,7 @@ func (s *Suite) Test_ShellLineChecker__variable_outside_quotes(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("dummy.mk", - MkRcsID, + MkCvsID, "GZIP=\t${ECHO} $$comment") mklines.Check() @@ -938,7 +1008,7 @@ func (s *Suite) Test_ShellLineChecker_CheckShellCommand__cd_inside_if(c *check.C t.SetUpVartypes() t.SetUpTool("echo", "ECHO", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "\t${RUN} if cd /bin; then echo \"/bin exists.\"; fi") @@ -955,7 +1025,7 @@ func (s *Suite) Test_ShellLineChecker_CheckShellCommand__negated_pipe(c *check.C t.SetUpTool("echo", "ECHO", AtRunTime) t.SetUpTool("test", "TEST", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "\t${RUN} if ! test -f /etc/passwd; then echo \"passwd is missing.\"; fi") @@ -971,7 +1041,7 @@ func (s *Suite) Test_ShellLineChecker_CheckShellCommand__subshell(c *check.C) { t.SetUpTool("echo", "ECHO", AtRunTime) t.SetUpTool("expr", "EXPR", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "pre-configure:", "\t@(echo ok)", @@ -998,7 +1068,7 @@ func (s *Suite) Test_ShellLineChecker_CheckShellCommand__case_patterns_from_vari t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "pre-configure:", "\tcase $$file in ${CHECK_PERMS_SKIP:@pattern@${pattern}) ;;@} *) continue; esac") @@ -1017,15 +1087,23 @@ func (s *Suite) Test_ShellLineChecker_checkHiddenAndSuppress(c *check.C) { t.SetUpTool("echo", "ECHO", AtRunTime) t.SetUpTool("ls", "LS", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "show-all-targets: .PHONY", "\t@echo 'hello'", - "\t@ls -l") + "\t@ls -l", + "", + "anything-message: .PHONY", + "\t@echo 'may be hidden'", + "\t@ls 'may be hidden'", + "", + "pre-configure:", + "\t@") mklines.Check() - // No warning about the hidden ls since the target name starts with "show-". + // No warning about the hidden ls since the target names start + // with "show-" or end with "-message". t.CheckOutputEmpty() } @@ -1034,7 +1112,7 @@ func (s *Suite) Test_ShellLineChecker_checkHiddenAndSuppress__no_tracing(c *chec t.SetUpTool("ls", "LS", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "pre-configure:", "\t@ls -l") @@ -1050,7 +1128,7 @@ func (s *Suite) Test_SimpleCommandChecker_handleForbiddenCommand(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "\t${RUN} mktexlsr; texconfig") @@ -1067,19 +1145,65 @@ func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable(c *check.C) { t.SetUpTool("perl", "PERL5", AtRunTime) t.SetUpTool("perl6", "PERL6", Nowhere) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "PERL5_VARS_CMD=\t${PERL5:Q}", - "PERL5_VARS_CMD=\t${PERL6:Q}") + "PERL5_VARS_CMD=\t${PERL6:Q}", + "", + "pre-configure:", + "\t${PERL5_VARS_CMD} -e 'print 12345'") mklines.Check() // FIXME: In PERL5:Q and PERL6:Q, the :Q is wrong. t.CheckOutputLines( - "WARN: Makefile:3: PERL5_VARS_CMD is defined but not used.", "WARN: Makefile:4: The \"${PERL6:Q}\" tool is used but not added to USE_TOOLS.") } +func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__parameterized(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + G.Pkg = NewPackage(t.File("category/package")) + t.FinishSetUp() + + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "MY_TOOL.i386=\t${PREFIX}/bin/tool-i386", + "MY_TOOL.x86_64=\t${PREFIX}/bin/tool-x86_64", + "", + "pre-configure:", + "\t${MY_TOOL.amd64} -e 'print 12345'", + "\t${UNKNOWN_TOOL}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:8: Unknown shell command \"${UNKNOWN_TOOL}\".", + "WARN: Makefile:8: UNKNOWN_TOOL is used but not defined.") +} + +func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__followed_by_literal(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + G.Pkg = NewPackage(t.File("category/package")) + t.FinishSetUp() + + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "QTDIR=\t${PREFIX}", + "", + "pre-configure:", + "\t${QTDIR}/bin/release") + + mklines.Check() + + t.CheckOutputEmpty() +} + // The package Makefile and other .mk files in a package directory // may use any shell commands defined by any included files. // But only if the package is checked as a whole. @@ -1099,7 +1223,7 @@ func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__from_package(c "", ".include \"extra.mk\"") t.CreateFileLines("category/package/extra.mk", - MkRcsID, + MkCvsID, "PYTHON_BIN=\tmy_cmd") t.FinishSetUp() @@ -1112,7 +1236,7 @@ func (s *Suite) Test_SimpleCommandChecker_handleComment(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("file.mk", - MkRcsID, + MkCvsID, "\t# comment; continuation") mklines.Check() @@ -1128,7 +1252,7 @@ func (s *Suite) Test_SimpleCommandChecker_checkInstallMulti(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("install.mk", - MkRcsID, + MkCvsID, "", "do-install:", "\t${INSTALL_PROGRAM_DIR} -m 0555 -g ${APACHE_GROUP} -o ${APACHE_USER} \\", @@ -1147,7 +1271,7 @@ func (s *Suite) Test_SimpleCommandChecker_checkPaxPe(c *check.C) { t.SetUpVartypes() t.SetUpTool("pax", "PAX", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "do-install:", "\t${RUN} pax -pe ${WRKSRC} ${DESTDIR}${PREFIX}", @@ -1166,7 +1290,7 @@ func (s *Suite) Test_SimpleCommandChecker_checkEchoN(c *check.C) { t.SetUpTool("echo", "ECHO", AtRunTime) t.SetUpTool("echo -n", "ECHO_N", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", "do-install:", "\t${RUN} ${ECHO} -n 'Computing...'", @@ -1182,22 +1306,27 @@ func (s *Suite) Test_SimpleCommandChecker_checkEchoN(c *check.C) { func (s *Suite) Test_ShellProgramChecker_checkConditionalCd(c *check.C) { t := s.Init(c) - t.SetUpTool("ls", "LS", AtRunTime) - t.SetUpTool("printf", "PRINTF", AtRunTime) + t.SetUpTool("ls", "", AtRunTime) + t.SetUpTool("printf", "", AtRunTime) + t.SetUpTool("wc", "", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "pre-configure:", "\t${RUN} while cd ..; do printf .; done", + "\t${RUN} while cd .. && cd ..; do printf .; done", // Unusual, therefore no warning. "\t${RUN} if cd ..; then printf .; fi", "\t${RUN} ! cd ..", - "\t${RUN} if cd .. && cd ..; then printf .; fi") // For code coverage + "\t${RUN} if cd ..; printf 'ok\\n'; then printf .; fi", + "\t${RUN} if cd .. | wc -l; then printf .; fi", // Unusual, therefore no warning. + "\t${RUN} if cd .. && cd ..; then printf .; fi") // Unusual, therefore no warning. mklines.Check() t.CheckOutputLines( "ERROR: Makefile:3: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", - "ERROR: Makefile:4: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", - "WARN: Makefile:5: The Solaris /bin/sh does not support negation of shell commands.") + "ERROR: Makefile:5: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", + "WARN: Makefile:6: The Solaris /bin/sh does not support negation of shell commands.", + "WARN: Makefile:8: The exitcode of \"cd\" at the left of the | operator is ignored.") } func (s *Suite) Test_SimpleCommandChecker_checkRegexReplace(c *check.C) { @@ -1207,7 +1336,7 @@ func (s *Suite) Test_SimpleCommandChecker_checkRegexReplace(c *check.C) { t.SetUpTool("pax", "PAX", AtRunTime) t.SetUpTool("sed", "SED", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "pre-configure:", "\t"+cmd) @@ -1252,6 +1381,59 @@ func (s *Suite) Test_SimpleCommandChecker_checkRegexReplace(c *check.C) { G.Testing = true } +func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpTool("awk", "AWK", AtRunTime) + t.SetUpTool("cp", "CP", AtRunTime) + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("mkdir", "MKDIR", AtRunTime) // This is actually "mkdir -p". + t.SetUpTool("unzip", "UNZIP_CMD", AtRunTime) + + test := func(shellCommand string, diagnostics ...string) { + mklines := t.NewMkLines("filename.mk", + "\t"+shellCommand) + ck := NewShellLineChecker(mklines, mklines.mklines[0]) + + mklines.ForEach(func(mkline *MkLine) { + ck.CheckShellCommandLine(ck.mkline.ShellCommand()) + }) + + t.CheckOutput(diagnostics) + } + + // AUTO_MKDIRS applies only when installing directories. + test("${RUN} ${INSTALL} -c ${WRKSRC}/file ${PREFIX}/bin/", + nil...) + + // TODO: Warn that ${INSTALL} -d can only handle a single directory. + test("${RUN} ${INSTALL} -m 0755 -d ${PREFIX}/first ${PREFIX}/second", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= first\" instead of \"${INSTALL} -d\".", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= second\" instead of \"${INSTALL} -d\".") + + G.Pkg = NewPackage(t.File("category/pkgbase")) + G.Pkg.Plist.Dirs["share/pkgbase"] = true + + // A directory that is found in the PLIST. + // TODO: Add a test for using this command inside a conditional; + // the note should not appear then. + test("${RUN} ${INSTALL_DATA_DIR} share/pkgbase ${PREFIX}/share/pkgbase", + "NOTE: filename.mk:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" "+ + "instead of \"${INSTALL_DATA_DIR}\".", + "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") + + // Directories from .for loops are too dynamic to be replaced with AUTO_MKDIRS. + // TODO: Expand simple .for loops. + test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/${dir}", + "WARN: filename.mk:1: dir is used but not defined.") + + // A directory that is not found in the PLIST would not be created by AUTO_MKDIRS, + // therefore only INSTALLATION_DIRS is suggested. + test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/share/other", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= share/other\" instead of \"${INSTALL_DATA_DIR}\".") +} + func (s *Suite) Test_ShellProgramChecker_checkSetE__simple_commands(c *check.C) { t := s.Init(c) @@ -1259,7 +1441,7 @@ func (s *Suite) Test_ShellProgramChecker_checkSetE__simple_commands(c *check.C) t.SetUpTool("rm", "", AtRunTime) t.SetUpTool("touch", "", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "pre-configure:", "\techo 1; echo 2; echo 3", "\techo 1; touch file; rm file", @@ -1278,7 +1460,7 @@ func (s *Suite) Test_ShellProgramChecker_checkSetE__compound_commands(c *check.C t.SetUpTool("echo", "", AtRunTime) t.SetUpTool("touch", "", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "pre-configure:", "\ttouch file; for f in file; do echo \"$$f\"; done", "\tfor f in file; do echo \"$$f\"; done; touch file", @@ -1298,7 +1480,7 @@ func (s *Suite) Test_ShellProgramChecker_checkSetE__no_tracing(c *check.C) { t.SetUpTool("touch", "", AtRunTime) mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "pre-configure:", "\ttouch 1; touch 2") t.DisableTracing() @@ -1314,6 +1496,8 @@ func (s *Suite) Test_ShellProgramChecker_canFail(c *check.C) { t := s.Init(c) t.SetUpVartypes() + t.SetUpTool("basename", "", AtRunTime) + t.SetUpTool("dirname", "", AtRunTime) t.SetUpTool("echo", "", AtRunTime) t.SetUpTool("env", "", AtRunTime) t.SetUpTool("grep", "GREP", AtRunTime) @@ -1321,36 +1505,86 @@ func (s *Suite) Test_ShellProgramChecker_canFail(c *check.C) { t.SetUpTool("touch", "", AtRunTime) t.SetUpTool("tr", "tr", AtRunTime) t.SetUpTool("true", "TRUE", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkRcsID, - "pre-configure:", - "\tsocklen=`${GREP} 'expr' ${WRKSRC}/config.h`; echo 'done.'", - "\tsocklen=`${GREP} 'expr' ${WRKSRC}/config.h || ${TRUE}`; echo 'done.'", - "\t${ECHO_MSG} \"Message\"; echo 'done.'", - "\t${FAIL_MSG} \"Failure\"; echo 'done.'", - "\tset -x; echo 'done.'", - "\techo 'input' | sed -e s,in,out,; echo 'done.'", - "\tsed -e s,in,out,; echo 'done.'", - "\tsed s,in,out,; echo 'done.'", - "\tgrep input; echo 'done.'", - "\ttouch file; echo 'done.'", - "\techo 'starting'; echo 'done.'", - "\techo 'logging' > log; echo 'done.'", - "\techo 'to stderr' 1>&2; echo 'done.'", - "\techo 'hello' | tr -d 'aeiou'", - "\tenv | grep '^PATH='") - mklines.Check() + test := func(cmd string, diagnostics ...string) { + mklines := t.NewMkLines("Makefile", + MkCvsID, + "pre-configure:", + "\t"+cmd+" ; echo 'done.'") - t.CheckOutputLines( + mklines.Check() + + t.CheckOutput(diagnostics) + } + + test("socklen=`${GREP} 'expr' ${WRKSRC}/config.h`", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"socklen=`${GREP} 'expr' ${WRKSRC}/config.h`\") to separate commands.") + + test("socklen=`${GREP} 'expr' ${WRKSRC}/config.h || ${TRUE}`", + nil...) + + test("socklen=$$(expr 16)", + "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"socklen=$$(expr 16)\") to separate commands.") + + test("socklen=$$(expr 16 || true)", + "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.") + + test("socklen=$$(expr 16 || ${TRUE})", + "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.") + + test("${ECHO_MSG} \"Message\"", + nil...) + + test("${FAIL_MSG} \"Failure\"", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"${FAIL_MSG} \\\"Failure\\\"\") to separate commands.") + + test("set -x", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"set -x\") to separate commands.") + + test("echo 'input' | sed -e s,in,out,", + nil...) + + test("sed -e s,in,out,", + nil...) + + test("sed s,in,out,", + nil...) + + test("grep input", + nil...) + + test("grep pattern file...", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"grep pattern file...\") to separate commands.") + + test("touch file", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"touch file\") to separate commands.") + + test("echo 'starting'", + nil...) + + test("echo 'logging' > log", "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"socklen=`${GREP} 'expr' ${WRKSRC}/config.h`\") to separate commands.", - "WARN: Makefile:6: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"${FAIL_MSG} \\\"Failure\\\"\") to separate commands.", - "WARN: Makefile:7: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"set -x\") to separate commands.", - "WARN: Makefile:12: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"touch file\") to separate commands.", - "WARN: Makefile:14: Please switch to \"set -e\" mode before using a semicolon "+ "(after \"echo 'logging'\") to separate commands.") + + test("echo 'to stderr' 1>&2", + nil...) + + test("echo 'hello' | tr -d 'aeiou'", + nil...) + + test("env | grep '^PATH='", + nil...) + + test("basename dir/file", + nil...) + + test("dirname dir/file", + nil...) } diff --git a/pkgtools/pkglint/files/shtokenizer.go b/pkgtools/pkglint/files/shtokenizer.go index a8b0cad7d7e..71bbd806065 100644 --- a/pkgtools/pkglint/files/shtokenizer.go +++ b/pkgtools/pkglint/files/shtokenizer.go @@ -6,7 +6,7 @@ type ShTokenizer struct { parser *MkParser } -func NewShTokenizer(line Line, text string, emitWarnings bool) *ShTokenizer { +func NewShTokenizer(line *Line, text string, emitWarnings bool) *ShTokenizer { // TODO: Switching to NewMkParser is nontrivial since emitWarnings must equal (line != nil). p := MkParser{line, textproc.NewLexer(text), emitWarnings} return &ShTokenizer{&p} @@ -428,7 +428,6 @@ func (p *ShTokenizer) ShToken() *ShToken { lexer := p.parser.lexer initialMark := lexer.Mark() - var atoms []*ShAtom for peek() != nil && peek().Type == shtSpace { skip() @@ -439,20 +438,20 @@ func (p *ShTokenizer) ShToken() *ShToken { return nil } - if atom := peek(); !atom.Type.IsWord() && atom.Quoting != shqSubsh { - return NewShToken(atom.MkText, atom) + if !curr.Type.IsWord() && q != shqSubsh { + return NewShToken(curr.MkText, curr) } + var atoms []*ShAtom for { mark := lexer.Mark() - atom := peek() - if atom != nil && (atom.Type.IsWord() || q != shqPlain || prevQ == shqSubsh) { - skip() - atoms = append(atoms, atom) - continue + peek() + if curr == nil || !curr.Type.IsWord() && q == shqPlain && prevQ != shqSubsh { + lexer.Reset(mark) + break } - lexer.Reset(mark) - break + atoms = append(atoms, curr) + skip() } if q != shqPlain { @@ -460,7 +459,6 @@ func (p *ShTokenizer) ShToken() *ShToken { return nil } - assertf(len(atoms) > 0, "ShTokenizer.ShToken") return NewShToken(lexer.Since(initialMark), atoms...) } diff --git a/pkgtools/pkglint/files/shtokenizer_test.go b/pkgtools/pkglint/files/shtokenizer_test.go index 38c7e44847a..78364bce396 100644 --- a/pkgtools/pkglint/files/shtokenizer_test.go +++ b/pkgtools/pkglint/files/shtokenizer_test.go @@ -580,7 +580,7 @@ func (s *Suite) Test_ShTokenizer__examples_from_fuzzing(c *check.C) { test := func(input string, diagnostics ...string) { mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "\t"+input) mklines.Check() t.CheckOutput(diagnostics) diff --git a/pkgtools/pkglint/files/shtypes.go b/pkgtools/pkglint/files/shtypes.go index 4580ab861bc..e12d320d46a 100644 --- a/pkgtools/pkglint/files/shtypes.go +++ b/pkgtools/pkglint/files/shtypes.go @@ -133,6 +133,8 @@ type ShToken struct { } func NewShToken(mkText string, atoms ...*ShAtom) *ShToken { + assert(mkText != "") + assert(len(atoms) > 0) return &ShToken{mkText, atoms} } diff --git a/pkgtools/pkglint/files/shtypes_test.go b/pkgtools/pkglint/files/shtypes_test.go index ca890e0ec37..aff0a6d3d03 100644 --- a/pkgtools/pkglint/files/shtypes_test.go +++ b/pkgtools/pkglint/files/shtypes_test.go @@ -29,6 +29,13 @@ func (s *Suite) Test_ShQuoting_String(c *check.C) { c.Check(shqDquotBacktSquot.String(), equals, "dbs") } +func (s *Suite) Test_NewShToken__no_atoms(c *check.C) { + t := s.Init(c) + + t.ExpectAssert(func() { NewShToken("", NewShAtom(shtText, "text", shqPlain)) }) + t.ExpectAssert(func() { NewShToken(" ", nil...) }) +} + func (s *Suite) Test_ShToken_String(c *check.C) { tokenizer := NewShTokenizer(dummyLine, "${ECHO} \"hello, world\"", false) diff --git a/pkgtools/pkglint/files/substcontext.go b/pkgtools/pkglint/files/substcontext.go index a648b181972..81e84c758ee 100644 --- a/pkgtools/pkglint/files/substcontext.go +++ b/pkgtools/pkglint/files/substcontext.go @@ -44,7 +44,7 @@ func (st *SubstContextStats) Or(other SubstContextStats) { st.seenTransform = st.seenTransform || other.seenTransform } -func (ctx *SubstContext) Process(mkline MkLine) { +func (ctx *SubstContext) Process(mkline *MkLine) { switch { case mkline.IsEmpty(): ctx.Finish(mkline) @@ -55,7 +55,7 @@ func (ctx *SubstContext) Process(mkline MkLine) { } } -func (ctx *SubstContext) Varassign(mkline MkLine) { +func (ctx *SubstContext) Varassign(mkline *MkLine) { if trace.Tracing { trace.Stepf("SubstContext.Varassign curr=%v all=%v", ctx.curr, ctx.inAllBranches) } @@ -181,7 +181,7 @@ func (ctx *SubstContext) Varassign(mkline MkLine) { } } -func (ctx *SubstContext) Directive(mkline MkLine) { +func (ctx *SubstContext) Directive(mkline *MkLine) { if ctx.id == "" { return } @@ -214,7 +214,7 @@ func (ctx *SubstContext) IsComplete() bool { return ctx.stage != "" && ctx.curr.seenFiles && ctx.curr.seenTransform } -func (ctx *SubstContext) Finish(mkline MkLine) { +func (ctx *SubstContext) Finish(mkline *MkLine) { if ctx.id == "" { return } @@ -233,21 +233,21 @@ func (ctx *SubstContext) Finish(mkline MkLine) { *ctx = *NewSubstContext() } -func (*SubstContext) dupString(mkline MkLine, pstr *string, varname, value string) { +func (*SubstContext) dupString(mkline *MkLine, pstr *string, varname, value string) { if *pstr != "" { mkline.Warnf("Duplicate definition of %q.", varname) } *pstr = value } -func (*SubstContext) dupBool(mkline MkLine, flag *bool, varname string, op MkOperator, value string) { +func (*SubstContext) dupBool(mkline *MkLine, flag *bool, varname string, op MkOperator, value string) { if *flag && op != opAssignAppend { mkline.Warnf("All but the first %q lines should use the \"+=\" operator.", varname) } *flag = true } -func (ctx *SubstContext) suggestSubstVars(mkline MkLine) { +func (ctx *SubstContext) suggestSubstVars(mkline *MkLine) { tokens, _ := splitIntoShellTokens(mkline.Line, mkline.Value()) for _, token := range tokens { diff --git a/pkgtools/pkglint/files/substcontext_test.go b/pkgtools/pkglint/files/substcontext_test.go index fbb07a8f867..5178f1c007a 100644 --- a/pkgtools/pkglint/files/substcontext_test.go +++ b/pkgtools/pkglint/files/substcontext_test.go @@ -305,7 +305,7 @@ func (s *Suite) Test_SubstContext__pre_patch(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("os.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+= os", "SUBST_STAGE.os= pre-patch", @@ -326,7 +326,7 @@ func (s *Suite) Test_SubstContext__post_patch(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("os.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+= os", "SUBST_STAGE.os= post-patch", @@ -395,7 +395,7 @@ func (s *Suite) Test_SubstContext__adjacent(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("os.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+= 1", "SUBST_STAGE.1= pre-configure", @@ -420,7 +420,7 @@ func (s *Suite) Test_SubstContext__do_patch(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("os.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+= os", "SUBST_STAGE.os= do-patch", @@ -443,7 +443,7 @@ func (s *Suite) Test_SubstContext__SUBST_VARS_defined_in_block(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("os.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+= os", "SUBST_STAGE.os= pre-configure", @@ -468,7 +468,7 @@ func (s *Suite) Test_SubstContext__SUBST_VARS_in_next_paragraph(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("os.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+= os", "SUBST_STAGE.os= pre-configure", @@ -491,7 +491,7 @@ func (s *Suite) Test_SubstContext__multiple_SUBST_VARS(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("os.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+= os", "SUBST_STAGE.os= pre-configure", @@ -513,7 +513,7 @@ func (s *Suite) Test_SubstContext_Directive__before_SUBST_CLASSES(c *check.C) { t.DisableTracing() // Just for branch coverage. mklines := t.NewMkLines("os.mk", - MkRcsID, + MkCvsID, "", ".if 0", ".endif", @@ -536,7 +536,7 @@ func (s *Suite) Test_SubstContext_suggestSubstVars(c *check.C) { t.SetUpTool("sh", "SH", AtRunTime) mklines := t.NewMkLines("subst.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+=\t\ttest", "SUBST_STAGE.test=\tpre-configure", @@ -626,7 +626,7 @@ func (s *Suite) Test_SubstContext_suggestSubstVars__plus(c *check.C) { t.SetUpTool("sh", "SH", AtRunTime) mklines := t.NewMkLines("subst.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+=\t\tgtk+", "SUBST_STAGE.gtk+ =\tpre-configure", @@ -667,7 +667,7 @@ func (s *Suite) Test_SubstContext_suggestSubstVars__autofix_realign_paragraph(c t.Chdir(".") mklines := t.SetUpFileMkLines("subst.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+=\t\tpfx", "SUBST_STAGE.pfx=\tpre-configure", @@ -694,7 +694,7 @@ func (s *Suite) Test_SubstContext_suggestSubstVars__autofix_realign_paragraph(c "with \"SUBST_VARS.pfx+=\\tPREFIX\".") t.CheckFileLinesDetab("subst.mk", - "# $NetBSD: substcontext_test.go,v 1.26 2019/06/10 19:51:57 rillig Exp $", + "# $NetBSD: substcontext_test.go,v 1.27 2019/06/30 20:56:19 rillig Exp $", "", "SUBST_CLASSES+= pfx", "SUBST_STAGE.pfx= pre-configure", @@ -710,7 +710,7 @@ func (s *Suite) Test_SubstContext_suggestSubstVars__autofix_plus_sed(c *check.C) t.Chdir(".") mklines := t.SetUpFileMkLines("subst.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+=\t\tpfx", "SUBST_STAGE.pfx=\tpre-configure", @@ -733,7 +733,7 @@ func (s *Suite) Test_SubstContext_suggestSubstVars__autofix_plus_sed(c *check.C) "with \"SUBST_VARS.pfx=\\t\\tPREFIX\".") t.CheckFileLinesDetab("subst.mk", - "# $NetBSD: substcontext_test.go,v 1.26 2019/06/10 19:51:57 rillig Exp $", + "# $NetBSD: substcontext_test.go,v 1.27 2019/06/30 20:56:19 rillig Exp $", "", "SUBST_CLASSES+= pfx", "SUBST_STAGE.pfx= pre-configure", @@ -752,7 +752,7 @@ func (s *Suite) Test_SubstContext_suggestSubstVars__autofix_plus_vars(c *check.C t.Chdir(".") mklines := t.SetUpFileMkLines("subst.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+=\tid", "SUBST_STAGE.id=\tpre-configure", @@ -767,7 +767,7 @@ func (s *Suite) Test_SubstContext_suggestSubstVars__autofix_plus_vars(c *check.C "with \"SUBST_VARS.id=\\tPREFIX\".") t.CheckFileLinesDetab("subst.mk", - "# $NetBSD: substcontext_test.go,v 1.26 2019/06/10 19:51:57 rillig Exp $", + "# $NetBSD: substcontext_test.go,v 1.27 2019/06/30 20:56:19 rillig Exp $", "", "SUBST_CLASSES+= id", "SUBST_STAGE.id= pre-configure", @@ -786,7 +786,7 @@ func (s *Suite) Test_SubstContext_suggestSubstVars__autofix_indentation(c *check t.Chdir(".") mklines := t.SetUpFileMkLines("subst.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+=\t\t\tfix-paths", "SUBST_STAGE.fix-paths=\t\tpre-configure", @@ -801,7 +801,7 @@ func (s *Suite) Test_SubstContext_suggestSubstVars__autofix_indentation(c *check "with \"SUBST_VARS.fix-paths=\\t\\tPREFIX\".") t.CheckFileLinesDetab("subst.mk", - "# $NetBSD: substcontext_test.go,v 1.26 2019/06/10 19:51:57 rillig Exp $", + "# $NetBSD: substcontext_test.go,v 1.27 2019/06/30 20:56:19 rillig Exp $", "", "SUBST_CLASSES+= fix-paths", "SUBST_STAGE.fix-paths= pre-configure", @@ -866,7 +866,7 @@ func (s *Suite) Test_SubstContext__unusual_variable_order(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("subst.mk", - MkRcsID, + MkCvsID, "", "SUBST_CLASSES+=\t\tid", "SUBST_SED.id=\t\t-e /deleteme/d", diff --git a/pkgtools/pkglint/files/tools.go b/pkgtools/pkglint/files/tools.go index 0aecc1e6b67..6222a3228c9 100644 --- a/pkgtools/pkglint/files/tools.go +++ b/pkgtools/pkglint/files/tools.go @@ -135,7 +135,7 @@ func NewTools() *Tools { // // After this tool is added to USE_TOOLS, it may be used by this name // (e.g. "awk") or by its variable (e.g. ${AWK}). -func (tr *Tools) Define(name, varname string, mkline MkLine) *Tool { +func (tr *Tools) Define(name, varname string, mkline *MkLine) *Tool { if trace.Tracing { trace.Stepf("Tools.Define: %q %q in %s", name, varname, mkline) } @@ -150,7 +150,7 @@ func (tr *Tools) Define(name, varname string, mkline MkLine) *Tool { } func (tr *Tools) def(name, varname string, mustUseVarForm bool, validity Validity, aliases []string) *Tool { - assertf(tr.IsValidToolName(name), "Invalid tool name %q", name) + assert(tr.IsValidToolName(name)) fresh := Tool{name, varname, mustUseVarForm, validity, aliases} @@ -225,7 +225,7 @@ func (tr *Tools) Trace() { // // If addToUseTools is true, a USE_TOOLS line makes a tool immediately // usable. This should only be done if the current line is unconditional. -func (tr *Tools) ParseToolLine(mklines MkLines, mkline MkLine, fromInfrastructure bool, addToUseTools bool) { +func (tr *Tools) ParseToolLine(mklines *MkLines, mkline *MkLine, fromInfrastructure bool, addToUseTools bool) { switch { case mkline.IsVarassign(): @@ -303,7 +303,7 @@ func (tr *Tools) addAlias(tool *Tool, alias string) { // This can be done only in the pkgsrc infrastructure files, where the // actual definition is assumed to be in some other file. In packages // though, this assumption cannot be made and pkglint needs to be strict. -func (tr *Tools) parseUseTools(mkline MkLine, createIfAbsent bool, addToUseTools bool) { +func (tr *Tools) parseUseTools(mkline *MkLine, createIfAbsent bool, addToUseTools bool) { value := mkline.Value() if containsVarRef(value) { return @@ -391,7 +391,7 @@ func (tr *Tools) Usable(tool *Tool, time ToolTime) bool { } func (tr *Tools) Fallback(other *Tools) { - assertf(tr.fallback == nil, "Tools.Fallback must only be called once.") + assert(tr.fallback == nil) // Must only be called once. tr.fallback = other } diff --git a/pkgtools/pkglint/files/tools_test.go b/pkgtools/pkglint/files/tools_test.go index f1e6554f689..1d7cec63635 100644 --- a/pkgtools/pkglint/files/tools_test.go +++ b/pkgtools/pkglint/files/tools_test.go @@ -40,7 +40,7 @@ func (s *Suite) Test_Tools_ParseToolLine__opsys(c *check.C) { t.SetUpTool("tool1", "", Nowhere) t.SetUpVartypes() t.CreateFileLines("Makefile", - MkRcsID, + MkCvsID, "", "USE_TOOLS.NetBSD+=\ttool1") @@ -55,7 +55,7 @@ func (s *Suite) Test_Tools_ParseToolLine__invalid_tool_name(c *check.C) { t.SetUpVartypes() mklines := t.NewMkLines("Makefile", - MkRcsID, + MkCvsID, "", ".for t in abc ${UNDEFINED}", "TOOLS_CREATE+=\t\t${t}", @@ -78,7 +78,7 @@ func (s *Suite) Test_Tools_parseUseTools(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("mk/triple-tool.mk", - MkRcsID, + MkCvsID, "", "USE_TOOLS+=\tunknown unknown unknown") t.FinishSetUp() @@ -128,7 +128,7 @@ func (s *Suite) Test_Tools__USE_TOOLS_predefined_sed(c *check.C) { t.CreateFileLines("mk/tools/defaults.mk", "_TOOLS_VARNAME.sed=\tSED") t.CreateFileLines("module.mk", - MkRcsID, + MkCvsID, "", "do-build:", "\t${SED} < input > output", @@ -174,12 +174,12 @@ func (s *Suite) Test_Tools__load_from_infrastructure(c *check.C) { dummyMklines := t.NewMkLines("dummy.mk") createMklines := t.NewMkLines("create.mk", - MkRcsID, + MkCvsID, "TOOLS_CREATE+= load", "TOOLS_CREATE+= run", "TOOLS_CREATE+= nowhere") - createMklines.ForEach(func(mkline MkLine) { + createMklines.ForEach(func(mkline *MkLine) { tools.ParseToolLine(createMklines, mkline, true, false) }) @@ -197,12 +197,12 @@ func (s *Suite) Test_Tools__load_from_infrastructure(c *check.C) { // The variable name RUN is reserved by pkgsrc, therefore RUN_CMD. varnamesMklines := t.NewMkLines("varnames.mk", - MkRcsID, + MkCvsID, "_TOOLS_VARNAME.load= LOAD", "_TOOLS_VARNAME.run= RUN_CMD", "_TOOLS_VARNAME.nowhere= NOWHERE") - varnamesMklines.ForEach(func(mkline MkLine) { + varnamesMklines.ForEach(func(mkline *MkLine) { tools.ParseToolLine(varnamesMklines, mkline, true, false) }) @@ -314,7 +314,7 @@ func (s *Suite) Test_Tools__builtin_mk(c *check.C) { // may be used in any file at load time. mklines := t.SetUpFileMkLines("category/package/builtin.mk", - MkRcsID, + MkCvsID, "", "VAR!= ${ECHO} 'too early'", "VAR!= ${LOAD} 'too early'", @@ -347,7 +347,7 @@ func (s *Suite) Test_Tools__implicit_definition_in_bsd_pkg_mk(c *check.C) { t.SetUpPkgsrc() t.SetUpCommandLine("-Wall,no-space") t.CreateFileLines("mk/tools/defaults.mk", - MkRcsID) // None + MkCvsID) // None t.CreateFileLines("mk/bsd.prefs.mk", "USE_TOOLS+= load") t.CreateFileLines("mk/bsd.pkg.mk", @@ -368,7 +368,7 @@ func (s *Suite) Test_Tools__both_prefs_and_pkg_mk(c *check.C) { t.SetUpPkgsrc() t.SetUpCommandLine("-Wall,no-space") t.CreateFileLines("mk/tools/defaults.mk", - MkRcsID) + MkCvsID) t.CreateFileLines("mk/bsd.prefs.mk", "USE_TOOLS+= both") t.CreateFileLines("mk/bsd.pkg.mk", @@ -458,7 +458,7 @@ func (s *Suite) Test_Tools__var(c *check.C) { t.FinishSetUp() mklines := t.NewMkLines("module.mk", - MkRcsID, + MkCvsID, "", "pre-configure:", "\t${LN} from to") @@ -552,16 +552,14 @@ func (s *Suite) Test_Tools_Fallback__called_twice(c *check.C) { tools.Fallback(fallback) - t.ExpectPanic( - func() { tools.Fallback(fallback) }, - "Pkglint internal error: Tools.Fallback must only be called once.") + t.ExpectAssert(func() { tools.Fallback(fallback) }) } func (s *Suite) Test_Tools__aliases(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("mk/tools/replace.mk", - MkRcsID, + MkCvsID, "TOOLS_CREATE+=\tsed", "TOOLS_PATH.sed=\t/bin/sed", "", @@ -570,7 +568,7 @@ func (s *Suite) Test_Tools__aliases(c *check.C) { "TOOLS_ALIASES.gsed=\tsed ${tool}") infraTools := NewTools() - mklines.ForEach(func(mkline MkLine) { + mklines.ForEach(func(mkline *MkLine) { infraTools.ParseToolLine(mklines, mkline, false, false) }) @@ -595,7 +593,7 @@ func (s *Suite) Test_Tools__aliases_in_for_loop(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("mk/tools/replace.mk", - MkRcsID, + MkCvsID, "_TOOLS_GREP=\tgrep egrep fgrep", "TOOLS_CREATE+=\tgrep egrep fgrep ggrep", ".for t in ${_TOOLS_GREP}", @@ -685,7 +683,7 @@ func (s *Suite) Test_Tools_IsValidToolName(c *check.C) { t.SetUpTool("[", "", AtRunTime) t.SetUpTool("echo -n", "ECHO_N", AtRunTime) mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, "", "USE_TOOLS+=\t[") diff --git a/pkgtools/pkglint/files/toplevel.go b/pkgtools/pkglint/files/toplevel.go index f8bdb12687e..8a72a372882 100644 --- a/pkgtools/pkglint/files/toplevel.go +++ b/pkgtools/pkglint/files/toplevel.go @@ -35,7 +35,7 @@ func CheckdirToplevel(dir string) { } } -func (ctx *Toplevel) checkSubdir(mkline MkLine) { +func (ctx *Toplevel) checkSubdir(mkline *MkLine) { subdir := mkline.Value() if mkline.IsCommentedVarassign() && (mkline.VarassignComment() == "#" || mkline.VarassignComment() == "") { diff --git a/pkgtools/pkglint/files/toplevel_test.go b/pkgtools/pkglint/files/toplevel_test.go index 9477b559067..c4608823466 100644 --- a/pkgtools/pkglint/files/toplevel_test.go +++ b/pkgtools/pkglint/files/toplevel_test.go @@ -6,7 +6,7 @@ func (s *Suite) Test_CheckdirToplevel(c *check.C) { t := s.Init(c) t.CreateFileLines("Makefile", - MkRcsID, + MkCvsID, "", "SUBDIR+= x11", "SUBDIR+=\tarchivers", @@ -39,7 +39,7 @@ func (s *Suite) Test_Toplevel_checkSubdir__sorting_x11(c *check.C) { t := s.Init(c) t.CreateFileLines("Makefile", - MkRcsID, + MkCvsID, "", "SUBDIR+=\tx11", "SUBDIR+=\tsysutils", @@ -60,7 +60,7 @@ func (s *Suite) Test_Toplevel_checkSubdir__commented_without_reason(c *check.C) t := s.Init(c) t.CreateFileLines("Makefile", - MkRcsID, + MkCvsID, "", "#SUBDIR+=\taaa", "#SUBDIR+=\tbbb\t#", @@ -81,15 +81,15 @@ func (s *Suite) Test_CheckdirToplevel__recursive(c *check.C) { t := s.Init(c) t.CreateFileLines("mk/misc/category.mk", - MkRcsID) + MkCvsID) t.SetUpPackage("category/package", "UNKNOWN=\tvalue") t.CreateFileLines("Makefile", - MkRcsID, + MkCvsID, "", "SUBDIR+=\tcategory") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tThe category", "", @@ -109,15 +109,15 @@ func (s *Suite) Test_CheckdirToplevel__recursive_inter_package(c *check.C) { t := s.Init(c) t.CreateFileLines("mk/misc/category.mk", - MkRcsID) + MkCvsID) t.SetUpPackage("category/package", "UNKNOWN=\tvalue") t.CreateFileLines("Makefile", - MkRcsID, + MkCvsID, "", "SUBDIR+=\tcategory") t.CreateFileLines("category/Makefile", - MkRcsID, + MkCvsID, "", "COMMENT=\tThe category", "", diff --git a/pkgtools/pkglint/files/trace/tracing.go b/pkgtools/pkglint/files/trace/tracing.go index 947f5ad2d09..926344cbdf5 100644 --- a/pkgtools/pkglint/files/trace/tracing.go +++ b/pkgtools/pkglint/files/trace/tracing.go @@ -78,7 +78,7 @@ func (t *Tracer) Call(args ...interface{}) func() { return t.traceCall(args...) } -// http://stackoverflow.com/questions/13476349/check-for-nil-and-nil-interface-in-go +// https://stackoverflow.com/questions/13476349/check-for-nil-and-nil-interface-in-go func isNil(a interface{}) bool { defer func() { _ = recover() diff --git a/pkgtools/pkglint/files/util.go b/pkgtools/pkglint/files/util.go index 68660eabf27..55a9338dc90 100644 --- a/pkgtools/pkglint/files/util.go +++ b/pkgtools/pkglint/files/util.go @@ -9,6 +9,7 @@ import ( "os" "path" "path/filepath" + "reflect" "regexp" "sort" "strconv" @@ -159,6 +160,30 @@ func assertNil(err error, format string, args ...interface{}) { } } +func assertNotNil(obj interface{}) { + + // https://stackoverflow.com/questions/13476349/check-for-nil-and-nil-interface-in-go + isNil := func() bool { + defer func() { _ = recover() }() + return reflect.ValueOf(obj).IsNil() + } + + if obj == nil || isNil() { + panic("Pkglint internal error: unexpected nil pointer") + } +} + +// assert checks that the condition is true. Otherwise it terminates the +// process with a fatal error message, prefixed with "Pkglint internal error". +// +// This method must only be used for programming errors. +// For runtime errors, use dummyLine.Fatalf. +func assert(cond bool) { + if !cond { + panic("Pkglint internal error") + } +} + // assertf checks that the condition is true. Otherwise it terminates the // process with a fatal error message, prefixed with "Pkglint internal error". // @@ -177,7 +202,7 @@ func isEmptyDir(filename string) bool { dirents, err := ioutil.ReadDir(filename) if err != nil { - return true + return true // XXX: Why not false? } for _, dirent := range dirents { @@ -233,46 +258,47 @@ func dirglob(dirname string) []string { // Checks whether a file is already committed to the CVS repository. func isCommitted(filename string) bool { - lines := G.loadCvsEntries(filename) - if lines == nil { - return false - } - needle := "/" + path.Base(filename) + "/" - for _, line := range lines.Lines { - if hasPrefix(line.Text, needle) { - return true - } - } - return false + entries := G.loadCvsEntries(filename) + _, found := entries[path.Base(filename)] + return found } +// isLocallyModified tests whether a file (not a directory) is modified, +// as seen by CVS. +// +// There is no corresponding test for Git (as used by pkgsrc-wip) since that +// is more difficult to implement than simply reading a CVS/Entries file. func isLocallyModified(filename string) bool { - baseName := path.Base(filename) - - lines := G.loadCvsEntries(filename) - if lines == nil { + entries := G.loadCvsEntries(filename) + entry, found := entries[path.Base(filename)] + if !found { return false } - for _, line := range lines.Lines { - fields := strings.Split(line.Text, "/") - if 3 < len(fields) && fields[1] == baseName { - st, err := os.Stat(filename) - if err != nil { - return true - } - - // Following http://cvsman.com/cvs-1.12.12/cvs_19.php, format both timestamps. - cvsModTime := fields[3] - fsModTime := st.ModTime().UTC().Format(time.ANSIC) - if trace.Tracing { - trace.Stepf("cvs.time=%q fs.time=%q", cvsModTime, fsModTime) - } + st, err := os.Stat(filename) + if err != nil { + return true + } - return cvsModTime != fsModTime - } + // Following http://cvsman.com/cvs-1.12.12/cvs_19.php, format both timestamps. + cvsModTime := entry.Timestamp + fsModTime := st.ModTime().UTC().Format(time.ANSIC) + if trace.Tracing { + trace.Stepf("cvs.time=%q fs.time=%q", cvsModTime, fsModTime) } - return false + + return cvsModTime != fsModTime +} + +// CvsEntry is one of the entries in a CVS/Entries file. +// +// See http://cvsman.com/cvs-1.12.12/cvs_19.php. +type CvsEntry struct { + Name string + Revision string + Timestamp string + Options string + TagDate string } // Returns the number of columns that a string occupies when printed with @@ -488,6 +514,34 @@ func cleanpath(filename string) string { return strings.Join(parts, "/") } +func pathContains(haystack, needle string) bool { + n0 := needle[0] + for i := 0; i < 1+len(haystack)-len(needle); i++ { + if haystack[i] == n0 && hasPrefix(haystack[i:], needle) { + if i == 0 || haystack[i-1] == '/' { + if i+len(needle) == len(haystack) || haystack[i+len(needle)] == '/' { + return true + } + } + } + } + return false +} + +func pathContainsDir(haystack, needle string) bool { + n0 := needle[0] + for i := 0; i < 1+len(haystack)-len(needle); i++ { + if haystack[i] == n0 && hasPrefix(haystack[i:], needle) { + if i == 0 || haystack[i-1] == '/' { + if i+len(needle) < len(haystack) && haystack[i+len(needle)] == '/' { + return true + } + } + } + } + return false +} + func containsVarRef(s string) bool { return contains(s, "${") } @@ -561,26 +615,26 @@ func (o *Once) check(key uint64) bool { // TODO: Merge this code with Var, which defines essentially the // same features. type Scope struct { - firstDef map[string]MkLine // TODO: Can this be removed? - lastDef map[string]MkLine + firstDef map[string]*MkLine // TODO: Can this be removed? + lastDef map[string]*MkLine value map[string]string - used map[string]MkLine + used map[string]*MkLine usedAtLoadTime map[string]bool fallback map[string]string } func NewScope() Scope { return Scope{ - make(map[string]MkLine), - make(map[string]MkLine), + make(map[string]*MkLine), + make(map[string]*MkLine), make(map[string]string), - make(map[string]MkLine), + make(map[string]*MkLine), make(map[string]bool), make(map[string]string)} } // Define marks the variable and its canonicalized form as defined. -func (s *Scope) Define(varname string, mkline MkLine) { +func (s *Scope) Define(varname string, mkline *MkLine) { def := func(name string) { if s.firstDef[name] == nil { s.firstDef[name] = mkline @@ -621,7 +675,7 @@ func (s *Scope) Fallback(varname string, value string) { } // Use marks the variable and its canonicalized form as used. -func (s *Scope) Use(varname string, line MkLine, time vucTime) { +func (s *Scope) Use(varname string, line *MkLine, time VucTime) { use := func(name string) { if s.used[name] == nil { s.used[name] = line @@ -629,7 +683,7 @@ func (s *Scope) Use(varname string, line MkLine, time vucTime) { trace.Step2("Using %q in %s", name, line.String()) } } - if time == vucTimeLoad { + if time == VucLoadTime { s.usedAtLoadTime[name] = true } } @@ -642,7 +696,7 @@ func (s *Scope) Use(varname string, line MkLine, time vucTime) { // - defined, // - mentioned in a commented variable assignment, // - mentioned in a documentation comment. -func (s *Scope) Mentioned(varname string) MkLine { +func (s *Scope) Mentioned(varname string) *MkLine { return s.firstDef[varname] } @@ -703,7 +757,7 @@ func (s *Scope) UsedAtLoadTime(varname string) bool { // value, and the including file later overrides that value. Or the other way // round: the including file sets a value first, and the included file then // assigns a default value using ?=. -func (s *Scope) FirstDefinition(varname string) MkLine { +func (s *Scope) FirstDefinition(varname string) *MkLine { mkline := s.firstDef[varname] if mkline != nil && mkline.IsVarassign() { lastLine := s.LastDefinition(varname) @@ -724,7 +778,7 @@ func (s *Scope) FirstDefinition(varname string) MkLine { // value, and the including file later overrides that value. Or the other way // round: the including file sets a value first, and the included file then // assigns a default value using ?=. -func (s *Scope) LastDefinition(varname string) MkLine { +func (s *Scope) LastDefinition(varname string) *MkLine { mkline := s.lastDef[varname] if mkline != nil && mkline.IsVarassign() { return mkline @@ -735,8 +789,8 @@ func (s *Scope) LastDefinition(varname string) MkLine { // Commented returns whether the variable has only been defined in commented // variable assignments. These are ignored by bmake but used heavily in // mk/defaults/mk.conf for documentation. -func (s *Scope) Commented(varname string) MkLine { - var mklines []MkLine +func (s *Scope) Commented(varname string) *MkLine { + var mklines []*MkLine if first := s.firstDef[varname]; first != nil { mklines = append(mklines, first) } @@ -759,7 +813,7 @@ func (s *Scope) Commented(varname string) MkLine { return nil } -func (s *Scope) FirstUse(varname string) MkLine { +func (s *Scope) FirstUse(varname string) *MkLine { return s.used[varname] } @@ -908,7 +962,7 @@ type fileCacheEntry struct { count int key string options LoadOptions - lines Lines + lines *Lines } func NewFileCache(size int) *FileCache { @@ -919,7 +973,7 @@ func NewFileCache(size int) *FileCache { 0} } -func (c *FileCache) Put(filename string, options LoadOptions, lines Lines) { +func (c *FileCache) Put(filename string, options LoadOptions, lines *Lines) { key := c.key(filename) entry := c.mapping[key] @@ -973,14 +1027,14 @@ func (c *FileCache) removeOldEntries() { } } -func (c *FileCache) Get(filename string, options LoadOptions) Lines { +func (c *FileCache) Get(filename string, options LoadOptions) *Lines { key := c.key(filename) entry, found := c.mapping[key] if found && entry.options == options { c.hits++ entry.count++ - lines := make([]Line, entry.lines.Len()) + lines := make([]*Line, entry.lines.Len()) for i, line := range entry.lines.Lines { lines[i] = NewLineMulti(filename, int(line.firstLine), int(line.lastLine), line.Text, line.raw) } @@ -1164,6 +1218,48 @@ func joinSkipEmptyOxford(conn string, elements ...string) string { return strings.Join(nonempty, ", ") } +type pathMatcher struct { + matchType pathMatchType + pattern string + originalPattern string +} + +func newPathMatcher(pattern string) *pathMatcher { + assert(strings.IndexByte(pattern, '[') == -1) + assert(strings.IndexByte(pattern, '?') == -1) + + stars := strings.Count(pattern, "*") + assert(stars == 0 || stars == 1) + switch { + case stars == 0: + return &pathMatcher{pmExact, pattern, pattern} + case pattern[0] == '*': + return &pathMatcher{pmSuffix, pattern[1:], pattern} + default: + assert(pattern[len(pattern)-1] == '*') + return &pathMatcher{pmPrefix, pattern[:len(pattern)-1], pattern} + } +} + +func (m pathMatcher) matches(subject string) bool { + switch m.matchType { + case pmPrefix: + return hasPrefix(subject, m.pattern) + case pmSuffix: + return hasSuffix(subject, m.pattern) + default: + return subject == m.pattern + } +} + +type pathMatchType uint8 + +const ( + pmExact pathMatchType = iota + pmPrefix + pmSuffix +) + // StringInterner collects commonly used strings to avoid wasting heap memory // by duplicated strings. type StringInterner struct { diff --git a/pkgtools/pkglint/files/util_test.go b/pkgtools/pkglint/files/util_test.go index abfef4d88cf..fc29cefdb9c 100644 --- a/pkgtools/pkglint/files/util_test.go +++ b/pkgtools/pkglint/files/util_test.go @@ -18,6 +18,19 @@ func (s *Suite) Test_assertNil(c *check.C) { "Pkglint internal error: Oops: unexpected error") } +func (s *Suite) Test_assertNotNil(c *check.C) { + t := s.Init(c) + + assertNotNil("this string is not nil") + + t.ExpectPanic( + func() { assertNotNil(nil) }, + "Pkglint internal error: unexpected nil pointer") + t.ExpectPanic( + func() { var ptr *string; assertNotNil(ptr) }, + "Pkglint internal error: unexpected nil pointer") +} + func (s *Suite) Test_YesNoUnknown_String(c *check.C) { c.Check(yes.String(), equals, "yes") c.Check(no.String(), equals, "no") @@ -175,6 +188,80 @@ func (s *Suite) Test_relpath__quick(c *check.C) { test("some/dir/.", ".", "../..") } +func (s *Suite) Test_pathContains(c *check.C) { + t := s.Init(c) + + test := func(haystack, needle string, expected bool) { + actual := pathContains(haystack, needle) + t.Check(actual, equals, expected) + } + + testPanic := func(haystack, needle string) { + t.c.Check( + func() { _ = pathContains(haystack, needle) }, + check.PanicMatches, + `runtime error: index out of range`) + } + + testPanic("", "") + testPanic("a", "") + testPanic("a/b/c", "") + + test("a", "a", true) + test("a", "b", false) + test("a", "A", false) + test("a/b/c", "a", true) + test("a/b/c", "b", true) + test("a/b/c", "c", true) + test("a/b/c", "a/b", true) + test("a/b/c", "b/c", true) + test("a/b/c", "a/b/c", true) + test("aa/bb/cc", "a/b", false) + test("aa/bb/cc", "a/bb", false) + test("aa/bb/cc", "aa/b", false) + test("aa/bb/cc", "aa/bb", true) + test("aa/bb/cc", "a", false) + test("aa/bb/cc", "b", false) + test("aa/bb/cc", "c", false) +} + +func (s *Suite) Test_pathContainsDir(c *check.C) { + t := s.Init(c) + + test := func(haystack, needle string, expected bool) { + actual := pathContainsDir(haystack, needle) + t.Check(actual, equals, expected) + } + + testPanic := func(haystack, needle string) { + t.c.Check( + func() { _ = pathContainsDir(haystack, needle) }, + check.PanicMatches, + `runtime error: index out of range`) + } + + testPanic("", "") + testPanic("a", "") + testPanic("a/b/c", "") + + test("a", "a", false) + test("a", "b", false) + test("a", "A", false) + test("a/b/c", "a", true) + test("a/b/c", "b", true) + test("a/b/c", "c", false) + test("a/b/c", "a/b", true) + test("a/b/c", "b/c", false) + test("a/b/c", "a/b/c", false) + test("aa/bb/cc", "a/b", false) + test("aa/bb/cc", "a/bb", false) + test("aa/bb/cc", "aa/b", false) + test("aa/bb/cc", "aa/bb", true) + test("aa/bb/cc", "a", false) + test("aa/bb/cc", "b", false) + test("aa/bb/cc", "c", false) +} + func (s *Suite) Test_fileExists(c *check.C) { t := s.Init(c) @@ -394,7 +481,6 @@ func (s *Suite) Test_isLocallyModified(c *check.C) { modified := t.CreateFileLines("modified") t.CreateFileLines("CVS/Entries", - "//", // Just for code coverage. "/unmodified//"+modTime.Format(time.ANSIC)+"//", "/modified//"+modTime.Format(time.ANSIC)+"//", "/enoent//"+modTime.Format(time.ANSIC)+"//") @@ -448,7 +534,7 @@ func (s *Suite) Test_Scope_Used(c *check.C) { scope := NewScope() mkline := t.NewMkLine("file.mk", 1, "\techo ${VAR.param}") - scope.Use("VAR.param", mkline, vucTimeRun) + scope.Use("VAR.param", mkline, VucRunTime) c.Check(scope.Used("VAR.param"), equals, true) c.Check(scope.Used("VAR.other"), equals, false) @@ -500,7 +586,7 @@ func (s *Suite) Test_Scope_LastValue(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("file.mk", - MkRcsID, + MkCvsID, "VAR=\tfirst", "VAR=\tsecond", ".if 1", @@ -637,7 +723,7 @@ func (s *Suite) Test_FileCache(c *check.C) { cache := NewFileCache(3) lines := t.NewLines("Makefile", - MkRcsID, + MkCvsID, "# line 2") c.Check(cache.Get("Makefile", 0), check.IsNil) @@ -648,13 +734,13 @@ func (s *Suite) Test_FileCache(c *check.C) { c.Check(cache.Get("Makefile", MustSucceed|LogErrors), check.IsNil) // Wrong LoadOptions. linesFromCache := cache.Get("Makefile", 0) - c.Check(linesFromCache.FileName, equals, "Makefile") + c.Check(linesFromCache.Filename, equals, "Makefile") c.Check(linesFromCache.Lines, check.HasLen, 2) c.Check(linesFromCache.Lines[0].Filename, equals, "Makefile") // Cache keys are normalized using path.Clean. linesFromCache2 := cache.Get("./Makefile", 0) - c.Check(linesFromCache2.FileName, equals, "./Makefile") + c.Check(linesFromCache2.Filename, equals, "./Makefile") c.Check(linesFromCache2.Lines, check.HasLen, 2) c.Check(linesFromCache2.Lines[0].Filename, equals, "./Makefile") @@ -700,7 +786,7 @@ func (s *Suite) Test_FileCache_removeOldEntries__branch_coverage(c *check.C) { G.Testing = false lines := t.NewLines("filename.mk", - MkRcsID) + MkCvsID) cache := NewFileCache(3) cache.Put("filename1.mk", 0, lines) cache.Put("filename2.mk", 0, lines) @@ -721,7 +807,7 @@ func (s *Suite) Test_FileCache_removeOldEntries__no_tracing(c *check.C) { t.DisableTracing() lines := t.NewLines("filename.mk", - MkRcsID) + MkCvsID) cache := NewFileCache(3) cache.Put("filename1.mk", 0, lines) cache.Put("filename2.mk", 0, lines) @@ -738,7 +824,7 @@ func (s *Suite) Test_FileCache_removeOldEntries__zero_capacity(c *check.C) { t := s.Init(c) lines := t.NewLines("filename.mk", - MkRcsID) + MkCvsID) cache := NewFileCache(1) cache.Put("filename1.mk", 0, lines) @@ -751,7 +837,7 @@ func (s *Suite) Test_FileCache_Evict__sort(c *check.C) { t := s.Init(c) lines := t.NewLines("filename.mk", - MkRcsID) + MkCvsID) cache := NewFileCache(10) cache.Put("filename0.mk", 0, lines) cache.Put("filename1.mk", 0, lines) @@ -939,6 +1025,56 @@ func (s *Suite) Test_joinSkipEmptyOxford(c *check.C) { "one, two, and three") } +func (s *Suite) Test_newPathMatcher(c *check.C) { + t := s.Init(c) + + test := func(pattern string, matchType pathMatchType, matchPattern string) { + c.Check(*newPathMatcher(pattern), equals, pathMatcher{matchType, matchPattern, pattern}) + } + + testPanic := func(pattern string) { + t.ExpectPanic( + func() { _ = newPathMatcher(pattern) }, + "Pkglint internal error") + } + + testPanic("*.[0123456]") + testPanic("file.???") + testPanic("*.???") + test("", pmExact, "") + test("exact", pmExact, "exact") + test("*.mk", pmSuffix, ".mk") + test("Makefile.*", pmPrefix, "Makefile.") + testPanic("*.*") + testPanic("**") + testPanic("a*b") + testPanic("[") + testPanic("malformed[") +} + +func (s *Suite) Test_pathMatcher_matches(c *check.C) { + + test := func(pattern string, subject string, expected bool) { + matcher := newPathMatcher(pattern) + c.Check(matcher.matches(subject), equals, expected) + } + + test("", "", true) + test("", "any", false) + test("exact", "exact", true) + test("exact", "different", false) + + test("*.mk", "filename.mk", true) + test("*.mk", "filename.txt", false) + test("*.mk", "filename.mkx", false) + test("*.mk", ".mk", true) + + test("Makefile.*", "Makefile", false) + test("Makefile.*", "Makefile.", true) + test("Makefile.*", "Makefile.txt", true) + test("Makefile.*", "makefile.txt", false) +} + func (s *Suite) Test_StringInterner(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/var.go b/pkgtools/pkglint/files/var.go index 38cc48a87fd..13f2a278e94 100644 --- a/pkgtools/pkglint/files/var.go +++ b/pkgtools/pkglint/files/var.go @@ -31,8 +31,8 @@ type Var struct { value string valueInfra string - readLocations []MkLine - writeLocations []MkLine + readLocations []*MkLine + writeLocations []*MkLine conditional bool conditionalVars StringSet @@ -106,7 +106,7 @@ func (v *Var) Constant() bool { // Variable assignments in the pkgsrc infrastructure are taken into account // for determining the constant value. func (v *Var) ConstantValue() string { - assertf(v.Constant(), "Variable must be constant.") + assert(v.Constant()) return v.constantValue } @@ -146,7 +146,7 @@ func (v *Var) ValueInfra() string { // are not listed. // // Variable uses in the pkgsrc infrastructure are taken into account. -func (v *Var) ReadLocations() []MkLine { +func (v *Var) ReadLocations() []*MkLine { return v.readLocations } @@ -156,11 +156,11 @@ func (v *Var) ReadLocations() []MkLine { // reachable in practice. // // Variable assignments in the pkgsrc infrastructure are taken into account. -func (v *Var) WriteLocations() []MkLine { +func (v *Var) WriteLocations() []*MkLine { return v.writeLocations } -func (v *Var) Read(mkline MkLine) { +func (v *Var) Read(mkline *MkLine) { v.readLocations = append(v.readLocations, mkline) v.constantState = [...]uint8{3, 2, 2, 3}[v.constantState] } @@ -169,8 +169,8 @@ func (v *Var) Read(mkline MkLine) { // Only standard assignments (VAR=value) are handled. // Side-effect assignments (${VAR::=value}) are not handled here since // they don't occur in practice. -func (v *Var) Write(mkline MkLine, conditional bool, conditionVarnames ...string) { - assertf(mkline.Varname() == v.Name, "wrong variable name") +func (v *Var) Write(mkline *MkLine, conditional bool, conditionVarnames ...string) { + assert(mkline.Varname() == v.Name) v.writeLocations = append(v.writeLocations, mkline) @@ -179,7 +179,7 @@ func (v *Var) Write(mkline MkLine, conditional bool, conditionVarnames ...string } v.conditionalVars.AddAll(conditionVarnames) - mkline.ForEachUsed(func(varUse *MkVarUse, time vucTime) { + mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { v.refs.Add(varUse.varname) }) v.refs.AddAll(conditionVarnames) @@ -192,7 +192,7 @@ func (v *Var) Write(mkline MkLine, conditional bool, conditionVarnames ...string v.updateConstantValue(mkline) } -func (v *Var) update(mkline MkLine, update *string) { +func (v *Var) update(mkline *MkLine, update *string) { firstWrite := len(v.writeLocations) == 1 if v.Conditional() && !firstWrite { return @@ -218,7 +218,7 @@ func (v *Var) update(mkline MkLine, update *string) { } } -func (v *Var) updateConstantValue(mkline MkLine) { +func (v *Var) updateConstantValue(mkline *MkLine) { if v.constantState == 3 { return } diff --git a/pkgtools/pkglint/files/var_test.go b/pkgtools/pkglint/files/var_test.go index 7dfc2968319..ebc5579322a 100644 --- a/pkgtools/pkglint/files/var_test.go +++ b/pkgtools/pkglint/files/var_test.go @@ -216,13 +216,13 @@ func (s *Suite) Test_Var_Write__conditional_without_variables(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("filename.mk", - MkRcsID, + MkCvsID, ".if exists(/usr/bin)", "VAR=\tvalue", ".endif") scope := NewRedundantScope() - mklines.ForEach(func(mkline MkLine) { + mklines.ForEach(func(mkline *MkLine) { if mkline.IsVarassign() { t.Check(scope.get("VAR").vari.Conditional(), equals, false) } @@ -239,11 +239,8 @@ func (s *Suite) Test_Var_Write__assertion(c *check.C) { t := s.Init(c) v := NewVar("VAR") - t.ExpectPanic( - func() { - v.Write(t.NewMkLine("filename.mk", 1, "OTHER=value"), false, nil...) - }, - "Pkglint internal error: wrong variable name") + t.ExpectAssert( + func() { v.Write(t.NewMkLine("filename.mk", 1, "OTHER=value"), false, nil...) }) } func (s *Suite) Test_Var_Value__conditional_write_after_unconditional(c *check.C) { @@ -321,7 +318,7 @@ func (s *Suite) Test_Var_ReadLocations(c *check.C) { mkline123 := t.NewMkLine("read.mk", 123, "OTHER=\t${VAR}") v.Read(mkline123) - t.Check(v.ReadLocations(), deepEquals, []MkLine{mkline123}) + t.Check(v.ReadLocations(), deepEquals, []*MkLine{mkline123}) mkline124 := t.NewMkLine("read.mk", 124, "OTHER=\t${VAR} ${VAR}") v.Read(mkline124) @@ -329,7 +326,7 @@ func (s *Suite) Test_Var_ReadLocations(c *check.C) { // For now, count every read of the variable. I'm not yet sure // whether that's the best way or whether to make the lines unique. - t.Check(v.ReadLocations(), deepEquals, []MkLine{mkline123, mkline124, mkline124}) + t.Check(v.ReadLocations(), deepEquals, []*MkLine{mkline123, mkline124, mkline124}) } func (s *Suite) Test_Var_WriteLocations(c *check.C) { @@ -342,7 +339,7 @@ func (s *Suite) Test_Var_WriteLocations(c *check.C) { mkline123 := t.NewMkLine("write.mk", 123, "VAR=\tvalue") v.Write(mkline123, false) - t.Check(v.WriteLocations(), deepEquals, []MkLine{mkline123}) + t.Check(v.WriteLocations(), deepEquals, []*MkLine{mkline123}) // Multiple writes from the same line may happen because of a .for loop. mkline125 := t.NewMkLine("write.mk", 125, "VAR+=\t${var}") @@ -351,7 +348,7 @@ func (s *Suite) Test_Var_WriteLocations(c *check.C) { // For now, count every write of the variable. I'm not yet sure // whether that's the best way or whether to make the lines unique. - t.Check(v.WriteLocations(), deepEquals, []MkLine{mkline123, mkline125, mkline125}) + t.Check(v.WriteLocations(), deepEquals, []*MkLine{mkline123, mkline125, mkline125}) } func (s *Suite) Test_Var_Refs(c *check.C) { diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go index 409c804d41f..3b28609b916 100644 --- a/pkgtools/pkglint/files/vardefs.go +++ b/pkgtools/pkglint/files/vardefs.go @@ -58,7 +58,7 @@ func (reg *VarTypeRegistry) DefineType(varcanon string, vartype *Vartype) { func (reg *VarTypeRegistry) Define(varname string, basicType *BasicType, options vartypeOptions, aclEntries ...ACLEntry) { m, varbase, varparam := match2(varname, `^([A-Z_.][A-Z0-9_]*|@)(|\*|\.\*)$`) - assertf(m, "invalid variable name") + assert(m) // invalid variable name vartype := NewVartype(basicType, options, aclEntries...) @@ -180,14 +180,6 @@ func (reg *VarTypeRegistry) pkgappendbl3(varname string, basicType *BasicType) { "Makefile, Makefile.*, *.mk: default, set, append, use") } -// Like pkgappend, but always needs a rationale. -func (reg *VarTypeRegistry) pkgappendrat(varname string, basicType *BasicType) { - reg.acl(varname, basicType, - PackageSettable|NeedsRationale, - "buildlink3.mk, builtin.mk: none", - "Makefile, Makefile.*, *.mk: default, set, append, use") -} - // Some package-defined variables may be modified in buildlink3.mk files. // These variables are typically related to compiling and linking files // from C and related languages. @@ -313,7 +305,7 @@ func (reg *VarTypeRegistry) cmdline(varname string, basicType *BasicType) { func (reg *VarTypeRegistry) infralist(varname string, basicType *BasicType) { reg.acllist(varname, basicType, List, - "*: append") + "*: set, append") } // compilerLanguages reads the available languages that are typically @@ -809,7 +801,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { reg.pkglist("BOOTSTRAP_DEPENDS", BtDependencyWithPath) reg.pkg("BOOTSTRAP_PKG", BtYesNo) // BROKEN should better be a list of messages instead of a simple string. - reg.pkgappendrat("BROKEN", BtMessage) + reg.pkgappend("BROKEN", BtMessage) reg.pkg("BROKEN_GETTEXT_DETECTION", BtYesNo) reg.pkglistrat("BROKEN_EXCEPT_ON_PLATFORM", BtMachinePlatformPattern) reg.pkglistrat("BROKEN_ON_PLATFORM", BtMachinePlatformPattern) @@ -1132,7 +1124,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { reg.pkgload("HAS_CONFIGURE", BtYes) reg.pkglist("HEADER_TEMPLATES", BtPathname) reg.pkg("HOMEPAGE", BtHomepage) - reg.pkg("ICON_THEMES", BtYes) + reg.pkgbl3("ICON_THEMES", BtYes) reg.acl("IGNORE_PKG.*", BtYes, PackageSettable, "*: set, use-loadtime") @@ -1232,16 +1224,20 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { reg.pkglist("MASTER_SITES", BtFetchURL) for _, filename := range []string{"mk/fetch/sites.mk", "mk/fetch/fetch.mk"} { - sitesMk := LoadMk(src.File(filename), NotEmpty) + loadOptions := NotEmpty | MustSucceed + if G.Testing { + loadOptions = NotEmpty + } + sitesMk := LoadMk(src.File(filename), loadOptions) if sitesMk != nil { - sitesMk.ForEach(func(mkline MkLine) { + sitesMk.ForEach(func(mkline *MkLine) { if mkline.IsVarassign() && hasPrefix(mkline.Varname(), "MASTER_SITE_") { reg.syslist(mkline.Varname(), BtFetchURL) } }) - } else { - // During tests, use t.SetUpMasterSite instead to declare these variables. } + + // During tests, use t.SetUpMasterSite instead to declare these variables. } reg.pkglist("MESSAGE_SRC", BtPathname) @@ -1670,11 +1666,13 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { reg.pkglist("_WRAP_EXTRA_ARGS.*", BtShellWord) reg.infralist("_VARGROUPS", BtIdentifier) - reg.infralist("_USER_VARS.*", BtIdentifier) - reg.infralist("_PKG_VARS.*", BtIdentifier) - reg.infralist("_SYS_VARS.*", BtIdentifier) - reg.infralist("_DEF_VARS.*", BtIdentifier) - reg.infralist("_USE_VARS.*", BtIdentifier) + reg.infralist("_USER_VARS.*", BtVariableName) + reg.infralist("_PKG_VARS.*", BtVariableName) + reg.infralist("_SYS_VARS.*", BtVariableName) + reg.infralist("_DEF_VARS.*", BtVariableName) + reg.infralist("_USE_VARS.*", BtVariableName) + reg.infralist("_SORTED_VARS.*", BtVariableNamePattern) + reg.infralist("_LISTED_VARS.*", BtVariableNamePattern) } func enum(values string) *BasicType { @@ -1692,7 +1690,7 @@ func enum(values string) *BasicType { func (reg *VarTypeRegistry) parseACLEntries(varname string, aclEntries ...string) []ACLEntry { - assertf(len(aclEntries) > 0, "At least one ACL entry must be given.") + assert(len(aclEntries) > 0) // TODO: Use separate rules for infrastructure files. // These rules would have the "infra:" prefix @@ -1727,8 +1725,7 @@ func (reg *VarTypeRegistry) parseACLEntries(varname string, aclEntries ...string } } for _, prev := range result { - matched, err := path.Match(prev.glob, glob) - assertNil(err, "Invalid ACL pattern %q for %q", glob, varname) + matched := prev.matcher.matches(glob) assertf(!matched, "Unreachable ACL pattern %q for %q.", glob, varname) } result = append(result, NewACLEntry(glob, permissions)) diff --git a/pkgtools/pkglint/files/vardefs_test.go b/pkgtools/pkglint/files/vardefs_test.go index b965b46b163..9ddfc80b35e 100644 --- a/pkgtools/pkglint/files/vardefs_test.go +++ b/pkgtools/pkglint/files/vardefs_test.go @@ -16,18 +16,18 @@ func (s *Suite) Test_VarTypeRegistry_enumFrom(c *check.C) { t := s.Init(c) t.CreateFileLines("editors/emacs/modules.mk", - MkRcsID, + MkCvsID, "", "_EMACS_VERSIONS_ALL= emacs31", "_EMACS_VERSIONS_ALL+= emacs29") t.CreateFileLines("mk/java-vm.mk", - MkRcsID, + MkCvsID, "", "_PKG_JVMS.8= openjdk8 oracle-jdk8", "_PKG_JVMS.7= ${_PKG_JVMS.8} openjdk7 sun-jdk7", "_PKG_JVMS.6= ${_PKG_JVMS.7} jdk16") t.CreateFileLines("mk/compiler.mk", - MkRcsID, + MkCvsID, "", "_COMPILERS= gcc ido mipspro-ucode \\", " sunpro", @@ -63,7 +63,7 @@ func (s *Suite) Test_VarTypeRegistry_enumFrom__no_tracing(c *check.C) { t := s.Init(c) t.CreateFileLines("mk/existing.mk", - MkRcsID, + MkCvsID, "VAR=\tfirst second") reg := NewVarTypeRegistry() t.DisableTracing() @@ -82,8 +82,8 @@ func (s *Suite) Test_VarTypeRegistry_enumFromDirs(c *check.C) { // To make the test useful, these directories must differ from the // PYPKGPREFIX default value in vardefs.go. - t.CreateFileLines("lang/python28/Makefile", MkRcsID) - t.CreateFileLines("lang/python33/Makefile", MkRcsID) + t.CreateFileLines("lang/python28/Makefile", MkCvsID) + t.CreateFileLines("lang/python33/Makefile", MkCvsID) t.SetUpVartypes() @@ -146,9 +146,7 @@ func (s *Suite) Test_VarTypeRegistry_parseACLEntries__invalid_arguments(c *check func() { parseACLEntries("VARNAME", "too: many: colons") }, "Pkglint internal error: ACL entry \"too: many: colons\" must have exactly 1 colon.") - t.ExpectPanic( - func() { parseACLEntries("VAR") }, - "Pkglint internal error: At least one ACL entry must be given.") + t.ExpectAssert(func() { parseACLEntries("VAR") }) } func (s *Suite) Test_VarTypeRegistry_Init__LP64PLATFORMS(c *check.C) { @@ -168,7 +166,7 @@ func (s *Suite) Test_VarTypeRegistry_Init__no_tracing(c *check.C) { t := s.Init(c) t.CreateFileLines("editors/emacs/modules.mk", - MkRcsID, + MkCvsID, "", "_EMACS_VERSIONS_ALL= emacs31", "_EMACS_VERSIONS_ALL+= emacs29") @@ -179,11 +177,22 @@ func (s *Suite) Test_VarTypeRegistry_Init__no_tracing(c *check.C) { t.CheckOutputEmpty() } +func (s *Suite) Test_VarTypeRegistry_Init__no_testing(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.Remove("mk/fetch/sites.mk") + G.Testing = false + t.ExpectFatal( + t.FinishSetUp, + "FATAL: ~/mk/fetch/sites.mk: Cannot be read.") +} + func (s *Suite) Test_VarTypeRegistry_Init__MASTER_SITES(c *check.C) { t := s.Init(c) t.CreateFileLines("mk/fetch/sites.mk", - MkRcsID, + MkCvsID, "", "MASTER_SITE_GITHUB=\thttps://github.com/", "", diff --git a/pkgtools/pkglint/files/vartype.go b/pkgtools/pkglint/files/vartype.go index ed432465cda..8586d00ca35 100644 --- a/pkgtools/pkglint/files/vartype.go +++ b/pkgtools/pkglint/files/vartype.go @@ -14,11 +14,6 @@ type Vartype struct { } func NewVartype(basicType *BasicType, options vartypeOptions, aclEntries ...ACLEntry) *Vartype { - for _, aclEntry := range aclEntries { - _, err := path.Match(aclEntry.glob, "") - assertNil(err, "path.Match") - } - return &Vartype{basicType, options, aclEntries} } @@ -47,12 +42,12 @@ const ( ) type ACLEntry struct { - glob string // Examples: "Makefile", "*.mk" + matcher *pathMatcher permissions ACLPermissions } func NewACLEntry(glob string, permissions ACLPermissions) ACLEntry { - return ACLEntry{glob, permissions} + return ACLEntry{newPathMatcher(glob), permissions} } type ACLPermissions uint8 @@ -109,7 +104,7 @@ func (vt *Vartype) NeedsRationale() bool { return vt.options&NeedsRationale func (vt *Vartype) EffectivePermissions(basename string) ACLPermissions { for _, aclEntry := range vt.aclEntries { - if m, _ := path.Match(aclEntry.glob, basename); m { + if aclEntry.matcher.matches(basename) { return aclEntry.permissions } } @@ -157,9 +152,9 @@ func (vt *Vartype) AlternativeFiles(perms ACLPermissions) string { for _, aclEntry := range vt.aclEntries { if aclEntry.permissions.Contains(perms) { - pos = append(pos, aclEntry.glob) + pos = append(pos, aclEntry.matcher.originalPattern) } else { - neg = append(neg, aclEntry.glob) + neg = append(neg, aclEntry.matcher.originalPattern) } } @@ -356,6 +351,7 @@ var ( BtURL = &BasicType{"URL", (*VartypeCheck).URL} BtUserGroupName = &BasicType{"UserGroupName", (*VartypeCheck).UserGroupName} BtVariableName = &BasicType{"VariableName", (*VartypeCheck).VariableName} + BtVariableNamePattern = &BasicType{"VariableNamePattern", (*VartypeCheck).VariableNamePattern} BtVersion = &BasicType{"Version", (*VartypeCheck).Version} BtWrapperReorder = &BasicType{"WrapperReorder", (*VartypeCheck).WrapperReorder} BtWrapperTransform = &BasicType{"WrapperTransform", (*VartypeCheck).WrapperTransform} diff --git a/pkgtools/pkglint/files/vartype_test.go b/pkgtools/pkglint/files/vartype_test.go index 8b23a2b397d..442f8227590 100644 --- a/pkgtools/pkglint/files/vartype_test.go +++ b/pkgtools/pkglint/files/vartype_test.go @@ -62,7 +62,7 @@ func (s *Suite) Test_Vartype_AlternativeFiles(c *check.C) { test( rules( "buildlink3.mk: set", - "special:b*.mk: set, append", + "special:*3.mk: set, append", "*.mk: set", "Makefile: set, append", "Makefile.*: set"), @@ -72,7 +72,7 @@ func (s *Suite) Test_Vartype_AlternativeFiles(c *check.C) { test( rules( "buildlink3.mk: set", - "special:b*.mk: set, append", + "special:*3.mk: set, append", "*.mk: set", "Makefile: set, append", "Makefile.*: set", @@ -99,12 +99,12 @@ func (s *Suite) Test_Vartype_AlternativeFiles(c *check.C) { test( rules( "buildlink3.mk: none", - "special:b*.mk: set", + "special:*3.mk: set", "*.mk: none", "Makefile: set", "Makefile.*: none", "*: set"), - "b*.mk, Makefile or *, but not buildlink3.mk, *.mk or Makefile.*") + "*3.mk, Makefile or *, but not buildlink3.mk, *.mk or Makefile.*") test( rules( diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go index 7c18bb2afcd..48d1cfd3c68 100644 --- a/pkgtools/pkglint/files/vartypecheck.go +++ b/pkgtools/pkglint/files/vartypecheck.go @@ -9,16 +9,16 @@ import ( // VartypeCheck groups together the various checks for variables of the different types. type VartypeCheck struct { - MkLines MkLines + MkLines *MkLines // Note: if "go vet" or "go test" complains about a "variable with invalid type", update to go1.11.4. // See https://github.com/golang/go/issues/28972. // That doesn't help though since pkglint contains these "more convoluted alias declarations" // mentioned in https://github.com/golang/go/commit/6971090515ba. - // Therefore MkLine is declared as *MkLineImpl here. + // Therefore MkLine is declared as *MkLine here. // Ideally the "more convoluted cyclic type declaration" should be broken up. - MkLine *MkLineImpl + MkLine *MkLine // The name of the variable being checked. // @@ -201,21 +201,19 @@ func (cv *VartypeCheck) CFlag() { if cv.Op == opUseMatch { return } + cflag := cv.Value switch { - case matches(cflag, `^-[DILOUWfgm]`), - hasPrefix(cflag, "-std="), - cflag == "-c99", - cflag == "-c", - cflag == "-no-integrated-as", - cflag == "-pthread", - hasPrefix(cflag, "`") && hasSuffix(cflag, "`"), - containsVarRef(cflag): - return - case hasPrefix(cflag, "-"): - cv.Warnf("Unknown compiler flag %q.", cflag) - default: - cv.Warnf("Compiler flag %q should start with a hyphen.", cflag) + case hasPrefix(cflag, "-l"), hasPrefix(cflag, "-L"): + cv.Warnf("%q is a linker flag and belong to LDFLAGS, LIBS or LDADD instead of %s.", + cflag, cv.Varname) + } + + if strings.Count(cflag, "\"")%2 != 0 { + cv.Warnf("Compiler flag %q has unbalanced double quotes.", cflag) + } + if strings.Count(cflag, "'")%2 != 0 { + cv.Warnf("Compiler flag %q has unbalanced single quotes.", cflag) } } @@ -612,34 +610,45 @@ func (cv *VartypeCheck) GccReqd() { func (cv *VartypeCheck) Homepage() { cv.URL() - if m, wrong, sitename, subdir := match3(cv.Value, `^(\$\{(MASTER_SITE\w+)(?::=([\w\-/]+))?\})`); m { - baseURL := G.Pkgsrc.MasterSiteVarToURL[sitename] - if sitename == "MASTER_SITES" && G.Pkg != nil { - if mkline := G.Pkg.vars.FirstDefinition("MASTER_SITES"); mkline != nil { - if masterSites := mkline.Value(); !containsVarRef(masterSites) { - baseURL = masterSites + m, wrong, sitename, subdir := match3(cv.Value, `^(\$\{(MASTER_SITE\w+)(?::=([\w\-/]+))?\})`) + if !m { + return + } + + baseURL := G.Pkgsrc.MasterSiteVarToURL[sitename] + if sitename == "MASTER_SITES" && G.Pkg != nil { + mkline := G.Pkg.vars.FirstDefinition("MASTER_SITES") + if mkline != nil { + if !containsVarRef(mkline.Value()) { + masterSites := cv.MkLine.ValueFields(mkline.Value()) + if len(masterSites) > 0 { + baseURL = masterSites[0] } } } + } - fixedURL := baseURL + subdir + fixedURL := baseURL + subdir - fix := cv.Autofix() - if baseURL != "" { - fix.Warnf("HOMEPAGE should not be defined in terms of MASTER_SITEs. Use %s directly.", fixedURL) - } else { - fix.Warnf("HOMEPAGE should not be defined in terms of MASTER_SITEs.") - } - fix.Explain( - "The HOMEPAGE is a single URL, while MASTER_SITES is a list of URLs.", - "As long as this list has exactly one element, this works, but as", - "soon as another site is added, the HOMEPAGE would not be a valid", - "URL anymore.", - "", - "Defining MASTER_SITES=${HOMEPAGE} is ok, though.") + fix := cv.Autofix() + if baseURL != "" { + fix.Warnf("HOMEPAGE should not be defined in terms of MASTER_SITEs. Use %s directly.", fixedURL) + } else { + fix.Warnf("HOMEPAGE should not be defined in terms of MASTER_SITEs.") + } + fix.Explain( + "The HOMEPAGE is a single URL, while MASTER_SITES is a list of URLs.", + "As long as this list has exactly one element, this works, but as", + "soon as another site is added, the HOMEPAGE would not be a valid", + "URL anymore.", + "", + "Defining MASTER_SITES=${HOMEPAGE} is ok, though.") + if baseURL != "" { fix.Replace(wrong, fixedURL) - fix.Apply() + } else { + fix.Anyway() } + fix.Apply() } // Identifier checks for valid identifiers in various contexts, limiting the @@ -679,6 +688,7 @@ func (cv *VartypeCheck) LdFlag() { if cv.Op == opUseMatch { return } + ldflag := cv.Value if m, rpathFlag := match1(ldflag, `^(-Wl,(?:-R|-rpath|--rpath))`); m { cv.Warnf("Please use \"${COMPILER_RPATH_FLAG}\" instead of %q.", rpathFlag) @@ -686,19 +696,12 @@ func (cv *VartypeCheck) LdFlag() { } switch { - case hasPrefix(ldflag, "-L"), - hasPrefix(ldflag, "-l"), - ldflag == "-pthread", - ldflag == "-static", - hasPrefix(ldflag, "-static-"), - hasPrefix(ldflag, "-Wl,-"), - hasPrefix(ldflag, "`") && hasSuffix(ldflag, "`"), - ldflag != cv.ValueNoVar: - return - case hasPrefix(ldflag, "-"): - cv.Warnf("Unknown linker flag %q.", cv.Value) - default: - cv.Warnf("Linker flag %q should start with a hyphen.", cv.Value) + case ldflag == "-P", + ldflag == "-E", + hasPrefix(ldflag, "-D"), + hasPrefix(ldflag, "-U"), + hasPrefix(ldflag, "-I"): + cv.Warnf("%q is a compiler flag and belongs on CFLAGS, CPPFLAGS, CXXFLAGS or FFLAGS instead of %s.", cv.Value, cv.Varname) } } @@ -791,7 +794,7 @@ func (cv *VartypeCheck) Option() { } if m, optname := match1(value, `^-?([a-z][-0-9a-z+]*)$`); m { - if !cv.MkLines.FirstTimeSlice("option:", optname) { + if !cv.MkLines.once.FirstTimeSlice("option:", optname) { return } @@ -820,7 +823,7 @@ func (cv *VartypeCheck) Pathlist() { value := cv.Value // Sometimes, variables called PATH contain a single pathname, - // especially those with auto-guessed type from MkLineImpl.VariableType. + // especially those with auto-guessed type from MkLine.VariableType. if !contains(value, ":") && cv.Guessed { cv.Pathname() return @@ -1095,7 +1098,7 @@ func (cv *VartypeCheck) SedCommands() { } func (cv *VartypeCheck) ShellCommand() { - if cv.Op == opUseMatch || cv.Op == opUseCompare { + if cv.Op == opUseMatch || cv.Op == opUseCompare || cv.Op == opAssignAppend { return } setE := true @@ -1200,6 +1203,7 @@ func (cv *VartypeCheck) UserGroupName() { // VariableName checks that the value is a valid variable name to be used in Makefiles. func (cv *VartypeCheck) VariableName() { + // TODO: sync with MkParser.Varname if cv.Value == cv.ValueNoVar && !matches(cv.Value, `^[A-Z_][0-9A-Z_]*(?:[.].*)?$`) { cv.Warnf("%q is not a valid variable name.", cv.Value) cv.Explain( @@ -1213,6 +1217,30 @@ func (cv *VartypeCheck) VariableName() { } } +func (cv *VartypeCheck) VariableNamePattern() { + if cv.Value != cv.ValueNoVar { + return + } + + // TODO: sync with MkParser.Varname + if matches(cv.Value, `^[*A-Z_][*0-9A-Z_]*(?:[.].*)?$`) { + return + } + + cv.Warnf("%q is not a valid variable name pattern.", cv.Value) + cv.Explain( + "Variable names are restricted to only uppercase letters and the", + "underscore in the basename, and arbitrary characters in the", + "parameterized part, following the dot.", + "", + "In addition to these characters, variable name patterns may use", + "the * placeholder.", + "", + "Examples:", + "* PKGNAME", + "* PKG_OPTIONS.gtk+-2.0") +} + func (cv *VartypeCheck) Version() { value := cv.Value diff --git a/pkgtools/pkglint/files/vartypecheck_test.go b/pkgtools/pkglint/files/vartypecheck_test.go index 434628bce99..5afd7d42779 100644 --- a/pkgtools/pkglint/files/vartypecheck_test.go +++ b/pkgtools/pkglint/files/vartypecheck_test.go @@ -101,12 +101,26 @@ func (s *Suite) Test_VartypeCheck_CFlag(c *check.C) { "-no-integrated-as", "-pthread", "`pkg-config`_plus") + vt.OutputEmpty() + + vt.Values( + "-L${PREFIX}/lib", + "-L${PREFIX}/lib64", + "-lncurses", + "-DMACRO=\\\"", + "-DMACRO=\\'") vt.Output( - "WARN: filename.mk:2: Compiler flag \"/W3\" should start with a hyphen.", - "WARN: filename.mk:3: Compiler flag \"target:sparc64\" should start with a hyphen.", - "WARN: filename.mk:5: Unknown compiler flag \"-XX:+PrintClassHistogramAfterFullGC\".", - "WARN: filename.mk:11: Compiler flag \"`pkg-config`_plus\" should start with a hyphen.") + "WARN: filename.mk:21: \"-L${PREFIX}/lib\" is a linker flag "+ + "and belong to LDFLAGS, LIBS or LDADD instead of CFLAGS.", + "WARN: filename.mk:22: \"-L${PREFIX}/lib64\" is a linker flag "+ + "and belong to LDFLAGS, LIBS or LDADD instead of CFLAGS.", + "WARN: filename.mk:23: \"-lncurses\" is a linker flag "+ + "and belong to LDFLAGS, LIBS or LDADD instead of CFLAGS.", + "WARN: filename.mk:24: Compiler flag \"-DMACRO=\\\\\\\"\" "+ + "has unbalanced double quotes.", + "WARN: filename.mk:25: Compiler flag \"-DMACRO=\\\\'\" "+ + "has unbalanced single quotes.") vt.Op(opUseMatch) vt.Values( @@ -420,7 +434,7 @@ func (s *Suite) Test_VartypeCheck_Enum__use_match(c *check.C) { t.SetUpCommandLine("-Wall", "--explain") mklines := t.NewMkLines("module.mk", - MkRcsID, + MkCvsID, "", ".if !empty(MACHINE_ARCH:Mi386) || ${MACHINE_ARCH} == i386", ".endif", @@ -740,16 +754,28 @@ func (s *Suite) Test_VartypeCheck_LdFlag(c *check.C) { "-static-something", "${LDFLAGS.NetBSD}", "-l${LIBNCURSES}", - "`pkg-config`_plus") + "`pkg-config`_plus", + "-DMACRO", + "-UMACRO", + "-P", + "-E", + "-I${PREFIX}/include") vt.Op(opUseMatch) vt.Values( "anything") vt.Output( - "WARN: filename.mk:4: Unknown linker flag \"-unknown\".", - "WARN: filename.mk:5: Linker flag \"no-hyphen\" should start with a hyphen.", "WARN: filename.mk:6: Please use \"${COMPILER_RPATH_FLAG}\" instead of \"-Wl,--rpath\".", - "WARN: filename.mk:12: Linker flag \"`pkg-config`_plus\" should start with a hyphen.") + "WARN: filename.mk:13: \"-DMACRO\" is a compiler flag "+ + "and belongs on CFLAGS, CPPFLAGS, CXXFLAGS or FFLAGS instead of LDFLAGS.", + "WARN: filename.mk:14: \"-UMACRO\" is a compiler flag "+ + "and belongs on CFLAGS, CPPFLAGS, CXXFLAGS or FFLAGS instead of LDFLAGS.", + "WARN: filename.mk:15: \"-P\" is a compiler flag "+ + "and belongs on CFLAGS, CPPFLAGS, CXXFLAGS or FFLAGS instead of LDFLAGS.", + "WARN: filename.mk:16: \"-E\" is a compiler flag "+ + "and belongs on CFLAGS, CPPFLAGS, CXXFLAGS or FFLAGS instead of LDFLAGS.", + "WARN: filename.mk:17: \"-I${PREFIX}/include\" is a compiler flag "+ + "and belongs on CFLAGS, CPPFLAGS, CXXFLAGS or FFLAGS instead of LDFLAGS.") } func (s *Suite) Test_VartypeCheck_License(c *check.C) { @@ -762,7 +788,7 @@ func (s *Suite) Test_VartypeCheck_License(c *check.C) { G.Pkg = NewPackage(t.File("category/package")) mklines := t.NewMkLines("perl5.mk", - MkRcsID, + MkCvsID, "PERL5_LICENSE= gnu-gpl-v2 OR artistic") // Also registers the PERL5_LICENSE variable in the package. mklines.collectDefinedVariables() @@ -1181,6 +1207,12 @@ func (s *Suite) Test_VartypeCheck_ShellCommand(c *check.C) { vt.Values("*") vt.OutputEmpty() + + vt.Varname("CC") + vt.Op(opAssignAppend) + vt.Values("-ggdb") + + vt.OutputEmpty() } func (s *Suite) Test_VartypeCheck_ShellCommands(c *check.C) { @@ -1346,6 +1378,23 @@ func (s *Suite) Test_VartypeCheck_VariableName(c *check.C) { "WARN: filename.mk:2: \"VarBase\" is not a valid variable name.") } +func (s *Suite) Test_VartypeCheck_VariableNamePattern(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).VariableNamePattern) + + vt.Varname("_SORTED_VARS.group") + vt.Values( + "VARBASE", + "VarBase", + "PKG_OPTIONS_VAR.pkgbase", + "${INDIRECT}", + "*_DIRS", + "VAR.*", + "***") + + vt.Output( + "WARN: filename.mk:2: \"VarBase\" is not a valid variable name pattern.") +} + func (s *Suite) Test_VartypeCheck_Version(c *check.C) { vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).Version) @@ -1565,7 +1614,7 @@ func (vt *VartypeCheckTester) Values(values ...string) { return varname + space + opStr + value } - test := func(mklines MkLines, mkline MkLine, value string) { + test := func(mklines *MkLines, mkline *MkLine, value string) { varname := vt.varname comment := "" if mkline.IsVarassign() { @@ -1599,10 +1648,10 @@ func (vt *VartypeCheckTester) Values(values ...string) { text := toText(value) line := vt.tester.NewLine(vt.filename, vt.lineno, text) - mklines := NewMkLines(NewLines(vt.filename, []Line{line})) + mklines := NewMkLines(NewLines(vt.filename, []*Line{line})) vt.lineno++ - mklines.ForEach(func(mkline MkLine) { test(mklines, mkline, value) }) + mklines.ForEach(func(mkline *MkLine) { test(mklines, mkline, value) }) } } |