diff options
author | rillig <rillig@pkgsrc.org> | 2019-11-17 01:26:25 +0000 |
---|---|---|
committer | rillig <rillig@pkgsrc.org> | 2019-11-17 01:26:25 +0000 |
commit | d0b8c28cda3d26676e33cfc1feadc7e0ab5ad8bd (patch) | |
tree | 9d972bca6e0655212ce0fbd3d20a50bd4a24d841 /pkgtools | |
parent | d248ab327d8df987bdaa3c0b8078d0a41597735f (diff) | |
download | pkgsrc-d0b8c28cda3d26676e33cfc1feadc7e0ab5ad8bd.tar.gz |
pkgtools/pkglint: update to 19.3.7
Changes since 19.3.6:
Improved variable value alignment.
Fixed wrong warning about comment lines that were interpreted as shell
commands before.
Warn when the first category of a package doesn't correspond to the
path in the filesystem. This affects 603 packages.
No longer warn about deprecated BUILDLINK_TRANSFORM.${OPSYS}. The
deprecation warning was meant for BUILDLINK_TRANSFORM.${pkgbase}, but
since pkglint cannot distinguish between these, the warnings were wrong.
Diffstat (limited to 'pkgtools')
67 files changed, 14031 insertions, 13139 deletions
diff --git a/pkgtools/pkglint/Makefile b/pkgtools/pkglint/Makefile index e5e2ce8c49f..ac5d68eff56 100644 --- a/pkgtools/pkglint/Makefile +++ b/pkgtools/pkglint/Makefile @@ -1,6 +1,6 @@ -# $NetBSD: Makefile,v 1.606 2019/11/04 18:44:21 rillig Exp $ +# $NetBSD: Makefile,v 1.607 2019/11/17 01:26:25 rillig Exp $ -PKGNAME= pkglint-19.3.6 +PKGNAME= pkglint-19.3.7 CATEGORIES= pkgtools DISTNAME= tools MASTER_SITES= ${MASTER_SITE_GITHUB:=golang/} diff --git a/pkgtools/pkglint/PLIST b/pkgtools/pkglint/PLIST index 00571ffb5cc..076c8201878 100644 --- a/pkgtools/pkglint/PLIST +++ b/pkgtools/pkglint/PLIST @@ -1,4 +1,4 @@ -@comment $NetBSD: PLIST,v 1.15 2019/10/26 09:51:47 rillig Exp $ +@comment $NetBSD: PLIST,v 1.16 2019/11/17 01:26:25 rillig Exp $ bin/pkglint gopkg/pkg/${GO_PLATFORM}/netbsd.org/pkglint.a gopkg/pkg/${GO_PLATFORM}/netbsd.org/pkglint/getopt.a @@ -31,6 +31,7 @@ gopkg/src/netbsd.org/pkglint/histogram/histogram.go gopkg/src/netbsd.org/pkglint/histogram/histogram_test.go gopkg/src/netbsd.org/pkglint/intqa/ideas.go gopkg/src/netbsd.org/pkglint/intqa/testnames.go +gopkg/src/netbsd.org/pkglint/intqa/testnames_test.go gopkg/src/netbsd.org/pkglint/licenses.go gopkg/src/netbsd.org/pkglint/licenses/licenses.go gopkg/src/netbsd.org/pkglint/licenses/licenses.y diff --git a/pkgtools/pkglint/files/alternatives.go b/pkgtools/pkglint/files/alternatives.go index 6b7122c9283..2a39394c5cc 100644 --- a/pkgtools/pkglint/files/alternatives.go +++ b/pkgtools/pkglint/files/alternatives.go @@ -25,7 +25,7 @@ func CheckFileAlternatives(filename string) { checkPlistAlternative := func(line *Line, alternative string) { relImplementation := strings.Replace(alternative, "@PREFIX@/", "", 1) plistName := replaceAll(relImplementation, `@(\w+)@`, "${$1}") - if plist.Files[plistName] != nil || G.Pkg.vars.Defined("ALTERNATIVES_SRC") { + if plist.Files[plistName] != nil || G.Pkg.vars.IsDefined("ALTERNATIVES_SRC") { return } diff --git a/pkgtools/pkglint/files/autofix.go b/pkgtools/pkglint/files/autofix.go index d31717849f3..62a50315f49 100644 --- a/pkgtools/pkglint/files/autofix.go +++ b/pkgtools/pkglint/files/autofix.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "netbsd.org/pkglint/regex" "os" + "path/filepath" "strconv" "strings" ) @@ -127,20 +128,20 @@ func (fix *Autofix) ReplaceAfter(prefix, from string, to string) { // ReplaceAt replaces the text "from" with "to", a single time. // But only if the text at the given position is indeed "from". -func (fix *Autofix) ReplaceAt(rawIndex int, textIndex int, from string, to string) { +func (fix *Autofix) ReplaceAt(rawIndex int, textIndex int, from string, to string) (modified bool, replaced string) { assert(from != to) fix.assertRealLine() if fix.skip() { - return + return false, "" } rawLine := fix.line.raw[rawIndex] if textIndex >= len(rawLine.textnl) || !hasPrefix(rawLine.textnl[textIndex:], from) { - return + return false, "" } - replaced := rawLine.textnl[:textIndex] + to + rawLine.textnl[textIndex+len(from):] + replaced = rawLine.textnl[:textIndex] + to + rawLine.textnl[textIndex+len(from):] if G.Logger.IsAutofix() { rawLine.textnl = replaced @@ -157,7 +158,7 @@ func (fix *Autofix) ReplaceAt(rawIndex int, textIndex int, from string, to strin } } fix.Describef(rawLine.Lineno, "Replacing %q with %q.", from, to) - return + return true, replaced } // ReplaceRegex replaces the first howOften or all occurrences (if negative) @@ -215,44 +216,6 @@ func (fix *Autofix) ReplaceRegex(from regex.Pattern, toText string, howOften int }) } -// Custom runs a custom fix action, unless the fix is skipped anyway -// because of the --only option. -// -// The fixer function must check whether it can actually fix something, -// and if so, call Describef to describe the actual fix. -// -// If showAutofix and autofix are both false, the fix must only be -// described by calling Describef. No observable modification must be done, -// not even in memory. -// -// If showAutofix is true but autofix is false, the fix should be done in -// memory as far as possible. For example, changing the text of Line.raw -// is appropriate, but changing files in the file system is not. -// -// Only if autofix is true, fixes other than modifying the current Line -// should be done persistently, such as changes to the file system. -// -// In any case, changes to the current Line will be written back to disk -// by SaveAutofixChanges, after fixing all the lines in the file at once. -func (fix *Autofix) Custom(fixer func(showAutofix, autofix bool)) { - // Contrary to the fixes that modify the line text, this one - // can be run even on dummy lines (like those standing for a - // file at whole), for example to fix the permissions of the file. - - if fix.skip() { - return - } - - fixer(G.Logger.Opts.ShowAutofix, G.Logger.Opts.Autofix) -} - -// Describef is used while Autofix.Custom is called to remember a description -// of the actual fix for logging it later when Apply is called. -// Describef may be called multiple times before calling Apply. -func (fix *Autofix) Describef(lineno int, format string, args ...interface{}) { - fix.actions = append(fix.actions, autofixAction{sprintf(format, args...), lineno}) -} - // InsertBefore prepends a line before the current line. // The newline is added internally. func (fix *Autofix) InsertBefore(text string) { @@ -298,6 +261,44 @@ func (fix *Autofix) Delete() { } } +// Custom runs a custom fix action, unless the fix is skipped anyway +// because of the --only option. +// +// The fixer function must check whether it can actually fix something, +// and if so, call Describef to describe the actual fix. +// +// If showAutofix and autofix are both false, the fix must only be +// described by calling Describef. No observable modification must be done, +// not even in memory. +// +// If showAutofix is true but autofix is false, the fix should be done in +// memory as far as possible. For example, changing the text of Line.raw +// is appropriate, but changing files in the file system is not. +// +// Only if autofix is true, fixes other than modifying the current Line +// should be done persistently, such as changes to the file system. +// +// In any case, changes to the current Line will be written back to disk +// by SaveAutofixChanges, after fixing all the lines in the file at once. +func (fix *Autofix) Custom(fixer func(showAutofix, autofix bool)) { + // Contrary to the fixes that modify the line text, this one + // can be run even on dummy lines (like those standing for a + // file at whole), for example to fix the permissions of the file. + + if fix.skip() { + return + } + + fixer(G.Logger.Opts.ShowAutofix, G.Logger.Opts.Autofix) +} + +// Describef is used while Autofix.Custom is called to remember a description +// of the actual fix for logging it later when Apply is called. +// Describef may be called multiple times before calling Apply. +func (fix *Autofix) Describef(lineno int, format string, args ...interface{}) { + fix.actions = append(fix.actions, autofixAction{sprintf(format, args...), lineno}) +} + // Anyway has the effect of showing the diagnostic even when nothing can // be fixed automatically. // @@ -373,6 +374,21 @@ func (fix *Autofix) Apply() { reset() } +func (fix *Autofix) setDiag(level *LogLevel, format string, args []interface{}) { + if G.Testing && format != SilentAutofixFormat { + assertf( + hasSuffix(format, "."), + "Autofix: format %q must end with a period.", + format) + } + assert(fix.level == nil) // Autofix can only have a single diagnostic. + assert(fix.diagFormat == "") // Autofix can only have a single diagnostic. + + fix.level = level + fix.diagFormat = format + fix.diagArgs = args +} + func (fix *Autofix) affectedLinenos() string { if len(fix.actions) == 0 { return fix.line.Linenos() @@ -401,21 +417,6 @@ func (fix *Autofix) affectedLinenos() string { } } -func (fix *Autofix) setDiag(level *LogLevel, format string, args []interface{}) { - if G.Testing && format != SilentAutofixFormat { - assertf( - hasSuffix(format, "."), - "Autofix: format %q must end with a period.", - format) - } - assert(fix.level == nil) // Autofix can only have a single diagnostic. - assert(fix.diagFormat == "") // Autofix can only have a single diagnostic. - - fix.level = level - fix.diagFormat = format - fix.diagArgs = args -} - // skip returns whether this autofix should be skipped because // its message is matched by one of the --only command line options. func (fix *Autofix) skip() bool { @@ -452,6 +453,12 @@ func SaveAutofixChanges(lines *Lines) (autofixed bool) { return } + if G.Testing { + abs := abspath(lines.Filename) + absTmp := abspath(filepath.ToSlash(os.TempDir())) + assertf(hasPrefix(abs, absTmp), "%q must be inside %q", abs, absTmp) + } + changes := make(map[string][]string) changed := make(map[string]bool) for _, line := range lines.Lines { diff --git a/pkgtools/pkglint/files/autofix_test.go b/pkgtools/pkglint/files/autofix_test.go index f336d9292c3..b29aee0e8da 100644 --- a/pkgtools/pkglint/files/autofix_test.go +++ b/pkgtools/pkglint/files/autofix_test.go @@ -7,16 +7,6 @@ import ( "strings" ) -func (s *Suite) Test_Autofix_Warnf__duplicate(c *check.C) { - t := s.Init(c) - - line := t.NewLine("DESCR", 1, "Description of the package") - - fix := line.Autofix() - fix.Warnf("Warning 1.") - t.ExpectAssert(func() { fix.Warnf("Warning 2.") }) -} - func (s *Suite) Test_Autofix__default_leaves_line_unchanged(c *check.C) { t := s.Init(c) @@ -81,165 +71,450 @@ func (s *Suite) Test_Autofix__show_autofix_modifies_line(c *check.C) { t.CheckEquals(fix.modified, true) } -func (s *Suite) Test_Autofix_ReplaceAfter__autofix_in_continuation_line(c *check.C) { +func (s *Suite) Test_Autofix__multiple_fixes(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--autofix", "--source") - mklines := t.SetUpFileMkLines("Makefile", - "# line 1 \\", - "continuation 1 \\", - "continuation 2") + t.SetUpCommandLine("--show-autofix", "--explain") - fix := mklines.lines.Lines[0].Autofix() - fix.Warnf("Line should be replaced with Row.") - fix.ReplaceAfter("", "line", "row") - fix.Apply() + line := t.NewLine("filename", 1, "original") + + c.Check(line.autofix, check.IsNil) + t.CheckDeepEquals(line.raw, t.NewRawLines(1, "original\n")) + { + fix := line.Autofix() + fix.Warnf(SilentAutofixFormat) + fix.ReplaceRegex(`(.)(.*)(.)`, "lriginao", 1) // XXX: the replacement should be "$3$2$1" + fix.Apply() + } + + c.Check(line.autofix, check.NotNil) + t.CheckDeepEquals(line.raw, t.NewRawLines(1, "original\n", "lriginao\n")) t.CheckOutputLines( - "AUTOFIX: ~/Makefile:1: Replacing \"line\" with \"row\".", - "-\t# line 1 \\", - "+\t# row 1 \\", - "\tcontinuation 1 \\", - "\tcontinuation 2") + "AUTOFIX: filename:1: Replacing \"original\" with \"lriginao\".") + + { + fix := line.Autofix() + fix.Warnf(SilentAutofixFormat) + fix.Replace("ig", "ug") + fix.Apply() + } + + c.Check(line.autofix, check.NotNil) + t.CheckDeepEquals(line.raw, t.NewRawLines(1, "original\n", "lruginao\n")) + t.CheckEquals(line.raw[0].textnl, "lruginao\n") + t.CheckOutputLines( + "AUTOFIX: filename:1: Replacing \"ig\" with \"ug\".") + + { + fix := line.Autofix() + fix.Warnf(SilentAutofixFormat) + fix.Replace("lruginao", "middle") + fix.Apply() + } + + c.Check(line.autofix, check.NotNil) + t.CheckDeepEquals(line.raw, t.NewRawLines(1, "original\n", "middle\n")) + t.CheckEquals(line.raw[0].textnl, "middle\n") + t.CheckOutputLines( + "AUTOFIX: filename:1: Replacing \"lruginao\" with \"middle\".") + + t.CheckEquals(line.raw[0].textnl, "middle\n") + t.CheckOutputEmpty() + + { + fix := line.Autofix() + fix.Warnf(SilentAutofixFormat) + fix.Delete() + fix.Apply() + } + + t.CheckEquals(line.Autofix().RawText(), "") + t.CheckOutputLines( + "AUTOFIX: filename:1: Deleting this line.") } -func (s *Suite) Test_Autofix_ReplaceAfter__autofix_several_times_in_continuation_line(c *check.C) { +// Up to 2018-11-25, pkglint in some cases logged only the source without +// a corresponding warning. +func (s *Suite) Test_Autofix__lonely_source(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--autofix", "--source") - mklines := t.SetUpFileMkLines("Makefile", - "# line 1 \\", - "continuation 1 \\", - "continuation 2") + t.SetUpCommandLine("-Wall", "--source") + G.Logger.Opts.LogVerbose = false // For realistic conditions; otherwise all diagnostics are logged. - fix := mklines.lines.Lines[0].Autofix() - fix.Warnf("N should be replaced with V.") - fix.ReplaceAfter("", "n", "v") - fix.Apply() + t.SetUpPackage("x11/xorg-cf-files", + ".include \"../../x11/xorgproto/buildlink3.mk\"") + t.SetUpPackage("x11/xorgproto", + "DISTNAME=\txorgproto-1.0") + t.CreateFileDummyBuildlink3("x11/xorgproto/buildlink3.mk") + t.CreateFileLines("x11/xorgproto/builtin.mk", + MkCvsID, + "", + "BUILTIN_PKG:=\txorgproto", + "", + "PRE_XORGPROTO_LIST_MISSING =\tapplewmproto", + "", + ".for id in ${PRE_XORGPROTO_LIST_MISSING}", + ".endfor") + t.Chdir(".") + t.FinishSetUp() - // Nothing is logged or fixed because the "n" appears more than once, - // and as of June 2019, pkglint doesn't know which occurrence is the - // correct one. - t.CheckOutputEmpty() + G.Check("x11/xorg-cf-files") + G.Check("x11/xorgproto") + + t.CheckOutputLines( + ">\tPRE_XORGPROTO_LIST_MISSING =\tapplewmproto", + "NOTE: x11/xorgproto/builtin.mk:5: Unnecessary space after variable name \"PRE_XORGPROTO_LIST_MISSING\".") } -func (s *Suite) Test_Autofix_ReplaceAfter__autofix_one_time(c *check.C) { +// Up to 2018-11-26, pkglint in some cases logged only the source without +// a corresponding warning. +func (s *Suite) Test_Autofix__lonely_source_2(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--autofix", "--source") + t.SetUpCommandLine("-Wall", "--source", "--explain") + G.Logger.Opts.LogVerbose = false // For realistic conditions; otherwise all diagnostics are logged. + + t.SetUpPackage("print/tex-bibtex8", + "MAKE_FLAGS+=\tCFLAGS=${CFLAGS.${PKGSRC_COMPILER}}") + t.Chdir(".") + t.FinishSetUp() + + G.Check("print/tex-bibtex8") + + t.CheckOutputLines( + ">\tMAKE_FLAGS+=\tCFLAGS=${CFLAGS.${PKGSRC_COMPILER}}", + "WARN: print/tex-bibtex8/Makefile:20: Please use ${CFLAGS.${PKGSRC_COMPILER}:Q} instead of ${CFLAGS.${PKGSRC_COMPILER}}.", + "", + "\tSee the pkgsrc guide, section \"Echoing a string exactly as-is\":", + "\thttps://www.NetBSD.org/docs/pkgsrc/pkgsrc.html#echo-literal", + "", + ">\tMAKE_FLAGS+=\tCFLAGS=${CFLAGS.${PKGSRC_COMPILER}}", + "WARN: print/tex-bibtex8/Makefile:20: The list variable PKGSRC_COMPILER should not be embedded in a word.", + "", + "\tWhen a list variable has multiple elements, this expression expands", + "\tto something unexpected:", + "", + "\tExample: ${MASTER_SITE_SOURCEFORGE}directory/ expands to", + "", + "\t\thttps://mirror1.sf.net/ https://mirror2.sf.net/directory/", + "", + "\tThe first URL is missing the directory. To fix this, write", + "\t\t${MASTER_SITE_SOURCEFORGE:=directory/}.", + "", + "\tExample: -l${LIBS} expands to", + "", + "\t\t-llib1 lib2", + "", + "\tThe second library is missing the -l. To fix this, write", + "\t${LIBS:S,^,-l,}.", + "") +} + +func (s *Suite) Test_Autofix__show_autofix_and_source_continuation_line(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--show-autofix", "--source") mklines := t.SetUpFileMkLines("Makefile", MkCvsID, - "VAR=\t$$(var) $(var)") + "# before \\", + "The old song \\", + "after") + line := mklines.lines.Lines[1] - mklines.Check() + fix := line.Autofix() + fix.Warnf("Using \"old\" is deprecated.") + fix.Replace("old", "new") + fix.Apply() - // Nothing is replaced since, as of June 2019, pkglint doesn't - // know which of the two "$(var)" should be replaced. - t.CheckOutputEmpty() + // Using a tab for indentation preserves the exact layout in the output + // since in pkgsrc Makefiles, tabs are also used in the middle of the line + // to align the variable values. Using a single space for indentation would + // make some of the lines appear misaligned in the pkglint output although + // they are correct in the Makefiles. + t.CheckOutputLines( + "WARN: ~/Makefile:3: Using \"old\" is deprecated.", + "AUTOFIX: ~/Makefile:3: Replacing \"old\" with \"new\".", + "\t# before \\", + "-\tThe old song \\", + "+\tThe new song \\", + "\tafter") } -func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix(c *check.C) { +// Demonstrates that without the --show-autofix option, diagnostics are +// shown even when they cannot be autofixed. +// +// This is typical when an autofix is provided for simple scenarios, +// but the code actually found is a little more complicated. +func (s *Suite) Test_Autofix__show_unfixable_diagnostics_in_default_mode(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--show-autofix") - lines := t.SetUpFileLines("Makefile", + t.SetUpCommandLine("--source") + lines := t.NewLines("Makefile", "line1", "line2", "line3") + lines.Lines[0].Warnf("This warning is shown since the --show-autofix option is not given.") + fix := lines.Lines[1].Autofix() - fix.Warnf("Something's wrong here.") - fix.ReplaceRegex(`.`, "X", -1) + fix.Warnf("This warning cannot be fixed and is therefore not shown.") + fix.Replace("XXX", "TODO") fix.Apply() - SaveAutofixChanges(lines) - t.CheckEquals(lines.Lines[1].raw[0].textnl, "XXXXX\n") - t.CheckFileLines("Makefile", - "line1", - "line2", - "line3") + fix.Warnf("This warning cannot be fixed automatically but should be shown anyway.") + fix.Replace("XXX", "TODO") + fix.Anyway() + fix.Apply() + + // If this warning should ever appear it is probably because fix.anyway is not reset properly. + fix.Warnf("This warning cannot be fixed and is therefore not shown.") + fix.Replace("XXX", "TODO") + fix.Apply() + + lines.Lines[2].Warnf("This warning is also shown.") + t.CheckOutputLines( - "WARN: ~/Makefile:2: Something's wrong here.", - "AUTOFIX: ~/Makefile:2: Replacing \"l\" with \"X\".", - "AUTOFIX: ~/Makefile:2: Replacing \"i\" with \"X\".", - "AUTOFIX: ~/Makefile:2: Replacing \"n\" with \"X\".", - "AUTOFIX: ~/Makefile:2: Replacing \"e\" with \"X\".", - "AUTOFIX: ~/Makefile:2: Replacing \"2\" with \"X\".") + ">\tline1", + "WARN: Makefile:1: This warning is shown since the --show-autofix option is not given.", + "", + ">\tline2", + "WARN: Makefile:2: This warning cannot be fixed automatically but should be shown anyway.", + "", + ">\tline3", + "WARN: Makefile:3: This warning is also shown.") } -func (s *Suite) Test_Autofix_ReplaceRegex__autofix(c *check.C) { +// Demonstrates that the --show-autofix option only shows those diagnostics +// that would be fixed. +func (s *Suite) Test_Autofix__suppress_unfixable_warnings_with_show_autofix(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--autofix", "--source") - lines := t.SetUpFileLines("Makefile", + t.SetUpCommandLine("--show-autofix", "--source") + lines := t.NewLines("Makefile", "line1", "line2", "line3") + lines.Lines[0].Warnf("This warning is not shown since it is not part of a fix.") + fix := lines.Lines[1].Autofix() fix.Warnf("Something's wrong here.") - fix.ReplaceRegex(`.`, "X", 3) + fix.ReplaceRegex(`.....`, "XXX", 1) + fix.Apply() + + fix.Warnf("Since XXX marks are usually not fixed, use TODO instead to draw attention.") + fix.Replace("XXX", "TODO") fix.Apply() + lines.Lines[2].Warnf("Neither is this warning shown.") + t.CheckOutputLines( - "AUTOFIX: ~/Makefile:2: Replacing \"l\" with \"X\".", - "AUTOFIX: ~/Makefile:2: Replacing \"i\" with \"X\".", - "AUTOFIX: ~/Makefile:2: Replacing \"n\" with \"X\".", + "WARN: Makefile:2: Something's wrong here.", + "AUTOFIX: Makefile:2: Replacing \"line2\" with \"XXX\".", "-\tline2", - "+\tXXXe2") + "+\tXXX", + "", + "WARN: Makefile:2: Since XXX marks are usually not fixed, use TODO instead to draw attention.", + "AUTOFIX: Makefile:2: Replacing \"XXX\" with \"TODO\".", + "-\tline2", + "+\tTODO") +} - // After calling fix.Apply above, the autofix is ready to be used again. - fix.Warnf("Use Y instead of X.") - fix.Replace("XXX", "YYY") +// If an Autofix doesn't do anything, it must not log any diagnostics. +func (s *Suite) Test_Autofix__noop_replace(c *check.C) { + t := s.Init(c) + + line := t.NewLine("Makefile", 14, "Original text") + + fix := line.Autofix() + fix.Warnf("All-uppercase words should not be used at all.") + fix.ReplaceRegex(`\b[A-Z]{3,}\b`, "---censored---", -1) + fix.Apply() + + // No output since there was no all-uppercase word in the text. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Autofix_Warnf__duplicate(c *check.C) { + t := s.Init(c) + + line := t.NewLine("DESCR", 1, "Description of the package") + + fix := line.Autofix() + fix.Warnf("Warning 1.") + t.ExpectAssert(func() { fix.Warnf("Warning 2.") }) +} + +func (s *Suite) Test_Autofix_Explain__without_explain_option(c *check.C) { + t := s.Init(c) + + line := t.NewLine("Makefile", 74, "line1") + + fix := line.Autofix() + fix.Warnf("Please write row instead of line.") + fix.Replace("line", "row") + fix.Explain("Explanation") fix.Apply() t.CheckOutputLines( - "AUTOFIX: ~/Makefile:2: Replacing \"XXX\" with \"YYY\".", - "-\tline2", - "+\tYYYe2") + "WARN: Makefile:74: Please write row instead of line.") + t.CheckEquals(G.Logger.explanationsAvailable, true) +} - SaveAutofixChanges(lines) +func (s *Suite) Test_Autofix_Explain__default(c *check.C) { + t := s.Init(c) - t.CheckFileLines("Makefile", - "line1", - "YYYe2", - "line3") + t.SetUpCommandLine("--explain") + line := t.NewLine("Makefile", 74, "line1") + + fix := line.Autofix() + fix.Warnf("Please write row instead of line.") + fix.Replace("line", "row") + fix.Explain("Explanation") + fix.Apply() + + t.CheckOutputLines( + "WARN: Makefile:74: Please write row instead of line.", + "", + "\tExplanation", + "") + t.CheckEquals(G.Logger.explanationsAvailable, true) } -func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix_and_source(c *check.C) { +func (s *Suite) Test_Autofix_Explain__show_autofix(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--show-autofix", "--source") - lines := t.SetUpFileLines("Makefile", - "line1", - "line2", - "line3") + t.SetUpCommandLine("--show-autofix", "--explain") + line := t.NewLine("Makefile", 74, "line1") - fix := lines.Lines[1].Autofix() - fix.Warnf("Something's wrong here.") - fix.ReplaceRegex(`.`, "X", -1) + fix := line.Autofix() + fix.Warnf("Please write row instead of line.") + fix.Replace("line", "row") + fix.Explain("Explanation") fix.Apply() - fix.Warnf("Use Y instead of X.") - fix.Replace("XXXXX", "YYYYY") + t.CheckOutputLines( + "WARN: Makefile:74: Please write row instead of line.", + "AUTOFIX: Makefile:74: Replacing \"line\" with \"row\".", + "", + "\tExplanation", + "") + t.CheckEquals(G.Logger.explanationsAvailable, true) +} + +func (s *Suite) Test_Autofix_Explain__autofix(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--autofix", "--explain") + line := t.NewLine("Makefile", 74, "line1") + + fix := line.Autofix() + fix.Warnf("Please write row instead of line.") + fix.Replace("line", "row") + fix.Explain("Explanation") fix.Apply() - SaveAutofixChanges(lines) + t.CheckOutputLines( + "AUTOFIX: Makefile:74: Replacing \"line\" with \"row\".") + t.CheckEquals(G.Logger.explanationsAvailable, false) // Not necessary. +} + +func (s *Suite) Test_Autofix_Explain__SilentAutofixFormat(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--explain") + line := t.NewLine("example.txt", 1, "Text") + + fix := line.Autofix() + fix.Warnf(SilentAutofixFormat) + t.ExpectAssert(func() { fix.Explain("Explanation for inserting a line before.") }) +} + +// To combine a silent diagnostic with an explanation, two separate autofixes +// are necessary. +func (s *Suite) Test_Autofix_Explain__silent_with_diagnostic(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--explain") + line := t.NewLine("example.txt", 1, "Text") + + fix := line.Autofix() + fix.Warnf(SilentAutofixFormat) + fix.InsertBefore("before") + fix.Apply() + + fix.Notef("This diagnostic is necessary for the following explanation.") + fix.Explain( + "When inserting multiple lines, Apply must be called in-between.", + "Otherwise the changes are not described to the human reader.") + fix.InsertAfter("after") + fix.Apply() t.CheckOutputLines( - "WARN: ~/Makefile:2: Something's wrong here.", - "AUTOFIX: ~/Makefile:2: Replacing \"l\" with \"X\".", - "AUTOFIX: ~/Makefile:2: Replacing \"i\" with \"X\".", - "AUTOFIX: ~/Makefile:2: Replacing \"n\" with \"X\".", - "AUTOFIX: ~/Makefile:2: Replacing \"e\" with \"X\".", - "AUTOFIX: ~/Makefile:2: Replacing \"2\" with \"X\".", - "-\tline2", - "+\tXXXXX", + "NOTE: example.txt:1: This diagnostic is necessary for the following explanation.", "", - "WARN: ~/Makefile:2: Use Y instead of X.", - "AUTOFIX: ~/Makefile:2: Replacing \"XXXXX\" with \"YYYYY\".", - "-\tline2", - "+\tYYYYY") + "\tWhen inserting multiple lines, Apply must be called in-between.", + "\tOtherwise the changes are not described to the human reader.", + "") + t.CheckEquals(fix.RawText(), "Text\n") +} + +func (s *Suite) Test_Autofix_ReplaceAfter__autofix_in_continuation_line(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--autofix", "--source") + mklines := t.SetUpFileMkLines("Makefile", + "# line 1 \\", + "continuation 1 \\", + "continuation 2") + + fix := mklines.lines.Lines[0].Autofix() + fix.Warnf("Line should be replaced with Row.") + fix.ReplaceAfter("", "line", "row") + fix.Apply() + + t.CheckOutputLines( + "AUTOFIX: ~/Makefile:1: Replacing \"line\" with \"row\".", + "-\t# line 1 \\", + "+\t# row 1 \\", + "\tcontinuation 1 \\", + "\tcontinuation 2") +} + +func (s *Suite) Test_Autofix_ReplaceAfter__autofix_several_times_in_continuation_line(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--autofix", "--source") + mklines := t.SetUpFileMkLines("Makefile", + "# line 1 \\", + "continuation 1 \\", + "continuation 2") + + fix := mklines.lines.Lines[0].Autofix() + fix.Warnf("N should be replaced with V.") + fix.ReplaceAfter("", "n", "v") + fix.Apply() + + // Nothing is logged or fixed because the "n" appears more than once, + // and as of June 2019, pkglint doesn't know which occurrence is the + // correct one. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Autofix_ReplaceAfter__autofix_one_time(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--autofix", "--source") + mklines := t.SetUpFileMkLines("Makefile", + MkCvsID, + "VAR=\t$$(var) $(var)") + + mklines.Check() + + // Nothing is replaced since, as of June 2019, pkglint doesn't + // know which of the two "$(var)" should be replaced. + t.CheckOutputEmpty() } // When an autofix replaces text, it does not touch those @@ -347,257 +622,108 @@ func (s *Suite) Test_Autofix_ReplaceAt(c *check.C) { nil...)) } -func (s *Suite) Test_SaveAutofixChanges(c *check.C) { +func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--autofix") - lines := t.SetUpFileLines("example.txt", - "line1 := value1", - "line2 := value2", - "line3 := value3") + t.SetUpCommandLine("--show-autofix") + lines := t.SetUpFileLines("Makefile", + "line1", + "line2", + "line3") fix := lines.Lines[1].Autofix() fix.Warnf("Something's wrong here.") - fix.ReplaceRegex(`...`, "XXX", 2) - fix.Apply() - - SaveAutofixChanges(lines) - - t.CheckOutputLines( - "AUTOFIX: ~/example.txt:2: Replacing \"lin\" with \"XXX\".", - "AUTOFIX: ~/example.txt:2: Replacing \"e2 \" with \"XXX\".") - t.CheckFileLines("example.txt", - "line1 := value1", - "XXXXXX:= value2", - "line3 := value3") -} - -func (s *Suite) Test_SaveAutofixChanges__no_changes_necessary(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--autofix") - lines := t.SetUpFileLines("DESCR", - "Line 1", - "Line 2") - - fix := lines.Lines[0].Autofix() - fix.Warnf("Dummy warning.") - fix.Replace("X", "Y") + fix.ReplaceRegex(`.`, "X", -1) fix.Apply() - - // Since nothing has been effectively changed, - // nothing needs to be saved. SaveAutofixChanges(lines) - // And therefore, no AUTOFIX action must appear in the log. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Autofix__multiple_fixes(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--show-autofix", "--explain") - - line := t.NewLine("filename", 1, "original") - - c.Check(line.autofix, check.IsNil) - t.CheckDeepEquals(line.raw, t.NewRawLines(1, "original\n")) - - { - fix := line.Autofix() - fix.Warnf(SilentAutofixFormat) - fix.ReplaceRegex(`(.)(.*)(.)`, "lriginao", 1) // XXX: the replacement should be "$3$2$1" - fix.Apply() - } - - c.Check(line.autofix, check.NotNil) - t.CheckDeepEquals(line.raw, t.NewRawLines(1, "original\n", "lriginao\n")) - t.CheckOutputLines( - "AUTOFIX: filename:1: Replacing \"original\" with \"lriginao\".") - - { - fix := line.Autofix() - fix.Warnf(SilentAutofixFormat) - fix.Replace("ig", "ug") - fix.Apply() - } - - c.Check(line.autofix, check.NotNil) - t.CheckDeepEquals(line.raw, t.NewRawLines(1, "original\n", "lruginao\n")) - t.CheckEquals(line.raw[0].textnl, "lruginao\n") - t.CheckOutputLines( - "AUTOFIX: filename:1: Replacing \"ig\" with \"ug\".") - - { - fix := line.Autofix() - fix.Warnf(SilentAutofixFormat) - fix.Replace("lruginao", "middle") - fix.Apply() - } - - c.Check(line.autofix, check.NotNil) - t.CheckDeepEquals(line.raw, t.NewRawLines(1, "original\n", "middle\n")) - t.CheckEquals(line.raw[0].textnl, "middle\n") - t.CheckOutputLines( - "AUTOFIX: filename:1: Replacing \"lruginao\" with \"middle\".") - - t.CheckEquals(line.raw[0].textnl, "middle\n") - t.CheckOutputEmpty() - - { - fix := line.Autofix() - fix.Warnf(SilentAutofixFormat) - fix.Delete() - fix.Apply() - } - - t.CheckEquals(line.Autofix().RawText(), "") + t.CheckEquals(lines.Lines[1].raw[0].textnl, "XXXXX\n") + t.CheckFileLines("Makefile", + "line1", + "line2", + "line3") t.CheckOutputLines( - "AUTOFIX: filename:1: Deleting this line.") + "WARN: ~/Makefile:2: Something's wrong here.", + "AUTOFIX: ~/Makefile:2: Replacing \"l\" with \"X\".", + "AUTOFIX: ~/Makefile:2: Replacing \"i\" with \"X\".", + "AUTOFIX: ~/Makefile:2: Replacing \"n\" with \"X\".", + "AUTOFIX: ~/Makefile:2: Replacing \"e\" with \"X\".", + "AUTOFIX: ~/Makefile:2: Replacing \"2\" with \"X\".") } -func (s *Suite) Test_Autofix_Explain__without_explain_option(c *check.C) { +func (s *Suite) Test_Autofix_ReplaceRegex__autofix(c *check.C) { t := s.Init(c) - line := t.NewLine("Makefile", 74, "line1") + t.SetUpCommandLine("--autofix", "--source") + lines := t.SetUpFileLines("Makefile", + "line1", + "line2", + "line3") - fix := line.Autofix() - fix.Warnf("Please write row instead of line.") - fix.Replace("line", "row") - fix.Explain("Explanation") + fix := lines.Lines[1].Autofix() + fix.Warnf("Something's wrong here.") + fix.ReplaceRegex(`.`, "X", 3) fix.Apply() t.CheckOutputLines( - "WARN: Makefile:74: Please write row instead of line.") - t.CheckEquals(G.Logger.explanationsAvailable, true) -} - -func (s *Suite) Test_Autofix_Explain__default(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--explain") - line := t.NewLine("Makefile", 74, "line1") + "AUTOFIX: ~/Makefile:2: Replacing \"l\" with \"X\".", + "AUTOFIX: ~/Makefile:2: Replacing \"i\" with \"X\".", + "AUTOFIX: ~/Makefile:2: Replacing \"n\" with \"X\".", + "-\tline2", + "+\tXXXe2") - fix := line.Autofix() - fix.Warnf("Please write row instead of line.") - fix.Replace("line", "row") - fix.Explain("Explanation") + // After calling fix.Apply above, the autofix is ready to be used again. + fix.Warnf("Use Y instead of X.") + fix.Replace("XXX", "YYY") fix.Apply() t.CheckOutputLines( - "WARN: Makefile:74: Please write row instead of line.", - "", - "\tExplanation", - "") - t.CheckEquals(G.Logger.explanationsAvailable, true) -} - -func (s *Suite) Test_Autofix_Explain__show_autofix(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--show-autofix", "--explain") - line := t.NewLine("Makefile", 74, "line1") + "AUTOFIX: ~/Makefile:2: Replacing \"XXX\" with \"YYY\".", + "-\tline2", + "+\tYYYe2") - fix := line.Autofix() - fix.Warnf("Please write row instead of line.") - fix.Replace("line", "row") - fix.Explain("Explanation") - fix.Apply() + SaveAutofixChanges(lines) - t.CheckOutputLines( - "WARN: Makefile:74: Please write row instead of line.", - "AUTOFIX: Makefile:74: Replacing \"line\" with \"row\".", - "", - "\tExplanation", - "") - t.CheckEquals(G.Logger.explanationsAvailable, true) + t.CheckFileLines("Makefile", + "line1", + "YYYe2", + "line3") } -func (s *Suite) Test_Autofix_Explain__autofix(c *check.C) { +func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix_and_source(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--autofix", "--explain") - line := t.NewLine("Makefile", 74, "line1") + t.SetUpCommandLine("--show-autofix", "--source") + lines := t.SetUpFileLines("Makefile", + "line1", + "line2", + "line3") - fix := line.Autofix() - fix.Warnf("Please write row instead of line.") - fix.Replace("line", "row") - fix.Explain("Explanation") + fix := lines.Lines[1].Autofix() + fix.Warnf("Something's wrong here.") + fix.ReplaceRegex(`.`, "X", -1) fix.Apply() - t.CheckOutputLines( - "AUTOFIX: Makefile:74: Replacing \"line\" with \"row\".") - t.CheckEquals(G.Logger.explanationsAvailable, false) // Not necessary. -} - -func (s *Suite) Test_Autofix_Explain__SilentAutofixFormat(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--explain") - line := t.NewLine("example.txt", 1, "Text") - - fix := line.Autofix() - fix.Warnf(SilentAutofixFormat) - t.ExpectAssert(func() { fix.Explain("Explanation for inserting a line before.") }) -} - -// To combine a silent diagnostic with an explanation, two separate autofixes -// are necessary. -func (s *Suite) Test_Autofix_Explain__silent_with_diagnostic(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--explain") - line := t.NewLine("example.txt", 1, "Text") - - fix := line.Autofix() - fix.Warnf(SilentAutofixFormat) - fix.InsertBefore("before") + fix.Warnf("Use Y instead of X.") + fix.Replace("XXXXX", "YYYYY") fix.Apply() - fix.Notef("This diagnostic is necessary for the following explanation.") - fix.Explain( - "When inserting multiple lines, Apply must be called in-between.", - "Otherwise the changes are not described to the human reader.") - fix.InsertAfter("after") - fix.Apply() + SaveAutofixChanges(lines) t.CheckOutputLines( - "NOTE: example.txt:1: This diagnostic is necessary for the following explanation.", + "WARN: ~/Makefile:2: Something's wrong here.", + "AUTOFIX: ~/Makefile:2: Replacing \"l\" with \"X\".", + "AUTOFIX: ~/Makefile:2: Replacing \"i\" with \"X\".", + "AUTOFIX: ~/Makefile:2: Replacing \"n\" with \"X\".", + "AUTOFIX: ~/Makefile:2: Replacing \"e\" with \"X\".", + "AUTOFIX: ~/Makefile:2: Replacing \"2\" with \"X\".", + "-\tline2", + "+\tXXXXX", "", - "\tWhen inserting multiple lines, Apply must be called in-between.", - "\tOtherwise the changes are not described to the human reader.", - "") - t.CheckEquals(fix.RawText(), "Text\n") -} - -func (s *Suite) Test_Autofix__show_autofix_and_source_continuation_line(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--show-autofix", "--source") - mklines := t.SetUpFileMkLines("Makefile", - MkCvsID, - "# before \\", - "The old song \\", - "after") - line := mklines.lines.Lines[1] - - fix := line.Autofix() - fix.Warnf("Using \"old\" is deprecated.") - fix.Replace("old", "new") - fix.Apply() - - // Using a tab for indentation preserves the exact layout in the output - // since in pkgsrc Makefiles, tabs are also used in the middle of the line - // to align the variable values. Using a single space for indentation would - // make some of the lines appear misaligned in the pkglint output although - // they are correct in the Makefiles. - t.CheckOutputLines( - "WARN: ~/Makefile:3: Using \"old\" is deprecated.", - "AUTOFIX: ~/Makefile:3: Replacing \"old\" with \"new\".", - "\t# before \\", - "-\tThe old song \\", - "+\tThe new song \\", - "\tafter") + "WARN: ~/Makefile:2: Use Y instead of X.", + "AUTOFIX: ~/Makefile:2: Replacing \"XXXXX\" with \"YYYYY\".", + "-\tline2", + "+\tYYYYY") } func (s *Suite) Test_Autofix_InsertBefore(c *check.C) { @@ -682,84 +808,53 @@ func (s *Suite) Test_Autofix_Delete__combined_with_insert(c *check.C) { "+\tbelow") } -// Demonstrates that without the --show-autofix option, diagnostics are -// shown even when they cannot be autofixed. +// When using Autofix.Custom, it is tricky to get all the details right. +// For best results, see the existing examples and the documentation. // -// This is typical when an autofix is provided for simple scenarios, -// but the code actually found is a little more complicated. -func (s *Suite) Test_Autofix__show_unfixable_diagnostics_in_default_mode(c *check.C) { +// Since this custom fix only operates on the text of the current line, +// it can handle both the --show-autofix and the --autofix cases using +// the same code. +func (s *Suite) Test_Autofix_Custom__in_memory(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--source") lines := t.NewLines("Makefile", "line1", "line2", "line3") - lines.Lines[0].Warnf("This warning is shown since the --show-autofix option is not given.") - - fix := lines.Lines[1].Autofix() - fix.Warnf("This warning cannot be fixed and is therefore not shown.") - fix.Replace("XXX", "TODO") - fix.Apply() - - fix.Warnf("This warning cannot be fixed automatically but should be shown anyway.") - fix.Replace("XXX", "TODO") - fix.Anyway() - fix.Apply() - - // If this warning should ever appear it is probably because fix.anyway is not reset properly. - fix.Warnf("This warning cannot be fixed and is therefore not shown.") - fix.Replace("XXX", "TODO") - fix.Apply() + doFix := func(line *Line) { + fix := line.Autofix() + fix.Warnf("Please write in ALL-UPPERCASE.") + fix.Custom(func(showAutofix, autofix bool) { + fix.Describef(int(line.firstLine), "Converting to uppercase") + if showAutofix || autofix { + line.Text = strings.ToUpper(line.Text) + } + }) + fix.Apply() + } - lines.Lines[2].Warnf("This warning is also shown.") + doFix(lines.Lines[0]) t.CheckOutputLines( - ">\tline1", - "WARN: Makefile:1: This warning is shown since the --show-autofix option is not given.", - "", - ">\tline2", - "WARN: Makefile:2: This warning cannot be fixed automatically but should be shown anyway.", - "", - ">\tline3", - "WARN: Makefile:3: This warning is also shown.") -} - -// Demonstrates that the --show-autofix option only shows those diagnostics -// that would be fixed. -func (s *Suite) Test_Autofix__suppress_unfixable_warnings_with_show_autofix(c *check.C) { - t := s.Init(c) + "WARN: Makefile:1: Please write in ALL-UPPERCASE.") - t.SetUpCommandLine("--show-autofix", "--source") - lines := t.NewLines("Makefile", - "line1", - "line2", - "line3") + t.SetUpCommandLine("--show-autofix") - lines.Lines[0].Warnf("This warning is not shown since it is not part of a fix.") + doFix(lines.Lines[1]) - fix := lines.Lines[1].Autofix() - fix.Warnf("Something's wrong here.") - fix.ReplaceRegex(`.....`, "XXX", 1) - fix.Apply() + t.CheckOutputLines( + "WARN: Makefile:2: Please write in ALL-UPPERCASE.", + "AUTOFIX: Makefile:2: Converting to uppercase") + t.CheckEquals(lines.Lines[1].Text, "LINE2") - fix.Warnf("Since XXX marks are usually not fixed, use TODO instead to draw attention.") - fix.Replace("XXX", "TODO") - fix.Apply() + t.SetUpCommandLine("--autofix") - lines.Lines[2].Warnf("Neither is this warning shown.") + doFix(lines.Lines[2]) t.CheckOutputLines( - "WARN: Makefile:2: Something's wrong here.", - "AUTOFIX: Makefile:2: Replacing \"line2\" with \"XXX\".", - "-\tline2", - "+\tXXX", - "", - "WARN: Makefile:2: Since XXX marks are usually not fixed, use TODO instead to draw attention.", - "AUTOFIX: Makefile:2: Replacing \"XXX\" with \"TODO\".", - "-\tline2", - "+\tTODO") + "AUTOFIX: Makefile:3: Converting to uppercase") + t.CheckEquals(lines.Lines[2].Text, "LINE3") } // With the default command line options, this warning is printed. @@ -791,107 +886,56 @@ func (s *Suite) Test_Autofix_Anyway__options(c *check.C) { "WARN: filename:3: This autofix doesn't match.") } -// If an Autofix doesn't do anything, it must not log any diagnostics. -func (s *Suite) Test_Autofix__noop_replace(c *check.C) { +func (s *Suite) Test_Autofix_Anyway__autofix_option(c *check.C) { t := s.Init(c) - line := t.NewLine("Makefile", 14, "Original text") + t.SetUpCommandLine("--autofix") + line := t.NewLine("filename", 5, "text") fix := line.Autofix() - fix.Warnf("All-uppercase words should not be used at all.") - fix.ReplaceRegex(`\b[A-Z]{3,}\b`, "---censored---", -1) + fix.Notef("This line is quite short.") + fix.Replace("not found", "needle") + fix.Anyway() fix.Apply() - // No output since there was no all-uppercase word in the text. + // The replacement text is not found, therefore the note should not be logged. + // Because of fix.Anyway, the note should be logged anyway. + // But because of the --autofix option, the note should not be logged. + // Therefore, in the end nothing is shown in this case. t.CheckOutputEmpty() } -// When using Autofix.Custom, it is tricky to get all the details right. -// For best results, see the existing examples and the documentation. -// -// Since this custom fix only operates on the text of the current line, -// it can handle both the --show-autofix and the --autofix cases using -// the same code. -func (s *Suite) Test_Autofix_Custom__in_memory(c *check.C) { - t := s.Init(c) - - lines := t.NewLines("Makefile", - "line1", - "line2", - "line3") - - doFix := func(line *Line) { - fix := line.Autofix() - fix.Warnf("Please write in ALL-UPPERCASE.") - fix.Custom(func(showAutofix, autofix bool) { - fix.Describef(int(line.firstLine), "Converting to uppercase") - if showAutofix || autofix { - line.Text = strings.ToUpper(line.Text) - } - }) - fix.Apply() - } - - doFix(lines.Lines[0]) - - t.CheckOutputLines( - "WARN: Makefile:1: Please write in ALL-UPPERCASE.") - - t.SetUpCommandLine("--show-autofix") - - doFix(lines.Lines[1]) - - t.CheckOutputLines( - "WARN: Makefile:2: Please write in ALL-UPPERCASE.", - "AUTOFIX: Makefile:2: Converting to uppercase") - t.CheckEquals(lines.Lines[1].Text, "LINE2") - - t.SetUpCommandLine("--autofix") - - doFix(lines.Lines[2]) - - t.CheckOutputLines( - "AUTOFIX: Makefile:3: Converting to uppercase") - t.CheckEquals(lines.Lines[2].Text, "LINE3") -} - -// Since the diagnostic doesn't contain the string "few", nothing happens. -func (s *Suite) Test_Autofix_skip(c *check.C) { +func (s *Suite) Test_Autofix_Anyway__autofix_and_show_autofix_options(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--only", "few", "--autofix") - - mklines := t.SetUpFileMkLines("filename", - "VAR=\t111 222 333 444 555 \\", - "666") - lines := mklines.lines - - fix := lines.Lines[0].Autofix() - fix.Warnf("Many.") - fix.Explain( - "Explanation.") - - // None of the following actions has any effect because of the --only option above. - fix.Replace("111", "___") - fix.ReplaceAfter(" ", "222", "___") - fix.ReplaceAt(0, 0, "VAR", "NEW") - fix.ReplaceRegex(`\d+`, "___", 1) - fix.InsertBefore("before") - fix.InsertAfter("after") - fix.Delete() - fix.Custom(func(showAutofix, autofix bool) {}) + t.SetUpCommandLine("--autofix", "--show-autofix") + line := t.NewLine("filename", 5, "text") + fix := line.Autofix() + fix.Notef("This line is quite short.") + fix.Replace("not found", "needle") + fix.Anyway() fix.Apply() - SaveAutofixChanges(lines) - + // The text to be replaced is not found. Because nothing is fixed here, + // there's no need to log anything. + // + // But fix.Anyway is set, therefore the diagnostic should be logged even + // though it cannot be fixed automatically. This comes handy in situations + // where simple cases can be fixed automatically and more complex cases + // (often involving special characters that need to be escaped properly) + // should nevertheless result in a diagnostics. + // + // The --autofix option is set, which means that the diagnostics don't + // get logged, only the actual fixes do. + // + // But then there's also the --show-autofix option, which logs the + // corresponding diagnostic for each autofix that actually changes + // something. But this autofix doesn't change anything, therefore even + // the --show-autofix doesn't have an effect. + // + // Therefore, in the end nothing is logged in this case. t.CheckOutputEmpty() - t.CheckFileLines("filename", - "VAR=\t111 222 333 444 555 \\", - "666") - t.CheckEquals(fix.RawText(), ""+ - "VAR=\t111 222 333 444 555 \\\n"+ - "666\n") } // Demonstrates how to filter log messages. @@ -969,58 +1013,6 @@ func (s *Suite) Test_Autofix_Apply__explanation_followed_by_note(c *check.C) { "NOTE: README.txt:123: A note without fix.") } -func (s *Suite) Test_Autofix_Anyway__autofix_option(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--autofix") - line := t.NewLine("filename", 5, "text") - - fix := line.Autofix() - fix.Notef("This line is quite short.") - fix.Replace("not found", "needle") - fix.Anyway() - fix.Apply() - - // The replacement text is not found, therefore the note should not be logged. - // Because of fix.Anyway, the note should be logged anyway. - // But because of the --autofix option, the note should not be logged. - // Therefore, in the end nothing is shown in this case. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Autofix_Anyway__autofix_and_show_autofix_options(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--autofix", "--show-autofix") - line := t.NewLine("filename", 5, "text") - - fix := line.Autofix() - fix.Notef("This line is quite short.") - fix.Replace("not found", "needle") - fix.Anyway() - fix.Apply() - - // The text to be replaced is not found. Because nothing is fixed here, - // there's no need to log anything. - // - // But fix.Anyway is set, therefore the diagnostic should be logged even - // though it cannot be fixed automatically. This comes handy in situations - // where simple cases can be fixed automatically and more complex cases - // (often involving special characters that need to be escaped properly) - // should nevertheless result in a diagnostics. - // - // The --autofix option is set, which means that the diagnostics don't - // get logged, only the actual fixes do. - // - // But then there's also the --show-autofix option, which logs the - // corresponding diagnostic for each autofix that actually changes - // something. But this autofix doesn't change anything, therefore even - // the --show-autofix doesn't have an effect. - // - // Therefore, in the end nothing is logged in this case. - t.CheckOutputEmpty() -} - // The --autofix option normally suppresses the diagnostics and just logs // the actual fixes. Adding the --show-autofix option logs both. func (s *Suite) Test_Autofix_Apply__autofix_and_show_autofix_options(c *check.C) { @@ -1164,6 +1156,35 @@ func (s *Suite) Test_Autofix_Apply__text_after_replacing_regex(c *check.C) { t.CheckEquals(mkline.Value(), "value") } +// Just for branch coverage. +func (s *Suite) Test_Autofix_setDiag__no_testing_mode(c *check.C) { + t := s.Init(c) + + line := t.NewLine("file.mk", 123, "text") + + G.Testing = false + + fix := line.Autofix() + fix.Notef("Note.") + fix.Replace("from", "to") + fix.Apply() + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Autofix_setDiag__bad_call_sequence(c *check.C) { + t := s.Init(c) + + line := t.NewLine("file.mk", 123, "text") + fix := line.Autofix() + fix.Notef("Note.") + + t.ExpectAssert(func() { fix.Notef("Note 2.") }) + + fix.level = nil // To cover the second assertion. + t.ExpectAssert(func() { fix.Notef("Note 2.") }) +} + // Pkglint tries to order the diagnostics from top to bottom. // Still, it could be possible that in a multiline the second line // gets a diagnostic before the first line. This only happens when @@ -1206,33 +1227,43 @@ func (s *Suite) Test_Autofix_affectedLinenos__reverse(c *check.C) { "+\t\tbbb") } -// Just for branch coverage. -func (s *Suite) Test_Autofix_setDiag__no_testing_mode(c *check.C) { +// Since the diagnostic doesn't contain the string "few", nothing happens. +func (s *Suite) Test_Autofix_skip(c *check.C) { t := s.Init(c) - line := t.NewLine("file.mk", 123, "text") - - G.Testing = false + t.SetUpCommandLine("--only", "few", "--autofix") - fix := line.Autofix() - fix.Notef("Note.") - fix.Replace("from", "to") - fix.Apply() + mklines := t.SetUpFileMkLines("filename", + "VAR=\t111 222 333 444 555 \\", + "666") + lines := mklines.lines - t.CheckOutputEmpty() -} + fix := lines.Lines[0].Autofix() + fix.Warnf("Many.") + fix.Explain( + "Explanation.") -func (s *Suite) Test_Autofix_setDiag__bad_call_sequence(c *check.C) { - t := s.Init(c) + // None of the following actions has any effect because of the --only option above. + fix.Replace("111", "___") + fix.ReplaceAfter(" ", "222", "___") + fix.ReplaceAt(0, 0, "VAR", "NEW") + fix.ReplaceRegex(`\d+`, "___", 1) + fix.InsertBefore("before") + fix.InsertAfter("after") + fix.Delete() + fix.Custom(func(showAutofix, autofix bool) {}) - line := t.NewLine("file.mk", 123, "text") - fix := line.Autofix() - fix.Notef("Note.") + fix.Apply() - t.ExpectAssert(func() { fix.Notef("Note 2.") }) + SaveAutofixChanges(lines) - fix.level = nil // To cover the second assertion. - t.ExpectAssert(func() { fix.Notef("Note 2.") }) + t.CheckOutputEmpty() + t.CheckFileLines("filename", + "VAR=\t111 222 333 444 555 \\", + "666") + t.CheckEquals(fix.RawText(), ""+ + "VAR=\t111 222 333 444 555 \\\n"+ + "666\n") } func (s *Suite) Test_Autofix_assertRealLine(c *check.C) { @@ -1320,81 +1351,50 @@ func (s *Suite) Test_SaveAutofixChanges__cannot_overwrite(c *check.C) { `ERROR: ~/file.txt.pkglint.tmp: Cannot overwrite with autofixed content: .*`) } -// Up to 2018-11-25, pkglint in some cases logged only the source without -// a corresponding warning. -func (s *Suite) Test_Autofix__lonely_source(c *check.C) { +func (s *Suite) Test_SaveAutofixChanges(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall", "--source") - G.Logger.Opts.LogVerbose = false // For realistic conditions; otherwise all diagnostics are logged. + t.SetUpCommandLine("--autofix") + lines := t.SetUpFileLines("example.txt", + "line1 := value1", + "line2 := value2", + "line3 := value3") - t.SetUpPackage("x11/xorg-cf-files", - ".include \"../../x11/xorgproto/buildlink3.mk\"") - t.SetUpPackage("x11/xorgproto", - "DISTNAME=\txorgproto-1.0") - t.CreateFileDummyBuildlink3("x11/xorgproto/buildlink3.mk") - t.CreateFileLines("x11/xorgproto/builtin.mk", - MkCvsID, - "", - "BUILTIN_PKG:=\txorgproto", - "", - "PRE_XORGPROTO_LIST_MISSING =\tapplewmproto", - "", - ".for id in ${PRE_XORGPROTO_LIST_MISSING}", - ".endfor") - t.Chdir(".") - t.FinishSetUp() + fix := lines.Lines[1].Autofix() + fix.Warnf("Something's wrong here.") + fix.ReplaceRegex(`...`, "XXX", 2) + fix.Apply() - G.Check("x11/xorg-cf-files") - G.Check("x11/xorgproto") + SaveAutofixChanges(lines) t.CheckOutputLines( - ">\tPRE_XORGPROTO_LIST_MISSING =\tapplewmproto", - "NOTE: x11/xorgproto/builtin.mk:5: Unnecessary space after variable name \"PRE_XORGPROTO_LIST_MISSING\".") + "AUTOFIX: ~/example.txt:2: Replacing \"lin\" with \"XXX\".", + "AUTOFIX: ~/example.txt:2: Replacing \"e2 \" with \"XXX\".") + t.CheckFileLines("example.txt", + "line1 := value1", + "XXXXXX:= value2", + "line3 := value3") } -// Up to 2018-11-26, pkglint in some cases logged only the source without -// a corresponding warning. -func (s *Suite) Test_Autofix__lonely_source_2(c *check.C) { +func (s *Suite) Test_SaveAutofixChanges__no_changes_necessary(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall", "--source", "--explain") - G.Logger.Opts.LogVerbose = false // For realistic conditions; otherwise all diagnostics are logged. + t.SetUpCommandLine("--autofix") + lines := t.SetUpFileLines("DESCR", + "Line 1", + "Line 2") - t.SetUpPackage("print/tex-bibtex8", - "MAKE_FLAGS+=\tCFLAGS=${CFLAGS.${PKGSRC_COMPILER}}") - t.Chdir(".") - t.FinishSetUp() + fix := lines.Lines[0].Autofix() + fix.Warnf("Dummy warning.") + fix.Replace("X", "Y") + fix.Apply() - G.Check("print/tex-bibtex8") + // Since nothing has been effectively changed, + // nothing needs to be saved. + SaveAutofixChanges(lines) - t.CheckOutputLines( - ">\tMAKE_FLAGS+=\tCFLAGS=${CFLAGS.${PKGSRC_COMPILER}}", - "WARN: print/tex-bibtex8/Makefile:20: Please use ${CFLAGS.${PKGSRC_COMPILER}:Q} instead of ${CFLAGS.${PKGSRC_COMPILER}}.", - "", - "\tSee the pkgsrc guide, section \"Echoing a string exactly as-is\":", - "\thttps://www.NetBSD.org/docs/pkgsrc/pkgsrc.html#echo-literal", - "", - ">\tMAKE_FLAGS+=\tCFLAGS=${CFLAGS.${PKGSRC_COMPILER}}", - "WARN: print/tex-bibtex8/Makefile:20: The list variable PKGSRC_COMPILER should not be embedded in a word.", - "", - "\tWhen a list variable has multiple elements, this expression expands", - "\tto something unexpected:", - "", - "\tExample: ${MASTER_SITE_SOURCEFORGE}directory/ expands to", - "", - "\t\thttps://mirror1.sf.net/ https://mirror2.sf.net/directory/", - "", - "\tThe first URL is missing the directory. To fix this, write", - "\t\t${MASTER_SITE_SOURCEFORGE:=directory/}.", - "", - "\tExample: -l${LIBS} expands to", - "", - "\t\t-llib1 lib2", - "", - "\tThe second library is missing the -l. To fix this, write", - "\t${LIBS:S,^,-l,}.", - "") + // And therefore, no AUTOFIX action must appear in the log. + t.CheckOutputEmpty() } // RawText returns the raw text of the fixed line, including line ends. diff --git a/pkgtools/pkglint/files/buildlink3_test.go b/pkgtools/pkglint/files/buildlink3_test.go index 865f6a4ddf3..89e618c1a70 100644 --- a/pkgtools/pkglint/files/buildlink3_test.go +++ b/pkgtools/pkglint/files/buildlink3_test.go @@ -334,38 +334,6 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__abi_api_versions(c *check.C) { "WARN: buildlink3.mk:9: ABI version \"1.6.0\" should be at least API version \"1.6.1\" (see line 8).") } -// As of October 2018, pkglint parses package dependencies a little -// different than the pkg_* tools. -// In all but two cases this works, this is one of the exceptions. -// The "{totem,totem-xine}" cannot be parsed, therefore the check skipped. -func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_api_versions_brace(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - t.CreateFileLines("multimedia/totem/Makefile") - mklines := t.SetUpFileMkLines("multimedia/totem/buildlink3.mk", - MkCvsID, - "", - "BUILDLINK_TREE+=\ttotem", - "", - ".if !defined(TOTEM_BUILDLINK3_MK)", - "TOTEM_BUILDLINK3_MK:=", - "", - "BUILDLINK_API_DEPENDS.totem+=\t{totem,totem-xine}>=1.4.0", - "BUILDLINK_ABI_DEPENDS.totem+=\ttotem>=2.32.0nb46", - "BUILDLINK_PKGSRCDIR.totem?=\t../../multimedia/totem", - "", - ".endif # TOTEM_BUILDLINK3_MK", - "", - "BUILDLINK_TREE+=\t-totem") - - CheckLinesBuildlink3Mk(mklines) - - // No warning about ABI "totem" and API "{totem,totem-xine}" - // because that case is explicitly not checked. - t.CheckOutputEmpty() -} - func (s *Suite) Test_CheckLinesBuildlink3Mk__missing_BUILDLINK_TREE_at_beginning(c *check.C) { t := s.Init(c) @@ -612,6 +580,20 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_unknown_variable(c *ch "(also in other variables in this file).") } +// Just for branch coverage. +func (s *Suite) Test_Buildlink3Checker_Check__no_tracing(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileDummyBuildlink3("category/package/buildlink3.mk") + t.DisableTracing() + t.FinishSetUp() + + G.Check(t.File("category/package/buildlink3.mk")) + + t.CheckOutputEmpty() +} + func (s *Suite) Test_Buildlink3Checker_checkUniquePkgbase(c *check.C) { t := s.Init(c) @@ -665,6 +647,25 @@ func (s *Suite) Test_Buildlink3Checker_checkUniquePkgbase(c *check.C) { nil...) } +func (s *Suite) Test_Buildlink3Checker_checkSecondParagraph__missing_mkbase(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "DISTNAME=\t# empty", + "PKGNAME=\t# empty, to force mkbase to be empty") + t.CreateFileDummyBuildlink3("category/package/buildlink3.mk") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + // There is no warning from buildlink3.mk about mismatched package names + // since that is only a follow-up error of being unable to parse the pkgbase. + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:3: As DISTNAME is not a valid package name, "+ + "please define the PKGNAME explicitly.", + "WARN: ~/category/package/Makefile:4: \"\" is not a valid package name.") +} + func (s *Suite) Test_Buildlink3Checker_checkMainPart__if_else_endif(c *check.C) { t := s.Init(c) @@ -680,6 +681,94 @@ func (s *Suite) Test_Buildlink3Checker_checkMainPart__if_else_endif(c *check.C) t.CheckOutputEmpty() } +// Since the buildlink3 checker does not use MkLines.ForEach, it has to keep +// track of the nesting depth of .if directives. +func (s *Suite) Test_Buildlink3Checker_checkMainPart__nested_if(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", + MkCvsID, + "", + "BUILDLINK_TREE+=\ths-X11", + "", + ".if !defined(HS_X11_BUILDLINK3_MK)", + "HS_X11_BUILDLINK3_MK:=", + "", + "BUILDLINK_API_DEPENDS.hs-X11+=\ths-X11>=1.6.1", + "BUILDLINK_ABI_DEPENDS.hs-X11+=\ths-X11>=1.6.1.2nb2", + "", + ".if ${OPSYS} == NetBSD", + ".endif", + "", + ".endif\t# HS_X11_BUILDLINK3_MK", + "", + "BUILDLINK_TREE+=\t-hs-X11") + + CheckLinesBuildlink3Mk(mklines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Buildlink3Checker_checkMainPart__comment_at_end_of_file(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", + MkCvsID, + "", + "BUILDLINK_TREE+=\ths-X11", + "", + ".if !defined(HS_X11_BUILDLINK3_MK)", + "HS_X11_BUILDLINK3_MK:=", + "", + "BUILDLINK_API_DEPENDS.hs-X11+=\ths-X11>=1.6.1", + "BUILDLINK_ABI_DEPENDS.hs-X11+=\ths-X11>=1.6.1.2nb2", + "", + ".endif\t# HS_X11_BUILDLINK3_MK", + "", + "BUILDLINK_TREE+=\t-hs-X11", + "", + "# the end") + + CheckLinesBuildlink3Mk(mklines) + + t.CheckOutputLines( + "WARN: ~/category/package/buildlink3.mk:14: The file should end here.") +} + +// As of October 2018, pkglint parses package dependencies a little +// different than the pkg_* tools. +// In all but two cases this works, this is one of the exceptions. +// The "{totem,totem-xine}" cannot be parsed, therefore the check skipped. +func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_api_versions_brace(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.CreateFileLines("multimedia/totem/Makefile") + mklines := t.SetUpFileMkLines("multimedia/totem/buildlink3.mk", + MkCvsID, + "", + "BUILDLINK_TREE+=\ttotem", + "", + ".if !defined(TOTEM_BUILDLINK3_MK)", + "TOTEM_BUILDLINK3_MK:=", + "", + "BUILDLINK_API_DEPENDS.totem+=\t{totem,totem-xine}>=1.4.0", + "BUILDLINK_ABI_DEPENDS.totem+=\ttotem>=2.32.0nb46", + "BUILDLINK_PKGSRCDIR.totem?=\t../../multimedia/totem", + "", + ".endif # TOTEM_BUILDLINK3_MK", + "", + "BUILDLINK_TREE+=\t-totem") + + CheckLinesBuildlink3Mk(mklines) + + // No warning about ABI "totem" and API "{totem,totem-xine}" + // because that case is explicitly not checked. + t.CheckOutputEmpty() +} + func (s *Suite) Test_Buildlink3Checker_checkVarassign__dependencies_with_path(c *check.C) { t := s.Init(c) @@ -817,92 +906,3 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__other_variables(c *check. "Only buildlink variables for \"package\", " + "not \"other\" may be set in this file.") } - -// Just for branch coverage. -func (s *Suite) Test_Buildlink3Checker_Check__no_tracing(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package") - t.CreateFileDummyBuildlink3("category/package/buildlink3.mk") - t.DisableTracing() - t.FinishSetUp() - - G.Check(t.File("category/package/buildlink3.mk")) - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Buildlink3Checker_checkSecondParagraph__missing_mkbase(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package", - "DISTNAME=\t# empty", - "PKGNAME=\t# empty, to force mkbase to be empty") - t.CreateFileDummyBuildlink3("category/package/buildlink3.mk") - t.FinishSetUp() - - G.Check(t.File("category/package")) - - // There is no warning from buildlink3.mk about mismatched package names - // since that is only a follow-up error of being unable to parse the pkgbase. - t.CheckOutputLines( - "WARN: ~/category/package/Makefile:3: As DISTNAME is not a valid package name, "+ - "please define the PKGNAME explicitly.", - "WARN: ~/category/package/Makefile:4: \"\" is not a valid package name.") -} - -// Since the buildlink3 checker does not use MkLines.ForEach, it has to keep -// track of the nesting depth of .if directives. -func (s *Suite) Test_Buildlink3Checker_checkMainPart__nested_if(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", - MkCvsID, - "", - "BUILDLINK_TREE+=\ths-X11", - "", - ".if !defined(HS_X11_BUILDLINK3_MK)", - "HS_X11_BUILDLINK3_MK:=", - "", - "BUILDLINK_API_DEPENDS.hs-X11+=\ths-X11>=1.6.1", - "BUILDLINK_ABI_DEPENDS.hs-X11+=\ths-X11>=1.6.1.2nb2", - "", - ".if ${OPSYS} == NetBSD", - ".endif", - "", - ".endif\t# HS_X11_BUILDLINK3_MK", - "", - "BUILDLINK_TREE+=\t-hs-X11") - - CheckLinesBuildlink3Mk(mklines) - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Buildlink3Checker_checkMainPart__comment_at_end_of_file(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", - MkCvsID, - "", - "BUILDLINK_TREE+=\ths-X11", - "", - ".if !defined(HS_X11_BUILDLINK3_MK)", - "HS_X11_BUILDLINK3_MK:=", - "", - "BUILDLINK_API_DEPENDS.hs-X11+=\ths-X11>=1.6.1", - "BUILDLINK_ABI_DEPENDS.hs-X11+=\ths-X11>=1.6.1.2nb2", - "", - ".endif\t# HS_X11_BUILDLINK3_MK", - "", - "BUILDLINK_TREE+=\t-hs-X11", - "", - "# the end") - - CheckLinesBuildlink3Mk(mklines) - - t.CheckOutputLines( - "WARN: ~/category/package/buildlink3.mk:14: The file should end here.") -} diff --git a/pkgtools/pkglint/files/check_test.go b/pkgtools/pkglint/files/check_test.go index e2e6181cde9..b0f1d69c1af 100644 --- a/pkgtools/pkglint/files/check_test.go +++ b/pkgtools/pkglint/files/check_test.go @@ -57,10 +57,8 @@ func (s *Suite) SetUpTest(c *check.C) { t := Tester{c: c, testName: c.TestName()} s.Tester = &t - G = NewPkglint() + G = NewPkglint(&t.stdout, &t.stderr) G.Testing = true - G.Logger.out = NewSeparatorWriter(&t.stdout) - G.Logger.err = NewSeparatorWriter(&t.stderr) trace.Out = &t.stdout // XXX: Maybe the tests can run a bit faster when they don't @@ -181,7 +179,7 @@ func (t *Tester) SetUpVartypes() { } func (t *Tester) SetUpMasterSite(varname string, urls ...string) { - if !G.Pkgsrc.vartypes.DefinedExact(varname) { + if !G.Pkgsrc.vartypes.IsDefinedExact(varname) { G.Pkgsrc.vartypes.DefineParse(varname, BtFetchURL, List|SystemProvided, "buildlink3.mk: none", @@ -420,9 +418,9 @@ line: for _, line := range makefileLines { assert(!hasSuffix(line, "\\")) // Continuation lines are not yet supported. - if m, prefix := match1(line, `^#?(\w+=)`); m { + if m, varname := match1(line, `^#?(\w+)[!+:?]?=`); m { for i, existingLine := range mlines[:19] { - if hasPrefix(strings.TrimPrefix(existingLine, "#"), prefix) { + if hasPrefix(strings.TrimPrefix(existingLine, "#"), varname+"=") { mlines[i] = line continue line } @@ -456,7 +454,7 @@ func (t *Tester) CreateFileLines(relativeFileName string, lines ...string) (file err := os.MkdirAll(path.Dir(filename), 0777) t.c.Assert(err, check.IsNil) - err = ioutil.WriteFile(filename, []byte(content.Bytes()), 0666) + err = ioutil.WriteFile(filename, content.Bytes(), 0666) t.c.Assert(err, check.IsNil) G.fileCache.Evict(filename) @@ -485,7 +483,7 @@ func (t *Tester) CreateFileDummyBuildlink3(relativeFileName string, customLines // see pkgtools/createbuildlink/files/createbuildlink, "package specific variables" upper := strings.Replace(strings.ToUpper(lower), "-", "_", -1) - width := tabWidth(sprintf("BUILDLINK_API_DEPENDS.%s+=\t", lower)) + width := tabWidthSlice("BUILDLINK_API_DEPENDS.", lower, "+=\t") aligned := func(format string, args ...interface{}) string { msg := sprintf(format, args...) @@ -917,7 +915,7 @@ func (t *Tester) Output() string { t.stdout.Reset() t.stderr.Reset() - if G.usable() { + if G.isUsable() { G.Logger.logged = Once{} if G.Logger.out != nil { // Necessary because Main resets the G variable. G.Logger.out.state = 0 // Prevent an empty line at the beginning of the next output. @@ -1130,7 +1128,7 @@ func (t *Tester) EnableSilentTracing() { // The diagnostics go to the in-memory buffer again, // ready to be checked with CheckOutputLines. func (t *Tester) DisableTracing() { - if G.usable() { + if G.isUsable() { G.Logger.out = NewSeparatorWriter(&t.stdout) } trace.Tracing = false @@ -1167,8 +1165,7 @@ func (t *Tester) CheckFileLinesDetab(relativeFileName string, lines ...string) { // This means that the test cases that follow do not have to use each of them, // and this in turn allows uninteresting test cases to be deleted during // development. -func (t *Tester) Use(functions ...interface{}) { -} +func (t *Tester) Use(...interface{}) {} func (t *Tester) Shquote(format string, rels ...string) string { var subs []interface{} diff --git a/pkgtools/pkglint/files/distinfo_test.go b/pkgtools/pkglint/files/distinfo_test.go index db97b7e88b8..d7662379231 100644 --- a/pkgtools/pkglint/files/distinfo_test.go +++ b/pkgtools/pkglint/files/distinfo_test.go @@ -36,6 +36,131 @@ func (s *Suite) Test_CheckLinesDistinfo__parse_errors(c *check.C) { "WARN: distinfo:9: Patch file \"patch-nonexistent\" does not exist in directory \"patches\".") } +func (s *Suite) Test_distinfoLinesChecker_parse__empty(c *check.C) { + t := s.Init(c) + + lines := t.SetUpFileLines("distinfo", + CvsID, + "") + + CheckLinesDistinfo(nil, lines) + + t.CheckOutputLines( + "NOTE: ~/distinfo:2: Trailing empty lines.") +} + +// When the distinfo file and the patches are placed in the same package, +// their diagnostics use short relative paths. +func (s *Suite) Test_distinfoLinesChecker_check__distinfo_and_patches_in_separate_directory(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "DISTINFO_FILE=\t../../other/common/distinfo", + "PATCHDIR=\t../../other/common/patches") + t.Remove("category/package/distinfo") + t.CreateFileLines("other/common/patches/CVS/Entries") + t.CreateFileDummyPatch("other/common/patches/patch-aa") + t.CreateFileDummyPatch("other/common/patches/patch-only-in-patches") + t.SetUpFileLines("other/common/distinfo", + CvsID, + "", + "SHA1 (patch-aa) = ...", + "SHA1 (patch-only-in-distinfo) = ...") + t.Chdir("category/package") + t.FinishSetUp() + + G.checkdirPackage(".") + + t.CheckOutputLines( + "ERROR: ../../other/common/distinfo:3: SHA1 hash of patches/patch-aa differs "+ + "(distinfo has ..., patch file has ebbf34b0641bcb508f17d5a27f2bf2a536d810ac).", + "WARN: ../../other/common/distinfo:4: Patch file \"patch-only-in-distinfo\" "+ + "does not exist in directory \"patches\".", + "ERROR: ../../other/common/distinfo: Patch \"patches/patch-only-in-patches\" "+ + "is not recorded. Run \""+confMake+" makepatchsum\".") +} + +func (s *Suite) Test_distinfoLinesChecker_check__manual_patches(c *check.C) { + t := s.Init(c) + + t.Chdir("category/package") + t.CreateFileLines("patches/manual-libtool.m4") + lines := t.SetUpFileLines("distinfo", + CvsID, + "", + "SHA1 (patch-aa) = ...") + + CheckLinesDistinfo(nil, lines) + + // When a distinfo file is checked on its own, without belonging to a package, + // the PATCHDIR is not known and therefore no diagnostics are logged. + t.CheckOutputEmpty() + + G.Pkg = NewPackage(".") + + CheckLinesDistinfo(G.Pkg, lines) + + // When a distinfo file is checked in the context of a package, + // the PATCHDIR is known, therefore the check is active. + t.CheckOutputLines( + "WARN: distinfo:3: Patch file \"patch-aa\" does not exist in directory \"patches\".") +} + +// PHP modules that are not PECL use the distinfo file from lang/php* but +// their own patches directory. Therefore the distinfo file refers to missing +// patches. Since this strange situation is caused by the pkgsrc +// infrastructure, there is nothing a package author can do about. +// +// XXX: Re-check the documentation for this test. +func (s *Suite) Test_distinfoLinesChecker_check__missing_php_patches(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.SetUpCommandLine("-Wall,no-space") + t.CreateFileLines("licenses/unknown-license") + t.CreateFileLines("lang/php/ext.mk", + MkCvsID, + "", + "PHPEXT_MK= # defined", + "PHPPKGSRCDIR= ../../lang/php72", + "LICENSE?= unknown-license", + "COMMENT?= Some PHP package", + "GENERATE_PLIST+=# none", + "", + ".if !defined(PECL_VERSION)", + "DISTINFO_FILE= ${.CURDIR}/${PHPPKGSRCDIR}/distinfo", + ".endif", + ".if defined(USE_PHP_EXT_PATCHES)", + "PATCHDIR= ${.CURDIR}/${PHPPKGSRCDIR}/patches", + ".endif") + t.CreateFileDummyPatch("lang/php72/patches/patch-php72") + t.CreateFileLines("lang/php72/distinfo", + CvsID, + "", + "SHA1 (patch-php72) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") + + t.CreateFileLines("archivers/php-bz2/Makefile", + MkCvsID, + "", + "USE_PHP_EXT_PATCHES= yes", + "", + ".include \"../../lang/php/ext.mk\"", + ".include \"../../mk/bsd.pkg.mk\"") + t.FinishSetUp() + + G.Check(t.File("archivers/php-bz2")) + + t.CreateFileLines("archivers/php-zlib/Makefile", + MkCvsID, + "", + ".include \"../../lang/php/ext.mk\"", + ".include \"../../mk/bsd.pkg.mk\"") + + G.Check(t.File("archivers/php-zlib")) + + t.CheckOutputEmpty() +} + func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__nonexistent_distfile_called_patch(c *check.C) { t := s.Init(c) @@ -124,90 +249,6 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__wrong_patch_algorithm "patch file has ebbf34b0641bcb508f17d5a27f2bf2a536d810ac).") } -func (s *Suite) Test_distinfoLinesChecker_parse__empty(c *check.C) { - t := s.Init(c) - - lines := t.SetUpFileLines("distinfo", - CvsID, - "") - - CheckLinesDistinfo(nil, lines) - - t.CheckOutputLines( - "NOTE: ~/distinfo:2: Trailing empty lines.") -} - -// When checking the complete pkgsrc tree, pkglint has all information it needs -// to check whether different packages use the same distfile but require -// different hashes for it. -// -// In such a case, typically one of the packages should put its distfiles into -// a DIST_SUBDIR. -func (s *Suite) Test_distinfoLinesChecker_checkGlobalDistfileMismatch(c *check.C) { - t := s.Init(c) - - t.SetUpPkgsrc() - t.SetUpPackage("category/package1") - t.SetUpPackage("category/package2") - t.CreateFileLines("category/package1/distinfo", - CvsID, - "", - "SHA512 (distfile-1.0.tar.gz) = 1234567811111111", - "SHA512 (distfile-1.1.tar.gz) = 1111111111111111", - "SHA512 (patch-4.2.tar.gz) = 1234567812345678") - t.CreateFileLines("category/package2/distinfo", - CvsID, - "", - "SHA512 (distfile-1.0.tar.gz) = 1234567822222222", - "SHA512 (distfile-1.1.tar.gz) = 1111111111111111", - "SHA512 (encoding-error.tar.gz) = 12345678abcdefgh") - t.CreateFileLines("Makefile", - MkCvsID, - "", - "COMMENT=\tThis is pkgsrc", - "", - "SUBDIR+=\tcategory") - t.CreateFileLines("category/Makefile", - MkCvsID, - "", - "COMMENT=\tUseful programs", - "", - "SUBDIR+=\tpackage1", - "SUBDIR+=\tpackage2", - "", - ".include \"../mk/misc/category.mk\"") - - t.Main("-r", "-Wall", "-Call", ".") - - t.CheckOutputLines( - "ERROR: ~/category/package1/distinfo:3: "+ - "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.0.tar.gz\", got SHA512.", - "ERROR: ~/category/package1/distinfo:4: "+ - "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.1.tar.gz\", got SHA512.", - "ERROR: ~/category/package1/distinfo:5: "+ - "Expected SHA1, RMD160, SHA512, Size checksums for \"patch-4.2.tar.gz\", got SHA512.", - - "ERROR: ~/category/package2/distinfo:3: "+ - "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.0.tar.gz\", got SHA512.", - "ERROR: ~/category/package2/distinfo:3: "+ - "The SHA512 hash for distfile-1.0.tar.gz is 1234567822222222, "+ - "which conflicts with 1234567811111111 in ../../category/package1/distinfo:3.", - "ERROR: ~/category/package2/distinfo:4: "+ - "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.1.tar.gz\", got SHA512.", - "ERROR: ~/category/package2/distinfo:5: "+ - "Expected SHA1, RMD160, SHA512, Size checksums for \"encoding-error.tar.gz\", got SHA512.", - "ERROR: ~/category/package2/distinfo:5: "+ - "The SHA512 hash for encoding-error.tar.gz contains a non-hex character.", - - "WARN: ~/licenses/gnu-gpl-v2: This license seems to be unused.", - "8 errors and 1 warning found.", - t.Shquote("(Run \"pkglint -e -r -Wall -Call %s\" to show explanations.)", ".")) - - // Ensure that hex.DecodeString does not waste memory here. - t.CheckEquals(len(G.InterPackage.hashes["SHA512:distfile-1.0.tar.gz"].hash), 8) - t.CheckEquals(cap(G.InterPackage.hashes["SHA512:distfile-1.0.tar.gz"].hash), 8) -} - func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__missing_patch_with_distfile_checksums(c *check.C) { t := s.Init(c) @@ -279,228 +320,6 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__missing_patch_with_wr "Expected SHA1, RMD160, SHA512, Size checksums for \"patch-aa\", got RMD160.") } -func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch__bad(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package") - t.Chdir("category/package") - t.CreateFileDummyPatch("patches/patch-aa") - t.CreateFileLines("CVS/Entries", - "/distinfo//modified//") - t.SetUpFileLines("distinfo", - CvsID, - "", - "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") - t.FinishSetUp() - - G.checkdirPackage(".") - - t.CheckOutputLines( - "WARN: distinfo:3: patches/patch-aa is registered in distinfo but not added to CVS.") -} - -func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch__good(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package") - t.Chdir("category/package") - t.CreateFileDummyPatch("patches/patch-aa") - t.CreateFileLines("CVS/Entries", - "/distinfo//modified//") - t.CreateFileLines("patches/CVS/Entries", - "/patch-aa//modified//") - t.SetUpFileLines("distinfo", - CvsID, - "", - "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") - t.FinishSetUp() - - G.checkdirPackage(".") - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_distinfoLinesChecker_checkUnrecordedPatches(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package") - t.Chdir("category/package") - t.CreateFileLines("patches/CVS/Entries") - t.CreateFileDummyPatch("patches/patch-aa") - t.CreateFileDummyPatch("patches/patch-src-Makefile") - t.SetUpFileLines("distinfo", - CvsID, - "", - "SHA1 (distfile.tar.gz) = ...", - "RMD160 (distfile.tar.gz) = ...", - "SHA512 (distfile.tar.gz) = ...", - "Size (distfile.tar.gz) = 1024 bytes") - t.FinishSetUp() - - G.checkdirPackage(".") - - t.CheckOutputLines( - "ERROR: distinfo: Patch \"patches/patch-aa\" is not recorded. Run \""+confMake+" makepatchsum\".", - "ERROR: distinfo: Patch \"patches/patch-src-Makefile\" is not recorded. Run \""+confMake+" makepatchsum\".") -} - -// The distinfo file and the patches are usually placed in the package -// directory. By defining PATCHDIR or DISTINFO_FILE, a package can define -// that they are somewhere else in pkgsrc. -func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1__relative_path_in_distinfo(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package", - "DISTINFO_FILE=\t../../other/common/distinfo", - "PATCHDIR=\t../../devel/patches/patches") - t.Remove("category/package/distinfo") - t.CreateFileLines("devel/patches/patches/CVS/Entries") - t.CreateFileDummyPatch("devel/patches/patches/patch-aa") - t.CreateFileDummyPatch("devel/patches/patches/patch-only-in-patches") - t.SetUpFileLines("other/common/distinfo", - CvsID, - "", - "SHA1 (patch-aa) = ...", - "SHA1 (patch-only-in-distinfo) = ...") - t.Chdir("category/package") - t.FinishSetUp() - - G.checkdirPackage(".") - - t.CheckOutputLines( - "ERROR: ../../other/common/distinfo:3: SHA1 hash of ../../devel/patches/patches/patch-aa differs "+ - "(distinfo has ..., patch file has ebbf34b0641bcb508f17d5a27f2bf2a536d810ac).", - "WARN: ../../other/common/distinfo:4: Patch file \"patch-only-in-distinfo\" "+ - "does not exist in directory \"../../devel/patches/patches\".", - "ERROR: ../../other/common/distinfo: Patch \"../../devel/patches/patches/patch-only-in-patches\" "+ - "is not recorded. Run \""+confMake+" makepatchsum\".") -} - -// When the distinfo file and the patches are placed in the same package, -// their diagnostics use short relative paths. -func (s *Suite) Test_CheckLinesDistinfo__distinfo_and_patches_in_separate_directory(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package", - "DISTINFO_FILE=\t../../other/common/distinfo", - "PATCHDIR=\t../../other/common/patches") - t.Remove("category/package/distinfo") - t.CreateFileLines("other/common/patches/CVS/Entries") - t.CreateFileDummyPatch("other/common/patches/patch-aa") - t.CreateFileDummyPatch("other/common/patches/patch-only-in-patches") - t.SetUpFileLines("other/common/distinfo", - CvsID, - "", - "SHA1 (patch-aa) = ...", - "SHA1 (patch-only-in-distinfo) = ...") - t.Chdir("category/package") - t.FinishSetUp() - - G.checkdirPackage(".") - - t.CheckOutputLines( - "ERROR: ../../other/common/distinfo:3: SHA1 hash of patches/patch-aa differs "+ - "(distinfo has ..., patch file has ebbf34b0641bcb508f17d5a27f2bf2a536d810ac).", - "WARN: ../../other/common/distinfo:4: Patch file \"patch-only-in-distinfo\" "+ - "does not exist in directory \"patches\".", - "ERROR: ../../other/common/distinfo: Patch \"patches/patch-only-in-patches\" "+ - "is not recorded. Run \""+confMake+" makepatchsum\".") -} - -func (s *Suite) Test_CheckLinesDistinfo__manual_patches(c *check.C) { - t := s.Init(c) - - t.Chdir("category/package") - t.CreateFileLines("patches/manual-libtool.m4") - lines := t.SetUpFileLines("distinfo", - CvsID, - "", - "SHA1 (patch-aa) = ...") - - CheckLinesDistinfo(nil, lines) - - // When a distinfo file is checked on its own, without belonging to a package, - // the PATCHDIR is not known and therefore no diagnostics are logged. - t.CheckOutputEmpty() - - G.Pkg = NewPackage(".") - - CheckLinesDistinfo(G.Pkg, lines) - - // When a distinfo file is checked in the context of a package, - // the PATCHDIR is known, therefore the check is active. - t.CheckOutputLines( - "WARN: distinfo:3: Patch file \"patch-aa\" does not exist in directory \"patches\".") -} - -// PHP modules that are not PECL use the distinfo file from lang/php* but -// their own patches directory. Therefore the distinfo file refers to missing -// patches. Since this strange situation is caused by the pkgsrc -// infrastructure, there is nothing a package author can do about. -// -// XXX: Re-check the documentation for this test. -func (s *Suite) Test_CheckLinesDistinfo__missing_php_patches(c *check.C) { - t := s.Init(c) - - t.SetUpPkgsrc() - t.SetUpCommandLine("-Wall,no-space") - t.CreateFileLines("licenses/unknown-license") - t.CreateFileLines("lang/php/ext.mk", - MkCvsID, - "", - "PHPEXT_MK= # defined", - "PHPPKGSRCDIR= ../../lang/php72", - "LICENSE?= unknown-license", - "COMMENT?= Some PHP package", - "GENERATE_PLIST+=# none", - "", - ".if !defined(PECL_VERSION)", - "DISTINFO_FILE= ${.CURDIR}/${PHPPKGSRCDIR}/distinfo", - ".endif", - ".if defined(USE_PHP_EXT_PATCHES)", - "PATCHDIR= ${.CURDIR}/${PHPPKGSRCDIR}/patches", - ".endif") - t.CreateFileDummyPatch("lang/php72/patches/patch-php72") - t.CreateFileLines("lang/php72/distinfo", - CvsID, - "", - "SHA1 (patch-php72) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") - - t.CreateFileLines("archivers/php-bz2/Makefile", - MkCvsID, - "", - "USE_PHP_EXT_PATCHES= yes", - "", - ".include \"../../lang/php/ext.mk\"", - ".include \"../../mk/bsd.pkg.mk\"") - t.FinishSetUp() - - G.Check(t.File("archivers/php-bz2")) - - t.CreateFileLines("archivers/php-zlib/Makefile", - MkCvsID, - "", - ".include \"../../lang/php/ext.mk\"", - ".include \"../../mk/bsd.pkg.mk\"") - - G.Check(t.File("archivers/php-zlib")) - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1(c *check.C) { - t := s.Init(c) - - G.Pkg = NewPackage(t.File("category/package")) - distinfoLine := t.NewLine(t.File("category/package/distinfo"), 5, "") - - checker := distinfoLinesChecker{G.Pkg, nil, "", false, nil, nil} - checker.checkPatchSha1(distinfoLine, "patch-nonexistent", "distinfo-sha1") - - t.CheckOutputLines( - "ERROR: ~/category/package/distinfo:5: Patch patch-nonexistent does not exist.") -} - // When there is at least one correct hash for a distfile and the distfile // has already been downloaded to pkgsrc/distfiles, which is the standard // distfiles location, running pkglint --autofix adds the missing hashes. @@ -770,3 +589,184 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__some_algorith "got RMD160, Size, SHA512.", "ERROR: ~/category/package/distinfo:3: Missing SHA1 hash for package-1.0.txt.") } + +func (s *Suite) Test_distinfoLinesChecker_checkUnrecordedPatches(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.CreateFileLines("patches/CVS/Entries") + t.CreateFileDummyPatch("patches/patch-aa") + t.CreateFileDummyPatch("patches/patch-src-Makefile") + t.SetUpFileLines("distinfo", + CvsID, + "", + "SHA1 (distfile.tar.gz) = ...", + "RMD160 (distfile.tar.gz) = ...", + "SHA512 (distfile.tar.gz) = ...", + "Size (distfile.tar.gz) = 1024 bytes") + t.FinishSetUp() + + G.checkdirPackage(".") + + t.CheckOutputLines( + "ERROR: distinfo: Patch \"patches/patch-aa\" is not recorded. Run \""+confMake+" makepatchsum\".", + "ERROR: distinfo: Patch \"patches/patch-src-Makefile\" is not recorded. Run \""+confMake+" makepatchsum\".") +} + +// When checking the complete pkgsrc tree, pkglint has all information it needs +// to check whether different packages use the same distfile but require +// different hashes for it. +// +// In such a case, typically one of the packages should put its distfiles into +// a DIST_SUBDIR. +func (s *Suite) Test_distinfoLinesChecker_checkGlobalDistfileMismatch(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.SetUpPackage("category/package1") + t.SetUpPackage("category/package2") + t.CreateFileLines("category/package1/distinfo", + CvsID, + "", + "SHA512 (distfile-1.0.tar.gz) = 1234567811111111", + "SHA512 (distfile-1.1.tar.gz) = 1111111111111111", + "SHA512 (patch-4.2.tar.gz) = 1234567812345678") + t.CreateFileLines("category/package2/distinfo", + CvsID, + "", + "SHA512 (distfile-1.0.tar.gz) = 1234567822222222", + "SHA512 (distfile-1.1.tar.gz) = 1111111111111111", + "SHA512 (encoding-error.tar.gz) = 12345678abcdefgh") + t.CreateFileLines("Makefile", + MkCvsID, + "", + "COMMENT=\tThis is pkgsrc", + "", + "SUBDIR+=\tcategory") + t.CreateFileLines("category/Makefile", + MkCvsID, + "", + "COMMENT=\tUseful programs", + "", + "SUBDIR+=\tpackage1", + "SUBDIR+=\tpackage2", + "", + ".include \"../mk/misc/category.mk\"") + + t.Main("-r", "-Wall", "-Call", ".") + + t.CheckOutputLines( + "ERROR: ~/category/package1/distinfo:3: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.0.tar.gz\", got SHA512.", + "ERROR: ~/category/package1/distinfo:4: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.1.tar.gz\", got SHA512.", + "ERROR: ~/category/package1/distinfo:5: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"patch-4.2.tar.gz\", got SHA512.", + + "ERROR: ~/category/package2/distinfo:3: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.0.tar.gz\", got SHA512.", + "ERROR: ~/category/package2/distinfo:3: "+ + "The SHA512 hash for distfile-1.0.tar.gz is 1234567822222222, "+ + "which conflicts with 1234567811111111 in ../../category/package1/distinfo:3.", + "ERROR: ~/category/package2/distinfo:4: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.1.tar.gz\", got SHA512.", + "ERROR: ~/category/package2/distinfo:5: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"encoding-error.tar.gz\", got SHA512.", + "ERROR: ~/category/package2/distinfo:5: "+ + "The SHA512 hash for encoding-error.tar.gz contains a non-hex character.", + + "WARN: ~/licenses/gnu-gpl-v2: This license seems to be unused.", + "8 errors and 1 warning found.", + t.Shquote("(Run \"pkglint -e -r -Wall -Call %s\" to show explanations.)", ".")) + + // Ensure that hex.DecodeString does not waste memory here. + t.CheckEquals(len(G.InterPackage.hashes["SHA512:distfile-1.0.tar.gz"].hash), 8) + t.CheckEquals(cap(G.InterPackage.hashes["SHA512:distfile-1.0.tar.gz"].hash), 8) +} + +func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch__bad(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.CreateFileDummyPatch("patches/patch-aa") + t.CreateFileLines("CVS/Entries", + "/distinfo//modified//") + t.SetUpFileLines("distinfo", + CvsID, + "", + "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") + t.FinishSetUp() + + G.checkdirPackage(".") + + t.CheckOutputLines( + "WARN: distinfo:3: patches/patch-aa is registered in distinfo but not added to CVS.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch__good(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.CreateFileDummyPatch("patches/patch-aa") + t.CreateFileLines("CVS/Entries", + "/distinfo//modified//") + t.CreateFileLines("patches/CVS/Entries", + "/patch-aa//modified//") + t.SetUpFileLines("distinfo", + CvsID, + "", + "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") + t.FinishSetUp() + + G.checkdirPackage(".") + + t.CheckOutputEmpty() +} + +// The distinfo file and the patches are usually placed in the package +// directory. By defining PATCHDIR or DISTINFO_FILE, a package can define +// that they are somewhere else in pkgsrc. +func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1__relative_path_in_distinfo(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "DISTINFO_FILE=\t../../other/common/distinfo", + "PATCHDIR=\t../../devel/patches/patches") + t.Remove("category/package/distinfo") + t.CreateFileLines("devel/patches/patches/CVS/Entries") + t.CreateFileDummyPatch("devel/patches/patches/patch-aa") + t.CreateFileDummyPatch("devel/patches/patches/patch-only-in-patches") + t.SetUpFileLines("other/common/distinfo", + CvsID, + "", + "SHA1 (patch-aa) = ...", + "SHA1 (patch-only-in-distinfo) = ...") + t.Chdir("category/package") + t.FinishSetUp() + + G.checkdirPackage(".") + + t.CheckOutputLines( + "ERROR: ../../other/common/distinfo:3: SHA1 hash of ../../devel/patches/patches/patch-aa differs "+ + "(distinfo has ..., patch file has ebbf34b0641bcb508f17d5a27f2bf2a536d810ac).", + "WARN: ../../other/common/distinfo:4: Patch file \"patch-only-in-distinfo\" "+ + "does not exist in directory \"../../devel/patches/patches\".", + "ERROR: ../../other/common/distinfo: Patch \"../../devel/patches/patches/patch-only-in-patches\" "+ + "is not recorded. Run \""+confMake+" makepatchsum\".") +} + +func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1(c *check.C) { + t := s.Init(c) + + G.Pkg = NewPackage(t.File("category/package")) + distinfoLine := t.NewLine(t.File("category/package/distinfo"), 5, "") + + checker := distinfoLinesChecker{G.Pkg, nil, "", false, nil, nil} + checker.checkPatchSha1(distinfoLine, "patch-nonexistent", "distinfo-sha1") + + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:5: Patch patch-nonexistent does not exist.") +} diff --git a/pkgtools/pkglint/files/files.go b/pkgtools/pkglint/files/files.go index 76745360d82..b54dde877c8 100644 --- a/pkgtools/pkglint/files/files.go +++ b/pkgtools/pkglint/files/files.go @@ -16,6 +16,14 @@ const ( LogErrors // ) +func LoadMk(filename string, options LoadOptions) *MkLines { + lines := Load(filename, options|Makefile) + if lines == nil { + return nil + } + return NewMkLines(lines) +} + func Load(filename string, options LoadOptions) *Lines { if fromCache := G.fileCache.Get(filename, options); fromCache != nil { return fromCache @@ -54,12 +62,34 @@ func Load(filename string, options LoadOptions) *Lines { return result } -func LoadMk(filename string, options LoadOptions) *MkLines { - lines := Load(filename, options|Makefile) - if lines == nil { - return nil +func convertToLogicalLines(filename string, rawText string, joinBackslashLines bool) *Lines { + var rawLines []*RawLine + for lineno, rawLine := range strings.SplitAfter(rawText, "\n") { + if rawLine != "" { + rawLines = append(rawLines, &RawLine{1 + lineno, rawLine, rawLine}) + } } - return NewMkLines(lines) + + var loglines []*Line + if joinBackslashLines { + for lineno := 0; lineno < len(rawLines); { + line, nextLineno := nextLogicalLine(filename, rawLines, lineno) + loglines = append(loglines, line) + lineno = nextLineno + } + } else { + for _, rawLine := range rawLines { + text := strings.TrimSuffix(rawLine.textnl, "\n") + logline := NewLine(filename, rawLine.Lineno, text, rawLine) + loglines = append(loglines, logline) + } + } + + if rawText != "" && !hasSuffix(rawText, "\n") { + loglines[len(loglines)-1].Errorf("File must end with a newline.") + } + + return NewLines(filename, loglines) } func nextLogicalLine(filename string, rawLines []*RawLine, index int) (*Line, int) { @@ -136,33 +166,3 @@ func matchContinuationLine(textnl string) (leadingWhitespace, text, trailingWhit text = textnl[leadingEnd:trailingStart] return } - -func convertToLogicalLines(filename string, rawText string, joinBackslashLines bool) *Lines { - var rawLines []*RawLine - for lineno, rawLine := range strings.SplitAfter(rawText, "\n") { - if rawLine != "" { - rawLines = append(rawLines, &RawLine{1 + lineno, rawLine, rawLine}) - } - } - - var loglines []*Line - if joinBackslashLines { - for lineno := 0; lineno < len(rawLines); { - line, nextLineno := nextLogicalLine(filename, rawLines, lineno) - loglines = append(loglines, line) - lineno = nextLineno - } - } else { - for _, rawLine := range rawLines { - text := strings.TrimSuffix(rawLine.textnl, "\n") - logline := NewLine(filename, rawLine.Lineno, text, rawLine) - loglines = append(loglines, logline) - } - } - - if rawText != "" && !hasSuffix(rawText, "\n") { - loglines[len(loglines)-1].Errorf("File must end with a newline.") - } - - return NewLines(filename, loglines) -} diff --git a/pkgtools/pkglint/files/files_test.go b/pkgtools/pkglint/files/files_test.go index 48ca6babd5f..82bee7eb9e1 100644 --- a/pkgtools/pkglint/files/files_test.go +++ b/pkgtools/pkglint/files/files_test.go @@ -4,6 +4,50 @@ import ( "gopkg.in/check.v1" ) +func (s *Suite) Test_Load(c *check.C) { + t := s.Init(c) + + nonexistent := t.File("nonexistent") + empty := t.CreateFileLines("empty") + oneLiner := t.CreateFileLines("one-liner", + "hello, world") + + t.Check(Load(nonexistent, 0), check.IsNil) + t.Check(Load(empty, 0).Lines, check.HasLen, 0) + t.CheckEquals(Load(oneLiner, 0).Lines[0].Text, "hello, world") + + t.CheckOutputEmpty() + + t.Check(Load(nonexistent, LogErrors), check.IsNil) + t.Check(Load(empty, LogErrors).Lines, check.HasLen, 0) + t.CheckEquals(Load(oneLiner, LogErrors).Lines[0].Text, "hello, world") + + t.CheckOutputLines( + "ERROR: ~/nonexistent: Cannot be read.") + + t.Check(Load(nonexistent, NotEmpty), check.IsNil) + t.Check(Load(empty, NotEmpty), check.IsNil) + t.CheckEquals(Load(oneLiner, NotEmpty).Lines[0].Text, "hello, world") + + t.CheckOutputEmpty() + + t.Check(Load(nonexistent, NotEmpty|LogErrors), check.IsNil) + t.Check(Load(empty, NotEmpty|LogErrors), check.IsNil) + t.CheckEquals(Load(oneLiner, NotEmpty|LogErrors).Lines[0].Text, "hello, world") + + t.CheckOutputLines( + "ERROR: ~/nonexistent: Cannot be read.", + "ERROR: ~/empty: Must not be empty.") + + t.ExpectFatal( + func() { Load(t.File("does-not-exist"), MustSucceed) }, + "FATAL: ~/does-not-exist: Cannot be read.") + + t.ExpectFatal( + func() { Load(t.File("empty"), MustSucceed|NotEmpty) }, + "FATAL: ~/empty: Must not be empty.") +} + func (s *Suite) Test_convertToLogicalLines__no_continuation(c *check.C) { t := s.Init(c) @@ -102,20 +146,6 @@ func (s *Suite) Test_convertToLogicalLines__comments(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_nextLogicalLine__commented_multi(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("filename.mk", - "#COMMENTED= \\", - "#\tcontinuation 1 \\", - "#\tcontinuation 2") - mkline := mklines.mklines[0] - - // The leading comments are stripped from the continuation lines as well. - t.CheckEquals(mkline.Value(), "continuation 1 \tcontinuation 2") - t.CheckEquals(mkline.HasComment(), false) -} - func (s *Suite) Test_convertToLogicalLines__missing_newline_at_eof(c *check.C) { t := s.Init(c) @@ -161,6 +191,20 @@ func (s *Suite) Test_convertToLogicalLines__missing_newline_at_eof_with_source(c "ERROR: filename:1: File must end with a newline.") } +func (s *Suite) Test_nextLogicalLine__commented_multi(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("filename.mk", + "#COMMENTED= \\", + "#\tcontinuation 1 \\", + "#\tcontinuation 2") + mkline := mklines.mklines[0] + + // The leading comments are stripped from the continuation lines as well. + t.CheckEquals(mkline.Value(), "continuation 1 \tcontinuation 2") + t.CheckEquals(mkline.HasComment(), false) +} + func (s *Suite) Test_matchContinuationLine(c *check.C) { t := s.Init(c) @@ -178,47 +222,3 @@ func (s *Suite) Test_matchContinuationLine(c *check.C) { t.CheckEquals(trailingWhitespace, " ") t.CheckEquals(continuation, "\\") } - -func (s *Suite) Test_Load(c *check.C) { - t := s.Init(c) - - nonexistent := t.File("nonexistent") - empty := t.CreateFileLines("empty") - oneLiner := t.CreateFileLines("one-liner", - "hello, world") - - t.Check(Load(nonexistent, 0), check.IsNil) - t.Check(Load(empty, 0).Lines, check.HasLen, 0) - t.CheckEquals(Load(oneLiner, 0).Lines[0].Text, "hello, world") - - t.CheckOutputEmpty() - - t.Check(Load(nonexistent, LogErrors), check.IsNil) - t.Check(Load(empty, LogErrors).Lines, check.HasLen, 0) - t.CheckEquals(Load(oneLiner, LogErrors).Lines[0].Text, "hello, world") - - t.CheckOutputLines( - "ERROR: ~/nonexistent: Cannot be read.") - - t.Check(Load(nonexistent, NotEmpty), check.IsNil) - t.Check(Load(empty, NotEmpty), check.IsNil) - t.CheckEquals(Load(oneLiner, NotEmpty).Lines[0].Text, "hello, world") - - t.CheckOutputEmpty() - - t.Check(Load(nonexistent, NotEmpty|LogErrors), check.IsNil) - t.Check(Load(empty, NotEmpty|LogErrors), check.IsNil) - t.CheckEquals(Load(oneLiner, NotEmpty|LogErrors).Lines[0].Text, "hello, world") - - t.CheckOutputLines( - "ERROR: ~/nonexistent: Cannot be read.", - "ERROR: ~/empty: Must not be empty.") - - t.ExpectFatal( - func() { Load(t.File("does-not-exist"), MustSucceed) }, - "FATAL: ~/does-not-exist: Cannot be read.") - - t.ExpectFatal( - func() { Load(t.File("empty"), MustSucceed|NotEmpty) }, - "FATAL: ~/empty: Must not be empty.") -} diff --git a/pkgtools/pkglint/files/getopt/getopt_test.go b/pkgtools/pkglint/files/getopt/getopt_test.go index 667b95b0c70..b8d0550e5d6 100644 --- a/pkgtools/pkglint/files/getopt/getopt_test.go +++ b/pkgtools/pkglint/files/getopt/getopt_test.go @@ -410,7 +410,7 @@ func (s *Suite) Test_Options_Help__with_flag_group(c *check.C) { } func (s *Suite) Test__test_names(c *check.C) { - ck := intqa.NewTestNameChecker(c) - ck.ShowWarnings(false) + ck := intqa.NewTestNameChecker(c.Errorf) + ck.Enable(intqa.EAll, -intqa.EMissingTest) ck.Check() } diff --git a/pkgtools/pkglint/files/histogram/histogram_test.go b/pkgtools/pkglint/files/histogram/histogram_test.go index a25d5b78641..31bdacac473 100644 --- a/pkgtools/pkglint/files/histogram/histogram_test.go +++ b/pkgtools/pkglint/files/histogram/histogram_test.go @@ -3,6 +3,7 @@ package histogram_test import ( "gopkg.in/check.v1" "netbsd.org/pkglint/histogram" + "netbsd.org/pkglint/intqa" "strings" "testing" ) @@ -26,3 +27,9 @@ func (s *Suite) Test_Histogram(c *check.C) { "caption 3 three\n"+ "caption 2 two\n") } + +func (s *Suite) Test__test_names(c *check.C) { + ck := intqa.NewTestNameChecker(c.Errorf) + ck.Enable(intqa.EAll, -intqa.EMissingTest) + ck.Check() +} diff --git a/pkgtools/pkglint/files/intqa/ideas.go b/pkgtools/pkglint/files/intqa/ideas.go index bdadbad23fb..2438b88a9d5 100644 --- a/pkgtools/pkglint/files/intqa/ideas.go +++ b/pkgtools/pkglint/files/intqa/ideas.go @@ -8,10 +8,5 @@ package intqa // XXX: All methods should be defined in the same file as their receiver type. // If that is not possible, there should only be a small list of exceptions. -// XXX: All tests should be in the same order as their corresponding elements in the -// main code. - -// XXX: All tests for a single testee should be grouped together. - // XXX: If there is a constructor for a type, only that constructor may be used // for constructing objects. All other forms (var x Type; x := &Type{}) should be forbidden. diff --git a/pkgtools/pkglint/files/intqa/testnames.go b/pkgtools/pkglint/files/intqa/testnames.go index 02f604739cc..3e3a71b40b6 100644 --- a/pkgtools/pkglint/files/intqa/testnames.go +++ b/pkgtools/pkglint/files/intqa/testnames.go @@ -6,7 +6,7 @@ import ( "go/ast" "go/parser" "go/token" - "gopkg.in/check.v1" + "io" "os" "path/filepath" "sort" @@ -14,87 +14,116 @@ import ( "unicode" ) -// TestNameChecker ensures that all test names follow a common naming scheme: -// -// Test_${Type}_${Method}__${description_using_underscores} -type TestNameChecker struct { - camelCase map[string]bool - ignore []string - warn bool - prefixes []testeePrefix - c *check.C - errors []string - warnings []string -} +type Error int -type testeePrefix struct { - prefix string - filename string -} +const ( + ENone Error = iota + EAll -// testeeElement is an element of the source code that can be tested. -// It is either a type, a function or a method. -// The test methods are also testeeElements. -type testeeElement struct { - File string // The file containing the testeeElement - Type string // The type, e.g. MkLine - Func string // The function or method name, e.g. Warnf + // A function or method does not have a corresponding test. + EMissingTest + + // The name of a test function does not correspond to a program + // element to be tested. + EMissingTestee - FullName string // Type + "." + Func + // The tests are not in the same order as their corresponding + // testees in the main code. + EOrder + + // The test method does not have a valid name. + EName + + // The file of the test method does not correspond to the + // file of the testee. + EFile +) + +// TestNameChecker ensures that all test names follow a common naming scheme: +// Test_${Type}_${Method}__${description_using_underscores} +// Each of the variable parts may be omitted. +type TestNameChecker struct { + errorf func(format string, args ...interface{}) - // Whether the testeeElement is a test or a testee - Test bool + ignoredFiles []string + order int - // For a test, its name without the description, - // otherwise the prefix (Type + "_" + Func) for the corresponding tests - Prefix string + testees []*testee + tests []*test + + errorsMask uint64 + errors []string + out io.Writer } -func NewTestNameChecker(c *check.C) *TestNameChecker { - return &TestNameChecker{c: c, camelCase: make(map[string]bool)} +// NewTestNameChecker creates a new checker. +// By default, all errors are disabled; call Enable to enable them. +func NewTestNameChecker(errorf func(format string, args ...interface{})) *TestNameChecker { + return &TestNameChecker{errorf: errorf, out: os.Stderr} } func (ck *TestNameChecker) IgnoreFiles(fileGlob string) { - ck.ignore = append(ck.ignore, fileGlob) + ck.ignoredFiles = append(ck.ignoredFiles, fileGlob) } -// AllowPrefix allows tests with the given prefix to appear in the test -// file corresponding to the given source file (which doesn't even have -// to exist). -// -// In all other cases, the tests may only be named after things from the -// main code that can actually be tested. -func (ck *TestNameChecker) AllowPrefix(prefix, sourceFileName string) { - ck.prefixes = append(ck.prefixes, testeePrefix{prefix, sourceFileName}) +func (ck *TestNameChecker) Enable(errors ...Error) { + for _, err := range errors { + if err == ENone { + ck.errorsMask = 0 + } else if err == EAll { + ck.errorsMask = ^uint64(0) + } else if err < 0 { + ck.errorsMask &= ^(uint64(1) << -uint(err)) + } else { + ck.errorsMask |= uint64(1) << uint(err) + } + } } -// AllowCamelCaseDescriptions allows the given strings to appear -// in the description part of a test name (Test_$Type_$Method__$description). -// In most cases the description should use snake case to allow for -// easier reading. -// -// When writing tests for combinations of several functions, it is most -// natural to mention one of these functions in the test name and the -// other in the test description. This is a typical use case. -func (ck *TestNameChecker) AllowCamelCaseDescriptions(descriptions ...string) { - for _, description := range descriptions { - ck.camelCase[description] = true - } +func (ck *TestNameChecker) Check() { + ck.load() + ck.checkTestees() + ck.checkTests() + ck.checkOrder() + ck.print() } -func (ck *TestNameChecker) ShowWarnings(warn bool) { ck.warn = warn } +// load loads all type, function and method names from the current package. +func (ck *TestNameChecker) load() { + fileSet := token.NewFileSet() + pkgs, err := parser.ParseDir(fileSet, ".", nil, 0) + if err != nil { + panic(err) + } -func (ck *TestNameChecker) addError(format string, args ...interface{}) { - ck.errors = append(ck.errors, "E: "+fmt.Sprintf(format, args...)) -} + var pkgnames []string + for pkgname := range pkgs { + pkgnames = append(pkgnames, pkgname) + } + sort.Strings(pkgnames) -func (ck *TestNameChecker) addWarning(format string, args ...interface{}) { - ck.warnings = append(ck.warnings, "W: "+fmt.Sprintf(format, args...)) + for _, pkgname := range pkgnames { + files := pkgs[pkgname].Files + + var filenames []string + for filename := range files { + filenames = append(filenames, filename) + } + sort.Strings(filenames) + + for _, filename := range filenames { + file := files[filename] + for _, decl := range file.Decls { + ck.loadDecl(decl, filename) + } + } + } + + ck.relate() } -// addElement adds a single type or function declaration -// to the known elements. -func (ck *TestNameChecker) addElement(elements *[]*testeeElement, decl ast.Decl, filename string) { +// loadDecl adds a single type or function declaration to the known elements. +func (ck *TestNameChecker) loadDecl(decl ast.Decl, filename string) { switch decl := decl.(type) { case *ast.GenDecl: @@ -102,7 +131,7 @@ func (ck *TestNameChecker) addElement(elements *[]*testeeElement, decl ast.Decl, switch spec := spec.(type) { case *ast.TypeSpec: typeName := spec.Name.Name - *elements = append(*elements, newElement(typeName, "", filename)) + ck.addCode(code{filename, typeName, "", ck.nextOrder()}) } } @@ -116,132 +145,149 @@ func (ck *TestNameChecker) addElement(elements *[]*testeeElement, decl ast.Decl, typeName = typeExpr.(*ast.Ident).Name } } - *elements = append(*elements, newElement(typeName, decl.Name.Name, filename)) + ck.addCode(code{filename, typeName, decl.Name.Name, ck.nextOrder()}) } } -// loadAllElements returns all type, function and method names -// from the current package, in the form FunctionName or -// TypeName.MethodName (omitting the * from the type name). -func (ck *TestNameChecker) loadAllElements() []*testeeElement { - fileSet := token.NewFileSet() - pkgs, err := parser.ParseDir(fileSet, ".", func(fi os.FileInfo) bool { return true }, 0) - if err != nil { - panic(err) - } +func (ck *TestNameChecker) addCode(code code) { + isTest := strings.HasSuffix(code.file, "_test.go") && + code.Type != "" && + strings.HasPrefix(code.Func, "Test") - var elements []*testeeElement - for _, pkg := range pkgs { - for filename, file := range pkg.Files { - for _, decl := range file.Decls { - ck.addElement(&elements, decl, filename) - } - } + if isTest { + ck.addTest(code) + } else { + ck.addTestee(code) } +} - sort.Slice(elements, func(i, j int) bool { return elements[i].Less(elements[j]) }) - - return elements +func (ck *TestNameChecker) addTestee(code code) { + ck.testees = append(ck.testees, &testee{code}) } -// collectTesteeByName generates a map containing the names of all -// testable elements, as used in the test names. Examples: -// -// Autofix -// Line_Warnf -// match5 -func (ck *TestNameChecker) collectTesteeByName(elements []*testeeElement) map[string]*testeeElement { - prefixes := make(map[string]*testeeElement) - for _, element := range elements { - if element.Prefix != "" { - prefixes[element.Prefix] = element - } +func (ck *TestNameChecker) addTest(code code) { + if !strings.HasPrefix(code.Func, "Test_") { + ck.addError( + EName, + "Test %q must start with %q.", + code.fullName(), "Test_") + return } - for _, p := range ck.prefixes { - prefixes[p.prefix] = newElement(p.prefix, "", p.filename) + parts := strings.SplitN(code.Func, "__", 2) + testeeName := strings.TrimPrefix(strings.TrimPrefix(parts[0], "Test"), "_") + descr := "" + if len(parts) > 1 { + if parts[1] == "" { + ck.addError( + EName, + "Test %q must not have a nonempty description.", + code.fullName()) + return + } + descr = parts[1] } - return prefixes + ck.tests = append(ck.tests, &test{code, testeeName, descr, nil}) } -func (ck *TestNameChecker) checkTestName(test *testeeElement, prefix string, descr string, testeeByName map[string]*testeeElement) { - testee := testeeByName[prefix] - if testee == nil { - ck.addError("Test %q for missing testee %q.", test.FullName, prefix) +func (ck *TestNameChecker) nextOrder() int { + id := ck.order + ck.order++ + return id +} - } else if !strings.HasSuffix(testee.File, "_test.go") { - correctTestFile := strings.TrimSuffix(testee.File, ".go") + "_test.go" - if correctTestFile != test.File { - ck.addError("Test %q for %q must be in %s instead of %s.", - test.FullName, testee.FullName, correctTestFile, test.File) - } +// relate connects the tests to their testees. +func (ck *TestNameChecker) relate() { + testeesByPrefix := make(map[string]*testee) + for _, testee := range ck.testees { + prefix := join(testee.Type, "_", testee.Func) + testeesByPrefix[prefix] = testee } - if isCamelCase(descr) && !ck.camelCase[descr] { - ck.addError("%s: Test description %q must not use CamelCase.", test.FullName, descr) + for _, test := range ck.tests { + test.testee = testeesByPrefix[test.testeeName] } } -func (ck *TestNameChecker) checkAll(elements []*testeeElement, testeeByName map[string]*testeeElement) { - testNames := make(map[string]bool) - - for _, element := range elements { - if element.Test { - method := element.Func - switch { - case strings.HasPrefix(method, "Test__"): - // OK - - case strings.HasPrefix(method, "Test_"): - refAndDescr := strings.SplitN(method[5:], "__", 2) - descr := "" - if len(refAndDescr) > 1 { - descr = refAndDescr[1] - } - testNames[refAndDescr[0]] = true - ck.checkTestName(element, refAndDescr[0], descr, testeeByName) +func (ck *TestNameChecker) checkTests() { + for _, test := range ck.tests { + ck.checkTestFile(test) + ck.checkTestName(test) + ck.checkTestTestee(test) + } +} - default: - ck.addError("Test name %q must contain an underscore.", element.FullName) - } - } +func (ck *TestNameChecker) checkTestFile(test *test) { + testee := test.testee + if testee == nil || testee.file == test.file { + return } - for _, element := range elements { - if !strings.HasSuffix(element.File, "_test.go") && !ck.isIgnored(element.File) { - if !testNames[element.Prefix] { - ck.addWarning("Missing unit test %q for %q.", - "Test_"+element.Prefix, element.FullName) - } - } + correctTestFile := strings.TrimSuffix(testee.file, ".go") + "_test.go" + if correctTestFile != test.file { + ck.addError( + EFile, + "Test %q for %q must be in %s instead of %s.", + test.fullName(), testee.fullName(), correctTestFile, test.file) } } -func (ck *TestNameChecker) Check() { - elements := ck.loadAllElements() - testeeByName := ck.collectTesteeByName(elements) - ck.checkAll(elements, testeeByName) +func (ck *TestNameChecker) checkTestTestee(test *test) { + testee := test.testee + if testee != nil || test.testeeName == "" { + return + } - for _, err := range ck.errors { - fmt.Println(err) + testeeName := strings.Replace(test.testeeName, "_", ".", -1) + ck.addError( + EMissingTestee, + "Missing testee %q for test %q.", + testeeName, test.fullName()) +} + +// checkTestName ensures that the method name does not accidentally +// end up in the description of the test. This could happen if there is a +// double underscore instead of a single underscore. +func (ck *TestNameChecker) checkTestName(test *test) { + testee := test.testee + if testee == nil { + return } - for _, warning := range ck.warnings { - if ck.warn { - fmt.Println(warning) - } + if testee.Type != "" && testee.Func != "" { + return } - if len(ck.errors) > 0 || (ck.warn && len(ck.warnings) > 0) { - ck.c.Errorf("%d %s and %d %s.", - len(ck.errors), - ifelseStr(len(ck.errors) == 1, "error", "errors"), - len(ck.warnings), - ifelseStr(len(ck.warnings) == 1, "warning", "warnings")) + if !isCamelCase(test.descr) { + return + } + + ck.addError( + EName, + "%s: Test description %q must not use CamelCase in the first word.", + test.fullName(), test.descr) +} + +func (ck *TestNameChecker) checkTestees() { + tested := make(map[*testee]bool) + for _, test := range ck.tests { + tested[test.testee] = true + } + + for _, testee := range ck.testees { + if tested[testee] || testee.Func == "" { + continue + } + + testName := "Test_" + join(testee.Type, "_", testee.Func) + ck.addError( + EMissingTest, + "Missing unit test %q for %q.", + testName, testee.fullName()) } } func (ck *TestNameChecker) isIgnored(filename string) bool { - for _, mask := range ck.ignore { + for _, mask := range ck.ignoredFiles { ok, err := filepath.Match(mask, filename) if err != nil { panic(err) @@ -253,47 +299,104 @@ func (ck *TestNameChecker) isIgnored(filename string) bool { return false } -func newElement(typeName, funcName, filename string) *testeeElement { - typeName = strings.TrimSuffix(typeName, "Impl") +// checkOrder ensures that the tests appear in the same order as their +// counterparts in the main code. +func (ck *TestNameChecker) checkOrder() { + maxOrderByFile := make(map[string]*test) - e := testeeElement{File: filename, Type: typeName, Func: funcName} - - e.FullName = e.Type + ifelseStr(e.Type != "" && e.Func != "", ".", "") + e.Func + for _, test := range ck.tests { + testee := test.testee + if testee == nil { + continue + } - e.Test = strings.HasSuffix(e.File, "_test.go") && e.Type != "" && strings.HasPrefix(e.Func, "Test") + maxOrder := maxOrderByFile[testee.file] + if maxOrder == nil || testee.order > maxOrder.testee.order { + maxOrderByFile[testee.file] = test + } - if e.Test { - e.Prefix = strings.Split(strings.TrimPrefix(e.Func, "Test"), "__")[0] - } else { - e.Prefix = e.Type + ifelseStr(e.Type != "" && e.Func != "", "_", "") + e.Func + if maxOrder != nil && testee.order < maxOrder.testee.order { + insertBefore := maxOrder + for _, before := range ck.tests { + if before.file == test.file && before.testee != nil && before.testee.order > testee.order { + insertBefore = before + break + } + } + ck.addError( + EOrder, + "Test %q should be ordered before %q.", + test.fullName(), insertBefore.fullName()) + } } +} - return &e +func (ck *TestNameChecker) addError(e Error, format string, args ...interface{}) { + if ck.errorsMask&(uint64(1)<<uint(e)) != 0 { + ck.errors = append(ck.errors, fmt.Sprintf(format, args...)) + } } -func (el *testeeElement) Less(other *testeeElement) bool { - switch { - case el.Type != other.Type: - return el.Type < other.Type - case el.Func != other.Func: - return el.Func < other.Func - default: - return el.File < other.File +func (ck *TestNameChecker) print() { + for _, msg := range ck.errors { + _, _ = fmt.Fprintln(ck.out, msg) + } + + errors := plural(len(ck.errors), "error", "errors") + if len(ck.errors) > 0 { + ck.errorf("%s.", errors) } } -func ifelseStr(cond bool, a, b string) string { - if cond { - return a +type code struct { + file string // The file containing the code + Type string // The type, e.g. MkLine + Func string // The function or method name, e.g. Warnf + order int // The relative order in the file +} + +func (c *code) fullName() string { return join(c.Type, ".", c.Func) } + +// testee is an element of the source code that can be tested. +// It is either a type, a function or a method. +type testee struct { + code +} + +type test struct { + code + + testeeName string // The method name without the "Test_" prefix and description + descr string // The part after the "__" in the method name + testee *testee +} + +func plural(n int, sg, pl string) string { + if n == 0 { + return "" } - return b + form := pl + if n == 1 { + form = sg + } + return fmt.Sprintf("%d %s", n, form) } func isCamelCase(str string) bool { for i := 0; i+1 < len(str); i++ { + if str[i] == '_' { + return false + } if unicode.IsLower(rune(str[i])) && unicode.IsUpper(rune(str[i+1])) { return true } } return false } + +func join(a, sep, b string) string { + if a == "" || b == "" { + sep = "" + } + return a + sep + b +} diff --git a/pkgtools/pkglint/files/intqa/testnames_test.go b/pkgtools/pkglint/files/intqa/testnames_test.go new file mode 100644 index 00000000000..1ea03193042 --- /dev/null +++ b/pkgtools/pkglint/files/intqa/testnames_test.go @@ -0,0 +1,261 @@ +package intqa + +import ( + "bytes" + "fmt" + "gopkg.in/check.v1" + "io/ioutil" + "testing" +) + +type Suite struct { + c *check.C + ck *TestNameChecker + summary string +} + +func Test(t *testing.T) { + check.Suite(&Suite{}) + check.TestingT(t) +} + +func (s *Suite) Init(c *check.C) *TestNameChecker { + errorf := func(format string, args ...interface{}) { + s.summary = fmt.Sprintf(format, args...) + } + + s.c = c + s.ck = NewTestNameChecker(errorf) + s.ck.Enable(EAll) + s.ck.out = ioutil.Discard + return s.ck +} + +func (s *Suite) TearDownTest(c *check.C) { + s.c = c + s.CheckErrors(nil...) + s.CheckSummary("") +} + +func (s *Suite) CheckErrors(errors ...string) { + s.c.Check(s.ck.errors, check.DeepEquals, errors) + s.ck.errors = nil +} + +func (s *Suite) CheckSummary(summary string) { + s.c.Check(s.summary, check.Equals, summary) + s.summary = "" +} + +func (s *Suite) Test_TestNameChecker_Check(c *check.C) { + ck := s.Init(c) + + ck.Check() + + s.CheckErrors( + "Missing unit test \"Test_NewTestNameChecker\" for \"NewTestNameChecker\".", + "Missing unit test \"Test_TestNameChecker_IgnoreFiles\" for \"TestNameChecker.IgnoreFiles\".", + "Missing unit test \"Test_TestNameChecker_Enable\" for \"TestNameChecker.Enable\".", + "Missing unit test \"Test_TestNameChecker_load\" for \"TestNameChecker.load\".", + "Missing unit test \"Test_TestNameChecker_loadDecl\" for \"TestNameChecker.loadDecl\".", + "Missing unit test \"Test_TestNameChecker_addCode\" for \"TestNameChecker.addCode\".", + "Missing unit test \"Test_TestNameChecker_addTestee\" for \"TestNameChecker.addTestee\".", + "Missing unit test \"Test_TestNameChecker_nextOrder\" for \"TestNameChecker.nextOrder\".", + "Missing unit test \"Test_TestNameChecker_relate\" for \"TestNameChecker.relate\".", + "Missing unit test \"Test_TestNameChecker_checkTests\" for \"TestNameChecker.checkTests\".", + "Missing unit test \"Test_TestNameChecker_checkTestees\" for \"TestNameChecker.checkTestees\".", + "Missing unit test \"Test_TestNameChecker_isIgnored\" for \"TestNameChecker.isIgnored\".", + "Missing unit test \"Test_TestNameChecker_addError\" for \"TestNameChecker.addError\".", + "Missing unit test \"Test_Test\" for \"Test\".", + "Missing unit test \"Test_Suite_Init\" for \"Suite.Init\".", + "Missing unit test \"Test_Suite_TearDownTest\" for \"Suite.TearDownTest\".", + "Missing unit test \"Test_Suite_CheckErrors\" for \"Suite.CheckErrors\".", + "Missing unit test \"Test_Suite_CheckSummary\" for \"Suite.CheckSummary\".", + "Missing unit test \"Test_Value_Method\" for \"Value.Method\".") + s.CheckSummary("19 errors.") +} + +func (s *Suite) Test_TestNameChecker_addTest(c *check.C) { + ck := s.Init(c) + + ck.addTest(code{"filename.go", "Type", "Method", 0}) + + s.CheckErrors( + "Test \"Type.Method\" must start with \"Test_\".") +} + +func (s *Suite) Test_TestNameChecker_addTest__empty_description(c *check.C) { + ck := s.Init(c) + + ck.addTest(code{"filename.go", "Suite", "Test_Method__", 0}) + + s.CheckErrors( + "Test \"Suite.Test_Method__\" must not have a nonempty description.") +} + +func (s *Suite) Test_TestNameChecker_checkTestFile__global(c *check.C) { + ck := s.Init(c) + + ck.checkTestFile(&test{ + code{"demo_test.go", "Suite", "Test__Global", 0}, + "", + "", + &testee{code{"other.go", "", "Global", 0}}}) + + s.CheckErrors( + "Test \"Suite.Test__Global\" for \"Global\" " + + "must be in other_test.go instead of demo_test.go.") +} + +func (s *Suite) Test_TestNameChecker_checkTestTestee__global(c *check.C) { + ck := s.Init(c) + + ck.checkTestTestee(&test{ + code{"demo_test.go", "Suite", "Test__Global", 0}, + "", + "", + nil}) + + s.CheckErrors( + nil...) +} + +func (s *Suite) Test_TestNameChecker_checkTestTestee__no_testee(c *check.C) { + ck := s.Init(c) + + ck.checkTestTestee(&test{ + code{"demo_test.go", "Suite", "Test_Missing", 0}, + "Missing", + "", + nil}) + + s.CheckErrors( + "Missing testee \"Missing\" for test \"Suite.Test_Missing\".") +} + +func (s *Suite) Test_TestNameChecker_checkTestTestee__testee_exists(c *check.C) { + ck := s.Init(c) + + ck.checkTestTestee(&test{ + code{"demo_test.go", "Suite", "Test_Missing", 0}, + "Missing", + "", + &testee{}}) + + s.CheckErrors( + nil...) +} + +func (s *Suite) Test_TestNameChecker_checkTestName__camel_case(c *check.C) { + ck := s.Init(c) + + ck.checkTestName(&test{ + code{"demo_test.go", "Suite", "Test_Missing__CamelCase", 0}, + "Missing", + "CamelCase", + &testee{}}) + + s.CheckErrors( + "Suite.Test_Missing__CamelCase: Test description \"CamelCase\" " + + "must not use CamelCase in the first word.") +} + +func (s *Suite) Test_TestNameChecker_checkOrder(c *check.C) { + ck := s.Init(c) + + ck.addTestee(code{"f.go", "T", "", 10}) + ck.addTestee(code{"f.go", "T", "M1", 11}) + ck.addTestee(code{"f.go", "T", "M2", 12}) + ck.addTestee(code{"f.go", "T", "M3", 13}) + ck.addTest(code{"f_test.go", "S", "Test_T_M1", 100}) // maxTestee = 11 + ck.addTest(code{"f_test.go", "S", "Test_T_M2", 101}) // maxTestee = 12 + ck.addTest(code{"f_test.go", "S", "Test_T", 102}) // testee 10 < maxTestee 12: insert before first [.testee > testee 10] == T_M1 + ck.addTest(code{"f_test.go", "S", "Test_T_M3", 103}) // maxTestee = 13 + ck.addTest(code{"f_test.go", "S", "Test_T__1", 104}) // testee < maxTestee: insert before first [testee > 10] + ck.addTest(code{"f_test.go", "S", "Test_T__2", 105}) // testee < maxTestee: insert before first [testee > 10] + ck.addTest(code{"f_test.go", "S", "Test_T_M2__1", 106}) // testee < maxTestee: insert before first [testee > 12] == T_M3 + ck.relate() + + ck.checkOrder() + + s.CheckErrors( + "Test \"S.Test_T\" should be ordered before \"S.Test_T_M1\".", + "Test \"S.Test_T__1\" should be ordered before \"S.Test_T_M1\".", + "Test \"S.Test_T__2\" should be ordered before \"S.Test_T_M1\".", + "Test \"S.Test_T_M2__1\" should be ordered before \"S.Test_T_M3\".") +} + +func (s *Suite) Test_TestNameChecker_print__empty(c *check.C) { + var out bytes.Buffer + ck := s.Init(c) + ck.out = &out + + ck.print() + + c.Check(out.String(), check.Equals, "") +} + +func (s *Suite) Test_TestNameChecker_print__errors(c *check.C) { + var out bytes.Buffer + ck := s.Init(c) + ck.out = &out + + ck.addError(EName, "1") + ck.print() + + c.Check(out.String(), check.Equals, "1\n") + s.CheckErrors("1") + s.CheckSummary("1 error.") +} + +func (s *Suite) Test_code_fullName(c *check.C) { + _ = s.Init(c) + + test := func(typeName, funcName, fullName string) { + code := code{"filename", typeName, funcName, 0} + c.Check(code.fullName(), check.Equals, fullName) + } + + test("Type", "", "Type") + test("", "Func", "Func") + test("Type", "Method", "Type.Method") +} + +func (s *Suite) Test_plural(c *check.C) { + _ = s.Init(c) + + c.Check(plural(0, "singular", "plural"), check.Equals, "") + c.Check(plural(1, "singular", "plural"), check.Equals, "1 singular") + c.Check(plural(2, "singular", "plural"), check.Equals, "2 plural") + c.Check(plural(1000, "singular", "plural"), check.Equals, "1000 plural") +} + +func (s *Suite) Test_isCamelCase(c *check.C) { + _ = s.Init(c) + + c.Check(isCamelCase(""), check.Equals, false) + c.Check(isCamelCase("Word"), check.Equals, false) + c.Check(isCamelCase("Ada_Case"), check.Equals, false) + c.Check(isCamelCase("snake_case"), check.Equals, false) + c.Check(isCamelCase("CamelCase"), check.Equals, true) + + // After the first underscore of the description, any CamelCase + // is ignored because there is no danger of confusing the method + // name with the description. + c.Check(isCamelCase("Word_CamelCase"), check.Equals, false) +} + +func (s *Suite) Test_join(c *check.C) { + _ = s.Init(c) + + c.Check(join("", " and ", ""), check.Equals, "") + c.Check(join("one", " and ", ""), check.Equals, "one") + c.Check(join("", " and ", "two"), check.Equals, "two") + c.Check(join("one", " and ", "two"), check.Equals, "one and two") +} + +type Value struct{} + +// Method has no star on the receiver, +// for code coverage of TestNameChecker.loadDecl. +func (Value) Method() {} diff --git a/pkgtools/pkglint/files/licenses/licenses_test.go b/pkgtools/pkglint/files/licenses/licenses_test.go index 168363c000b..9d23592b368 100644 --- a/pkgtools/pkglint/files/licenses/licenses_test.go +++ b/pkgtools/pkglint/files/licenses/licenses_test.go @@ -132,8 +132,7 @@ func Test(t *testing.T) { } func (s *Suite) Test__test_names(c *check.C) { - ck := intqa.NewTestNameChecker(c) - ck.IgnoreFiles("*yacc.go") - ck.ShowWarnings(false) + ck := intqa.NewTestNameChecker(c.Errorf) + ck.Enable(intqa.EAll, -intqa.EMissingTest) ck.Check() } diff --git a/pkgtools/pkglint/files/linelexer_test.go b/pkgtools/pkglint/files/linelexer_test.go index b93c53ca374..7696125beec 100644 --- a/pkgtools/pkglint/files/linelexer_test.go +++ b/pkgtools/pkglint/files/linelexer_test.go @@ -4,6 +4,21 @@ import ( "gopkg.in/check.v1" ) +func (s *Suite) Test_LinesLexer_SkipPrefix(c *check.C) { + t := s.Init(c) + + lines := t.NewLines("file.txt", + "line 1", + "line 2") + llex := NewLinesLexer(lines) + + t.CheckEquals(llex.SkipPrefix("line 1"), true) + t.CheckEquals(llex.SkipPrefix("line 1"), false) + t.CheckEquals(llex.SkipPrefix("line 2"), true) + t.CheckEquals(llex.SkipPrefix("line 2"), false) + t.CheckEquals(llex.SkipPrefix(""), false) +} + func (s *Suite) Test_LinesLexer_SkipEmptyOrNote__beginning_of_file(c *check.C) { t := s.Init(c) @@ -34,18 +49,3 @@ func (s *Suite) Test_LinesLexer_SkipEmptyOrNote__end_of_file(c *check.C) { t.CheckOutputLines( "NOTE: file.txt:2: Empty line expected after this line.") } - -func (s *Suite) Test_LinesLexer_SkipPrefix(c *check.C) { - t := s.Init(c) - - lines := t.NewLines("file.txt", - "line 1", - "line 2") - llex := NewLinesLexer(lines) - - t.CheckEquals(llex.SkipPrefix("line 1"), true) - t.CheckEquals(llex.SkipPrefix("line 1"), false) - t.CheckEquals(llex.SkipPrefix("line 2"), true) - t.CheckEquals(llex.SkipPrefix("line 2"), false) - t.CheckEquals(llex.SkipPrefix(""), false) -} diff --git a/pkgtools/pkglint/files/logging.go b/pkgtools/pkglint/files/logging.go index 4ee831a7165..0d25f3ef73f 100644 --- a/pkgtools/pkglint/files/logging.go +++ b/pkgtools/pkglint/files/logging.go @@ -55,33 +55,6 @@ var ( var dummyLine = NewLineMulti("", 0, 0, "", nil) -// IsAutofix returns whether one of the --show-autofix or --autofix options is active. -func (l *Logger) IsAutofix() bool { return l.Opts.Autofix || l.Opts.ShowAutofix } - -// Relevant decides and remembers whether the given diagnostic is relevant and should be logged. -// -// The result of the decision affects all log items until Relevant is called for the next time. -func (l *Logger) Relevant(format string) bool { - relevant := l.shallBeLogged(format) - l.suppressDiag = !relevant - l.suppressExpl = !relevant - return relevant -} - -func (l *Logger) FirstTime(filename, linenos, msg string) bool { - if l.Opts.LogVerbose { - return true - } - - if !l.logged.FirstTimeSlice(path.Clean(filename), linenos, msg) { - l.suppressDiag = true - l.suppressExpl = true - return false - } - - return true -} - // Explain outputs an explanation for the preceding diagnostic // if the --explain option is given. Otherwise it just records // that an explanation is available. @@ -119,71 +92,6 @@ func (l *Logger) Explain(explanation ...string) { l.out.WriteLine("") } -func (l *Logger) ShowSummary(args []string) { - if l.Opts.Quiet || l.Opts.Autofix { - return - } - - if l.Opts.ShowSource { - l.out.Separate() - } - - if l.errors != 0 || l.warnings != 0 { - num := func(n int, singular, plural string) string { - if n == 0 { - return "" - } else if n == 1 { - return sprintf("%d %s", n, singular) - } else { - return sprintf("%d %s", n, plural) - } - } - - l.out.Write(sprintf("%s found.\n", - joinSkipEmptyCambridge("and", - num(l.errors, "error", "errors"), - num(l.warnings, "warning", "warnings"), - num(l.notes, "note", "notes")))) - } else { - l.out.WriteLine("Looks fine.") - } - - commandLine := func(arg string) string { - argv := append([]string{args[0], arg}, args[1:]...) - for i := range argv { - argv[i] = shquote(argv[i]) - } - return strings.Join(argv, " ") - } - - if l.explanationsAvailable && !l.Opts.Explain { - l.out.WriteLine(sprintf("(Run \"%s\" to show explanations.)", commandLine("-e"))) - } - if l.autofixAvailable { - if !l.Opts.ShowAutofix { - l.out.WriteLine(sprintf("(Run \"%s\" to show what can be fixed automatically.)", commandLine("-fs"))) - } - l.out.WriteLine(sprintf("(Run \"%s\" to automatically fix some issues.)", commandLine("-F"))) - } -} - -// shallBeLogged tests whether a diagnostic with the given format should -// be logged. -// -// It only inspects the --only arguments; duplicates are handled in Logger.Logf. -func (l *Logger) shallBeLogged(format string) bool { - if len(G.Opts.LogOnly) == 0 { - return true - } - - for _, substr := range G.Opts.LogOnly { - if contains(format, substr) { - return true - } - } - return false -} - // Diag logs a diagnostic. These are filtered by the --only command line option, // and duplicates are suppressed unless the --log-verbose command line option is given. // @@ -218,6 +126,47 @@ func (l *Logger) Diag(line *Line, level *LogLevel, format string, args ...interf l.Logf(level, filename, linenos, format, msg) } +func (l *Logger) FirstTime(filename, linenos, msg string) bool { + if l.Opts.LogVerbose { + return true + } + + if !l.logged.FirstTimeSlice(path.Clean(filename), linenos, msg) { + l.suppressDiag = true + l.suppressExpl = true + return false + } + + return true +} + +// Relevant decides and remembers whether the given diagnostic is relevant and should be logged. +// +// The result of the decision affects all log items until Relevant is called for the next time. +func (l *Logger) Relevant(format string) bool { + relevant := l.shallBeLogged(format) + l.suppressDiag = !relevant + l.suppressExpl = !relevant + return relevant +} + +// shallBeLogged tests whether a diagnostic with the given format should +// be logged. +// +// It only inspects the --only arguments; duplicates are handled in Logger.Logf. +func (l *Logger) shallBeLogged(format string) bool { + if len(G.Opts.LogOnly) == 0 { + return true + } + + for _, substr := range G.Opts.LogOnly { + if contains(format, substr) { + return true + } + } + return false +} + func (l *Logger) showSource(line *Line) { if !G.Logger.Opts.ShowSource { return @@ -278,6 +227,9 @@ func (l *Logger) showSource(line *Line) { } } +// IsAutofix returns whether one of the --show-autofix or --autofix options is active. +func (l *Logger) IsAutofix() bool { return l.Opts.Autofix || l.Opts.ShowAutofix } + func (l *Logger) Logf(level *LogLevel, filename, lineno, format, msg string) { if l.suppressDiag { l.suppressDiag = false @@ -349,6 +301,54 @@ func (l *Logger) Errorf(location string, format string, args ...interface{}) { l.err.Write(escapePrintable(diag)) } +func (l *Logger) ShowSummary(args []string) { + if l.Opts.Quiet || l.Opts.Autofix { + return + } + + if l.Opts.ShowSource { + l.out.Separate() + } + + if l.errors != 0 || l.warnings != 0 { + num := func(n int, singular, plural string) string { + if n == 0 { + return "" + } else if n == 1 { + return sprintf("%d %s", n, singular) + } else { + return sprintf("%d %s", n, plural) + } + } + + l.out.Write(sprintf("%s found.\n", + joinSkipEmptyCambridge("and", + num(l.errors, "error", "errors"), + num(l.warnings, "warning", "warnings"), + num(l.notes, "note", "notes")))) + } else { + l.out.WriteLine("Looks fine.") + } + + commandLine := func(arg string) string { + argv := append([]string{args[0], arg}, args[1:]...) + for i := range argv { + argv[i] = shquote(argv[i]) + } + return strings.Join(argv, " ") + } + + if l.explanationsAvailable && !l.Opts.Explain { + l.out.WriteLine(sprintf("(Run \"%s\" to show explanations.)", commandLine("-e"))) + } + if l.autofixAvailable { + if !l.Opts.ShowAutofix { + l.out.WriteLine(sprintf("(Run \"%s\" to show what can be fixed automatically.)", commandLine("-fs"))) + } + l.out.WriteLine(sprintf("(Run \"%s\" to automatically fix some issues.)", commandLine("-F"))) + } +} + // SeparatorWriter writes output, occasionally separated by an // empty line. This is used for separating the diagnostics when // --source is combined with --show-autofix, where each @@ -360,7 +360,7 @@ type SeparatorWriter struct { } func NewSeparatorWriter(out io.Writer) *SeparatorWriter { - return &SeparatorWriter{out: out, state: 3} + return &SeparatorWriter{out, 3, bytes.Buffer{}} } func (wr *SeparatorWriter) WriteLine(text string) { @@ -384,11 +384,6 @@ func (wr *SeparatorWriter) Separate() { } } -func (wr *SeparatorWriter) Flush() { - _, _ = io.Copy(wr.out, &wr.line) - wr.line.Reset() -} - func (wr *SeparatorWriter) write(b byte) { if b == '\n' { if wr.state == 1 { @@ -408,3 +403,8 @@ func (wr *SeparatorWriter) write(b byte) { wr.state = 1 wr.line.WriteByte(b) } + +func (wr *SeparatorWriter) Flush() { + _, _ = io.Copy(wr.out, &wr.line) + wr.line.Reset() +} diff --git a/pkgtools/pkglint/files/logging_test.go b/pkgtools/pkglint/files/logging_test.go index b4a4ed9ae21..5ee4c6372a0 100644 --- a/pkgtools/pkglint/files/logging_test.go +++ b/pkgtools/pkglint/files/logging_test.go @@ -6,118 +6,162 @@ import ( "strings" ) -// Calling Logf without further preparation just logs the message. -// Suppressing duplicate messages or filtering messages happens -// in other methods of the Logger, namely Relevant, FirstTime, Diag. -func (s *Suite) Test_Logger_Logf(c *check.C) { +func (s *Suite) Test_Logger_Explain__only(c *check.C) { t := s.Init(c) - var sw strings.Builder - logger := Logger{out: NewSeparatorWriter(&sw)} + t.SetUpCommandLine("--only", "interesting", "--explain") + line := t.NewLine("Makefile", 27, "The old song") - logger.Logf(Error, "filename", "3", "Blue should be %s.", "Blue should be orange.") + // Neither the warning nor the corresponding explanation are logged. + line.Warnf("Filtered warning.") + line.Explain("Explanation for the above warning.") - t.CheckEquals(sw.String(), ""+ - "ERROR: filename:3: Blue should be orange.\n") + line.Notef("What an interesting line.") + line.Explain("This explanation is logged.") + + t.CheckOutputLines( + "NOTE: Makefile:27: What an interesting line.", + "", + "\tThis explanation is logged.", + "") } -// Logf doesn't filter duplicates, but Diag does. -func (s *Suite) Test_Logger_Logf__duplicates(c *check.C) { +func (s *Suite) Test_Logger_Explain__show_autofix(c *check.C) { t := s.Init(c) - var sw strings.Builder - logger := Logger{out: NewSeparatorWriter(&sw)} + t.SetUpCommandLine("--explain", "--show-autofix") + line := t.NewLine("Makefile", 27, "The old song") - logger.Logf(Error, "filename", "3", "Blue should be %s.", "Blue should be orange.") - logger.Logf(Error, "filename", "3", "Blue should be %s.", "Blue should be orange.") + line.Warnf("Warning without fix.") + line.Explain( + "Explanation for warning without fix.") - t.CheckEquals(sw.String(), ""+ - "ERROR: filename:3: Blue should be orange.\n"+ - "ERROR: filename:3: Blue should be orange.\n") + fix := line.Autofix() + fix.Warnf("Warning with fix.") + fix.Explain( + "Explanation for warning with fix.") + fix.Replace("old", "new") + fix.Apply() + + // Since the warning without fix doesn't fix anything, it is filtered out. + // So is the corresponding explanation. + t.CheckOutputLines( + "WARN: Makefile:27: Warning with fix.", + "AUTOFIX: Makefile:27: Replacing \"old\" with \"new\".", + "", + "\tExplanation for warning with fix.", + "") } -// Ensure that suppressing a diagnostic doesn't influence later calls to Logf. -func (s *Suite) Test_Logger_Logf__mixed_with_Diag(c *check.C) { +func (s *Suite) Test_Logger_Explain__show_autofix_and_source(c *check.C) { t := s.Init(c) - var sw strings.Builder - logger := Logger{out: NewSeparatorWriter(&sw)} - line := t.NewLine("filename", 3, "Text") + t.SetUpCommandLine("--explain", "--show-autofix", "--source") + line := t.NewLine("Makefile", 27, "The old song") - logger.Logf(Error, "filename", "3", "Logf output 1.", "Logf output 1.") - logger.Diag(line, Error, "Diag %s.", "1") - logger.Logf(Error, "filename", "3", "Logf output 2.", "Logf output 2.") - logger.Diag(line, Error, "Diag %s.", "1") // Duplicate, therefore suppressed - logger.Logf(Error, "filename", "3", "Logf output 3.", "Logf output 3.") + line.Warnf("Warning without fix.") + line.Explain( + "Explanation for warning without fix.") - t.CheckEquals(sw.String(), ""+ - "ERROR: filename:3: Logf output 1.\n"+ - "ERROR: filename:3: Diag 1.\n"+ - "ERROR: filename:3: Logf output 2.\n"+ - "ERROR: filename:3: Logf output 3.\n") + fix := line.Autofix() + fix.Warnf("Warning with fix.") + fix.Explain( + "Explanation for warning with fix.") + fix.Replace("old", "new") + fix.Apply() + + // Since the warning without fix doesn't fix anything, it is filtered out. + // So is the corresponding explanation. + t.CheckOutputLines( + "WARN: Makefile:27: Warning with fix.", + "AUTOFIX: Makefile:27: Replacing \"old\" with \"new\".", + "-\tThe old song", + "+\tThe new song", + "", + "\tExplanation for warning with fix.", + "") } -func (s *Suite) Test_Logger_Logf__production(c *check.C) { +// When the --autofix option is given, the warnings are not shown, therefore it doesn't +// make sense to show the explanation for the warning. +func (s *Suite) Test_Logger_Explain__autofix_and_source(c *check.C) { t := s.Init(c) - var sw strings.Builder - logger := Logger{out: NewSeparatorWriter(&sw)} + t.SetUpCommandLine("--explain", "--autofix", "--source") + line := t.NewLine("Makefile", 27, "The old song") - // In production mode, the checks for the diagnostic messages are - // turned off, for performance reasons. The unit tests provide - // enough coverage. - G.Testing = false - logger.Logf(Error, "filename", "3", "diagnostic", "message") + line.Warnf("Warning without fix.") + line.Explain( + "Explanation for warning without fix.") - t.CheckEquals(sw.String(), ""+ - "ERROR: filename:3: message\n") + fix := line.Autofix() + fix.Warnf("Warning with fix.") + fix.Explain( + "Explanation for warning with fix.") + fix.Replace("old", "new") + fix.Apply() + + // Since the warning without fix doesn't fix anything, it is filtered out. + // So is the corresponding explanation. + t.CheckOutputLines( + "AUTOFIX: Makefile:27: Replacing \"old\" with \"new\".", + "-\tThe old song", + "+\tThe new song") } -func (s *Suite) Test_Logger_Logf__profiling(c *check.C) { +// When an explanation consists of multiple paragraphs, it contains some empty lines. +// When printing these lines, there is no need to write the tab that is used for indenting +// the normal lines. +// +// Since pkglint likes to complain about trailing whitespace, it should not generate it itself. +func (s *Suite) Test_Logger_Explain__empty_lines(c *check.C) { t := s.Init(c) - line := t.NewLine("filename", 123, "text") - - G.Opts.Profiling = true - G.Logger.histo = histogram.New() - line.Warnf("Warning.") + t.SetUpCommandLine("--explain") + line := t.NewLine("Makefile", 27, "The old song") - G.Logger.histo.PrintStats(G.Logger.out.out, "loghisto", -1) + line.Warnf("A normal warning.") + line.Explain( + "Paragraph 1 of the explanation.", + "", + "Paragraph 2 of the explanation.") t.CheckOutputLines( - "WARN: filename:123: Warning.", - "loghisto 1 Warning.") + "WARN: Makefile:27: A normal warning.", + "", + "\tParagraph 1 of the explanation.", + "", + "\tParagraph 2 of the explanation.", + "") } -func (s *Suite) Test_Logger_Logf__profiling_autofix(c *check.C) { +// In an explanation, it can happen that the pkgsrc directory is mentioned. +// While pkgsrc does not support either PKGSRCDIR or PREFIX or really any +// other directory name to contain spaces, during pkglint development this +// may happen because the pkgsrc root is in the temporary directory. +// +// In this situation, the ~ placeholder must still be properly substituted. +func (s *Suite) Test_Logger_Explain__line_wrapped_temporary_directory(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--show-autofix", "--source", "--explain") - line := t.NewLine("filename", 123, "text") - - G.Opts.Profiling = true - G.Logger.histo = histogram.New() - - fix := line.Autofix() - fix.Notef("Autofix note.") - fix.Explain( - "Autofix explanation.") - fix.Replace("text", "replacement") - fix.Apply() + t.SetUpCommandLine("--explain") + filename := t.File("filename.mk") + mkline := t.NewMkLine(filename, 123, "") - // The AUTOFIX line is not counted in the histogram although - // it uses the same code path as the other messages. - G.Logger.histo.PrintStats(G.Logger.out.out, "loghisto", -1) + mkline.Notef("Just a note to get the below explanation.") + G.Logger.Explain( + sprintf("%[1]s %[1]s %[1]s %[1]s %[1]s %[1]q", filename)) - t.CheckOutputLines( - "NOTE: filename:123: Autofix note.", - "AUTOFIX: filename:123: Replacing \"text\" with \"replacement\".", - "-\ttext", - "+\treplacement", - "", - "\tAutofix explanation.", + t.CheckOutputLinesIgnoreSpace( + "NOTE: ~/filename.mk:123: Just a note to get the below explanation.", "", - "loghisto 1 Autofix note.") + "\t~/filename.mk", + "\t~/filename.mk", + "\t~/filename.mk", + "\t~/filename.mk", + "\t~/filename.mk", + "\t\"~/filename.mk\"", + "") } // Diag filters duplicate messages, unlike Logf. @@ -166,6 +210,96 @@ func (s *Suite) Test_Logger_Diag__explanation(c *check.C) { "\n") } +func (s *Suite) Test_Logger_Diag__show_source(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--show-autofix", "--source") + line := t.NewLine("filename", 123, "text") + + fix := line.Autofix() + fix.Notef("Diagnostics can show the differences in autofix mode.") + fix.InsertBefore("new line before") + fix.InsertAfter("new line after") + fix.Apply() + + t.CheckOutputLines( + "NOTE: filename:123: Diagnostics can show the differences in autofix mode.", + "AUTOFIX: filename:123: Inserting a line \"new line before\" before this line.", + "AUTOFIX: filename:123: Inserting a line \"new line after\" after this line.", + "+\tnew line before", + ">\ttext", + "+\tnew line after") +} + +func (s *Suite) Test_Logger_Diag__show_source_with_whole_file(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--source") + line := NewLineWhole("filename") + + line.Warnf("This line does not have any RawLine attached.") + + t.CheckOutputLines( + "WARN: filename: This line does not have any RawLine attached.") +} + +// Ensures that when two packages produce a warning in the same file, both the +// warning and the corresponding source code are logged only once. +func (s *Suite) Test_Logger_Diag__source_duplicates(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("category/dependency/patches/patch-aa", + CvsID, + "", + "--- old file", + "+++ new file", + "@@ -1,1 +1,1 @@", + "-old line", + "+new line") + t.SetUpPackage("category/package1", + "PATCHDIR=\t../../category/dependency/patches") + t.SetUpPackage("category/package2", + "PATCHDIR=\t../../category/dependency/patches") + + t.Main("--source", "-Wall", "category/package1", "category/package2") + + t.CheckOutputLines( + "ERROR: ~/category/package1/distinfo: "+ + "Patch \"../../category/dependency/patches/patch-aa\" is not recorded. "+ + "Run \""+confMake+" makepatchsum\".", + "", + ">\t--- old file", + "ERROR: ~/category/dependency/patches/patch-aa:3: "+ + "Each patch must be documented.", + "", + "ERROR: ~/category/package2/distinfo: "+ + "Patch \"../../category/dependency/patches/patch-aa\" is not recorded. "+ + "Run \""+confMake+" makepatchsum\".", + "", + "3 errors found.", + t.Shquote("(Run \"pkglint -e --source -Wall %s %s\" to show explanations.)", + "category/package1", "category/package2")) +} + +func (s *Suite) Test_Logger_shallBeLogged(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine( /* none */ ) + + t.CheckEquals(G.Logger.shallBeLogged("Options should not contain whitespace."), true) + + t.SetUpCommandLine("--only", "whitespace") + + t.CheckEquals(G.Logger.shallBeLogged("Options should not contain whitespace."), true) + t.CheckEquals(G.Logger.shallBeLogged("Options should not contain space."), false) + + t.SetUpCommandLine( /* none again */ ) + + t.CheckEquals(G.Logger.shallBeLogged("Options should not contain whitespace."), true) + t.CheckEquals(G.Logger.shallBeLogged("Options should not contain space."), true) +} + // Since the --source option generates multi-line diagnostics, // they are separated by an empty line. // @@ -179,7 +313,7 @@ func (s *Suite) Test_Logger_Diag__explanation(c *check.C) { // to first show the code and then show the diagnostic. This allows // the diagnostics to underline the relevant part of the source code // and reminds of the squiggly line used for spellchecking. -func (s *Suite) Test_Logger__show_source_separator(c *check.C) { +func (s *Suite) Test_Logger_showSource__separator(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--source") @@ -209,7 +343,7 @@ func (s *Suite) Test_Logger__show_source_separator(c *check.C) { "WARN: ~/DESCR:3: Using \"third\" is deprecated.") } -func (s *Suite) Test_Logger__show_source_with_explanation(c *check.C) { +func (s *Suite) Test_Logger_showSource__with_explanation(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--source", "--explain") @@ -250,7 +384,7 @@ func (s *Suite) Test_Logger__show_source_with_explanation(c *check.C) { // if there are several diagnostics for the same line. In this case though, // there is an explanation between the diagnostics, and because it may get // quite long, it's better to repeat the source code once again. -func (s *Suite) Test_Logger__show_source_with_explanation_in_same_line(c *check.C) { +func (s *Suite) Test_Logger_showSource__with_explanation_in_same_line(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--source", "--explain") @@ -283,7 +417,7 @@ func (s *Suite) Test_Logger__show_source_with_explanation_in_same_line(c *check. // When there is no explanation after the first diagnostic, it is not // necessary to repeat the source code again for the second diagnostic. -func (s *Suite) Test_Logger__show_source_without_explanation_in_same_line(c *check.C) { +func (s *Suite) Test_Logger_showSource__without_explanation_in_same_line(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--source", "--explain") @@ -315,7 +449,7 @@ func (s *Suite) Test_Logger__show_source_without_explanation_in_same_line(c *che // the "Replacing" message. Since these are shown in diff style, they // must be kept together. And since the "+" line must be below the "Replacing" // line, this order of lines seems to be the most intuitive. -func (s *Suite) Test__show_source_separator_show_autofix(c *check.C) { +func (s *Suite) Test_Logger_showSource__separator_show_autofix(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--source", "--show-autofix") @@ -348,7 +482,7 @@ func (s *Suite) Test__show_source_separator_show_autofix(c *check.C) { "+\tThe bronze medal line") } -func (s *Suite) Test__show_source_separator_show_autofix_with_explanation(c *check.C) { +func (s *Suite) Test_Logger_showSource__separator_show_autofix_with_explanation(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--source", "--show-autofix", "--explain") @@ -393,7 +527,7 @@ func (s *Suite) Test__show_source_separator_show_autofix_with_explanation(c *che // // TODO: Giving the diagnostics again would be useful, but the warning and // error counters should not be affected, as well as the exitcode. -func (s *Suite) Test__show_source_separator_autofix(c *check.C) { +func (s *Suite) Test_Logger_showSource__separator_autofix(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--source", "--autofix") @@ -424,164 +558,260 @@ func (s *Suite) Test__show_source_separator_autofix(c *check.C) { "+\tThe bronze medal line") } -func (s *Suite) Test_Logger_Explain__only(c *check.C) { +// Calling Logf without further preparation just logs the message. +// Suppressing duplicate messages or filtering messages happens +// in other methods of the Logger, namely Relevant, FirstTime, Diag. +func (s *Suite) Test_Logger_Logf(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--only", "interesting", "--explain") - line := t.NewLine("Makefile", 27, "The old song") + var sw strings.Builder + logger := Logger{out: NewSeparatorWriter(&sw)} - // Neither the warning nor the corresponding explanation are logged. - line.Warnf("Filtered warning.") - line.Explain("Explanation for the above warning.") + logger.Logf(Error, "filename", "3", "Blue should be %s.", "Blue should be orange.") - line.Notef("What an interesting line.") - line.Explain("This explanation is logged.") + t.CheckEquals(sw.String(), ""+ + "ERROR: filename:3: Blue should be orange.\n") +} - t.CheckOutputLines( - "NOTE: Makefile:27: What an interesting line.", - "", - "\tThis explanation is logged.", - "") +// Logf doesn't filter duplicates, but Diag does. +func (s *Suite) Test_Logger_Logf__duplicates(c *check.C) { + t := s.Init(c) + + var sw strings.Builder + logger := Logger{out: NewSeparatorWriter(&sw)} + + logger.Logf(Error, "filename", "3", "Blue should be %s.", "Blue should be orange.") + logger.Logf(Error, "filename", "3", "Blue should be %s.", "Blue should be orange.") + + t.CheckEquals(sw.String(), ""+ + "ERROR: filename:3: Blue should be orange.\n"+ + "ERROR: filename:3: Blue should be orange.\n") } -func (s *Suite) Test_Logger_Explain__show_autofix(c *check.C) { +// Ensure that suppressing a diagnostic doesn't influence later calls to Logf. +func (s *Suite) Test_Logger_Logf__mixed_with_Diag(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--explain", "--show-autofix") - line := t.NewLine("Makefile", 27, "The old song") + var sw strings.Builder + logger := Logger{out: NewSeparatorWriter(&sw)} + line := t.NewLine("filename", 3, "Text") - line.Warnf("Warning without fix.") - line.Explain( - "Explanation for warning without fix.") + logger.Logf(Error, "filename", "3", "Logf output 1.", "Logf output 1.") + logger.Diag(line, Error, "Diag %s.", "1") + logger.Logf(Error, "filename", "3", "Logf output 2.", "Logf output 2.") + logger.Diag(line, Error, "Diag %s.", "1") // Duplicate, therefore suppressed + logger.Logf(Error, "filename", "3", "Logf output 3.", "Logf output 3.") - fix := line.Autofix() - fix.Warnf("Warning with fix.") - fix.Explain( - "Explanation for warning with fix.") - fix.Replace("old", "new") - fix.Apply() + t.CheckEquals(sw.String(), ""+ + "ERROR: filename:3: Logf output 1.\n"+ + "ERROR: filename:3: Diag 1.\n"+ + "ERROR: filename:3: Logf output 2.\n"+ + "ERROR: filename:3: Logf output 3.\n") +} - // Since the warning without fix doesn't fix anything, it is filtered out. - // So is the corresponding explanation. - t.CheckOutputLines( - "WARN: Makefile:27: Warning with fix.", - "AUTOFIX: Makefile:27: Replacing \"old\" with \"new\".", - "", - "\tExplanation for warning with fix.", - "") +func (s *Suite) Test_Logger_Logf__production(c *check.C) { + t := s.Init(c) + + var sw strings.Builder + logger := Logger{out: NewSeparatorWriter(&sw)} + + // In production mode, the checks for the diagnostic messages are + // turned off, for performance reasons. The unit tests provide + // enough coverage. + G.Testing = false + logger.Logf(Error, "filename", "3", "diagnostic", "message") + + t.CheckEquals(sw.String(), ""+ + "ERROR: filename:3: message\n") } -func (s *Suite) Test_Logger_Explain__show_autofix_and_source(c *check.C) { +func (s *Suite) Test_Logger_Logf__profiling(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--explain", "--show-autofix", "--source") - line := t.NewLine("Makefile", 27, "The old song") + line := t.NewLine("filename", 123, "text") - line.Warnf("Warning without fix.") - line.Explain( - "Explanation for warning without fix.") + G.Opts.Profiling = true + G.Logger.histo = histogram.New() + line.Warnf("Warning.") - fix := line.Autofix() - fix.Warnf("Warning with fix.") - fix.Explain( - "Explanation for warning with fix.") - fix.Replace("old", "new") - fix.Apply() + G.Logger.histo.PrintStats(G.Logger.out.out, "loghisto", -1) - // Since the warning without fix doesn't fix anything, it is filtered out. - // So is the corresponding explanation. t.CheckOutputLines( - "WARN: Makefile:27: Warning with fix.", - "AUTOFIX: Makefile:27: Replacing \"old\" with \"new\".", - "-\tThe old song", - "+\tThe new song", - "", - "\tExplanation for warning with fix.", - "") + "WARN: filename:123: Warning.", + "loghisto 1 Warning.") } -// When the --autofix option is given, the warnings are not shown, therefore it doesn't -// make sense to show the explanation for the warning. -func (s *Suite) Test_Logger_Explain__autofix_and_source(c *check.C) { +func (s *Suite) Test_Logger_Logf__profiling_autofix(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--explain", "--autofix", "--source") - line := t.NewLine("Makefile", 27, "The old song") + t.SetUpCommandLine("--show-autofix", "--source", "--explain") + line := t.NewLine("filename", 123, "text") - line.Warnf("Warning without fix.") - line.Explain( - "Explanation for warning without fix.") + G.Opts.Profiling = true + G.Logger.histo = histogram.New() fix := line.Autofix() - fix.Warnf("Warning with fix.") + fix.Notef("Autofix note.") fix.Explain( - "Explanation for warning with fix.") - fix.Replace("old", "new") + "Autofix explanation.") + fix.Replace("text", "replacement") fix.Apply() - // Since the warning without fix doesn't fix anything, it is filtered out. - // So is the corresponding explanation. + // The AUTOFIX line is not counted in the histogram although + // it uses the same code path as the other messages. + G.Logger.histo.PrintStats(G.Logger.out.out, "loghisto", -1) + t.CheckOutputLines( - "AUTOFIX: Makefile:27: Replacing \"old\" with \"new\".", - "-\tThe old song", - "+\tThe new song") + "NOTE: filename:123: Autofix note.", + "AUTOFIX: filename:123: Replacing \"text\" with \"replacement\".", + "-\ttext", + "+\treplacement", + "", + "\tAutofix explanation.", + "", + "loghisto 1 Autofix note.") } -// When an explanation consists of multiple paragraphs, it contains some empty lines. -// When printing these lines, there is no need to write the tab that is used for indenting -// the normal lines. +// In rare cases, the explanations for the same warning may differ +// when they appear in different contexts. In such a case, if the +// warning is suppressed, the explanation must not appear on its own. // -// Since pkglint likes to complain about trailing whitespace, it should not generate it itself. -func (s *Suite) Test_Logger_Explain__empty_lines(c *check.C) { +// An example of this was (until November 2018) DESTDIR in the check +// for absolute pathnames. +func (s *Suite) Test_Logger_Logf__duplicate_messages(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--explain") - line := t.NewLine("Makefile", 27, "The old song") + G.Logger.Opts.LogVerbose = false + line := t.NewLine("README.txt", 123, "text") - line.Warnf("A normal warning.") - line.Explain( - "Paragraph 1 of the explanation.", - "", - "Paragraph 2 of the explanation.") + // Is logged because it is the first appearance of this warning. + line.Warnf("The warning.") + line.Explain("Explanation 1") + + // Is suppressed because the warning is the same as above and LogVerbose + // has been set to false for this test. + line.Warnf("The warning.") + line.Explain("Explanation 2") t.CheckOutputLines( - "WARN: Makefile:27: A normal warning.", - "", - "\tParagraph 1 of the explanation.", + "WARN: README.txt:123: The warning.", "", - "\tParagraph 2 of the explanation.", + "\tExplanation 1", "") } -// In an explanation, it can happen that the pkgsrc directory is mentioned. -// While pkgsrc does not support either PKGSRCDIR or PREFIX or really any -// other directory name to contain spaces, during pkglint development this -// may happen because the pkgsrc root is in the temporary directory. -// -// In this situation, the ~ placeholder must still be properly substituted. -func (s *Suite) Test_Logger_Explain__line_wrapped_temporary_directory(c *check.C) { +func (s *Suite) Test_Logger_Logf__duplicate_explanations(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--explain") - filename := t.File("filename.mk") - mkline := t.NewMkLine(filename, 123, "") + line := t.NewLine("README.txt", 123, "text") - mkline.Notef("Just a note to get the below explanation.") + // In rare cases, different diagnostics may have the same explanation. + line.Warnf("Warning 1.") + line.Explain("Explanation") + line.Warnf("Warning 2.") + line.Explain("Explanation") // Is suppressed. + + t.CheckOutputLines( + "WARN: README.txt:123: Warning 1.", + "", + "\tExplanation", + "", + "WARN: README.txt:123: Warning 2.") +} + +func (s *Suite) Test_Logger_Logf__gcc_format(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--gcc-output-format") + + logger := &G.Logger + logger.Logf(Note, "filename", "123", "Both filename and line number.", "Both filename and line number.") + logger.Logf(Note, "", "123", "No filename, only line number.", "No filename, only line number.") + logger.Logf(Note, "filename", "", "Filename without line number.", "Filename without line number.") + logger.Logf(Note, "", "", "Neither filename nor line number.", "Neither filename nor line number.") + + t.CheckOutputLines( + "filename:123: note: Both filename and line number.", + "note: No filename, only line number.", + "filename: note: Filename without line number.", + "note: Neither filename nor line number.") +} + +func (s *Suite) Test_Logger_Logf__traditional_format(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--gcc-output-format=no") + + logger := &G.Logger + logger.Logf(Note, "filename", "123", "Both filename and line number.", "Both filename and line number.") + logger.Logf(Note, "", "123", "No filename, only line number.", "No filename, only line number.") + logger.Logf(Note, "filename", "", "Filename without line number.", "Filename without line number.") + logger.Logf(Note, "", "", "Neither filename nor line number.", "Neither filename nor line number.") + + t.CheckOutputLines( + "NOTE: filename:123: Both filename and line number.", + "NOTE: No filename, only line number.", + "NOTE: filename: Filename without line number.", + "NOTE: Neither filename nor line number.") +} + +// Ensures that pkglint never destroys the terminal emulator by sending unintended escape sequences. +func (s *Suite) Test_Logger_Logf__strange_characters(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--gcc-output-format", "--source", "--explain") + + G.Logger.Logf(Note, "filename", "123", "Format.", "Unicode \U0001F645 and ANSI \x1B are never logged.") G.Logger.Explain( - sprintf("%[1]s %[1]s %[1]s %[1]s %[1]s %[1]q", filename)) + "Even a \u0007 in the explanation is silent.") - t.CheckOutputLinesIgnoreSpace( - "NOTE: ~/filename.mk:123: Just a note to get the below explanation.", + t.CheckOutputLines( + "filename:123: note: Unicode <U+1F645> and ANSI <U+001B> are never logged.", "", - "\t~/filename.mk", - "\t~/filename.mk", - "\t~/filename.mk", - "\t~/filename.mk", - "\t~/filename.mk", - "\t\"~/filename.mk\"", + "\tEven a <U+0007> in the explanation is silent.", "") } +// Even if verbose logging is disabled, the "Replacing" diagnostics +// must not be filtered for duplicates since each of them modifies the line. +func (s *Suite) Test_Logger_Logf__duplicate_autofix(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--explain", "--autofix") + G.Logger.Opts.LogVerbose = false // See SetUpTest + line := t.NewLine("README.txt", 123, "text") + + fix := line.Autofix() + fix.Warnf("T should always be uppercase.") + fix.ReplaceRegex(`t`, "T", -1) + fix.Apply() + + t.CheckOutputLines( + "AUTOFIX: README.txt:123: Replacing \"t\" with \"T\".", + "AUTOFIX: README.txt:123: Replacing \"t\" with \"T\".") +} + +func (s *Suite) Test_Logger_Logf__panic(c *check.C) { + t := s.Init(c) + + t.ExpectPanic( + func() { G.Logger.Logf(Error, "filename", "13", "No period", "No period") }, + "Pkglint internal error: Diagnostic format \"No period\" must end in a period.") +} + +func (s *Suite) Test_Logger_Errorf__gcc_format(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--gcc-output-format") + + G.Logger.Errorf("filename", "Cannot be opened for %s.", "reading") + + t.CheckOutputLines( + "filename: error: Cannot be opened for reading.") +} + func (s *Suite) Test_Logger_ShowSummary__explanations_with_only(c *check.C) { t := s.Init(c) @@ -780,254 +1010,61 @@ func (s *Suite) Test_Logger_ShowSummary__quoting(c *check.C) { "(Run \"pkglint -e --only 'string with '\\''quotes'\\'''\" to show explanations.)") } -// In rare cases, the explanations for the same warning may differ -// when they appear in different contexts. In such a case, if the -// warning is suppressed, the explanation must not appear on its own. -// -// An example of this was (until November 2018) DESTDIR in the check -// for absolute pathnames. -func (s *Suite) Test_Logger_Logf__duplicate_messages(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--explain") - G.Logger.Opts.LogVerbose = false - line := t.NewLine("README.txt", 123, "text") - - // Is logged because it is the first appearance of this warning. - line.Warnf("The warning.") - line.Explain("Explanation 1") - - // Is suppressed because the warning is the same as above and LogVerbose - // has been set to false for this test. - line.Warnf("The warning.") - line.Explain("Explanation 2") - - t.CheckOutputLines( - "WARN: README.txt:123: The warning.", - "", - "\tExplanation 1", - "") -} - -func (s *Suite) Test_Logger_Logf__duplicate_explanations(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--explain") - line := t.NewLine("README.txt", 123, "text") - - // In rare cases, different diagnostics may have the same explanation. - line.Warnf("Warning 1.") - line.Explain("Explanation") - line.Warnf("Warning 2.") - line.Explain("Explanation") // Is suppressed. - - t.CheckOutputLines( - "WARN: README.txt:123: Warning 1.", - "", - "\tExplanation", - "", - "WARN: README.txt:123: Warning 2.") -} - -func (s *Suite) Test_Logger_Logf__gcc_format(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--gcc-output-format") - - logger := &G.Logger - logger.Logf(Note, "filename", "123", "Both filename and line number.", "Both filename and line number.") - logger.Logf(Note, "", "123", "No filename, only line number.", "No filename, only line number.") - logger.Logf(Note, "filename", "", "Filename without line number.", "Filename without line number.") - logger.Logf(Note, "", "", "Neither filename nor line number.", "Neither filename nor line number.") - - t.CheckOutputLines( - "filename:123: note: Both filename and line number.", - "note: No filename, only line number.", - "filename: note: Filename without line number.", - "note: Neither filename nor line number.") -} - -func (s *Suite) Test_Logger_Logf__traditional_format(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--gcc-output-format=no") - - logger := &G.Logger - logger.Logf(Note, "filename", "123", "Both filename and line number.", "Both filename and line number.") - logger.Logf(Note, "", "123", "No filename, only line number.", "No filename, only line number.") - logger.Logf(Note, "filename", "", "Filename without line number.", "Filename without line number.") - logger.Logf(Note, "", "", "Neither filename nor line number.", "Neither filename nor line number.") - - t.CheckOutputLines( - "NOTE: filename:123: Both filename and line number.", - "NOTE: No filename, only line number.", - "NOTE: filename: Filename without line number.", - "NOTE: Neither filename nor line number.") -} - -func (s *Suite) Test_Logger_Errorf__gcc_format(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--gcc-output-format") - - G.Logger.Errorf("filename", "Cannot be opened for %s.", "reading") - - t.CheckOutputLines( - "filename: error: Cannot be opened for reading.") -} - -// Ensures that pkglint never destroys the terminal emulator by sending unintended escape sequences. -func (s *Suite) Test_Logger_Logf__strange_characters(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--gcc-output-format", "--source", "--explain") - - G.Logger.Logf(Note, "filename", "123", "Format.", "Unicode \U0001F645 and ANSI \x1B are never logged.") - G.Logger.Explain( - "Even a \u0007 in the explanation is silent.") - - t.CheckOutputLines( - "filename:123: note: Unicode <U+1F645> and ANSI <U+001B> are never logged.", - "", - "\tEven a <U+0007> in the explanation is silent.", - "") -} - -func (s *Suite) Test_Logger_Diag__show_source(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--show-autofix", "--source") - line := t.NewLine("filename", 123, "text") - - fix := line.Autofix() - fix.Notef("Diagnostics can show the differences in autofix mode.") - fix.InsertBefore("new line before") - fix.InsertAfter("new line after") - fix.Apply() - - t.CheckOutputLines( - "NOTE: filename:123: Diagnostics can show the differences in autofix mode.", - "AUTOFIX: filename:123: Inserting a line \"new line before\" before this line.", - "AUTOFIX: filename:123: Inserting a line \"new line after\" after this line.", - "+\tnew line before", - ">\ttext", - "+\tnew line after") -} - -func (s *Suite) Test_Logger_Diag__show_source_with_whole_file(c *check.C) { +func (s *Suite) Test_SeparatorWriter(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--source") - line := NewLineWhole("filename") + var sb strings.Builder + wr := NewSeparatorWriter(&sb) - line.Warnf("This line does not have any RawLine attached.") + wr.WriteLine("a") + wr.WriteLine("b") - t.CheckOutputLines( - "WARN: filename: This line does not have any RawLine attached.") -} + t.CheckEquals(sb.String(), "a\nb\n") -// Ensures that when two packages produce a warning in the same file, both the -// warning and the corresponding source code are logged only once. -func (s *Suite) Test_Logger_Diag__source_duplicates(c *check.C) { - t := s.Init(c) + wr.Separate() - t.SetUpPkgsrc() - t.CreateFileLines("category/dependency/patches/patch-aa", - CvsID, - "", - "--- old file", - "+++ new file", - "@@ -1,1 +1,1 @@", - "-old line", - "+new line") - t.SetUpPackage("category/package1", - "PATCHDIR=\t../../category/dependency/patches") - t.SetUpPackage("category/package2", - "PATCHDIR=\t../../category/dependency/patches") + t.CheckEquals(sb.String(), "a\nb\n") - t.Main("--source", "-Wall", "category/package1", "category/package2") + wr.WriteLine("c") - t.CheckOutputLines( - "ERROR: ~/category/package1/distinfo: "+ - "Patch \"../../category/dependency/patches/patch-aa\" is not recorded. "+ - "Run \""+confMake+" makepatchsum\".", - "", - ">\t--- old file", - "ERROR: ~/category/dependency/patches/patch-aa:3: "+ - "Each patch must be documented.", - "", - "ERROR: ~/category/package2/distinfo: "+ - "Patch \"../../category/dependency/patches/patch-aa\" is not recorded. "+ - "Run \""+confMake+" makepatchsum\".", - "", - "3 errors found.", - t.Shquote("(Run \"pkglint -e --source -Wall %s %s\" to show explanations.)", - "category/package1", "category/package2")) + t.CheckEquals(sb.String(), "a\nb\n\nc\n") } -func (s *Suite) Test_Logger_shallBeLogged(c *check.C) { +func (s *Suite) Test_SeparatorWriter_Separate(c *check.C) { t := s.Init(c) - t.SetUpCommandLine( /* none */ ) - - t.CheckEquals(G.Logger.shallBeLogged("Options should not contain whitespace."), true) - - t.SetUpCommandLine("--only", "whitespace") - - t.CheckEquals(G.Logger.shallBeLogged("Options should not contain whitespace."), true) - t.CheckEquals(G.Logger.shallBeLogged("Options should not contain space."), false) - - t.SetUpCommandLine( /* none again */ ) - - t.CheckEquals(G.Logger.shallBeLogged("Options should not contain whitespace."), true) - t.CheckEquals(G.Logger.shallBeLogged("Options should not contain space."), true) -} + var sb strings.Builder + wr := NewSeparatorWriter(&sb) -// Even if verbose logging is disabled, the "Replacing" diagnostics -// must not be filtered for duplicates since each of them modifies the line. -func (s *Suite) Test_Logger_Logf__duplicate_autofix(c *check.C) { - t := s.Init(c) + wr.WriteLine("a") + wr.Separate() - t.SetUpCommandLine("--explain", "--autofix") - G.Logger.Opts.LogVerbose = false // See SetUpTest - line := t.NewLine("README.txt", 123, "text") + t.CheckEquals(sb.String(), "a\n") - fix := line.Autofix() - fix.Warnf("T should always be uppercase.") - fix.ReplaceRegex(`t`, "T", -1) - fix.Apply() + // The call to Separate had requested an empty line. That empty line + // can either be given explicitly (like here), or it will be written + // implicitly before the next non-newline character. + wr.WriteLine("") + wr.Separate() - t.CheckOutputLines( - "AUTOFIX: README.txt:123: Replacing \"t\" with \"T\".", - "AUTOFIX: README.txt:123: Replacing \"t\" with \"T\".") -} + t.CheckEquals(sb.String(), "a\n\n") -func (s *Suite) Test_Logger_Logf__panic(c *check.C) { - t := s.Init(c) + wr.WriteLine("c") + wr.Separate() - t.ExpectPanic( - func() { G.Logger.Logf(Error, "filename", "13", "No period", "No period") }, - "Pkglint internal error: Diagnostic format \"No period\" must end in a period.") + t.CheckEquals(sb.String(), "a\n\nc\n") } -func (s *Suite) Test_SeparatorWriter(c *check.C) { +func (s *Suite) Test_SeparatorWriter_Separate__at_the_beginning(c *check.C) { t := s.Init(c) var sb strings.Builder wr := NewSeparatorWriter(&sb) - wr.WriteLine("a") - wr.WriteLine("b") - - t.CheckEquals(sb.String(), "a\nb\n") - wr.Separate() + wr.WriteLine("a") - t.CheckEquals(sb.String(), "a\nb\n") - - wr.WriteLine("c") - - t.CheckEquals(sb.String(), "a\nb\n\nc\n") + t.CheckEquals(sb.String(), "a\n") } func (s *Suite) Test_SeparatorWriter_Flush(c *check.C) { @@ -1061,40 +1098,3 @@ func (s *Suite) Test_SeparatorWriter_Flush(c *check.C) { t.CheckEquals(sb.String(), "ab\n\nc") } - -func (s *Suite) Test_SeparatorWriter_Separate(c *check.C) { - t := s.Init(c) - - var sb strings.Builder - wr := NewSeparatorWriter(&sb) - - wr.WriteLine("a") - wr.Separate() - - t.CheckEquals(sb.String(), "a\n") - - // The call to Separate had requested an empty line. That empty line - // can either be given explicitly (like here), or it will be written - // implicitly before the next non-newline character. - wr.WriteLine("") - wr.Separate() - - t.CheckEquals(sb.String(), "a\n\n") - - wr.WriteLine("c") - wr.Separate() - - t.CheckEquals(sb.String(), "a\n\nc\n") -} - -func (s *Suite) Test_SeparatorWriter_Separate__at_the_beginning(c *check.C) { - t := s.Init(c) - - var sb strings.Builder - wr := NewSeparatorWriter(&sb) - - wr.Separate() - wr.WriteLine("a") - - t.CheckEquals(sb.String(), "a\n") -} diff --git a/pkgtools/pkglint/files/mkline.go b/pkgtools/pkglint/files/mkline.go index f2df0a3ddb5..83b1c35ffb7 100644 --- a/pkgtools/pkglint/files/mkline.go +++ b/pkgtools/pkglint/files/mkline.go @@ -678,8 +678,8 @@ func (mkline *MkLine) VariableNeedsQuoting(mklines *MkLines, varuse *MkVarUse, v } if !vartype.basicType.NeedsQ() { - if !vartype.List() { - if vartype.Guessed() { + if !vartype.IsList() { + if vartype.IsGuessed() { return unknown } return no @@ -691,7 +691,7 @@ func (mkline *MkLine) VariableNeedsQuoting(mklines *MkLines, varuse *MkVarUse, v // A shell word may appear as part of a shell word, for example COMPILER_RPATH_FLAG. if vuc.IsWordPart && vuc.quoting == VucQuotPlain { - if !vartype.List() && vartype.basicType == BtShellWord { + if !vartype.IsList() && vartype.basicType == BtShellWord { return no } } @@ -1056,7 +1056,7 @@ type indentationLevel struct { checkedFiles []string } -func (ind *Indentation) Empty() bool { +func (ind *Indentation) IsEmpty() bool { return len(ind.levels) == 0 } @@ -1095,7 +1095,7 @@ func (ind *Indentation) Push(mkline *MkLine, indent int, condition string) { // // Variables named *_MK are ignored since they are usually not interesting. func (ind *Indentation) AddVar(varname string) { - if hasSuffix(varname, "_MK") || ind.Empty() { + if hasSuffix(varname, "_MK") || ind.IsEmpty() { return } @@ -1210,17 +1210,17 @@ func (ind *Indentation) TrackAfter(mkline *MkLine) { case "elif": // Handled here instead of TrackBefore to allow the action to access the previous condition. - if !ind.Empty() { + if !ind.IsEmpty() { ind.top().args = args } case "else": - if !ind.Empty() { + if !ind.IsEmpty() { ind.top().mkline.SetHasElseBranch(mkline) } case "endfor", "endif": - if !ind.Empty() { // Can only be false in unbalanced files. + if !ind.IsEmpty() { // Can only be false in unbalanced files. ind.Pop() } } @@ -1248,11 +1248,11 @@ func (ind *Indentation) TrackAfter(mkline *MkLine) { } func (ind *Indentation) CheckFinish(filename string) { - if ind.Empty() { + if ind.IsEmpty() { return } eofLine := NewLineEOF(filename) - for !ind.Empty() { + for !ind.IsEmpty() { openingMkline := ind.top().mkline eofLine.Errorf(".%s from %s must be closed.", openingMkline.Directive(), eofLine.RefTo(openingMkline.Line)) ind.Pop() diff --git a/pkgtools/pkglint/files/mkline_test.go b/pkgtools/pkglint/files/mkline_test.go index 89f40f79bfa..b13ac378a94 100644 --- a/pkgtools/pkglint/files/mkline_test.go +++ b/pkgtools/pkglint/files/mkline_test.go @@ -4,6 +4,37 @@ import ( "gopkg.in/check.v1" ) +func (s *Suite) Test_MkLine__shell_varuse_in_backt_dquot(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpTool("grep", "GREP", AtRunTime) + mklines := t.NewMkLines("x11/motif/Makefile", + MkCvsID, + "post-patch:", + "\tfiles=`${GREP} -l \".fB$${name}.fP(3)\" *.3`") + + mklines.Check() + + // Just ensure that there are no parse errors. + t.CheckOutputEmpty() +} + +// PR 51696, security/py-pbkdf2/Makefile, r1.2 +func (s *Suite) Test_MkLine__comment_in_comment(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("Makefile", + MkCvsID, + "COMMENT=\tPKCS#5 v2.0 PBKDF2 Module") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:2: The # character starts a Makefile comment.") +} + func (s *Suite) Test_MkLine_Varparam(c *check.C) { t := s.Init(c) @@ -69,199 +100,464 @@ func (s *Suite) Test_MkLine_FirstLineContainsValue(c *check.C) { false) } -// Up to July 2019, there was a method MkLine.IsMultiAligned, which has -// been replaced by VaralignBlock. The test cases were still useful, -// therefore they were kept. -func (s *Suite) Test_MkLine__aligned(c *check.C) { +// Demonstrates how a simple condition is structured internally. +// For most of the checks, using cond.Walk is the simplest way to go. +func (s *Suite) Test_MkLine_Cond(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("Makefile", 2, ".if ${VAR} == Value") + + cond := mkline.Cond() + + t.CheckEquals(cond.Compare.Left.Var.varname, "VAR") + t.CheckEquals(cond.Compare.Right.Str, "Value") + t.CheckEquals(mkline.Cond(), cond) +} + +// Ensures that the conditional variables of a line can be set even +// after initializing the MkLine. +// +// If this test should fail, it is probably because mkLineDirective +// is not a pointer type anymore. +// +// See https://github.com/golang/go/issues/28045. +func (s *Suite) Test_MkLine_ConditionalVars(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("Makefile", 45, ".include \"../../category/package/buildlink3.mk\"") + + c.Check(mkline.ConditionalVars(), check.HasLen, 0) + + mkline.SetConditionalVars([]string{"OPSYS"}) + + t.CheckDeepEquals(mkline.ConditionalVars(), []string{"OPSYS"}) +} + +func (s *Suite) Test_MkLine_Tokenize__commented_varassign(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("filename.mk", 123, "#VAR=\tvalue ${VAR} suffix text") + + t.Check(mkline.Tokenize(mkline.Value(), false), check.HasLen, 3) +} + +func (s *Suite) Test_MkLine_ValueSplit(c *check.C) { + t := s.Init(c) + + test := func(value string, expected ...string) { + mkline := t.NewMkLine("Makefile", 1, "PATH=\t"+value) + split := mkline.ValueSplit(value, ":") + t.CheckDeepEquals(split, expected) + } + + test("Platform-independent C# compiler #5", + "Platform-independent C# compiler #5") + + // This warning refers to the #5 since it starts a word, but not to the C#. + t.CheckOutputLines( + "WARN: Makefile:1: The # character starts a Makefile comment.") + + test("/bin", + "/bin") + + test("/bin:/sbin", + "/bin", + "/sbin") + + test("${DESTDIR}/bin:/bin/${SUBDIR}", + "${DESTDIR}/bin", + "/bin/${SUBDIR}") + + test("/bin:${DESTDIR}${PREFIX}:${DESTDIR:S,/,\\:,:S,:,:,}/sbin", + "/bin", + "${DESTDIR}${PREFIX}", + "${DESTDIR:S,/,\\:,:S,:,:,}/sbin") + + test("${VAR:Udefault}::${VAR2}two:words", + "${VAR:Udefault}", + "", + "${VAR2}two", + "words") + +} + +func (s *Suite) Test_MkLine_ValueSplit__invalid_argument(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("filename.mk", 123, "VAR=\tvalue") + + t.ExpectAssert(func() { mkline.ValueSplit("value", "") }) +} + +func (s *Suite) Test_MkLine_ValueFields(c *check.C) { t := s.Init(c) - test := func(data ...interface{}) { - var lineTexts []string - for _, text := range data[:len(data)-1] { - lineTexts = append(lineTexts, text.(string)) + test := func(value string, expected ...string) { + mkline := t.NewMkLine("Makefile", 1, "VAR=\t"+value) + split := mkline.ValueFields(value) + t.CheckDeepEquals(split, expected) + } + + test("one two\t\t${THREE:Uthree:Nsome \tspaces}", + "one", + "two", + "${THREE:Uthree:Nsome \tspaces}") + + // The example from the ValueFields documentation. + test("${VAR:Udefault value} ${VAR2}two words;;; 'word three'", + "${VAR:Udefault value}", + "${VAR2}two", + "words;;;", + "'word three'") + + test("\"double quotes\" group words", + "\"double quotes\"", + "group", + "words") + + test("\"unfinished", + nil...) // the rest is silently discarded + + test("'single quotes' group words", + "'single quotes'", + "group", + "words") + + test("'unfinished", + nil...) // the rest is silently discarded + + // This is how it works in bmake. + test("'\\' ' end", + "'\\'") // the "' end" is silently discarded + + // This is how it works in pkglint. + test("'\\' end", + "'\\'", + "end") + + test("`backticks do not group words`", + "`backticks", + "do", + "not", + "group", + "words`") + + test("plain${VAR}plain", + "plain${VAR}plain") + + test("\"${DOUBLE}\" \"\\${DOUBLE}\"", + "\"${DOUBLE}\"", + "\"\\${DOUBLE}\"") + + test("'${SINGLE}' '\\${SINGLE}'", + "'${SINGLE}'", + "'\\${SINGLE}'") + + test("\"\"''\"\"", + "\"\"''\"\"") + + test("$@ $<", + "$@", + "$<") +} + +// Before 2018-11-26, this test panicked. +func (s *Suite) Test_MkLine_ValueFields__adjacent_vars(c *check.C) { + t := s.Init(c) + + test := func(value string, expected ...string) { + mkline := t.NewMkLine("Makefile", 1, "") + split := mkline.ValueFields(value) + t.CheckDeepEquals(split, expected) + } + + test("\t; ${RM} ${WRKSRC}", + ";", + "${RM}", + "${WRKSRC}") +} + +func (s *Suite) Test_MkLine_ValueFields__compared_to_splitIntoShellTokens(c *check.C) { + t := s.Init(c) + url := "http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=" + mkline := t.NewMkLine("filename.mk", 123, "MASTER_SITES=\t"+url) + + words, rest := splitIntoShellTokens(dummyLine, url) // Doesn't really make sense + + t.CheckDeepEquals(words, []string{ + "http://registry.gimp.org/file/fix-ca.c?action=download", + "&", + "id=9884", + "&", + "file="}) + t.CheckEquals(rest, "") + + words = mkline.ValueFields(url) + + t.CheckDeepEquals(words, []string{url}) + + words = mkline.ValueFields("a b \"c c c\" d;;d;; \"e\"''`` 'rest") + + t.CheckDeepEquals(words, []string{"a", "b", "\"c c c\"", "d;;d;;", "\"e\"''``"}) + // TODO: c.Check(rest, equals, "'rest") +} + +func (s *Suite) Test_MkLine_ValueTokens(c *check.C) { + t := s.Init(c) + b := NewMkTokenBuilder() + text := b.TextToken + varUseText := b.VaruseTextToken + tokens := b.Tokens + + test := func(value string, expected []*MkToken, diagnostics ...string) { + mkline := t.NewMkLine("Makefile", 1, "PATH=\t"+value) + actualTokens, _ := mkline.ValueTokens() + t.CheckDeepEquals(actualTokens, expected) + t.CheckOutput(diagnostics) + } + + t.Use(text, varUseText, tokens, test) + + test("#empty", + tokens()) + + test("value", + tokens(text("value"))) + + test("value ${VAR} rest", + tokens( + text("value "), + varUseText("${VAR}", "VAR"), + text(" rest"))) + + test("value # comment", + tokens( + text("value"))) + + test("value ${UNFINISHED", + tokens( + text("value "), + varUseText("${UNFINISHED", "UNFINISHED")), + + "WARN: Makefile:1: Missing closing \"}\" for \"UNFINISHED\".") +} + +func (s *Suite) Test_MkLine_ValueTokens__parse_error(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("filename.mk", 123, "VAR=\t$") + + tokens, rest := mkline.ValueTokens() + + t.Check(tokens, check.IsNil) + t.CheckEquals(rest, "$") + + // Returns the same values, this time from the cache. + tokens, rest = mkline.ValueTokens() + + t.Check(tokens, check.IsNil) + t.CheckEquals(rest, "$") +} + +func (s *Suite) Test_MkLine_ValueTokens__caching(c *check.C) { + t := s.Init(c) + b := NewMkTokenBuilder() + + mkline := t.NewMkLine("Makefile", 1, "PATH=\tvalue ${UNFINISHED") + valueTokens, rest := mkline.ValueTokens() + + t.CheckDeepEquals(valueTokens, + b.Tokens( + b.TextToken("value "), + b.VaruseTextToken("${UNFINISHED", "UNFINISHED"))) + t.CheckEquals(rest, "") + t.CheckOutputLines( + "WARN: Makefile:1: Missing closing \"}\" for \"UNFINISHED\".") + + // This time the slice is taken from the cache. + tokens2, rest2 := mkline.ValueTokens() + + t.CheckEquals(&tokens2[0], &valueTokens[0]) + t.CheckEquals(rest2, rest) +} + +func (s *Suite) Test_MkLine_ValueTokens__caching_parse_error(c *check.C) { + t := s.Init(c) + b := NewMkTokenBuilder() + + mkline := t.NewMkLine("Makefile", 1, "PATH=\t${UNFINISHED") + valueTokens, rest := mkline.ValueTokens() + + t.CheckDeepEquals(valueTokens, b.Tokens(b.VaruseTextToken("${UNFINISHED", "UNFINISHED"))) + t.CheckEquals(rest, "") + t.CheckOutputLines( + "WARN: Makefile:1: Missing closing \"}\" for \"UNFINISHED\".") + + // This time the slice is taken from the cache. + tokens2, rest2 := mkline.ValueTokens() + + t.CheckEquals(&tokens2[0], &valueTokens[0]) + t.CheckEquals(rest2, rest) +} + +func (s *Suite) Test_MkLine_ValueTokens__warnings(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", + MkCvsID, + "ROUND=\t$(ROUND)") + + mklines.mklines[1].ValueTokens() + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:2: Please use curly braces {} instead of round parentheses () for ROUND.") +} + +func (s *Suite) Test_MkLine_Fields__varassign(c *check.C) { + t := s.Init(c) + + test := func(value string, expected ...string) { + mkline := t.NewMkLine("Makefile", 1, "PATH=\t"+value) + fields := mkline.Fields() + t.CheckDeepEquals(fields, expected) + + // Repeated calls get the cached value. + if len(fields) > 0 { + cached := mkline.Fields() + t.CheckEquals(&cached[0], &fields[0]) } - expected := data[len(data)-1].(bool) + } + + test("# empty", + nil...) + + test("word", + "word") - mklines := t.NewMkLines("filename.mk", - lineTexts...) - assert(len(mklines.mklines) == 1) + test("word '${VAR}single ${VAR}' \"\t\"", + "word", + "'${VAR}single ${VAR}'", + "\"\t\"") +} - var varalign VaralignBlock - varalign.Process(mklines.mklines[0]) - varalign.Finish() +func (s *Suite) Test_MkLine_Fields__for(c *check.C) { + t := s.Init(c) - output := t.Output() - if expected { - t.CheckEquals(output, "") - } else if output == "" { - t.Check(output, check.Not(check.Equals), "") + test := func(value string, expected ...string) { + mkline := t.NewMkLine("Makefile", 1, ".for "+value) + fields := mkline.Fields() + t.CheckDeepEquals(fields, expected) + + // Repeated calls get the cached value. + if len(fields) > 0 { + cached := mkline.Fields() + t.CheckEquals(&cached[0], &fields[0]) } } - // The first line uses a space for indentation, which is typical of - // the outlier line in VaralignBlock. - // - // The second line starts in column 0, which is too far to the left. - // For a human reader the second line looks like a variable assignment - // of its own. - test( - "CONFIGURE_ENV+= \\", - "AWK=${AWK:Q}", - false) + // Unrealistic, but needed for full code coverage. + test("# empty", + nil...) - // The second line is indented and therefore visually distinct from - // a Makefile assignment line. Everything's fine. - test( - "CONFIGURE_ENV+= \\", - "\tAWK=${AWK:Q}", - true) + // Still unrealistic. + test("i in # empty", + "i", + "in") - // The first line may also use a tab instead of a space for indentation. - // This is typical of variable assignments whose name is short enough - // to be aligned with the other lines. - test( - "CONFIGURE_ENV+=\t\\", - "AWK=${AWK:Q}", - false) - test( - "CONFIGURE_ENV+=\t\\", - "\tAWK=${AWK:Q}", - true) + test("i in word '${VAR}single ${VAR}' \"\t\"", + "i", + "in", + "word", + "'${VAR}single ${VAR}'", + "\"\t\"") +} - // The first line contains a value, and the second line has the same - // indentation as the first line. This looks nicely aligned. - test( - "CONFIGURE_ENV+=\tAWK=${AWK:Q} \\", - "\t\tSED=${SED:Q}", - true) +func (s *Suite) Test_MkLine_Fields__semicolons(c *check.C) { + t := s.Init(c) - // The second line is indented less than the first line. This looks - // confusing to the human reader because the actual values do not - // appear in a rectangular shape in the source code. - test( - "VAR.param=\tvalue \\", - "\t10........20........30........40........50........60...4", - false) + mkline := t.NewMkLine("filename.mk", 123, "VAR=\tword1 word2;;;") + words := mkline.Fields() - // The second line is indented with a single tab because otherwise - // it would be longer than 72 characters. In this case it is ok to - // use the smaller indentation. - test( - "VAR.param=\tvalue \\", - "\t10........20........30........40........50........60....5", - true) + t.CheckDeepEquals(words, []string{"word1", "word2;;;"}) +} - // Having the continuation line in column 0 looks even more confusing. - test( - "CONFIGURE_ENV+=\tAWK=${AWK:Q} \\", - "SED=${SED:Q}", - false) +func (s *Suite) Test_MkLine_Fields__varuse_with_embedded_space(c *check.C) { + t := s.Init(c) - // Longer continuation lines may use internal indentation to represent - // AWK or shell code. - test( - "GENERATE_PLIST+=\t/pattern/ { \\", - "\t\t\t action(); \\", - "\t\t\t}", - true) + mkline := t.NewMkLine("filename.mk", 123, "VAR=\t${VAR:S/ /_/g}") - // If any of the continuation lines is indented less than the first - // line, it looks confusing. - test( - "GENERATE_PLIST+=\t/pattern/ { \\", - "\t action(); \\", - "\t}", - false) + words := mkline.Fields() - // If the first line is empty, the indentation may start in column 8, - // and the continuation lines have to be indented as least as far to - // the right as the second line. - test( - "GENERATE_PLIST+= \\", - "\t/pattern/ { \\", - "\t action(); \\", - "\t}", - true) + t.CheckDeepEquals(words, []string{"${VAR:S/ /_/g}"}) +} - // The very last line is indented at column 0, therefore the whole - // line is not indented properly. - test( - "GENERATE_PLIST+= \\", - "\t/pattern/ { \\", - "\t action(); \\", - "}", - false) +func (s *Suite) Test_MkLine_ResolveVarsInRelativePath(c *check.C) { + t := s.Init(c) - // If there is no visible variable value at all, pkglint must not crash. - // This case doesn't occur in practice since the code is usually - // succinct enough to avoid these useless lines. - // - // The first line is empty, the second line is indented to column 8 and - // the remaining lines are all indented by at least 8, therefore the - // alignment is correct. - // - // A theoretical use case might be to have a long explaining comment - // in the continuation lines, but that is not possible syntactically. - // In the line "VAR= value \# comment", the \# is interpreted as - // an escaped number sign, and not as a continuation marker followed - // by a comment. In the line "VAR= value \ # comment", the backslash - // is not a continuation marker as well, since it is not the very - // last character of the line. - test( - "CONFIGURE_ENV+= \\", - "\t\\", - "\t\\", - "\t# nothing", - true) + t.CreateFileLines("lang/lua53/Makefile") + t.CreateFileLines("lang/php72/Makefile") + t.CreateFileLines("emulators/suse100_base/Makefile") + t.CreateFileLines("lang/python36/Makefile") + mklines := t.SetUpFileMkLines("Makefile", + MkCvsID) + mkline := mklines.mklines[0] - // Commented variable assignments can also be tested for alignment. - test( - "#CONFIGURE_ENV+= \\", - "\tvalue", - true) + test := func(before string, after string) { + t.CheckEquals(mkline.ResolveVarsInRelativePath(before), after) + } - // In commented multilines, bmake doesn't care whether the - // continuation lines does or doesn't start with a comment character. - // For human readers though, it is confusing to omit the leading - // comment character. - // - // For determining whether a multiline is aligned, the initial comment - // character is ignored. - test( - "#CONFIGURE_ENV+= \\", - "#\tvalue", - true) + test("", ".") + test("${PKGSRCDIR}", ".") + test("${LUA_PKGSRCDIR}", "../../lang/lua53") + test("${PHPPKGSRCDIR}", "../../lang/php72") + test("${SUSE_DIR_PREFIX}", "suse100") + test("${PYPKGSRCDIR}", "../../lang/python36") + test("${PYPACKAGE}", "python36") + test("${FILESDIR}", "${FILESDIR}") + test("${PKGDIR}", "${PKGDIR}") - // The indentation of the continuation line is neither 8 nor the - // indentation of the first line. Therefore the line is not aligned. - test( - "#CONFIGURE_ENV+= value1 \\", - "#\t\tvalue2", - false) + G.Pkg = NewPackage(t.File("category/package")) + + test("${FILESDIR}", "files") + test("${PKGDIR}", ".") + + // Just for branch coverage. + G.Testing = false + test("${PKGSRCDIR}", "../..") } -// Demonstrates how a simple condition is structured internally. -// For most of the checks, using cond.Walk is the simplest way to go. -func (s *Suite) Test_MkLine_Cond(c *check.C) { +func (s *Suite) Test_MkLine_ResolveVarsInRelativePath__directory_depth(c *check.C) { t := s.Init(c) - mkline := t.NewMkLine("Makefile", 2, ".if ${VAR} == Value") + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("multimedia/totem/filename.mk", + MkCvsID, + "BUILDLINK_PKGSRCDIR.totem?=\t../../multimedia/totem") - cond := mkline.Cond() + mklines.Check() - t.CheckEquals(cond.Compare.Left.Var.varname, "VAR") - t.CheckEquals(cond.Compare.Right.Str, "Value") - t.CheckEquals(mkline.Cond(), cond) + t.CheckOutputLines( + "WARN: ~/multimedia/totem/filename.mk:2: "+ + "The variable BUILDLINK_PKGSRCDIR.totem should not be given a default value in this file; "+ + "it would be ok in buildlink3.mk.", + "ERROR: ~/multimedia/totem/filename.mk:2: Relative path \"../../multimedia/totem/Makefile\" does not exist.") } -func (s *Suite) Test_VarUseContext_String(c *check.C) { +// Just for code coverage +func (s *Suite) Test_MkLine_ResolveVarsInRelativePath__without_tracing(c *check.C) { t := s.Init(c) + t.DisableTracing() t.SetUpVartypes() - vartype := G.Pkgsrc.VariableType(nil, "PKGNAME") - vuc := VarUseContext{vartype, VucUnknownTime, VucQuotBackt, false} + mklines := t.SetUpFileMkLines("buildlink3.mk", + MkCvsID, + "BUILDLINK_PKGSRCDIR.totem?=\t../../${PKGPATH.multimedia/totem}") - t.CheckEquals(vuc.String(), "(Pkgname (package-settable) time:unknown quoting:backt wordpart:false)") + mklines.Check() + + t.CheckOutputLines( + "WARN: ~/buildlink3.mk:2: PKGPATH.multimedia/totem is used but not defined.") } func (s *Suite) Test_MkLine_VariableNeedsQuoting__unknown_rhs(c *check.C) { @@ -839,490 +1135,124 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__uncovered_cases(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_MkLine__shell_varuse_in_backt_dquot(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - t.SetUpTool("grep", "GREP", AtRunTime) - mklines := t.NewMkLines("x11/motif/Makefile", - MkCvsID, - "post-patch:", - "\tfiles=`${GREP} -l \".fB$${name}.fP(3)\" *.3`") - - mklines.Check() - - // Just ensure that there are no parse errors. - t.CheckOutputEmpty() -} - -// PR 51696, security/py-pbkdf2/Makefile, r1.2 -func (s *Suite) Test_MkLine__comment_in_comment(c *check.C) { +func (s *Suite) Test_MkLine_ForEachUsed(c *check.C) { t := s.Init(c) - t.SetUpVartypes() mklines := t.NewMkLines("Makefile", MkCvsID, - "COMMENT=\tPKCS#5 v2.0 PBKDF2 Module") - - mklines.Check() - - t.CheckOutputLines( - "WARN: Makefile:2: The # character starts a Makefile comment.") -} - -// Ensures that the conditional variables of a line can be set even -// after initializing the MkLine. -// -// If this test should fail, it is probably because mkLineDirective -// is not a pointer type anymore. -// -// See https://github.com/golang/go/issues/28045. -func (s *Suite) Test_MkLine_ConditionalVars(c *check.C) { - t := s.Init(c) - - mkline := t.NewMkLine("Makefile", 45, ".include \"../../category/package/buildlink3.mk\"") - - c.Check(mkline.ConditionalVars(), check.HasLen, 0) - - mkline.SetConditionalVars([]string{"OPSYS"}) - - t.CheckDeepEquals(mkline.ConditionalVars(), []string{"OPSYS"}) -} - -func (s *Suite) Test_MkLine_ValueSplit(c *check.C) { - t := s.Init(c) - - test := func(value string, expected ...string) { - mkline := t.NewMkLine("Makefile", 1, "PATH=\t"+value) - split := mkline.ValueSplit(value, ":") - t.CheckDeepEquals(split, expected) - } - - test("Platform-independent C# compiler #5", - "Platform-independent C# compiler #5") - - // This warning refers to the #5 since it starts a word, but not to the C#. - t.CheckOutputLines( - "WARN: Makefile:1: The # character starts a Makefile comment.") - - test("/bin", - "/bin") - - test("/bin:/sbin", - "/bin", - "/sbin") - - test("${DESTDIR}/bin:/bin/${SUBDIR}", - "${DESTDIR}/bin", - "/bin/${SUBDIR}") - - test("/bin:${DESTDIR}${PREFIX}:${DESTDIR:S,/,\\:,:S,:,:,}/sbin", - "/bin", - "${DESTDIR}${PREFIX}", - "${DESTDIR:S,/,\\:,:S,:,:,}/sbin") - - test("${VAR:Udefault}::${VAR2}two:words", - "${VAR:Udefault}", + "VAR=\t${VALUE} # ${varassign.comment}", + ".if ${OPSYS:M${endianness}} == ${Hello:L} # ${if.comment}", + ".for var in one ${two} three # ${for.comment}", + "# ${empty.comment}", + "${TARGETS}: ${SOURCES} # ${dependency.comment}", + ".include \"${OTHER_FILE}\"", "", - "${VAR2}two", - "words") - -} - -func (s *Suite) Test_MkLine_ValueSplit__invalid_argument(c *check.C) { - t := s.Init(c) - - mkline := t.NewMkLine("filename.mk", 123, "VAR=\tvalue") - - t.ExpectAssert(func() { mkline.ValueSplit("value", "") }) -} - -func (s *Suite) Test_MkLine_Fields__varassign(c *check.C) { - t := s.Init(c) - - test := func(value string, expected ...string) { - mkline := t.NewMkLine("Makefile", 1, "PATH=\t"+value) - fields := mkline.Fields() - t.CheckDeepEquals(fields, expected) - - // Repeated calls get the cached value. - if len(fields) > 0 { - cached := mkline.Fields() - t.CheckEquals(&cached[0], &fields[0]) - } - } - - test("# empty", - nil...) - - test("word", - "word") - - test("word '${VAR}single ${VAR}' \"\t\"", - "word", - "'${VAR}single ${VAR}'", - "\"\t\"") -} - -func (s *Suite) Test_MkLine_Fields__for(c *check.C) { - t := s.Init(c) - - test := func(value string, expected ...string) { - mkline := t.NewMkLine("Makefile", 1, ".for "+value) - fields := mkline.Fields() - t.CheckDeepEquals(fields, expected) - - // Repeated calls get the cached value. - if len(fields) > 0 { - cached := mkline.Fields() - t.CheckEquals(&cached[0], &fields[0]) - } - } - - // Unrealistic, but needed for full code coverage. - test("# empty", - nil...) - - // Still unrealistic. - test("i in # empty", - "i", - "in") - - test("i in word '${VAR}single ${VAR}' \"\t\"", - "i", - "in", - "word", - "'${VAR}single ${VAR}'", - "\"\t\"") -} - -func (s *Suite) Test_MkLine_Fields__semicolons(c *check.C) { - t := s.Init(c) - - mkline := t.NewMkLine("filename.mk", 123, "VAR=\tword1 word2;;;") - words := mkline.Fields() - - t.CheckDeepEquals(words, []string{"word1", "word2;;;"}) -} - -func (s *Suite) Test_MkLine_Fields__varuse_with_embedded_space(c *check.C) { - t := s.Init(c) - - mkline := t.NewMkLine("filename.mk", 123, "VAR=\t${VAR:S/ /_/g}") - - words := mkline.Fields() - - t.CheckDeepEquals(words, []string{"${VAR:S/ /_/g}"}) -} - -func (s *Suite) Test_MkLine_ValueFields(c *check.C) { - t := s.Init(c) - - test := func(value string, expected ...string) { - mkline := t.NewMkLine("Makefile", 1, "VAR=\t"+value) - split := mkline.ValueFields(value) - t.CheckDeepEquals(split, expected) - } - - test("one two\t\t${THREE:Uthree:Nsome \tspaces}", - "one", - "two", - "${THREE:Uthree:Nsome \tspaces}") - - // The example from the ValueFields documentation. - test("${VAR:Udefault value} ${VAR2}two words;;; 'word three'", - "${VAR:Udefault value}", - "${VAR2}two", - "words;;;", - "'word three'") - - test("\"double quotes\" group words", - "\"double quotes\"", - "group", - "words") - - test("\"unfinished", - nil...) // the rest is silently discarded - - test("'single quotes' group words", - "'single quotes'", - "group", - "words") - - test("'unfinished", - nil...) // the rest is silently discarded - - // This is how it works in bmake. - test("'\\' ' end", - "'\\'") // the "' end" is silently discarded - - // This is how it works in pkglint. - test("'\\' end", - "'\\'", - "end") - - test("`backticks do not group words`", - "`backticks", - "do", - "not", - "group", - "words`") - - test("plain${VAR}plain", - "plain${VAR}plain") - - test("\"${DOUBLE}\" \"\\${DOUBLE}\"", - "\"${DOUBLE}\"", - "\"\\${DOUBLE}\"") - - test("'${SINGLE}' '\\${SINGLE}'", - "'${SINGLE}'", - "'\\${SINGLE}'") - - test("\"\"''\"\"", - "\"\"''\"\"") - - test("$@ $<", - "$@", - "$<") -} - -// Before 2018-11-26, this test panicked. -func (s *Suite) Test_MkLine_ValueFields__adjacent_vars(c *check.C) { - t := s.Init(c) + "\t${VAR.${param}}", + "\t${VAR}and${VAR2}", + "\t${VAR:M${pattern}}", + "\t$(ROUND_PARENTHESES)", + "\t$$shellvar", + "\t$< $@ $x") - test := func(value string, expected ...string) { - mkline := t.NewMkLine("Makefile", 1, "") - split := mkline.ValueFields(value) - t.CheckDeepEquals(split, expected) + var varnames []string + for _, mkline := range mklines.mklines { + mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { + varnames = append(varnames, time.String()+" "+varUse.varname) + }) } - test("\t; ${RM} ${WRKSRC}", - ";", - "${RM}", - "${WRKSRC}") -} - -func (s *Suite) Test_MkLine_ValueFields__compared_to_splitIntoShellTokens(c *check.C) { - t := s.Init(c) - url := "http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=" - mkline := t.NewMkLine("filename.mk", 123, "MASTER_SITES=\t"+url) - - words, rest := splitIntoShellTokens(dummyLine, url) // Doesn't really make sense - - t.CheckDeepEquals(words, []string{ - "http://registry.gimp.org/file/fix-ca.c?action=download", - "&", - "id=9884", - "&", - "file="}) - t.CheckEquals(rest, "") - - words = mkline.ValueFields(url) - - t.CheckDeepEquals(words, []string{url}) - - words = mkline.ValueFields("a b \"c c c\" d;;d;; \"e\"''`` 'rest") + t.CheckDeepEquals(varnames, []string{ + "run VALUE", + "load OPSYS", + "load endianness", + // "Hello" is not a variable name, the :L modifier makes it an expression. + "load two", + "load TARGETS", + "load SOURCES", + "load OTHER_FILE", - t.CheckDeepEquals(words, []string{"a", "b", "\"c c c\"", "d;;d;;", "\"e\"''``"}) - // TODO: c.Check(rest, equals, "'rest") + "run VAR.${param}", + "run param", + "run VAR", + "run VAR2", + "run VAR", + "run pattern", + "run ROUND_PARENTHESES", + // Shell variables are ignored here. + "run <", + "run @", + "run x"}) + t.CheckOutputLines( + "WARN: Makefile:12: Please use curly braces {} instead of round parentheses () for ROUND_PARENTHESES.", + "WARN: Makefile:14: $x is ambiguous. Use ${x} if you mean a Make variable or $$x if you mean a shell variable.") } -func (s *Suite) Test_MkLine_ValueTokens(c *check.C) { +func (s *Suite) Test_MkLine_UnquoteShell(c *check.C) { t := s.Init(c) - b := NewMkTokenBuilder() - text := b.TextToken - varUseText := b.VaruseTextToken - tokens := b.Tokens - test := func(value string, expected []*MkToken, diagnostics ...string) { - mkline := t.NewMkLine("Makefile", 1, "PATH=\t"+value) - actualTokens, _ := mkline.ValueTokens() - t.CheckDeepEquals(actualTokens, expected) + test := func(input, output string, diagnostics ...string) { + mkline := t.NewMkLine("filename.mk", 1, "") + unquoted := mkline.UnquoteShell(input, true) + t.CheckEquals(unquoted, output) t.CheckOutput(diagnostics) } - t.Use(text, varUseText, tokens, test) - - test("#empty", - tokens()) - - test("value", - tokens(text("value"))) - - test("value ${VAR} rest", - tokens( - text("value "), - varUseText("${VAR}", "VAR"), - text(" rest"))) - - test("value # comment", - tokens( - text("value"))) - - test("value ${UNFINISHED", - tokens( - text("value "), - varUseText("${UNFINISHED", "UNFINISHED")), - - "WARN: Makefile:1: Missing closing \"}\" for \"UNFINISHED\".") -} - -func (s *Suite) Test_MkLine_ValueTokens__parse_error(c *check.C) { - t := s.Init(c) - - mkline := t.NewMkLine("filename.mk", 123, "VAR=\t$") - - tokens, rest := mkline.ValueTokens() - - t.Check(tokens, check.IsNil) - t.CheckEquals(rest, "$") - - // Returns the same values, this time from the cache. - tokens, rest = mkline.ValueTokens() - - t.Check(tokens, check.IsNil) - t.CheckEquals(rest, "$") -} - -func (s *Suite) Test_MkLine_ValueTokens__caching(c *check.C) { - t := s.Init(c) - b := NewMkTokenBuilder() - - mkline := t.NewMkLine("Makefile", 1, "PATH=\tvalue ${UNFINISHED") - valueTokens, rest := mkline.ValueTokens() - - t.CheckDeepEquals(valueTokens, - b.Tokens( - b.TextToken("value "), - b.VaruseTextToken("${UNFINISHED", "UNFINISHED"))) - t.CheckEquals(rest, "") - t.CheckOutputLines( - "WARN: Makefile:1: Missing closing \"}\" for \"UNFINISHED\".") - - // This time the slice is taken from the cache. - tokens2, rest2 := mkline.ValueTokens() - - t.CheckEquals(&tokens2[0], &valueTokens[0]) - t.CheckEquals(rest2, rest) -} - -func (s *Suite) Test_MkLine_ValueTokens__caching_parse_error(c *check.C) { - t := s.Init(c) - b := NewMkTokenBuilder() - - mkline := t.NewMkLine("Makefile", 1, "PATH=\t${UNFINISHED") - valueTokens, rest := mkline.ValueTokens() - - t.CheckDeepEquals(valueTokens, b.Tokens(b.VaruseTextToken("${UNFINISHED", "UNFINISHED"))) - t.CheckEquals(rest, "") - t.CheckOutputLines( - "WARN: Makefile:1: Missing closing \"}\" for \"UNFINISHED\".") - - // This time the slice is taken from the cache. - tokens2, rest2 := mkline.ValueTokens() - - t.CheckEquals(&tokens2[0], &valueTokens[0]) - t.CheckEquals(rest2, rest) -} - -func (s *Suite) Test_MkLine_ValueTokens__warnings(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("Makefile", - MkCvsID, - "ROUND=\t$(ROUND)") - - mklines.mklines[1].ValueTokens() - mklines.Check() - - t.CheckOutputLines( - "WARN: Makefile:2: Please use curly braces {} instead of round parentheses () for ROUND.") -} - -func (s *Suite) Test_MkLine_Tokenize__commented_varassign(c *check.C) { - t := s.Init(c) - - mkline := t.NewMkLine("filename.mk", 123, "#VAR=\tvalue ${VAR} suffix text") + test("", "") + test("plain", "plain") + test("plain words", "plain words") + test("\"dquot\"", "dquot") + test("\"dquot \\\"escaped\\\\\"", "dquot \"escaped\\") + test("'squot \\\"escaped\\\\'", "squot \\\"escaped\\\\") + test("'squot,''squot'", "squot,squot") + test("\"dquot,\"'squot'", "dquot,squot") + test("\"'\",'\"'", "',\"") + test("\\\" \\\\", "\" \\") - t.Check(mkline.Tokenize(mkline.Value(), false), check.HasLen, 3) -} + // UnquoteShell does not parse shell variable expansions or even subshells. + // It therefore must cope with unexpected input and make the best out of it. -func (s *Suite) Test_MkLine_ResolveVarsInRelativePath(c *check.C) { - t := s.Init(c) + test("\\", "") + test("\"\\", "") + test("'", "") - t.CreateFileLines("lang/lua53/Makefile") - t.CreateFileLines("lang/php72/Makefile") - t.CreateFileLines("emulators/suse100_base/Makefile") - t.CreateFileLines("lang/python36/Makefile") - mklines := t.SetUpFileMkLines("Makefile", - MkCvsID) - mkline := mklines.mklines[0] + test("\"$(\"", "$(\"") - test := func(before string, after string) { - t.CheckEquals(mkline.ResolveVarsInRelativePath(before), after) - } + test("`", "`") - test("", ".") - test("${PKGSRCDIR}", ".") - test("${LUA_PKGSRCDIR}", "../../lang/lua53") - test("${PHPPKGSRCDIR}", "../../lang/php72") - test("${SUSE_DIR_PREFIX}", "suse100") - test("${PYPKGSRCDIR}", "../../lang/python36") - test("${PYPACKAGE}", "python36") - test("${FILESDIR}", "${FILESDIR}") - test("${PKGDIR}", "${PKGDIR}") + // Quotes inside a varuse are not unquoted. + test("${VAR}", "${VAR}") + test("${VAR:S,',',g}", "${VAR:S,',',g}") - G.Pkg = NewPackage(t.File("category/package")) + test("\"*?[\"", "*?[") + test("'*?['", "*?[") - test("${FILESDIR}", "files") - test("${PKGDIR}", ".") + test("*?[", "*?[", + "WARN: filename.mk:1: The \"*\" in the word \"*?[\" may lead to unintended file globbing.", + "WARN: filename.mk:1: The \"?\" in the word \"*?[\" may lead to unintended file globbing.", + "WARN: filename.mk:1: The \"[\" in the word \"*?[\" may lead to unintended file globbing.") - // Just for branch coverage. - G.Testing = false - test("${PKGSRCDIR}", "../..") + test("'single'*\"double\"", "single*double", + "WARN: filename.mk:1: The \"*\" in the word \"'single'*\\\"double\\\"\" "+ + "may lead to unintended file globbing.") } -func (s *Suite) Test_MkLine_ResolveVarsInRelativePath__directory_depth(c *check.C) { +func (s *Suite) Test_NewMkOperator(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - mklines := t.SetUpFileMkLines("multimedia/totem/filename.mk", - MkCvsID, - "BUILDLINK_PKGSRCDIR.totem?=\t../../multimedia/totem") - - mklines.Check() + t.CheckEquals(NewMkOperator(":="), opAssignEval) + t.CheckEquals(NewMkOperator("="), opAssign) - t.CheckOutputLines( - "WARN: ~/multimedia/totem/filename.mk:2: "+ - "The variable BUILDLINK_PKGSRCDIR.totem should not be given a default value in this file; "+ - "it would be ok in buildlink3.mk.", - "ERROR: ~/multimedia/totem/filename.mk:2: Relative path \"../../multimedia/totem/Makefile\" does not exist.") + c.Check(func() { NewMkOperator("???") }, check.Panics, "Invalid operator: ???") } -// Just for code coverage -func (s *Suite) Test_MkLine_ResolveVarsInRelativePath__without_tracing(c *check.C) { +func (s *Suite) Test_VarUseContext_String(c *check.C) { t := s.Init(c) - t.DisableTracing() t.SetUpVartypes() - mklines := t.SetUpFileMkLines("buildlink3.mk", - MkCvsID, - "BUILDLINK_PKGSRCDIR.totem?=\t../../${PKGPATH.multimedia/totem}") - - mklines.Check() - - t.CheckOutputLines( - "WARN: ~/buildlink3.mk:2: PKGPATH.multimedia/totem is used but not defined.") -} - -func (s *Suite) Test_NewMkOperator(c *check.C) { - t := s.Init(c) - - t.CheckEquals(NewMkOperator(":="), opAssignEval) - t.CheckEquals(NewMkOperator("="), opAssign) + vartype := G.Pkgsrc.VariableType(nil, "PKGNAME") + vuc := VarUseContext{vartype, VucUnknownTime, VucQuotBackt, false} - c.Check(func() { NewMkOperator("???") }, check.Panics, "Invalid operator: ???") + t.CheckEquals(vuc.String(), "(Pkgname (package-settable) time:unknown quoting:backt wordpart:false)") } func (s *Suite) Test_Indentation(c *check.C) { @@ -1444,44 +1374,6 @@ func (s *Suite) Test_Indentation_RememberUsedVariables(c *check.C) { t.CheckDeepEquals(ind.Varnames(), []string{"PKGREVISION"}) } -func (s *Suite) Test_Indentation_TrackAfter__checked_files(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("file.mk", - MkCvsID, - "", - ".if make(other.mk)", - ". include \"other.mk\"", - ".endif", - "", - ".if exists(checked.mk)", - ". include \"checked.mk\"", - ".elif exists(other-checked.mk)", - ". include \"other-checked.mk\"", - ".endif") - - mklines.Check() - - t.CheckOutputLines( - "ERROR: file.mk:4: Relative path \"other.mk\" does not exist.") -} - -func (s *Suite) Test_Indentation_TrackAfter__lonely_else(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("file.mk", - MkCvsID, - "", - ".else") - - mklines.Check() - - // Surprisingly, pkglint doesn't report an error about this trivial bug. - // This will be caught by bmake, though. Therefore the only purpose of - // this test is the branch coverage in the "top.mkline != nil" case. - t.CheckOutputEmpty() -} - func (s *Suite) Test_Indentation_Varnames__repetition(c *check.C) { t := s.Init(c) @@ -1507,105 +1399,42 @@ func (s *Suite) Test_Indentation_Varnames__repetition(c *check.C) { "conditionally in buildlink3.mk:14 (depending on OPSYS).") } -func (s *Suite) Test_MkLine_ForEachUsed(c *check.C) { +func (s *Suite) Test_Indentation_TrackAfter__checked_files(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("Makefile", + mklines := t.NewMkLines("file.mk", MkCvsID, - "VAR=\t${VALUE} # ${varassign.comment}", - ".if ${OPSYS:M${endianness}} == ${Hello:L} # ${if.comment}", - ".for var in one ${two} three # ${for.comment}", - "# ${empty.comment}", - "${TARGETS}: ${SOURCES} # ${dependency.comment}", - ".include \"${OTHER_FILE}\"", "", - "\t${VAR.${param}}", - "\t${VAR}and${VAR2}", - "\t${VAR:M${pattern}}", - "\t$(ROUND_PARENTHESES)", - "\t$$shellvar", - "\t$< $@ $x") - - var varnames []string - for _, mkline := range mklines.mklines { - mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { - varnames = append(varnames, time.String()+" "+varUse.varname) - }) - } + ".if make(other.mk)", + ". include \"other.mk\"", + ".endif", + "", + ".if exists(checked.mk)", + ". include \"checked.mk\"", + ".elif exists(other-checked.mk)", + ". include \"other-checked.mk\"", + ".endif") - t.CheckDeepEquals(varnames, []string{ - "run VALUE", - "load OPSYS", - "load endianness", - // "Hello" is not a variable name, the :L modifier makes it an expression. - "load two", - "load TARGETS", - "load SOURCES", - "load OTHER_FILE", + mklines.Check() - "run VAR.${param}", - "run param", - "run VAR", - "run VAR2", - "run VAR", - "run pattern", - "run ROUND_PARENTHESES", - // Shell variables are ignored here. - "run <", - "run @", - "run x"}) t.CheckOutputLines( - "WARN: Makefile:12: Please use curly braces {} instead of round parentheses () for ROUND_PARENTHESES.", - "WARN: Makefile:14: $x is ambiguous. Use ${x} if you mean a Make variable or $$x if you mean a shell variable.") + "ERROR: file.mk:4: Relative path \"other.mk\" does not exist.") } -func (s *Suite) Test_MkLine_UnquoteShell(c *check.C) { +func (s *Suite) Test_Indentation_TrackAfter__lonely_else(c *check.C) { t := s.Init(c) - test := func(input, output string, diagnostics ...string) { - mkline := t.NewMkLine("filename.mk", 1, "") - unquoted := mkline.UnquoteShell(input, true) - t.CheckEquals(unquoted, output) - t.CheckOutput(diagnostics) - } - - test("", "") - test("plain", "plain") - test("plain words", "plain words") - test("\"dquot\"", "dquot") - test("\"dquot \\\"escaped\\\\\"", "dquot \"escaped\\") - test("'squot \\\"escaped\\\\'", "squot \\\"escaped\\\\") - test("'squot,''squot'", "squot,squot") - test("\"dquot,\"'squot'", "dquot,squot") - test("\"'\",'\"'", "',\"") - test("\\\" \\\\", "\" \\") - - // UnquoteShell does not parse shell variable expansions or even subshells. - // It therefore must cope with unexpected input and make the best out of it. - - test("\\", "") - test("\"\\", "") - test("'", "") - - test("\"$(\"", "$(\"") - - test("`", "`") - - // Quotes inside a varuse are not unquoted. - test("${VAR}", "${VAR}") - test("${VAR:S,',',g}", "${VAR:S,',',g}") - - test("\"*?[\"", "*?[") - test("'*?['", "*?[") + mklines := t.NewMkLines("file.mk", + MkCvsID, + "", + ".else") - test("*?[", "*?[", - "WARN: filename.mk:1: The \"*\" in the word \"*?[\" may lead to unintended file globbing.", - "WARN: filename.mk:1: The \"?\" in the word \"*?[\" may lead to unintended file globbing.", - "WARN: filename.mk:1: The \"[\" in the word \"*?[\" may lead to unintended file globbing.") + mklines.Check() - test("'single'*\"double\"", "single*double", - "WARN: filename.mk:1: The \"*\" in the word \"'single'*\\\"double\\\"\" "+ - "may lead to unintended file globbing.") + // Surprisingly, pkglint doesn't report an error about this trivial bug. + // This will be caught by bmake, though. Therefore the only purpose of + // this test is the branch coverage in the "top.mkline != nil" case. + t.CheckOutputEmpty() } func (s *Suite) Test_MatchMkInclude(c *check.C) { diff --git a/pkgtools/pkglint/files/mklinechecker.go b/pkgtools/pkglint/files/mklinechecker.go index ef95ef53894..b58040d4b79 100644 --- a/pkgtools/pkglint/files/mklinechecker.go +++ b/pkgtools/pkglint/files/mklinechecker.go @@ -53,275 +53,184 @@ func (ck MkLineChecker) checkEmptyContinuation() { } } -func (ck MkLineChecker) checkComment() { - mkline := ck.MkLine - - if hasPrefix(mkline.Text, "# url2pkg-marker") { - mkline.Errorf("This comment indicates unfinished work (url2pkg).") - } +func (ck MkLineChecker) checkVarassign() { + ck.checkVarassignLeft() + ck.checkVarassignOp() + ck.checkVarassignRight() } -func (ck MkLineChecker) checkShellCommand() { - mkline := ck.MkLine - - shellCommand := mkline.ShellCommand() - if G.Opts.WarnSpace && hasPrefix(mkline.Text, "\t\t") { - lexer := textproc.NewLexer(mkline.raw[0].textnl) - tabs := lexer.NextBytesFunc(func(b byte) bool { return b == '\t' }) - - fix := mkline.Autofix() - fix.Notef("Shell programs should be indented with a single tab.") - fix.Explain( - "The first tab in the line marks the line as a shell command.", - "Since every line of shell commands starts with a completely new shell environment,", - "there is no need to indent some of the commands,", - "or to use more horizontal space than necessary.") +// checkVarassignLeft checks everything to the left of the assignment operator. +func (ck MkLineChecker) checkVarassignLeft() { + varname := ck.MkLine.Varname() + if hasPrefix(varname, "_") && !G.Infrastructure && G.Pkgsrc.vartypes.Canon(varname) == nil { + ck.MkLine.Warnf("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", varname) + } - for i, raw := range mkline.Line.raw { - if hasPrefix(raw.textnl, tabs) { - fix.ReplaceAt(i, 0, tabs, "\t") - } - } - fix.Apply() + ck.checkVarassignLeftNotUsed() + ck.checkVarassignLeftDeprecated() + ck.checkVarassignLeftBsdPrefs() + if !ck.checkVarassignLeftUserSettable() { + ck.checkVarassignLeftPermissions() } + ck.checkVarassignLeftRationale() - ck.checkText(shellCommand) - NewShellLineChecker(ck.MkLines, mkline).CheckShellCommandLine(shellCommand) + ck.checkTextVarUse( + ck.MkLine.Varname(), + NewVartype(BtVariableName, NoVartypeOptions, NewACLEntry("*", aclpAll)), + VucLoadTime) } -func (ck MkLineChecker) checkInclude() { - if trace.Tracing { - defer trace.Call0()() - } +// checkVarassignLeftNotUsed checks whether the left-hand side of a variable +// assignment is not used. If it is unused and also doesn't have a predefined +// data type, it may be a spelling mistake. +func (ck MkLineChecker) checkVarassignLeftNotUsed() { + varname := ck.MkLine.Varname() + varcanon := varnameCanon(varname) - mkline := ck.MkLine - if mkline.Indent() != "" { - ck.checkDirectiveIndentation(ck.MkLines.indentation.Depth("include")) + if ck.MkLine.Op() == opAssignEval && varname == strings.ToLower(varname) { + if trace.Tracing { + trace.Step1("%s might be unused unless it is an argument to a procedure file.", varname) + } + return } - includedFile := mkline.IncludedFile() - mustExist := mkline.MustExist() - if trace.Tracing { - trace.Step2("includingFile=%s includedFile=%s", mkline.Filename, includedFile) + if ck.MkLines.vars.IsUsedSimilar(varname) { + return } - ck.CheckRelativePath(includedFile, mustExist) - - switch { - case hasSuffix(includedFile, "/Makefile"): - mkline.Errorf("Other Makefiles must not be included directly.") - mkline.Explain( - "To include portions of another Makefile, extract the common parts", - "and put them into a Makefile.common or a Makefile fragment called", - "module.mk or similar.", - "After that, both this one and the other package should include the newly created file.") - - case IsPrefs(includedFile): - if mkline.Basename == "buildlink3.mk" && includedFile == "../../mk/bsd.prefs.mk" { - fix := mkline.Autofix() - fix.Notef("For efficiency reasons, please include bsd.fast.prefs.mk instead of bsd.prefs.mk.") - fix.Replace("bsd.prefs.mk", "bsd.fast.prefs.mk") - fix.Apply() - } - - case hasSuffix(includedFile, "pkgtools/x11-links/buildlink3.mk"): - fix := mkline.Autofix() - fix.Errorf("%s must not be included directly. Include \"../../mk/x11.buildlink3.mk\" instead.", includedFile) - fix.Replace("pkgtools/x11-links/buildlink3.mk", "mk/x11.buildlink3.mk") - fix.Apply() - case hasSuffix(includedFile, "graphics/jpeg/buildlink3.mk"): - fix := mkline.Autofix() - fix.Errorf("%s must not be included directly. Include \"../../mk/jpeg.buildlink3.mk\" instead.", includedFile) - fix.Replace("graphics/jpeg/buildlink3.mk", "mk/jpeg.buildlink3.mk") - fix.Apply() - - case hasSuffix(includedFile, "/intltool/buildlink3.mk"): - mkline.Warnf("Please write \"USE_TOOLS+= intltool\" instead of this line.") - - case hasSuffix(includedFile, "/builtin.mk"): - if mkline.Basename != "hacks.mk" && !mkline.HasRationale() { - fix := mkline.Autofix() - fix.Errorf("%s must not be included directly. Include \"%s/buildlink3.mk\" instead.", includedFile, path.Dir(includedFile)) - fix.Replace("builtin.mk", "buildlink3.mk") - fix.Apply() - } + if G.Pkg != nil && G.Pkg.vars.IsUsedSimilar(varname) { + return } -} - -func (ck MkLineChecker) checkDirective(forVars map[string]bool, ind *Indentation) { - mkline := ck.MkLine - directive := mkline.Directive() - args := mkline.Args() - - expectedDepth := ind.Depth(directive) - ck.checkDirectiveIndentation(expectedDepth) - - if directive == "endfor" || directive == "endif" { - ck.checkDirectiveEnd(ind) + vartypes := G.Pkgsrc.vartypes + if vartypes.IsDefinedExact(varname) || vartypes.IsDefinedExact(varcanon) { + return } - needsArgument := false - switch directive { - case - "if", "ifdef", "ifndef", "elif", - "for", "undef", - "error", "warning", "info", - "export", "export-env", "unexport", "unexport-env": - needsArgument = true + deprecated := G.Pkgsrc.Deprecated + if deprecated[varname] != "" || deprecated[varcanon] != "" { + return } - switch { - case needsArgument && args == "": - mkline.Errorf("\".%s\" requires arguments.", directive) - - case !needsArgument && args != "": - if directive == "else" { - mkline.Errorf("\".%s\" does not take arguments. If you meant \"else if\", use \".elif\".", directive) - } else { - mkline.Errorf("\".%s\" does not take arguments.", directive) - } - - case directive == "if" || directive == "elif": - ck.checkDirectiveCond() - - case directive == "ifdef" || directive == "ifndef": - mkline.Warnf("The \".%s\" directive is deprecated. Please use \".if %sdefined(%s)\" instead.", - directive, condStr(directive == "ifdef", "", "!"), args) + if !ck.MkLines.once.FirstTimeSlice("defined but not used: ", varname) { + return + } - case directive == "for": - ck.checkDirectiveFor(forVars, ind) + ck.MkLine.Warnf("%s is defined but not used.", varname) + ck.MkLine.Explain( + "This might be a simple typo.", + "", + "If a package provides a file containing several related variables", + "(such as module.mk, app.mk, extension.mk), that file may define", + "variables that look unused since they are only used by other packages.", + "These variables should be documented at the head of the file;", + "see mk/subst.mk for an example of such a documentation comment.") +} - case directive == "undef": - for _, varname := range mkline.Fields() { - if forVars[varname] { - mkline.Notef("Using \".undef\" after a \".for\" loop is unnecessary.") - } - } +func (ck MkLineChecker) checkVarassignLeftDeprecated() { + varname := ck.MkLine.Varname() + if fix := G.Pkgsrc.Deprecated[varname]; fix != "" { + ck.MkLine.Warnf("Definition of %s is deprecated. %s", varname, fix) + } else if fix = G.Pkgsrc.Deprecated[varnameCanon(varname)]; fix != "" { + ck.MkLine.Warnf("Definition of %s is deprecated. %s", varname, fix) } } -func (ck MkLineChecker) checkDirectiveEnd(ind *Indentation) { +func (ck MkLineChecker) checkVarassignLeftBsdPrefs() { mkline := ck.MkLine - directive := mkline.Directive() - comment := mkline.DirectiveComment() - if ind.Empty() { - mkline.Errorf("Unmatched .%s.", directive) + switch mkline.Varcanon() { + case "BUILDLINK_PKGSRCDIR.*", + "BUILDLINK_DEPMETHOD.*", + "BUILDLINK_ABI_DEPENDS.*", + "BUILDLINK_INCDIRS.*", + "BUILDLINK_LIBDIRS.*": return } - if comment == "" { + if !G.Opts.WarnExtra || + G.Infrastructure || + mkline.Op() != opAssignDefault || + ck.MkLines.Tools.SeenPrefs || + !ck.MkLines.once.FirstTime("include bsd.prefs.mk before using ?=") { return } - if directive == "endif" { - if args := ind.Args(); !contains(args, comment) { - mkline.Warnf("Comment %q does not match condition %q.", comment, args) - } + // Package-settable variables may use the ?= operator before including + // bsd.prefs.mk in situations like the following: + // + // Makefile: LICENSE= package-license + // .include "module.mk" + // module.mk: LICENSE?= default-license + // + vartype := G.Pkgsrc.VariableType(nil, mkline.Varname()) + if vartype != nil && vartype.IsPackageSettable() { + return } - if directive == "endfor" { - if args := ind.Args(); !contains(args, comment) { - mkline.Warnf("Comment %q does not match loop %q.", comment, args) - } - } + mkline.Warnf("Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".") + mkline.Explain( + "The ?= operator is used to provide a default value to a variable.", + "In pkgsrc, many variables can be set by the pkgsrc user in the", + "mk.conf file.", + "This file must be included explicitly.", + "If a ?= operator appears before mk.conf has been included,", + "it will not care about the user's preferences,", + "which can result in unexpected behavior.", + "", + "The easiest way to include the mk.conf file is by including the", + "bsd.prefs.mk file, which will take care of everything.") } -func (ck MkLineChecker) checkDirectiveFor(forVars map[string]bool, indentation *Indentation) { +// checkVarassignLeftUserSettable checks whether a package defines a +// variable that is marked as user-settable since it is defined in +// mk/defaults/mk.conf. +func (ck MkLineChecker) checkVarassignLeftUserSettable() bool { mkline := ck.MkLine - args := mkline.Args() - - if m, vars, _ := match2(args, `^([^\t ]+(?:[\t ]*[^\t ]+)*?)[\t ]+in[\t ]+(.*)$`); m { - for _, forvar := range strings.Fields(vars) { - indentation.AddVar(forvar) - if !G.Infrastructure && hasPrefix(forvar, "_") { - mkline.Warnf("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", forvar) - } - - if matches(forvar, `^[_a-z][_a-z0-9]*$`) { - // Fine. - } else if matches(forvar, `^[A-Z_a-z][0-9A-Z_a-z]*$`) { - mkline.Warnf("The variable name %q in the .for loop should not contain uppercase letters.", forvar) - } else { - mkline.Errorf("Invalid variable name %q.", forvar) - } - - forVars[forvar] = true - } + varname := mkline.Varname() - // XXX: The type BtUnknown is very unspecific here. For known variables - // or constant values this could probably be improved. - // - // The guessed flag could also be determined more correctly. As of November 2018, - // running pkglint over the whole pkgsrc tree did not produce any different result - // whether guessed was true or false. - forLoopType := NewVartype(btForLoop, List, NewACLEntry("*", aclpAllRead)) - forLoopContext := VarUseContext{forLoopType, VucLoadTime, VucQuotPlain, false} - mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { - ck.CheckVaruse(varUse, &forLoopContext) - }) + defaultMkline := G.Pkgsrc.UserDefinedVars.Mentioned(varname) + if defaultMkline == nil { + return false } -} + defaultValue := defaultMkline.Value() -func (ck MkLineChecker) checkDirectiveIndentation(expectedDepth int) { - if !G.Opts.WarnSpace { - return - } - mkline := ck.MkLine - indent := mkline.Indent() - if expected := strings.Repeat(" ", expectedDepth); indent != expected { - fix := mkline.Line.Autofix() - fix.Notef("This directive should be indented by %d spaces.", expectedDepth) - fix.ReplaceRegex(regex.Pattern(`^\.`+indent), "."+expected, 1) - fix.Apply() + // A few of the user-settable variables can also be set by packages. + // That's an unfortunate situation since there is no definite source + // of truth, but luckily only a few variables make use of it. + vartype := G.Pkgsrc.VariableType(ck.MkLines, varname) + if vartype.IsPackageSettable() { + return true } -} -func (ck MkLineChecker) checkDependencyRule(allowedTargets map[string]bool) { - mkline := ck.MkLine - targets := mkline.ValueFields(mkline.Targets()) - sources := mkline.ValueFields(mkline.Sources()) + switch { + case mkline.HasComment(): + // Assume that the comment contains a rationale for disabling + // this particular check. - for _, source := range sources { - if source == ".PHONY" { - for _, target := range targets { - allowedTargets[target] = true - } - } - } - for _, target := range targets { - if target == ".PHONY" { - for _, source := range sources { - allowedTargets[source] = true - } - } - } + case mkline.Op() == opAssignAppend: + mkline.Warnf("Packages should not append to user-settable %s.", varname) - for _, target := range targets { - ck.checkDependencyTarget(target, allowedTargets) - } -} + case defaultValue != mkline.Value(): + mkline.Warnf( + "Package sets user-defined %q to %q, which differs "+ + "from the default value %q from mk/defaults/mk.conf.", + varname, mkline.Value(), defaultValue) -func (ck MkLineChecker) checkDependencyTarget(target string, allowedTargets map[string]bool) { - if target == ".PHONY" || - target == ".ORDER" || - NewMkParser(nil, target).VarUse() != nil || - allowedTargets[target] { - return + case defaultMkline.IsCommentedVarassign(): + // Since the variable assignment is commented out in + // mk/defaults/mk.conf, the package has to define it. + + default: + mkline.Notef("Redundant definition for %s from mk/defaults/mk.conf.", varname) + if !ck.MkLines.Tools.SeenPrefs { + mkline.Explain( + "Instead of defining the variable redundantly, it suffices to include", + "../../mk/bsd.prefs.mk, which provides all user-settable variables.") + } } - mkline := ck.MkLine - mkline.Warnf("Undeclared target %q.", target) - mkline.Explain( - "To define a custom target in a package, declare it like this:", - "", - "\t.PHONY: my-target", - "", - "To define a custom target that creates a file (should be rarely needed),", - "declare it like this:", - "", - "\t${.CURDIR}/my-file:") + return true } // checkVarassignLeftPermissions checks the permissions for the left-hand side @@ -472,6 +381,27 @@ func (ck MkLineChecker) checkVarassignLeftRationale() { "* has it been reported upstream?") } +func (ck MkLineChecker) checkTextVarUse(text string, vartype *Vartype, time VucTime) { + if !contains(text, "$") { + return + } + + if trace.Tracing { + defer trace.Call(vartype, time)() + } + + tokens := NewMkParser(nil, text).MkTokens() + for i, token := range tokens { + if token.Varuse != nil { + spaceLeft := i-1 < 0 || matches(tokens[i-1].Text, `[\t ]$`) + spaceRight := i+1 >= len(tokens) || matches(tokens[i+1].Text, `^[\t ]`) + isWordPart := !(spaceLeft && spaceRight) + vuc := VarUseContext{vartype, time, VucQuotPlain, isWordPart} + ck.CheckVaruse(token.Varuse, &vuc) + } + } +} + // CheckVaruse checks a single use of a variable in a specific context. func (ck MkLineChecker) CheckVaruse(varuse *MkVarUse, vuc *VarUseContext) { mkline := ck.MkLine @@ -496,52 +426,17 @@ func (ck MkLineChecker) CheckVaruse(varuse *MkVarUse, vuc *VarUseContext) { ck.checkTextVarUse(varname, vartype, vuc.time) } -func (ck MkLineChecker) checkVarUseVarname(varuse *MkVarUse) { - if varuse.varname == "@" { - ck.MkLine.Warnf("Please use %q instead of %q.", "${.TARGET}", "$@") - ck.MkLine.Explain( - "It is more readable and prevents confusion with the shell variable", - "of the same name.") - } - - if varuse.varname == "LOCALBASE" && !G.Infrastructure { - fix := ck.MkLine.Autofix() - fix.Warnf("Please use PREFIX instead of LOCALBASE.") - fix.ReplaceRegex(`\$\{LOCALBASE\b`, "${PREFIX", 1) - fix.Apply() - } -} - -func (ck MkLineChecker) checkVarUseBuildDefs(varname string) { - if !(G.Pkgsrc.UserDefinedVars.Defined(varname) && !G.Pkgsrc.IsBuildDef(varname)) { - return - } - - if !(!ck.MkLines.buildDefs[varname] && ck.MkLines.once.FirstTimeSlice("BUILD_DEFS", varname)) { - return - } - - ck.MkLine.Warnf("The user-defined variable %s is used but not added to BUILD_DEFS.", varname) - ck.MkLine.Explain( - "When a pkgsrc package is built, many things can be configured by the", - "pkgsrc user in the mk.conf file.", - "All these configurations should be recorded in the binary package", - "so the package can be reliably rebuilt.", - "The BUILD_DEFS variable contains a list of all these", - "user-settable variables, so please add your variable to it, too.") -} - func (ck MkLineChecker) checkVaruseUndefined(vartype *Vartype, varname string) { switch { case !G.Opts.WarnExtra, // Well-known variables are probably defined by the infrastructure. - vartype != nil && !vartype.Guessed(), - ck.MkLines.vars.DefinedSimilar(varname), + vartype != nil && !vartype.IsGuessed(), + ck.MkLines.vars.IsDefinedSimilar(varname), ck.MkLines.forVars[varname], ck.MkLines.vars.Mentioned(varname) != nil, - G.Pkg != nil && G.Pkg.vars.DefinedSimilar(varname), + G.Pkg != nil && G.Pkg.vars.IsDefinedSimilar(varname), containsVarRef(varname), - G.Pkgsrc.vartypes.DefinedCanon(varname), + G.Pkgsrc.vartypes.IsDefinedCanon(varname), varname == "": return } @@ -567,7 +462,7 @@ func (ck MkLineChecker) checkVaruseModifiers(varuse *MkVarUse, vartype *Vartype) } func (ck MkLineChecker) checkVaruseModifiersSuffix(varuse *MkVarUse, vartype *Vartype) { - if varuse.modifiers[0].IsSuffixSubst() && vartype != nil && !vartype.List() { + if varuse.modifiers[0].IsSuffixSubst() && vartype != nil && !vartype.IsList() { ck.MkLine.Warnf("The :from=to modifier should only be used with lists, not with %s.", varuse.varname) ck.MkLine.Explain( "Instead of (for example):", @@ -604,6 +499,22 @@ func (ck MkLineChecker) checkVaruseModifiersRange(varuse *MkVarUse) { } } +func (ck MkLineChecker) checkVarUseVarname(varuse *MkVarUse) { + if varuse.varname == "@" { + ck.MkLine.Warnf("Please use %q instead of %q.", "${.TARGET}", "$@") + ck.MkLine.Explain( + "It is more readable and prevents confusion with the shell variable", + "of the same name.") + } + + if varuse.varname == "LOCALBASE" && !G.Infrastructure { + fix := ck.MkLine.Autofix() + fix.Warnf("Please use PREFIX instead of LOCALBASE.") + fix.ReplaceRegex(`\$\{LOCALBASE\b`, "${PREFIX", 1) + fix.Apply() + } +} + // checkVarusePermissions checks the permissions when a variable is used, // be it in a variable assignment, in a shell command, a conditional, or // somewhere else. @@ -635,7 +546,7 @@ func (ck MkLineChecker) checkVarusePermissions(varname string, vartype *Vartype, return } - if vartype.Guessed() { + if vartype.IsGuessed() { return } @@ -717,7 +628,7 @@ func (ck MkLineChecker) warnVarusePermissions( // Some of the guessed variables may be used at load time. But since the // variable type and these permissions are guessed, pkglint should not // issue the following warning, since it is often wrong. - if vucVartype.Guessed() { + if vucVartype.IsGuessed() { return } @@ -839,12 +750,12 @@ func (ck MkLineChecker) checkVarUseQuoting(varUse *MkVarUse, vartype *Vartype, v // since the GNU configure scripts cannot handle these space characters. // // When doing checks outside a package, the :M* modifier is needed for safety. - needMstar := (G.Pkg == nil || G.Pkg.vars.Defined("GNU_CONFIGURE")) && + needMstar := (G.Pkg == nil || G.Pkg.vars.IsDefined("GNU_CONFIGURE")) && matches(varname, `^(?:.*_)?(?:CFLAGS|CPPFLAGS|CXXFLAGS|FFLAGS|LDFLAGS|LIBS)$`) mkline := ck.MkLine if mod == ":M*:Q" && !needMstar { - if !vartype.Guessed() { + if !vartype.IsGuessed() { mkline.Notef("The :M* modifier is not needed here.") } @@ -860,7 +771,7 @@ func (ck MkLineChecker) checkVarUseQuoting(varUse *MkVarUse, vartype *Vartype, v } varinfo := G.Pkg.redundant.vars[varname] - if varinfo == nil || !varinfo.vari.Constant() { + if varinfo == nil || !varinfo.vari.IsConstant() { return false } @@ -868,11 +779,11 @@ func (ck MkLineChecker) checkVarUseQuoting(varUse *MkVarUse, vartype *Vartype, v return len(mkline.ValueFields(value)) == 1 } - if vartype.List() && isSingleWordConstant() { + if vartype.IsList() && isSingleWordConstant() { // Do not warn in this special case, which typically occurs // for BUILD_DIRS or similar package-settable variables. - } else if vartype.List() { + } else if vartype.IsList() { mkline.Warnf("The list variable %s should not be embedded in a word.", varname) mkline.Explain( "When a list variable has multiple elements, this expression expands", @@ -962,6 +873,25 @@ func (ck MkLineChecker) checkVarUseQuoting(varUse *MkVarUse, vartype *Vartype, v } } +func (ck MkLineChecker) checkVarUseBuildDefs(varname string) { + if !(G.Pkgsrc.UserDefinedVars.IsDefined(varname) && !G.Pkgsrc.IsBuildDef(varname)) { + return + } + + if !(!ck.MkLines.buildDefs[varname] && ck.MkLines.once.FirstTimeSlice("BUILD_DEFS", varname)) { + return + } + + ck.MkLine.Warnf("The user-defined variable %s is used but not added to BUILD_DEFS.", varname) + ck.MkLine.Explain( + "When a pkgsrc package is built, many things can be configured by the", + "pkgsrc user in the mk.conf file.", + "All these configurations should be recorded in the binary package", + "so the package can be reliably rebuilt.", + "The BUILD_DEFS variable contains a list of all these", + "user-settable variables, so please add your variable to it, too.") +} + func (ck MkLineChecker) checkVaruseDeprecated(varuse *MkVarUse) { varname := varuse.varname instead := G.Pkgsrc.Deprecated[varname] @@ -973,57 +903,6 @@ func (ck MkLineChecker) checkVaruseDeprecated(varuse *MkVarUse) { } } -func (ck MkLineChecker) checkVarassignDecreasingVersions() { - mkline := ck.MkLine - strVersions := mkline.Fields() - intVersions := make([]int, len(strVersions)) - for i, strVersion := range strVersions { - iver, err := strconv.Atoi(strVersion) - if err != nil || !(iver > 0) { - mkline.Errorf("Value %q for %s must be a positive integer.", strVersion, mkline.Varname()) - return - } - intVersions[i] = iver - } - - for i, ver := range intVersions { - if i > 0 && ver >= intVersions[i-1] { - mkline.Warnf("The values for %s should be in decreasing order (%d before %d).", - mkline.Varname(), ver, intVersions[i-1]) - mkline.Explain( - "If they aren't, it may be possible that needless versions of", - "packages are installed.") - } - } -} - -func (ck MkLineChecker) checkVarassign() { - ck.checkVarassignLeft() - ck.checkVarassignOp() - ck.checkVarassignRight() -} - -// checkVarassignLeft checks everything to the left of the assignment operator. -func (ck MkLineChecker) checkVarassignLeft() { - varname := ck.MkLine.Varname() - if hasPrefix(varname, "_") && !G.Infrastructure && G.Pkgsrc.vartypes.Canon(varname) == nil { - ck.MkLine.Warnf("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", varname) - } - - ck.checkVarassignLeftNotUsed() - ck.checkVarassignLeftDeprecated() - ck.checkVarassignLeftBsdPrefs() - if !ck.checkVarassignLeftUserSettable() { - ck.checkVarassignLeftPermissions() - } - ck.checkVarassignLeftRationale() - - ck.checkTextVarUse( - ck.MkLine.Varname(), - NewVartype(BtVariableName, NoVartypeOptions, NewACLEntry("*", aclpAll)), - VucLoadTime) -} - func (ck MkLineChecker) checkVarassignOp() { ck.checkVarassignOpShell() } @@ -1043,7 +922,7 @@ func (ck MkLineChecker) checkVarassignOpShell() { // Authors of builtin.mk files usually know what they're doing. return - case G.Pkg == nil || G.Pkg.vars.UsedAtLoadTime(mkline.Varname()): + case G.Pkg == nil || G.Pkg.vars.IsUsedAtLoadTime(mkline.Varname()): return } @@ -1095,137 +974,140 @@ func (ck MkLineChecker) checkVarassignRight() { ck.checkVarassignRightVaruse() } -func (ck MkLineChecker) checkVarassignLeftDeprecated() { - varname := ck.MkLine.Varname() - if fix := G.Pkgsrc.Deprecated[varname]; fix != "" { - ck.MkLine.Warnf("Definition of %s is deprecated. %s", varname, fix) - } else if fix = G.Pkgsrc.Deprecated[varnameCanon(varname)]; fix != "" { - ck.MkLine.Warnf("Definition of %s is deprecated. %s", varname, fix) - } -} - -// checkVarassignLeftNotUsed checks whether the left-hand side of a variable -// assignment is not used. If it is unused and also doesn't have a predefined -// data type, it may be a spelling mistake. -func (ck MkLineChecker) checkVarassignLeftNotUsed() { - varname := ck.MkLine.Varname() - varcanon := varnameCanon(varname) - - if ck.MkLine.Op() == opAssignEval && varname == strings.ToLower(varname) { - if trace.Tracing { - trace.Step1("%s might be unused unless it is an argument to a procedure file.", varname) - } - return - } - - if ck.MkLines.vars.UsedSimilar(varname) { - return - } - - if G.Pkg != nil && G.Pkg.vars.UsedSimilar(varname) { - return +// checkText checks the given text (which is typically the right-hand side of a variable +// assignment or a shell command). +// +// Note: checkTextVarUse cannot be called here since it needs to know the context where it is included. +// Maybe that context should be added here as parameters. +func (ck MkLineChecker) checkText(text string) { + if trace.Tracing { + defer trace.Call1(text)() } - vartypes := G.Pkgsrc.vartypes - if vartypes.DefinedExact(varname) || vartypes.DefinedExact(varcanon) { - return - } + ck.checkTextWrksrcDotDot(text) + ck.checkTextRpath(text) +} - deprecated := G.Pkgsrc.Deprecated - if deprecated[varname] != "" || deprecated[varcanon] != "" { - return +func (ck MkLineChecker) checkTextWrksrcDotDot(text string) { + if contains(text, "${WRKSRC}/..") { + ck.MkLine.Warnf("Building the package should take place entirely inside ${WRKSRC}, not \"${WRKSRC}/..\".") + ck.MkLine.Explain( + "WRKSRC should be defined so that there is no need to do anything", + "outside of this directory.", + "", + "Example:", + "", + "\tWRKSRC=\t${WRKDIR}", + "\tCONFIGURE_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src", + "\tBUILD_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src ${WRKSRC}/cmd", + "", + seeGuide("Directories used during the build process", "build.builddirs")) } +} - if !ck.MkLines.once.FirstTimeSlice("defined but not used: ", varname) { - return +// checkTextPath checks for literal -Wl,--rpath options. +// +// Note: A simple -R is not detected, as the rate of false positives is too high. +func (ck MkLineChecker) checkTextRpath(text string) { + if m, flag := match1(text, `(-Wl,--rpath,|-Wl,-rpath-link,|-Wl,-rpath,|-Wl,-R\b)`); m { + ck.MkLine.Warnf("Please use ${COMPILER_RPATH_FLAG} instead of %q.", flag) } - - ck.MkLine.Warnf("%s is defined but not used.", varname) - ck.MkLine.Explain( - "This might be a simple typo.", - "", - "If a package provides a file containing several related variables", - "(such as module.mk, app.mk, extension.mk), that file may define", - "variables that look unused since they are only used by other packages.", - "These variables should be documented at the head of the file;", - "see mk/subst.mk for an example of such a documentation comment.") } -// checkVarassignRightVaruse checks that in a variable assignment, -// each variable used on the right-hand side of the assignment operator -// has the correct data type and quoting. -func (ck MkLineChecker) checkVarassignRightVaruse() { +// comment is an empty string for no comment, or "#" + the actual comment otherwise. +func (ck MkLineChecker) checkVartype(varname string, op MkOperator, value, comment string) { if trace.Tracing { - defer trace.Call0()() + defer trace.Call(varname, op, value, comment)() } mkline := ck.MkLine - op := mkline.Op() + vartype := G.Pkgsrc.VariableType(ck.MkLines, varname) - time := VucRunTime - if op == opAssignEval || op == opAssignShell { - time = VucLoadTime + if op == opAssignAppend { + // XXX: MayBeAppendedTo also depends on the current file, see checkVarusePermissions. + // These checks may be combined. + if vartype != nil && !vartype.MayBeAppendedTo() { + mkline.Warnf("The \"+=\" operator should only be used with lists, not with %s.", varname) + } } - vartype := G.Pkgsrc.VariableType(ck.MkLines, mkline.Varname()) - if op == opAssignShell { - vartype = shellCommandsType - } + switch { + case vartype == nil: + if trace.Tracing { + trace.Step1("Unchecked variable assignment for %s.", varname) + } - if vartype != nil && vartype.IsShell() { - ck.checkVarassignVaruseShell(vartype, time) - } else { // XXX: This else looks as if it should be omitted. - ck.checkTextVarUse(ck.MkLine.Value(), vartype, time) - } -} + case op == opAssignShell: + if trace.Tracing { + trace.Step1("Unchecked use of !=: %q", value) + } -func (ck MkLineChecker) checkTextVarUse(text string, vartype *Vartype, time VucTime) { - if !contains(text, "$") { - return - } + case !vartype.IsList(): + ck.CheckVartypeBasic(varname, vartype.basicType, op, value, comment, vartype.IsGuessed()) - if trace.Tracing { - defer trace.Call(vartype, time)() - } + case value == "": + break - tokens := NewMkParser(nil, text).MkTokens() - for i, token := range tokens { - if token.Varuse != nil { - spaceLeft := i-1 < 0 || matches(tokens[i-1].Text, `[\t ]$`) - spaceRight := i+1 >= len(tokens) || matches(tokens[i+1].Text, `^[\t ]`) - isWordPart := !(spaceLeft && spaceRight) - vuc := VarUseContext{vartype, time, VucQuotPlain, isWordPart} - ck.CheckVaruse(token.Varuse, &vuc) + default: + words := mkline.ValueFields(value) + if len(words) > 1 && vartype.IsOnePerLine() { + mkline.Warnf("%s should only get one item per line.", varname) + mkline.Explain( + "Use the += operator to append each of the items.", + "", + "Or, enclose the words in quotes to group them.") + } + if vartype.basicType == BtCategory { + ck.checkVarassignRightCategory() + } + for _, word := range words { + ck.CheckVartypeBasic(varname, vartype.basicType, op, word, comment, vartype.IsGuessed()) } } } -// checkVarassignVaruseShell is very similar to checkVarassignRightVaruse, they just differ -// in the way they determine isWordPart. -func (ck MkLineChecker) checkVarassignVaruseShell(vartype *Vartype, time VucTime) { +// CheckVartypeBasic checks a single list element of the given type. +// +// For some variables (like `BuildlinkDepth`), `op` influences the valid values. +// The `comment` parameter comes from a variable assignment, when a part of the line is commented out. +func (ck MkLineChecker) CheckVartypeBasic(varname string, checker *BasicType, op MkOperator, value, comment string, guessed bool) { if trace.Tracing { - defer trace.Call(vartype, time)() + defer trace.Call(varname, checker.name, op, value, comment, guessed)() } - isWordPart := func(tokens []*ShAtom, i int) bool { - if i-1 >= 0 && tokens[i-1].Type.IsWord() { - return true - } - if i+1 < len(tokens) && tokens[i+1].Type.IsWord() { - return true - } - return false - } + mkline := ck.MkLine + valueNoVar := mkline.WithoutMakeVariables(value) + ctx := VartypeCheck{ck.MkLines, mkline, varname, op, value, valueNoVar, comment, guessed} + checker.checker(&ctx) +} +func (ck MkLineChecker) checkVarassignRightCategory() { mkline := ck.MkLine - atoms := NewShTokenizer(mkline.Line, mkline.Value(), false).ShAtoms() - for i, atom := range atoms { - if varuse := atom.VarUse(); varuse != nil { - wordPart := isWordPart(atoms, i) - vuc := VarUseContext{vartype, time, atom.Quoting.ToVarUseContext(), wordPart} - ck.CheckVaruse(varuse, &vuc) - } + if mkline.Op() != opAssign && mkline.Op() != opAssignDefault { + return } + + categories := mkline.ValueFields(mkline.Value()) + actual := categories[0] + expected := path.Base(path.Dir(path.Dir(mkline.Filename))) + if expected == "." { + expected = path.Base(path.Dir(path.Dir(G.Pkgsrc.ToRel(mkline.Filename)))) + } + if expected == "wip" || actual == expected { + return + } + + fix := mkline.Autofix() + fix.Warnf("The primary category should be %q, not %q.", expected, actual) + fix.Explain( + "The primary category of a package should be its location in the", + "pkgsrc directory tree, to make it easy to find the package.", + "All other categories may be added after this primary category.") + if len(categories) > 1 && categories[1] == expected { + fix.Replace(categories[0]+" "+categories[1], categories[1]+" "+categories[0]) + } + fix.Anyway() + fix.Apply() } func (ck MkLineChecker) checkVarassignMisc() { @@ -1276,6 +1158,30 @@ func (ck MkLineChecker) checkVarassignMisc() { ck.checkVarassignMiscRedundantInstallationDirs() } +func (ck MkLineChecker) checkVarassignDecreasingVersions() { + mkline := ck.MkLine + strVersions := mkline.Fields() + intVersions := make([]int, len(strVersions)) + for i, strVersion := range strVersions { + iver, err := strconv.Atoi(strVersion) + if err != nil || !(iver > 0) { + mkline.Errorf("Value %q for %s must be a positive integer.", strVersion, mkline.Varname()) + return + } + intVersions[i] = iver + } + + for i, ver := range intVersions { + if i > 0 && ver >= intVersions[i-1] { + mkline.Warnf("The values for %s should be in decreasing order (%d before %d).", + mkline.Varname(), ver, intVersions[i-1]) + mkline.Explain( + "If they aren't, it may be possible that needless versions of", + "packages are installed.") + } + } +} + func (ck MkLineChecker) checkVarassignMiscRedundantInstallationDirs() { mkline := ck.MkLine varname := mkline.Varname() @@ -1297,204 +1203,331 @@ func (ck MkLineChecker) checkVarassignMiscRedundantInstallationDirs() { } } -func (ck MkLineChecker) checkVarassignLeftBsdPrefs() { +// checkVarassignRightVaruse checks that in a variable assignment, +// each variable used on the right-hand side of the assignment operator +// has the correct data type and quoting. +func (ck MkLineChecker) checkVarassignRightVaruse() { + if trace.Tracing { + defer trace.Call0()() + } + mkline := ck.MkLine + op := mkline.Op() - switch mkline.Varcanon() { - case "BUILDLINK_PKGSRCDIR.*", - "BUILDLINK_DEPMETHOD.*", - "BUILDLINK_ABI_DEPENDS.*", - "BUILDLINK_INCDIRS.*", - "BUILDLINK_LIBDIRS.*": - return + time := VucRunTime + if op == opAssignEval || op == opAssignShell { + time = VucLoadTime } - if !G.Opts.WarnExtra || - G.Infrastructure || - mkline.Op() != opAssignDefault || - ck.MkLines.Tools.SeenPrefs || - !ck.MkLines.once.FirstTime("include bsd.prefs.mk before using ?=") { - return + vartype := G.Pkgsrc.VariableType(ck.MkLines, mkline.Varname()) + if op == opAssignShell { + vartype = shellCommandsType } - // Package-settable variables may use the ?= operator before including - // bsd.prefs.mk in situations like the following: - // - // Makefile: LICENSE= package-license - // .include "module.mk" - // module.mk: LICENSE?= default-license - // - vartype := G.Pkgsrc.VariableType(nil, mkline.Varname()) - if vartype != nil && vartype.PackageSettable() { - return + if vartype != nil && vartype.IsShell() { + ck.checkVarassignVaruseShell(vartype, time) + } else { // XXX: This else looks as if it should be omitted. + ck.checkTextVarUse(ck.MkLine.Value(), vartype, time) } +} - mkline.Warnf("Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".") - mkline.Explain( - "The ?= operator is used to provide a default value to a variable.", - "In pkgsrc, many variables can be set by the pkgsrc user in the", - "mk.conf file.", - "This file must be included explicitly.", - "If a ?= operator appears before mk.conf has been included,", - "it will not care about the user's preferences,", - "which can result in unexpected behavior.", - "", - "The easiest way to include the mk.conf file is by including the", - "bsd.prefs.mk file, which will take care of everything.") +// checkVarassignVaruseShell is very similar to checkVarassignRightVaruse, they just differ +// in the way they determine isWordPart. +func (ck MkLineChecker) checkVarassignVaruseShell(vartype *Vartype, time VucTime) { + if trace.Tracing { + defer trace.Call(vartype, time)() + } + + isWordPart := func(tokens []*ShAtom, i int) bool { + if i-1 >= 0 && tokens[i-1].Type.IsWord() { + return true + } + if i+1 < len(tokens) && tokens[i+1].Type.IsWord() { + return true + } + return false + } + + mkline := ck.MkLine + atoms := NewShTokenizer(mkline.Line, mkline.Value(), false).ShAtoms() + for i, atom := range atoms { + if varuse := atom.VarUse(); varuse != nil { + wordPart := isWordPart(atoms, i) + vuc := VarUseContext{vartype, time, atom.Quoting.ToVarUseContext(), wordPart} + ck.CheckVaruse(varuse, &vuc) + } + } } -// checkVarassignLeftUserSettable checks whether a package defines a -// variable that is marked as user-settable since it is defined in -// mk/defaults/mk.conf. -func (ck MkLineChecker) checkVarassignLeftUserSettable() bool { +func (ck MkLineChecker) checkShellCommand() { mkline := ck.MkLine - varname := mkline.Varname() - defaultMkline := G.Pkgsrc.UserDefinedVars.Mentioned(varname) - if defaultMkline == nil { - return false + shellCommand := mkline.ShellCommand() + if G.Opts.WarnSpace && hasPrefix(mkline.Text, "\t\t") { + lexer := textproc.NewLexer(mkline.raw[0].textnl) + tabs := lexer.NextBytesFunc(func(b byte) bool { return b == '\t' }) + + fix := mkline.Autofix() + fix.Notef("Shell programs should be indented with a single tab.") + fix.Explain( + "The first tab in the line marks the line as a shell command.", + "Since every line of shell commands starts with a completely new shell environment,", + "there is no need to indent some of the commands,", + "or to use more horizontal space than necessary.") + + for i, raw := range mkline.Line.raw { + if hasPrefix(raw.textnl, tabs) { + fix.ReplaceAt(i, 0, tabs, "\t") + } + } + fix.Apply() } - defaultValue := defaultMkline.Value() - // A few of the user-settable variables can also be set by packages. - // That's an unfortunate situation since there is no definite source - // of truth, but luckily only a few variables make use of it. - vartype := G.Pkgsrc.VariableType(ck.MkLines, varname) - if vartype.PackageSettable() { - return true + ck.checkText(shellCommand) + NewShellLineChecker(ck.MkLines, mkline).CheckShellCommandLine(shellCommand) +} + +func (ck MkLineChecker) checkComment() { + mkline := ck.MkLine + + if hasPrefix(mkline.Text, "# url2pkg-marker") { + mkline.Errorf("This comment indicates unfinished work (url2pkg).") + } +} + +func (ck MkLineChecker) checkInclude() { + if trace.Tracing { + defer trace.Call0()() + } + + mkline := ck.MkLine + if mkline.Indent() != "" { + ck.checkDirectiveIndentation(ck.MkLines.indentation.Depth("include")) + } + + includedFile := mkline.IncludedFile() + mustExist := mkline.MustExist() + if trace.Tracing { + trace.Step2("includingFile=%s includedFile=%s", mkline.Filename, includedFile) } + ck.CheckRelativePath(includedFile, mustExist) switch { - case mkline.HasComment(): - // Assume that the comment contains a rationale for disabling - // this particular check. + case hasSuffix(includedFile, "/Makefile"): + mkline.Errorf("Other Makefiles must not be included directly.") + mkline.Explain( + "To include portions of another Makefile, extract the common parts", + "and put them into a Makefile.common or a Makefile fragment called", + "module.mk or similar.", + "After that, both this one and the other package should include the newly created file.") - case mkline.Op() == opAssignAppend: - mkline.Warnf("Packages should not append to user-settable %s.", varname) + case IsPrefs(includedFile): + if mkline.Basename == "buildlink3.mk" && includedFile == "../../mk/bsd.prefs.mk" { + fix := mkline.Autofix() + fix.Notef("For efficiency reasons, please include bsd.fast.prefs.mk instead of bsd.prefs.mk.") + fix.Replace("bsd.prefs.mk", "bsd.fast.prefs.mk") + fix.Apply() + } - case defaultValue != mkline.Value(): - mkline.Warnf( - "Package sets user-defined %q to %q, which differs "+ - "from the default value %q from mk/defaults/mk.conf.", - varname, mkline.Value(), defaultValue) + case hasSuffix(includedFile, "pkgtools/x11-links/buildlink3.mk"): + fix := mkline.Autofix() + fix.Errorf("%s must not be included directly. Include \"../../mk/x11.buildlink3.mk\" instead.", includedFile) + fix.Replace("pkgtools/x11-links/buildlink3.mk", "mk/x11.buildlink3.mk") + fix.Apply() - case defaultMkline.IsCommentedVarassign(): - // Since the variable assignment is commented out in - // mk/defaults/mk.conf, the package has to define it. + case hasSuffix(includedFile, "graphics/jpeg/buildlink3.mk"): + fix := mkline.Autofix() + fix.Errorf("%s must not be included directly. Include \"../../mk/jpeg.buildlink3.mk\" instead.", includedFile) + fix.Replace("graphics/jpeg/buildlink3.mk", "mk/jpeg.buildlink3.mk") + fix.Apply() - default: - mkline.Notef("Redundant definition for %s from mk/defaults/mk.conf.", varname) - if !ck.MkLines.Tools.SeenPrefs { - mkline.Explain( - "Instead of defining the variable redundantly, it suffices to include", - "../../mk/bsd.prefs.mk, which provides all user-settable variables.") + case hasSuffix(includedFile, "/intltool/buildlink3.mk"): + mkline.Warnf("Please write \"USE_TOOLS+= intltool\" instead of this line.") + + case hasSuffix(includedFile, "/builtin.mk"): + if mkline.Basename != "hacks.mk" && !mkline.HasRationale() { + fix := mkline.Autofix() + fix.Errorf("%s must not be included directly. Include \"%s/buildlink3.mk\" instead.", includedFile, path.Dir(includedFile)) + fix.Replace("builtin.mk", "buildlink3.mk") + fix.Apply() } } +} - return true +func (ck MkLineChecker) checkDirectiveIndentation(expectedDepth int) { + if !G.Opts.WarnSpace { + return + } + mkline := ck.MkLine + indent := mkline.Indent() + if expected := strings.Repeat(" ", expectedDepth); indent != expected { + fix := mkline.Line.Autofix() + fix.Notef("This directive should be indented by %d spaces.", expectedDepth) + fix.ReplaceRegex(regex.Pattern(`^\.`+indent), "."+expected, 1) + fix.Apply() + } } -// comment is an empty string for no comment, or "#" + the actual comment otherwise. -func (ck MkLineChecker) checkVartype(varname string, op MkOperator, value, comment string) { +// CheckRelativePath checks a relative path that leads to the directory of another package +// or to a subdirectory thereof or a file within there. +func (ck MkLineChecker) CheckRelativePath(relativePath string, mustExist bool) { if trace.Tracing { - defer trace.Call(varname, op, value, comment)() + defer trace.Call(relativePath, mustExist)() } mkline := ck.MkLine - vartype := G.Pkgsrc.VariableType(ck.MkLines, varname) + if !G.Wip && contains(relativePath, "/wip/") { + mkline.Errorf("A main pkgsrc package must not depend on a pkgsrc-wip package.") + } - if op == opAssignAppend { - // XXX: MayBeAppendedTo also depends on the current file, see checkVarusePermissions. - // These checks may be combined. - if vartype != nil && !vartype.MayBeAppendedTo() { - mkline.Warnf("The \"+=\" operator should only be used with lists, not with %s.", varname) + resolvedPath := mkline.ResolveVarsInRelativePath(relativePath) + if containsVarRef(resolvedPath) { + return + } + + if filepath.IsAbs(resolvedPath) { + mkline.Errorf("The path %q must be relative.", resolvedPath) + return + } + + abs := joinPath(path.Dir(mkline.Filename), resolvedPath) + if _, err := os.Stat(abs); err != nil { + if mustExist && !ck.MkLines.indentation.HasExists(resolvedPath) { + mkline.Errorf("Relative path %q does not exist.", resolvedPath) } + return } switch { - case vartype == nil: - if trace.Tracing { - trace.Step1("Unchecked variable assignment for %s.", varname) - } + case !hasPrefix(resolvedPath, "../"): + break - case op == opAssignShell: - if trace.Tracing { - trace.Step1("Unchecked use of !=: %q", value) - } + case hasPrefix(resolvedPath, "../../mk/"): + // From a package to the infrastructure. - case !vartype.List(): - ck.CheckVartypeBasic(varname, vartype.basicType, op, value, comment, vartype.Guessed()) + case matches(resolvedPath, `^\.\./\.\./[^./][^/]*/[^/]`): + // From a package to another package. - case value == "": - break + case hasPrefix(resolvedPath, "../mk/") && relpath(path.Dir(mkline.Filename), G.Pkgsrc.File(".")) == "..": + // For category Makefiles. + // TODO: Or from a pkgsrc wip package to wip/mk. - default: - words := mkline.ValueFields(value) - if len(words) > 1 && vartype.OnePerLine() { - mkline.Warnf("%s should only get one item per line.", varname) - mkline.Explain( - "Use the += operator to append each of the items.", - "", - "Or, enclose the words in quotes to group them.") - } - for _, word := range words { - ck.CheckVartypeBasic(varname, vartype.basicType, op, word, comment, vartype.Guessed()) + case matches(resolvedPath, `^\.\./[^./][^/]*/[^/]`): + if G.Wip && contains(resolvedPath, "/mk/") { + mkline.Warnf("References to the pkgsrc-wip infrastructure should look like \"../../wip/mk\", not \"../mk\".") + } else { + mkline.Warnf("References to other packages should look like \"../../category/package\", not \"../package\".") } + mkline.ExplainRelativeDirs() } } -// CheckVartypeBasic checks a single list element of the given type. +// CheckRelativePkgdir checks a reference from one pkgsrc package to another. +// These references should always have the form ../../category/package. // -// For some variables (like `BuildlinkDepth`), `op` influences the valid values. -// The `comment` parameter comes from a variable assignment, when a part of the line is commented out. -func (ck MkLineChecker) CheckVartypeBasic(varname string, checker *BasicType, op MkOperator, value, comment string, guessed bool) { +// When used in DEPENDS or similar variables, these directories could theoretically +// also be relative to the pkgsrc root, which would save a few keystrokes. +// This, however, is not implemented in pkgsrc and suggestions regarding this topic +// have not been made in the last two decades on the public mailing lists. +// While being a bit redundant, the current scheme works well. +// +// When used in .include directives, the relative package directories must be written +// with the leading ../.. anyway, so the benefit might not be too big at all. +func (ck MkLineChecker) CheckRelativePkgdir(pkgdir string) { if trace.Tracing { - defer trace.Call(varname, checker.name, op, value, comment, guessed)() + defer trace.Call1(pkgdir)() } mkline := ck.MkLine - valueNoVar := mkline.WithoutMakeVariables(value) - ctx := VartypeCheck{ck.MkLines, mkline, varname, op, value, valueNoVar, comment, guessed} - checker.checker(&ctx) + ck.CheckRelativePath(pkgdir+"/Makefile", true) + pkgdir = mkline.ResolveVarsInRelativePath(pkgdir) + + if !matches(pkgdir, `^\.\./\.\./([^./][^/]*/[^./][^/]*)$`) && !containsVarRef(pkgdir) { + mkline.Warnf("%q is not a valid relative package directory.", pkgdir) + mkline.Explain( + "A relative pathname always starts with \"../../\", followed", + "by a category, a slash and a the directory name of the package.", + "For example, \"../../misc/screen\" is a valid relative pathname.") + } } -// checkText checks the given text (which is typically the right-hand side of a variable -// assignment or a shell command). -// -// Note: checkTextVarUse cannot be called here since it needs to know the context where it is included. -// Maybe that context should be added here as parameters. -func (ck MkLineChecker) checkText(text string) { - if trace.Tracing { - defer trace.Call1(text)() +func (ck MkLineChecker) checkDirective(forVars map[string]bool, ind *Indentation) { + mkline := ck.MkLine + + directive := mkline.Directive() + args := mkline.Args() + + expectedDepth := ind.Depth(directive) + ck.checkDirectiveIndentation(expectedDepth) + + if directive == "endfor" || directive == "endif" { + ck.checkDirectiveEnd(ind) } - ck.checkTextWrksrcDotDot(text) - ck.checkTextRpath(text) -} + needsArgument := false + switch directive { + case + "if", "ifdef", "ifndef", "elif", + "for", "undef", + "error", "warning", "info", + "export", "export-env", "unexport", "unexport-env": + needsArgument = true + } -func (ck MkLineChecker) checkTextWrksrcDotDot(text string) { - if contains(text, "${WRKSRC}/..") { - ck.MkLine.Warnf("Building the package should take place entirely inside ${WRKSRC}, not \"${WRKSRC}/..\".") - ck.MkLine.Explain( - "WRKSRC should be defined so that there is no need to do anything", - "outside of this directory.", - "", - "Example:", - "", - "\tWRKSRC=\t${WRKDIR}", - "\tCONFIGURE_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src", - "\tBUILD_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src ${WRKSRC}/cmd", - "", - seeGuide("Directories used during the build process", "build.builddirs")) + switch { + case needsArgument && args == "": + mkline.Errorf("\".%s\" requires arguments.", directive) + + case !needsArgument && args != "": + if directive == "else" { + mkline.Errorf("\".%s\" does not take arguments. If you meant \"else if\", use \".elif\".", directive) + } else { + mkline.Errorf("\".%s\" does not take arguments.", directive) + } + + case directive == "if" || directive == "elif": + ck.checkDirectiveCond() + + case directive == "ifdef" || directive == "ifndef": + mkline.Warnf("The \".%s\" directive is deprecated. Please use \".if %sdefined(%s)\" instead.", + directive, condStr(directive == "ifdef", "", "!"), args) + + case directive == "for": + ck.checkDirectiveFor(forVars, ind) + + case directive == "undef": + for _, varname := range mkline.Fields() { + if forVars[varname] { + mkline.Notef("Using \".undef\" after a \".for\" loop is unnecessary.") + } + } } } -// checkTextPath checks for literal -Wl,--rpath options. -// -// Note: A simple -R is not detected, as the rate of false positives is too high. -func (ck MkLineChecker) checkTextRpath(text string) { - if m, flag := match1(text, `(-Wl,--rpath,|-Wl,-rpath-link,|-Wl,-rpath,|-Wl,-R\b)`); m { - ck.MkLine.Warnf("Please use ${COMPILER_RPATH_FLAG} instead of %q.", flag) +func (ck MkLineChecker) checkDirectiveEnd(ind *Indentation) { + mkline := ck.MkLine + directive := mkline.Directive() + comment := mkline.DirectiveComment() + + if ind.IsEmpty() { + mkline.Errorf("Unmatched .%s.", directive) + return + } + + if comment == "" { + return + } + + if directive == "endif" { + if args := ind.Args(); !contains(args, comment) { + mkline.Warnf("Comment %q does not match condition %q.", comment, args) + } + } + + if directive == "endfor" { + if args := ind.Args(); !contains(args, comment) { + mkline.Warnf("Comment %q does not match loop %q.", comment, args) + } } } @@ -1623,7 +1656,7 @@ func (ck MkLineChecker) simplifyCondition(varuse *MkVarUse, fromEmpty bool, notE switch { case !exact, vartype == nil, - vartype.List(), + vartype.IsList(), textproc.NewLexer(pattern).NextBytesSet(mkCondLiteralChars) != pattern: continue } @@ -1646,17 +1679,6 @@ func (ck MkLineChecker) simplifyCondition(varuse *MkVarUse, fromEmpty bool, notE } } -func (ck MkLineChecker) checkCompareVarStr(varname, op, value string) { - ck.checkVartype(varname, opUseCompare, value, "") - - if varname == "PKGSRC_COMPILER" { - ck.MkLine.Warnf("Use ${PKGSRC_COMPILER:%s%s} instead of the %s operator.", condStr(op == "==", "M", "N"), value, op) - ck.MkLine.Explain( - "The PKGSRC_COMPILER can be a list of chained compilers, e.g. \"ccache distcc clang\".", - "Therefore, comparing it using == or != leads to wrong results in these cases.") - } -} - func (ck MkLineChecker) checkDirectiveCondCompare(left *MkCondTerm, op string, right *MkCondTerm) { switch { case left.Var != nil && right.Var == nil && right.Num == "": @@ -1696,85 +1718,95 @@ func (ck MkLineChecker) checkDirectiveCondCompareVarStr(varuse *MkVarUse, op str } } -// CheckRelativePkgdir checks a reference from one pkgsrc package to another. -// These references should always have the form ../../category/package. -// -// When used in DEPENDS or similar variables, these directories could theoretically -// also be relative to the pkgsrc root, which would save a few keystrokes. -// This, however, is not implemented in pkgsrc and suggestions regarding this topic -// have not been made in the last two decades on the public mailing lists. -// While being a bit redundant, the current scheme works well. -// -// When used in .include directives, the relative package directories must be written -// with the leading ../.. anyway, so the benefit might not be too big at all. -func (ck MkLineChecker) CheckRelativePkgdir(pkgdir string) { - if trace.Tracing { - defer trace.Call1(pkgdir)() +func (ck MkLineChecker) checkCompareVarStr(varname, op, value string) { + ck.checkVartype(varname, opUseCompare, value, "") + + if varname == "PKGSRC_COMPILER" { + ck.MkLine.Warnf("Use ${PKGSRC_COMPILER:%s%s} instead of the %s operator.", condStr(op == "==", "M", "N"), value, op) + ck.MkLine.Explain( + "The PKGSRC_COMPILER can be a list of chained compilers, e.g. \"ccache distcc clang\".", + "Therefore, comparing it using == or != leads to wrong results in these cases.") } +} +func (ck MkLineChecker) checkDirectiveFor(forVars map[string]bool, indentation *Indentation) { mkline := ck.MkLine - ck.CheckRelativePath(pkgdir+"/Makefile", true) - pkgdir = mkline.ResolveVarsInRelativePath(pkgdir) + args := mkline.Args() - if !matches(pkgdir, `^\.\./\.\./([^./][^/]*/[^./][^/]*)$`) && !containsVarRef(pkgdir) { - mkline.Warnf("%q is not a valid relative package directory.", pkgdir) - mkline.Explain( - "A relative pathname always starts with \"../../\", followed", - "by a category, a slash and a the directory name of the package.", - "For example, \"../../misc/screen\" is a valid relative pathname.") - } -} + if m, vars, _ := match2(args, `^([^\t ]+(?:[\t ]*[^\t ]+)*?)[\t ]+in[\t ]+(.*)$`); m { + for _, forvar := range strings.Fields(vars) { + indentation.AddVar(forvar) + if !G.Infrastructure && hasPrefix(forvar, "_") { + mkline.Warnf("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", forvar) + } -// CheckRelativePath checks a relative path that leads to the directory of another package -// or to a subdirectory thereof or a file within there. -func (ck MkLineChecker) CheckRelativePath(relativePath string, mustExist bool) { - if trace.Tracing { - defer trace.Call(relativePath, mustExist)() + if matches(forvar, `^[_a-z][_a-z0-9]*$`) { + // Fine. + } else if matches(forvar, `^[A-Z_a-z][0-9A-Z_a-z]*$`) { + mkline.Warnf("The variable name %q in the .for loop should not contain uppercase letters.", forvar) + } else { + mkline.Errorf("Invalid variable name %q.", forvar) + } + + forVars[forvar] = true + } + + // XXX: The type BtUnknown is very unspecific here. For known variables + // or constant values this could probably be improved. + // + // The guessed flag could also be determined more correctly. As of November 2018, + // running pkglint over the whole pkgsrc tree did not produce any different result + // whether guessed was true or false. + forLoopType := NewVartype(btForLoop, List, NewACLEntry("*", aclpAllRead)) + forLoopContext := VarUseContext{forLoopType, VucLoadTime, VucQuotPlain, false} + mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { + ck.CheckVaruse(varUse, &forLoopContext) + }) } +} +func (ck MkLineChecker) checkDependencyRule(allowedTargets map[string]bool) { mkline := ck.MkLine - if !G.Wip && contains(relativePath, "/wip/") { - mkline.Errorf("A main pkgsrc package must not depend on a pkgsrc-wip package.") - } + targets := mkline.ValueFields(mkline.Targets()) + sources := mkline.ValueFields(mkline.Sources()) - resolvedPath := mkline.ResolveVarsInRelativePath(relativePath) - if containsVarRef(resolvedPath) { - return + for _, source := range sources { + if source == ".PHONY" { + for _, target := range targets { + allowedTargets[target] = true + } + } + } + for _, target := range targets { + if target == ".PHONY" { + for _, source := range sources { + allowedTargets[source] = true + } + } } - if filepath.IsAbs(resolvedPath) { - mkline.Errorf("The path %q must be relative.", resolvedPath) - return + for _, target := range targets { + ck.checkDependencyTarget(target, allowedTargets) } +} - abs := joinPath(path.Dir(mkline.Filename), resolvedPath) - if _, err := os.Stat(abs); err != nil { - if mustExist && !ck.MkLines.indentation.HasExists(resolvedPath) { - mkline.Errorf("Relative path %q does not exist.", resolvedPath) - } +func (ck MkLineChecker) checkDependencyTarget(target string, allowedTargets map[string]bool) { + if target == ".PHONY" || + target == ".ORDER" || + NewMkParser(nil, target).VarUse() != nil || + allowedTargets[target] { return } - switch { - case !hasPrefix(resolvedPath, "../"): - break - - case hasPrefix(resolvedPath, "../../mk/"): - // From a package to the infrastructure. - - case matches(resolvedPath, `^\.\./\.\./[^./][^/]*/[^/]`): - // From a package to another package. - - case hasPrefix(resolvedPath, "../mk/") && relpath(path.Dir(mkline.Filename), G.Pkgsrc.File(".")) == "..": - // For category Makefiles. - // TODO: Or from a pkgsrc wip package to wip/mk. - - case matches(resolvedPath, `^\.\./[^./][^/]*/[^/]`): - if G.Wip && contains(resolvedPath, "/mk/") { - mkline.Warnf("References to the pkgsrc-wip infrastructure should look like \"../../wip/mk\", not \"../mk\".") - } else { - mkline.Warnf("References to other packages should look like \"../../category/package\", not \"../package\".") - } - mkline.ExplainRelativeDirs() - } + mkline := ck.MkLine + mkline.Warnf("Undeclared target %q.", target) + mkline.Explain( + "To define a custom target in a package, declare it like this:", + "", + "\t.PHONY: my-target", + "", + "To define a custom target that creates a file (should be rarely needed),", + "declare it like this:", + "", + "\t${.CURDIR}/my-file:") } diff --git a/pkgtools/pkglint/files/mklinechecker_test.go b/pkgtools/pkglint/files/mklinechecker_test.go index d400f484a29..7698b5e4b03 100644 --- a/pkgtools/pkglint/files/mklinechecker_test.go +++ b/pkgtools/pkglint/files/mklinechecker_test.go @@ -5,6 +5,106 @@ import ( "runtime" ) +// PR pkg/46570, item 2 +func (s *Suite) Test_MkLineChecker__unclosed_varuse(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", + MkCvsID, + "EGDIRS=\t${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:2: Missing closing \"}\" for \"EGDIR/pam.d\".", + "WARN: Makefile:2: Invalid part \"/pam.d\" after variable name \"EGDIR\".", + "WARN: Makefile:2: Missing closing \"}\" for \"EGDIR/dbus-1/system.d ${EGDIR/pam.d\".", + "WARN: Makefile:2: Invalid part \"/dbus-1/system.d ${EGDIR/pam.d\" after variable name \"EGDIR\".", + "WARN: Makefile:2: Missing closing \"}\" for \"EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".", + "WARN: Makefile:2: Invalid part \"/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\" after variable name \"EGDIR\".", + "WARN: Makefile:2: EGDIRS is defined but not used.", + "WARN: Makefile:2: EGDIR/pam.d is used but not defined.") +} + +func (s *Suite) Test_MkLineChecker_Check__url2pkg(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "# url2pkg-marker") + + mklines.Check() + + t.CheckOutputLines( + "ERROR: filename.mk:2: This comment indicates unfinished work (url2pkg).") +} + +func (s *Suite) Test_MkLineChecker_Check__buildlink3_include_prefs(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + t.CreateFileLines("mk/bsd.prefs.mk") + t.CreateFileLines("mk/bsd.fast.prefs.mk") + mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", + MkCvsID, + ".include \"../../mk/bsd.prefs.mk\"", + ".include \"../../mk/bsd.fast.prefs.mk\"") + + // If the buildlink3.mk file doesn't actually exist, resolving the + // relative path fails since that depends on the actual file system, + // not on syntactical paths; see os.Stat in CheckRelativePath. + // + // TODO: Refactor relpath to be independent of a filesystem. + + mklines.Check() + + t.CheckOutputLines( + "NOTE: ~/category/package/buildlink3.mk:2: For efficiency reasons, " + + "please include bsd.fast.prefs.mk instead of bsd.prefs.mk.") +} + +func (s *Suite) Test_MkLineChecker_Check__warn_varuse_LOCALBASE(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("options.mk", + MkCvsID, + "PKGNAME=\t${LOCALBASE}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: options.mk:2: Please use PREFIX instead of LOCALBASE.") +} + +func (s *Suite) Test_MkLineChecker_Check__varuse_modifier_L(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("x11/xkeyboard-config/Makefile", + MkCvsID, + "FILES_SUBST+=\tXKBCOMP_SYMLINK=${${XKBBASE}/xkbcomp:L:Q}", + "FILES_SUBST+=\tXKBCOMP_SYMLINK=${${XKBBASE}/xkbcomp:Q}") + + mklines.Check() + + // In line 2, don't warn that ${XKBBASE}/xkbcomp is used but not defined. + // This is because the :L modifier interprets everything before as an expression + // instead of a variable name. + // + // In line 3 the :L modifier is missing, therefore ${XKBBASE}/xkbcomp is the + // name of another variable, and that variable is not known. Only XKBBASE is known. + // + // In line 3, warn about the invalid "/" as part of the variable name. + t.CheckOutputLines( + "WARN: x11/xkeyboard-config/Makefile:3: "+ + "Invalid part \"/xkbcomp\" after variable name \"${XKBBASE}\".", + "WARN: x11/xkeyboard-config/Makefile:3: XKBBASE is used but not defined.") +} + func (s *Suite) Test_MkLineChecker_checkEmptyContinuation(c *check.C) { t := s.Init(c) @@ -25,36 +125,69 @@ func (s *Suite) Test_MkLineChecker_checkEmptyContinuation(c *check.C) { "WARN: ~/filename.mk:3: This line looks empty but continues the previous line.") } -func (s *Suite) Test_MkLineChecker_checkShellCommand__indentation(c *check.C) { +// Pkglint once interpreted all lists as consisting of shell tokens, +// splitting this URL at the ampersand. +func (s *Suite) Test_MkLineChecker_checkVarassign__URL_with_shell_special_characters(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall", "--autofix") - mklines := t.SetUpFileMkLines("filename.mk", + G.Pkg = NewPackage(t.File("graphics/gimp-fix-ca")) + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", MkCvsID, - "", - "do-install:", - "\t\techo 'unnecessarily indented'", - "\t\tfor var in 1 2 3; do \\", - "\t\t\techo \"$$var\"; \\", - "\t echo \"spaces\"; \\", - "\t\tdone") + "MASTER_SITES=\thttp://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=") mklines.Check() - t.CheckOutputLines( - "AUTOFIX: ~/filename.mk:4: Replacing \"\\t\\t\" with \"\\t\".", - "AUTOFIX: ~/filename.mk:5: Replacing \"\\t\\t\" with \"\\t\".", - "AUTOFIX: ~/filename.mk:6: Replacing \"\\t\\t\" with \"\\t\".", - "AUTOFIX: ~/filename.mk:8: Replacing \"\\t\\t\" with \"\\t\".") - t.CheckFileLinesDetab("filename.mk", + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLineChecker_checkVarassign__list(c *check.C) { + t := s.Init(c) + + t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") + t.SetUpVartypes() + t.SetUpCommandLine("-Wall", "--explain") + mklines := t.NewMkLines("filename.mk", MkCvsID, + "SITES.distfile=\t-${MASTER_SITE_GITHUB:=project/}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: filename.mk:2: The list variable MASTER_SITE_GITHUB should not be embedded in a word.", "", - "do-install:", - " echo 'unnecessarily indented'", - " for var in 1 2 3; do \\", - " echo \"$$var\"; \\", - " echo \"spaces\"; \\", // not changed - " done") + "\tWhen a list variable has multiple elements, this expression expands", + "\tto something unexpected:", + "", + "\tExample: ${MASTER_SITE_SOURCEFORGE}directory/ expands to", + "", + "\t\thttps://mirror1.sf.net/ https://mirror2.sf.net/directory/", + "", + "\tThe first URL is missing the directory. To fix this, write", + "\t\t${MASTER_SITE_SOURCEFORGE:=directory/}.", + "", + "\tExample: -l${LIBS} expands to", + "", + "\t\t-llib1 lib2", + "", + "\tThe second library is missing the -l. To fix this, write", + "\t${LIBS:S,^,-l,}.", + "") +} + +func (s *Suite) Test_MkLineChecker_checkVarassign(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + mklines := t.NewMkLines("Makefile", + MkCvsID, + "ac_cv_libpari_libs+=\t-L${BUILDLINK_PREFIX.pari}/lib") // From math/clisp-pari/Makefile, rev. 1.8 + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:2: ac_cv_libpari_libs is defined but not used.") } func (s *Suite) Test_MkLineChecker_checkVarassignLeft(c *check.C) { @@ -73,6 +206,38 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeft(c *check.C) { "(_VARNAME) are reserved for internal pkgsrc use.") } +// Files from the pkgsrc infrastructure may define and use variables +// whose name starts with an underscore. +func (s *Suite) Test_MkLineChecker_checkVarassignLeft__infrastructure(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("mk/infra.mk", + MkCvsID, + "_VARNAME=\t\tvalue", + "_SORTED_VARS.group=\tVARNAME") + t.FinishSetUp() + + G.Check(t.File("mk/infra.mk")) + + t.CheckOutputLines( + "WARN: ~/mk/infra.mk:2: _VARNAME is defined but not used.") +} + +func (s *Suite) Test_MkLineChecker_checkVarassignLeft__documented_underscore(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("category/package/filename.mk", + MkCvsID, + "_SORTED_VARS.group=\tVARNAME") + t.FinishSetUp() + + G.Check(t.File("category/package/filename.mk")) + + t.CheckOutputEmpty() +} + func (s *Suite) Test_MkLineChecker_checkVarassignLeftNotUsed__procedure_call(c *check.C) { t := s.Init(c) @@ -143,36 +308,18 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftNotUsed__infra(c *check.C) "WARN: ~/category/package/Makefile:22: UNDOCUMENTED is used but not defined.") } -// Files from the pkgsrc infrastructure may define and use variables -// whose name starts with an underscore. -func (s *Suite) Test_MkLineChecker_checkVarassignLeft__infrastructure(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVarassignLeftBsdPrefs__vartype_nil(c *check.C) { t := s.Init(c) - t.SetUpPkgsrc() - t.CreateFileLines("mk/infra.mk", + mklines := t.NewMkLines("builtin.mk", MkCvsID, - "_VARNAME=\t\tvalue", - "_SORTED_VARS.group=\tVARNAME") - t.FinishSetUp() + "VAR_SH?=\tvalue") - G.Check(t.File("mk/infra.mk")) + mklines.Check() t.CheckOutputLines( - "WARN: ~/mk/infra.mk:2: _VARNAME is defined but not used.") -} - -func (s *Suite) Test_MkLineChecker_checkVarassignLeft__documented_underscore(c *check.C) { - t := s.Init(c) - - t.SetUpPkgsrc() - t.CreateFileLines("category/package/filename.mk", - MkCvsID, - "_SORTED_VARS.group=\tVARNAME") - t.FinishSetUp() - - G.Check(t.File("category/package/filename.mk")) - - t.CheckOutputEmpty() + "WARN: builtin.mk:2: VAR_SH is defined but not used.", + "WARN: builtin.mk:2: Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".") } func (s *Suite) Test_MkLineChecker_checkVarassignLeftUserSettable(c *check.C) { @@ -290,182 +437,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftUserSettable__vartype_nil(c "WARN: Makefile:20: USER_SETTABLE is defined but not used.") } -func (s *Suite) Test_MkLineChecker_checkVarassignLeftBsdPrefs__vartype_nil(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("builtin.mk", - MkCvsID, - "VAR_SH?=\tvalue") - - mklines.Check() - - t.CheckOutputLines( - "WARN: builtin.mk:2: VAR_SH is defined but not used.", - "WARN: builtin.mk:2: Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".") -} - -func (s *Suite) Test_MkLineChecker_Check__url2pkg(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "# url2pkg-marker") - - mklines.Check() - - t.CheckOutputLines( - "ERROR: filename.mk:2: This comment indicates unfinished work (url2pkg).") -} - -func (s *Suite) Test_MkLineChecker_Check__buildlink3_include_prefs(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - - t.CreateFileLines("mk/bsd.prefs.mk") - t.CreateFileLines("mk/bsd.fast.prefs.mk") - mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", - MkCvsID, - ".include \"../../mk/bsd.prefs.mk\"", - ".include \"../../mk/bsd.fast.prefs.mk\"") - - // If the buildlink3.mk file doesn't actually exist, resolving the - // relative path fails since that depends on the actual file system, - // not on syntactical paths; see os.Stat in CheckRelativePath. - // - // TODO: Refactor relpath to be independent of a filesystem. - - mklines.Check() - - t.CheckOutputLines( - "NOTE: ~/category/package/buildlink3.mk:2: For efficiency reasons, " + - "please include bsd.fast.prefs.mk instead of bsd.prefs.mk.") -} - -func (s *Suite) Test_MkLineChecker_checkInclude(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - - t.CreateFileLines("pkgtools/x11-links/buildlink3.mk") - t.CreateFileLines("graphics/jpeg/buildlink3.mk") - t.CreateFileLines("devel/intltool/buildlink3.mk") - t.CreateFileLines("devel/intltool/builtin.mk") - mklines := t.SetUpFileMkLines("category/package/filename.mk", - MkCvsID, - "", - ".include \"../../pkgtools/x11-links/buildlink3.mk\"", - ".include \"../../graphics/jpeg/buildlink3.mk\"", - ".include \"../../devel/intltool/buildlink3.mk\"", - ".include \"../../devel/intltool/builtin.mk\"") - - mklines.Check() - - t.CheckOutputLines( - "ERROR: ~/category/package/filename.mk:3: "+ - "../../pkgtools/x11-links/buildlink3.mk must not be included directly. "+ - "Include \"../../mk/x11.buildlink3.mk\" instead.", - "ERROR: ~/category/package/filename.mk:4: "+ - "../../graphics/jpeg/buildlink3.mk must not be included directly. "+ - "Include \"../../mk/jpeg.buildlink3.mk\" instead.", - "WARN: ~/category/package/filename.mk:5: "+ - "Please write \"USE_TOOLS+= intltool\" instead of this line.", - "ERROR: ~/category/package/filename.mk:6: "+ - "../../devel/intltool/builtin.mk must not be included directly. "+ - "Include \"../../devel/intltool/buildlink3.mk\" instead.") -} - -func (s *Suite) Test_MkLineChecker_checkInclude__Makefile(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines(t.File("Makefile"), - MkCvsID, - ".include \"../../other/package/Makefile\"") - - mklines.Check() - - t.CheckOutputLines( - "ERROR: ~/Makefile:2: Relative path \"../../other/package/Makefile\" does not exist.", - "ERROR: ~/Makefile:2: Other Makefiles must not be included directly.") -} - -func (s *Suite) Test_MkLineChecker_checkInclude__Makefile_exists(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("other/existing/Makefile") - t.SetUpPackage("category/package", - ".include \"../../other/existing/Makefile\"", - ".include \"../../other/not-found/Makefile\"") - t.FinishSetUp() - - G.checkdirPackage(t.File("category/package")) - - t.CheckOutputLines( - "ERROR: ~/category/package/Makefile:21: Cannot read \"../../other/not-found/Makefile\".") -} - -func (s *Suite) Test_MkLineChecker_checkInclude__hacks(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package") - t.CreateFileLines("category/package/hacks.mk", - MkCvsID, - ".include \"../../category/package/nonexistent.mk\"", - ".include \"../../category/package/builtin.mk\"") - t.CreateFileLines("category/package/builtin.mk", - MkCvsID) - t.FinishSetUp() - - G.checkdirPackage(t.File("category/package")) - - // The purpose of this "nonexistent" diagnostic is only to show that - // hacks.mk is indeed parsed and checked. - t.CheckOutputLines( - "ERROR: ~/category/package/hacks.mk:2: " + - "Relative path \"../../category/package/nonexistent.mk\" does not exist.") -} - -func (s *Suite) Test_MkLineChecker_checkInclude__builtin_mk(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package", - ".include \"../../category/package/builtin.mk\"", - ".include \"../../category/package/builtin.mk\" # ok") - t.CreateFileLines("category/package/builtin.mk", - MkCvsID) - t.FinishSetUp() - - G.checkdirPackage(t.File("category/package")) - - t.CheckOutputLines( - "ERROR: ~/category/package/Makefile:20: " + - "../../category/package/builtin.mk must not be included directly. " + - "Include \"../../category/package/buildlink3.mk\" instead.") -} - -func (s *Suite) Test_MkLineChecker_checkInclude__builtin_mk_rationale(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package", - "# I have good reasons for including this file directly.", - ".include \"../../category/package/builtin.mk\"", - "", - ".include \"../../category/package/builtin.mk\"") - t.CreateFileLines("category/package/builtin.mk", - MkCvsID) - t.FinishSetUp() - - G.checkdirPackage(t.File("category/package")) - - t.CheckOutputLines( - "ERROR: ~/category/package/Makefile:23: " + - "../../category/package/builtin.mk must not be included directly. " + - "Include \"../../category/package/buildlink3.mk\" instead.") -} - -func (s *Suite) Test_MkLineChecker__permissions_in_hacks_mk(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__hacks_mk(c *check.C) { t := s.Init(c) t.SetUpVartypes() @@ -482,528 +454,6 @@ func (s *Suite) Test_MkLineChecker__permissions_in_hacks_mk(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_MkLineChecker_checkDirective(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - - mklines := t.NewMkLines("category/package/filename.mk", - MkCvsID, - "", - ".for", - ".endfor", - "", - ".if", - ".else don't", - ".endif invalid-arg", - "", - ".ifdef FNAME_MK", - ".endif", - ".ifndef FNAME_MK", - ".endif", - "", - ".for var in a b c", - ".endfor", - ".undef var unrelated") - - mklines.Check() - - t.CheckOutputLines( - "ERROR: category/package/filename.mk:3: \".for\" requires arguments.", - "ERROR: category/package/filename.mk:6: \".if\" requires arguments.", - "ERROR: category/package/filename.mk:7: \".else\" does not take arguments. "+ - "If you meant \"else if\", use \".elif\".", - "ERROR: category/package/filename.mk:8: \".endif\" does not take arguments.", - "WARN: category/package/filename.mk:10: The \".ifdef\" directive is deprecated. "+ - "Please use \".if defined(FNAME_MK)\" instead.", - "WARN: category/package/filename.mk:12: The \".ifndef\" directive is deprecated. "+ - "Please use \".if !defined(FNAME_MK)\" instead.", - "NOTE: category/package/filename.mk:17: Using \".undef\" after a \".for\" loop is unnecessary.") -} - -func (s *Suite) Test_MkLineChecker_checkDirective__for_loop_varname(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "", - ".for VAR in a b c", // Should be lowercase. - ".endfor", - "", - ".for _var_ in a b c", // Should be written without underscores. - ".endfor", - "", - ".for .var. in a b c", // Should be written without dots. - ".endfor", - "", - ".for ${VAR} in a b c", // The variable name really must be an identifier. - ".endfor") - - mklines.Check() - - t.CheckOutputLines( - "WARN: filename.mk:3: The variable name \"VAR\" in the .for loop should not contain uppercase letters.", - "WARN: filename.mk:6: Variable names starting with an underscore (_var_) are reserved for internal pkgsrc use.", - "ERROR: filename.mk:9: Invalid variable name \".var.\".", - "ERROR: filename.mk:12: Invalid variable name \"${VAR}\".") -} - -func (s *Suite) Test_MkLineChecker_checkDirectiveEnd__ending_comments(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("opsys.mk", - MkCvsID, - "", - ".for i in 1 2 3 4 5", - ". if ${OPSYS} == NetBSD", - ". if ${MACHINE_ARCH} == x86_64", - ". if ${OS_VERSION:M8.*}", - ". endif # MACHINE_ARCH", // Wrong, should be OS_VERSION. - ". endif # OS_VERSION", // Wrong, should be MACHINE_ARCH. - ". endif # OPSYS", // Correct. - ".endfor # j", // Wrong, should be i. - "", - ".if ${PKG_OPTIONS:Moption}", - ".endif # option", // Correct. - "", - ".if ${PKG_OPTIONS:Moption}", - ".endif # opti", // This typo goes unnoticed since "opti" is a substring of the condition. - "", - ".if ${OPSYS} == NetBSD", - ".elif ${OPSYS} == FreeBSD", - ".endif # NetBSD", // Wrong, should be FreeBSD from the .elif. - "", - ".for ii in 1 2", - ". for jj in 1 2", - ". endfor # ii", // Note: a simple "i" would not generate a warning because it is found in the word "in". - ".endfor # ii") - - // See MkLineChecker.checkDirective - mklines.Check() - - t.CheckOutputLines( - "WARN: opsys.mk:7: Comment \"MACHINE_ARCH\" does not match condition \"${OS_VERSION:M8.*}\".", - "WARN: opsys.mk:8: Comment \"OS_VERSION\" does not match condition \"${MACHINE_ARCH} == x86_64\".", - "WARN: opsys.mk:10: Comment \"j\" does not match loop \"i in 1 2 3 4 5\".", - "WARN: opsys.mk:12: Unknown option \"option\".", - "WARN: opsys.mk:20: Comment \"NetBSD\" does not match condition \"${OPSYS} == FreeBSD\".", - "WARN: opsys.mk:24: Comment \"ii\" does not match loop \"jj in 1 2\".") -} - -// After removing the dummy indentation in commit d5a926af, -// there was a panic: runtime error: index out of range, -// in wip/jacorb-lib/buildlink3.mk. -func (s *Suite) Test_MkLineChecker_checkDirectiveEnd__unbalanced(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "", - ".endfor # comment", - ".endif # comment") - - mklines.Check() - - t.CheckOutputLines( - "ERROR: filename.mk:3: Unmatched .endfor.", - "ERROR: filename.mk:4: Unmatched .endif.") -} - -func (s *Suite) Test_MkLineChecker_checkDirectiveFor(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("for.mk", - MkCvsID, - ".for dir in ${PATH:C,:, ,g}", - ".endfor", - "", - ".for dir in ${PATH}", - ".endfor", - "", - ".for dir in ${PATH:M*/bin}", - ".endfor") - - mklines.Check() - - t.CheckOutputLines( - // No warning about a missing :Q in line 2 since the :C modifier - // converts the colon-separated list into a space-separated list, - // as required by the .for loop. - - // This warning is correct since PATH is separated by colons, not by spaces. - "WARN: for.mk:5: Please use ${PATH:Q} instead of ${PATH}.", - - // This warning is also correct since the :M modifier doesn't change the - // word boundaries. - "WARN: for.mk:8: Please use ${PATH:M*/bin:Q} instead of ${PATH:M*/bin}.") -} - -func (s *Suite) Test_MkLineChecker_checkDirectiveFor__infrastructure(c *check.C) { - t := s.Init(c) - - t.SetUpPkgsrc() - t.CreateFileLines("mk/file.mk", - MkCvsID, - ".for i = 1 2 3", // The "=" should rather be "in". - ".endfor", - "", - ".for _i_ in 1 2 3", // Underscores are only allowed in infrastructure files. - ".endfor") - t.FinishSetUp() - - G.Check(t.File("mk/file.mk")) - - // Pkglint doesn't care about trivial syntax errors like the "=" instead - // of "in" above; bmake will already catch these. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLineChecker_checkDependencyRule(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - - mklines := t.NewMkLines("category/package/filename.mk", - MkCvsID, - "", - ".PHONY: target-1", - "target-2: .PHONY", - ".ORDER: target-1 target-2", - "target-1:", - "target-2:", - "target-3:", - "${_COOKIE.test}:") - - mklines.Check() - - t.CheckOutputLines( - "WARN: category/package/filename.mk:8: Undeclared target \"target-3\".") -} - -func (s *Suite) Test_MkLineChecker_checkVartype__simple_type(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - - // Since COMMENT is defined in vardefs.go its type is certain instead of guessed. - vartype := G.Pkgsrc.VariableType(nil, "COMMENT") - - c.Assert(vartype, check.NotNil) - t.CheckEquals(vartype.basicType.name, "Comment") - t.CheckEquals(vartype.Guessed(), false) - t.CheckEquals(vartype.List(), false) - - mklines := t.NewMkLines("Makefile", - MkCvsID, - "COMMENT=\tA nice package") - mklines.Check() - - t.CheckOutputLines( - "WARN: Makefile:2: COMMENT should not begin with \"A\".") -} - -func (s *Suite) Test_MkLineChecker_checkVartype(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "DISTNAME=\tgcc-${GCC_VERSION}") - - mklines.vars.Define("GCC_VERSION", mklines.mklines[1]) - mklines.Check() - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLineChecker_checkVartype__append_to_non_list(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "DISTNAME+=\tsuffix", - "COMMENT=\tComment for", - "COMMENT+=\tthe package") - - mklines.Check() - - t.CheckOutputLines( - "WARN: filename.mk:2: The variable DISTNAME should not be appended to "+ - "(only set, or given a default value) in this file.", - "WARN: filename.mk:2: The \"+=\" operator should only be used with lists, not with DISTNAME.") -} - -func (s *Suite) Test_MkLineChecker_checkVartype__no_tracing(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "UNKNOWN=\tvalue", - "CUR_DIR!=\tpwd") - t.DisableTracing() - - mklines.Check() - - t.CheckOutputLines( - "WARN: filename.mk:2: UNKNOWN is defined but not used.", - "WARN: filename.mk:3: CUR_DIR is defined but not used.") -} - -func (s *Suite) Test_MkLineChecker_checkVartype__one_per_line(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "PKG_FAIL_REASON+=\tSeveral words are wrong.", - "PKG_FAIL_REASON+=\t\"Properly quoted\"", - "PKG_FAIL_REASON+=\t# none") - t.DisableTracing() - - mklines.Check() - - t.CheckOutputLines( - "WARN: filename.mk:2: PKG_FAIL_REASON should only get one item per line.") -} - -// Pkglint once interpreted all lists as consisting of shell tokens, -// splitting this URL at the ampersand. -func (s *Suite) Test_MkLineChecker_checkVarassign__URL_with_shell_special_characters(c *check.C) { - t := s.Init(c) - - G.Pkg = NewPackage(t.File("graphics/gimp-fix-ca")) - t.SetUpVartypes() - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "MASTER_SITES=\thttp://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=") - - mklines.Check() - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLineChecker_checkVarassign__list(c *check.C) { - t := s.Init(c) - - t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") - t.SetUpVartypes() - t.SetUpCommandLine("-Wall", "--explain") - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "SITES.distfile=\t-${MASTER_SITE_GITHUB:=project/}") - - mklines.Check() - - t.CheckOutputLines( - "WARN: filename.mk:2: The list variable MASTER_SITE_GITHUB should not be embedded in a word.", - "", - "\tWhen a list variable has multiple elements, this expression expands", - "\tto something unexpected:", - "", - "\tExample: ${MASTER_SITE_SOURCEFORGE}directory/ expands to", - "", - "\t\thttps://mirror1.sf.net/ https://mirror2.sf.net/directory/", - "", - "\tThe first URL is missing the directory. To fix this, write", - "\t\t${MASTER_SITE_SOURCEFORGE:=directory/}.", - "", - "\tExample: -l${LIBS} expands to", - "", - "\t\t-llib1 lib2", - "", - "\tThe second library is missing the -l. To fix this, write", - "\t${LIBS:S,^,-l,}.", - "") -} - -func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - - test := func(cond string, output ...string) { - mklines := t.NewMkLines("filename.mk", - cond) - mklines.ForEach(func(mkline *MkLine) { - MkLineChecker{mklines, mkline}.checkDirectiveCond() - }) - t.CheckOutput(output) - } - - test( - ".if !empty(PKGSRC_COMPILER:Mmycc)", - "WARN: filename.mk:1: The pattern \"mycc\" cannot match any of "+ - "{ ccache ccc clang distcc f2c gcc hp icc ido "+ - "mipspro mipspro-ucode pcc sunpro xlc } for PKGSRC_COMPILER.") - - test( - ".elif ${A} != ${B}", - "WARN: filename.mk:1: A is used but not defined.", - "WARN: filename.mk:1: B is used but not defined.") - - test(".if ${HOMEPAGE} == \"mailto:someone@example.org\"", - "WARN: filename.mk:1: \"mailto:someone@example.org\" is not a valid URL.", - "WARN: filename.mk:1: HOMEPAGE should not be used at load time in any file.") - - test(".if !empty(PKGSRC_RUN_TEST:M[Y][eE][sS])", - "WARN: filename.mk:1: PKGSRC_RUN_TEST should be matched "+ - "against \"[yY][eE][sS]\" or \"[nN][oO]\", not \"[Y][eE][sS]\".") - - test(".if !empty(IS_BUILTIN.Xfixes:M[yY][eE][sS])") - - test(".if !empty(${IS_BUILTIN.Xfixes:M[yY][eE][sS]})", - "WARN: filename.mk:1: The empty() function takes a variable name as parameter, "+ - "not a variable expression.") - - test(".if ${PKGSRC_COMPILER} == \"msvc\"", - "WARN: filename.mk:1: \"msvc\" is not valid for PKGSRC_COMPILER. "+ - "Use one of { ccache ccc clang distcc f2c gcc hp icc ido mipspro mipspro-ucode pcc sunpro xlc } instead.", - "WARN: filename.mk:1: Use ${PKGSRC_COMPILER:Mmsvc} instead of the == operator.") - - test(".if ${PKG_LIBTOOL:Mlibtool}", - "NOTE: filename.mk:1: PKG_LIBTOOL should be compared using == instead of matching against \":Mlibtool\".", - "WARN: filename.mk:1: PKG_LIBTOOL should not be used at load time in any file.") - - test(".if ${MACHINE_PLATFORM:MUnknownOS-*-*} || ${MACHINE_ARCH:Mx86}", - "WARN: filename.mk:1: "+ - "The pattern \"UnknownOS\" cannot match any of "+ - "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ - "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ - "} for the operating system part of MACHINE_PLATFORM.", - "WARN: filename.mk:1: "+ - "The pattern \"x86\" cannot match any of "+ - "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm "+ - "earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb "+ - "earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 "+ - "m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax "+ - "powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ - "} for MACHINE_ARCH.", - "NOTE: filename.mk:1: MACHINE_ARCH should be compared using == instead of matching against \":Mx86\".") - - // Doesn't occur in practice since it is surprising that the ! applies - // to the comparison operator, and not to one of its arguments. - test(".if !${VAR} == value", - "WARN: filename.mk:1: VAR is used but not defined.") - - // Doesn't occur in practice since this string can never be empty. - test(".if !\"${VAR}str\"", - "WARN: filename.mk:1: VAR is used but not defined.") - - // Doesn't occur in practice since !${VAR} && !${VAR2} is more idiomatic. - test(".if !\"${VAR}${VAR2}\"", - "WARN: filename.mk:1: VAR is used but not defined.", - "WARN: filename.mk:1: VAR2 is used but not defined.") - - // Just for code coverage; always evaluates to true. - test(".if \"string\"", - nil...) - - // Code coverage for checkVar. - test(".if ${OPSYS} || ${MACHINE_ARCH}", - nil...) - - test(".if ${VAR}", - "WARN: filename.mk:1: VAR is used but not defined.") - - test(".if ${VAR} == 3", - "WARN: filename.mk:1: VAR is used but not defined.") - - test(".if \"value\" == ${VAR}", - "WARN: filename.mk:1: VAR is used but not defined.") - - test(".if ${MASTER_SITES:Mftp://*} == \"ftp://netbsd.org/\"", - "WARN: filename.mk:1: Invalid variable modifier \"//*\" for \"MASTER_SITES\".", - "WARN: filename.mk:1: \"ftp\" is not a valid URL.", - "WARN: filename.mk:1: MASTER_SITES should not be used at load time in any file.", - "WARN: filename.mk:1: Invalid variable modifier \"//*\" for \"MASTER_SITES\".") -} - -func (s *Suite) Test_MkLineChecker_checkDirectiveCondCompare(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - - test := func(cond string, output ...string) { - mklines := t.NewMkLines("filename.mk", - cond) - mklines.ForEach(func(mkline *MkLine) { - MkLineChecker{mklines, mkline}.checkDirectiveCond() - }) - t.CheckOutput(output) - } - - // As of July 2019, pkglint doesn't have specific checks for comparing - // variables to numbers. - test(".if ${VAR} > 0", - "WARN: filename.mk:1: VAR is used but not defined.") - - // For string comparisons, the checks from vartypecheck.go are - // performed. - test(".if ${DISTNAME} == \"<>\"", - "WARN: filename.mk:1: The filename \"<>\" contains the invalid characters \"<>\".", - "WARN: filename.mk:1: DISTNAME should not be used at load time in any file.") - - // This type of comparison doesn't occur in practice since it is - // overly verbose. - test(".if \"${BUILD_DIRS}str\" == \"str\"", - // TODO: why should it not be used? In a .for loop it sounds pretty normal. - "WARN: filename.mk:1: BUILD_DIRS should not be used at load time in any file.") - - // This is a shorthand for defined(VAR), but it is not used in practice. - test(".if VAR", - "WARN: filename.mk:1: Invalid condition, unrecognized part: \"VAR\".") - - // Calling a function with braces instead of parentheses is syntactically - // invalid. Pkglint is stricter than bmake in this situation. - // - // Bmake reads the "empty{VAR}" as a variable name. It then checks whether - // this variable is defined. It is not, of course, therefore the expression - // is false. The ! in front of it negates this false, which makes the whole - // condition true. - // - // See https://mail-index.netbsd.org/tech-pkg/2019/07/07/msg021539.html - test(".if !empty{VAR}", - "WARN: filename.mk:1: Invalid condition, unrecognized part: \"empty{VAR}\".") -} - -func (s *Suite) Test_MkLineChecker_checkDirectiveCond__tracing(c *check.C) { - t := s.Init(c) - - t.EnableTracingToLog() - mklines := t.NewMkLines("filename.mk", - ".if ${VAR:Mpattern1:Mpattern2} == comparison") - - mklines.ForEach(func(mkline *MkLine) { - MkLineChecker{mklines, mkline}.checkDirectiveCond() - }) - - t.CheckOutputLinesMatching(`^WARN|checkCompare`, - "TRACE: 1 checkCompareVarStr ${VAR:Mpattern1:Mpattern2} == comparison", - "WARN: filename.mk:1: VAR is used but not defined.") -} - -func (s *Suite) Test_MkLineChecker_checkVarassign(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - - mklines := t.NewMkLines("Makefile", - MkCvsID, - "ac_cv_libpari_libs+=\t-L${BUILDLINK_PREFIX.pari}/lib") // From math/clisp-pari/Makefile, rev. 1.8 - - mklines.Check() - - t.CheckOutputLines( - "WARN: Makefile:2: ac_cv_libpari_libs is defined but not used.") -} - func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions(c *check.C) { t := s.Init(c) @@ -1099,6 +549,40 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__infrastructure t.CheckOutputEmpty() } +func (s *Suite) Test_MkLineChecker_explainPermissions(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "--explain") + t.SetUpVartypes() + + mklines := t.NewMkLines("buildlink3.mk", + MkCvsID, + "AUTO_MKDIRS=\tyes") + + mklines.Check() + + t.CheckOutputLines( + "WARN: buildlink3.mk:2: The variable AUTO_MKDIRS should not be set in this file; "+ + "it would be ok in Makefile, Makefile.* or *.mk, "+ + "but not buildlink3.mk or builtin.mk.", + "", + "\tThe allowed actions for a variable are determined based on the file", + "\tname in which the variable is used or defined. The rules for", + "\tAUTO_MKDIRS are:", + "", + "\t* in buildlink3.mk, it should not be accessed at all", + "\t* in builtin.mk, it should not be accessed at all", + "\t* in Makefile, it may be set, given a default value, or used", + "\t* in Makefile.*, it may be set, given a default value, or used", + "\t* in *.mk, it may be set, given a default value, or used", + // TODO: Add a check for infrastructure permissions + // when the "infra:" prefix is added. + "", + "\tIf these rules seem to be incorrect, please ask on the", + "\ttech-pkg@NetBSD.org mailing list.", + "") +} + func (s *Suite) Test_MkLineChecker_checkVarassignLeftRationale(c *check.C) { t := s.Init(c) @@ -1190,96 +674,310 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftRationale(c *check.C) { nil...) } -func (s *Suite) Test_MkLineChecker_checkVarassignOpShell(c *check.C) { +// The ${VARNAME:=suffix} expression should only be used with lists. +// It typically appears in MASTER_SITE definitions. +func (s *Suite) Test_MkLineChecker_CheckVaruse__eq_nonlist(c *check.C) { t := s.Init(c) - t.SetUpTool("uname", "UNAME", AfterPrefsMk) - t.SetUpTool("echo", "", AtRunTime) + t.SetUpVartypes() + t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") + mklines := t.SetUpFileMkLines("options.mk", + MkCvsID, + "WRKSRC=\t\t${WRKDIR:=/subdir}", + "MASTER_SITES=\t${MASTER_SITE_GITHUB:=organization/}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: ~/options.mk:2: The :from=to modifier should only be used with lists, not with WRKDIR.") +} + +func (s *Suite) Test_MkLineChecker_CheckVaruse__for(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") + mklines := t.SetUpFileMkLines("options.mk", + MkCvsID, + ".for var in a b c", + "\t: ${var}", + ".endfor") + + mklines.Check() + + t.CheckOutputEmpty() +} + +// When a parameterized variable is defined in the pkgsrc infrastructure, +// it does not generate a warning about being "used but not defined". +// Even if the variable parameter differs, like .Linux and .SunOS in this +// case. This pattern is typical for pkgsrc, therefore pkglint doesn't +// check that the variable names match exactly. +func (s *Suite) Test_MkLineChecker_CheckVaruse__varcanon(c *check.C) { + t := s.Init(c) + b := NewMkTokenBuilder() + t.SetUpPkgsrc() - t.SetUpPackage("category/package", - ".include \"standalone.mk\"") - t.CreateFileLines("category/package/standalone.mk", + t.CreateFileLines("mk/sys-vars.mk", MkCvsID, - "", - ".include \"../../mk/bsd.prefs.mk\"", - "", - "OPSYS_NAME!=\t${UNAME}", - ".if ${OPSYS_NAME} == \"NetBSD\"", - ".endif", - "", - "OS_NAME!=\t${UNAME}", - "", - "MUST_BE_EARLY!=\techo 123 # must be evaluated early", - "", - "show-package-vars: .PHONY", - "\techo OS_NAME=${OS_NAME:Q}", - "\techo MUST_BE_EARLY=${MUST_BE_EARLY:Q}") + "CPPPATH.Linux=\t/usr/bin/cpp") t.FinishSetUp() - G.Check(t.File("category/package/standalone.mk")) + mklines := t.NewMkLines("module.mk", + MkCvsID, + "COMMENT=\t${CPPPATH.SunOS}") + + ck := MkLineChecker{mklines, mklines.mklines[1]} + + ck.CheckVaruse(b.VarUse("CPPPATH.SunOS"), &VarUseContext{ + vartype: &Vartype{ + basicType: BtPathname, + options: Guessed, + aclEntries: nil, + }, + time: VucRunTime, + quoting: VucQuotPlain, + IsWordPart: false, + }) + + t.CheckOutputEmpty() +} + +// Any variable that is defined in the pkgsrc infrastructure in mk/**/*.mk is +// considered defined, and no "used but not defined" warning is logged for it. +// +// See Pkgsrc.loadUntypedVars. +func (s *Suite) Test_MkLineChecker_CheckVaruse__defined_in_infrastructure(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("mk/deeply/nested/infra.mk", + MkCvsID, + "INFRA_VAR?=\tvalue") + t.FinishSetUp() + mklines := t.SetUpFileMkLines("category/package/module.mk", + MkCvsID, + "do-fetch:", + "\t: ${INFRA_VAR} ${UNDEFINED}") + + mklines.Check() - // There is no warning about any variable since no package is currently - // being checked, therefore pkglint cannot decide whether the variable - // is used a load time. t.CheckOutputLines( - "WARN: ~/category/package/standalone.mk:14: Please use \"${ECHO}\" instead of \"echo\".", - "WARN: ~/category/package/standalone.mk:15: Please use \"${ECHO}\" instead of \"echo\".") + "WARN: ~/category/package/module.mk:3: UNDEFINED is used but not defined.") +} - t.SetUpCommandLine("-Wall", "--explain") - G.Check(t.File("category/package")) +func (s *Suite) Test_MkLineChecker_CheckVaruse__build_defs(c *check.C) { + t := s.Init(c) + + // XXX: This paragraph should not be necessary since VARBASE and X11_TYPE + // are also defined in vardefs.go. + t.SetUpPkgsrc() + t.CreateFileLines("mk/defaults/mk.conf", + "VARBASE?= /usr/pkg/var") + t.SetUpCommandLine("-Wall,no-space") + t.FinishSetUp() + + mklines := t.SetUpFileMkLines("options.mk", + MkCvsID, + "COMMENT= ${VARBASE} ${X11_TYPE}", + "PKG_FAIL_REASON+= ${VARBASE} ${X11_TYPE}", + "BUILD_DEFS+= X11_TYPE") + + mklines.Check() - // There is no warning for OPSYS_NAME since that variable is used at - // load time. In such a case the command has to be executed anyway, - // and executing it exactly once is the best thing to do. - // - // There is no warning for MUST_BE_EARLY since the comment provides the - // reason that this command really has to be executed at load time. t.CheckOutputLines( - "NOTE: ~/category/package/standalone.mk:9: Consider the :sh modifier instead of != for \"${UNAME}\".", - "", - "\tFor variable assignments using the != operator, the shell command is", - "\trun every time the file is parsed. In some cases this is too early,", - "\tand the command may not yet be installed. In other cases the command", - "\tis executed more often than necessary. Most commands don't need to", - "\tbe executed for \"make clean\", for example.", - "", - "\tThe :sh modifier defers execution until the variable value is", - "\tactually needed. On the other hand, this means the command is", - "\texecuted each time the variable is evaluated.", - "", - "\tExample:", - "", - "\t\tEARLY_YEAR!= date +%Y", - "", - "\t\tLATE_YEAR_CMD= date +%Y", - "\t\tLATE_YEAR= ${LATE_YEAR_CMD:sh}", + "WARN: ~/options.mk:2: The user-defined variable VARBASE is used but not added to BUILD_DEFS.", + "WARN: ~/options.mk:3: PKG_FAIL_REASON should only get one item per line.") +} + +// The LOCALBASE variable may be defined and used in the infrastructure. +// It is always equivalent to PREFIX and only exists for historic reasons. +func (s *Suite) Test_MkLineChecker_CheckVaruse__LOCALBASE_in_infrastructure(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("mk/infra.mk", + MkCvsID, + "LOCALBASE?=\t${PREFIX}", + "DEFAULT_PREFIX=\t${LOCALBASE}") + t.FinishSetUp() + + G.Check(t.File("mk/infra.mk")) + + // No warnings about LOCALBASE being used; the infrastructure files may + // do this. In packages though, LOCALBASE is deprecated. + + // There is no warning about DEFAULT_PREFIX being "defined but not used" + // since Pkgsrc.loadUntypedVars calls Pkgsrc.vartypes.DefineType, which + // registers that variable globally. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLineChecker_CheckVaruse__user_defined_variable_and_BUILD_DEFS(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("mk/defaults/mk.conf", + "VARBASE?=\t${PREFIX}/var", + "PYTHON_VER?=\t36") + mklines := t.NewMkLines("file.mk", + MkCvsID, + "BUILD_DEFS+=\tPYTHON_VER", + "\t: ${VARBASE}", + "\t: ${VARBASE}", + "\t: ${PYTHON_VER}") + t.FinishSetUp() + + mklines.Check() + + t.CheckOutputLines( + "WARN: file.mk:3: The user-defined variable VARBASE is used but not added to BUILD_DEFS.") +} + +func (s *Suite) Test_MkLineChecker_CheckVaruse__deprecated_PKG_DEBUG(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + G.Pkgsrc.initDeprecatedVars() + + mklines := t.NewMkLines("module.mk", + MkCvsID, + "\t${_PKG_SILENT}${_PKG_DEBUG} :") + + mklines.Check() + + t.CheckOutputLines( + "WARN: module.mk:2: Use of _PKG_SILENT and _PKG_DEBUG is deprecated. Use ${RUN} instead.") +} + +func (s *Suite) Test_MkLineChecker_checkVaruseUndefined(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("mk/infra.mk", + MkCvsID, + "#", + "# User-settable variables:", + "#", + "# DOCUMENTED", "", - "\t\t# or, in a single line:", - "\t\tLATE_YEAR= ${date +%Y:L:sh}", + "ASSIGNED=\tassigned", + "#COMMENTED=\tcommented") + t.FinishSetUp() + + mklines := t.NewMkLines("filename.mk", + MkCvsID, "", - "\tTo suppress this note, provide an explanation in a comment at the", - "\tend of the line, or force the variable to be evaluated at load time,", - "\tby using it at the right-hand side of the := operator, or in an .if", - "\tor .for directive.", + "do-build:", + "\t: ${ASSIGNED} ${COMMENTED} ${DOCUMENTED} ${UNKNOWN}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: filename.mk:4: UNKNOWN is used but not defined.") +} + +// PR 46570, item "15. net/uucp/Makefile has a make loop" +func (s *Suite) Test_MkLineChecker_checkVaruseUndefined__indirect_variables(c *check.C) { + t := s.Init(c) + + t.SetUpTool("echo", "ECHO", AfterPrefsMk) + mklines := t.NewMkLines("net/uucp/Makefile", + MkCvsID, + "\techo ${UUCP_${var}}") + + mklines.Check() + + // No warning about UUCP_${var} being used but not defined. + // + // Normally, parameterized variables use a dot instead of an underscore as separator. + // This is one of the few other cases. Pkglint doesn't warn about dynamic variable + // names like UUCP_${var} or SITES_${distfile}. + // + // It does warn about simple variable names though, like ${var} in this example. + t.CheckOutputLines( + "WARN: net/uucp/Makefile:2: var is used but not defined.") +} + +// Documented variables are declared as both defined and used since, as +// of April 2019, pkglint doesn't yet interpret the "Package-settable +// variables" comment. +func (s *Suite) Test_MkLineChecker_checkVaruseUndefined__documented(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + mklines := t.NewMkLines("interpreter.mk", + MkCvsID, + "#", + "# Package-settable variables:", + "#", + "# REPLACE_INTERP", + "#\tThe list of files whose interpreter will be corrected.", "", - "WARN: ~/category/package/standalone.mk:14: Please use \"${ECHO}\" instead of \"echo\".", - "WARN: ~/category/package/standalone.mk:15: Please use \"${ECHO}\" instead of \"echo\".") + "REPLACE_INTERPRETER+=\tinterp", + "REPLACE.interp.old=\t.*/interp", + "REPLACE.interp.new=\t${PREFIX}/bin/interp", + "REPLACE_FILES.interp=\t${REPLACE_INTERP}") + + mklines.Check() + + t.CheckOutputEmpty() } -func (s *Suite) Test_MkLineChecker_checkVarassignRightVaruse(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVaruseModifiersSuffix(c *check.C) { t := s.Init(c) t.SetUpVartypes() + mklines := t.NewMkLines("file.mk", + MkCvsID, + "\t: ${HOMEPAGE:=subdir/:Q}", // wrong + "\t: ${BUILD_DIRS:=subdir/}", // correct + "\t: ${BIN_PROGRAMS:=.exe}") // unknown since BIN_PROGRAMS doesn't have a type - mklines := t.NewMkLines("module.mk", + mklines.Check() + + t.CheckOutputLines( + "WARN: file.mk:2: The :from=to modifier should only be used with lists, not with HOMEPAGE.", + "WARN: file.mk:4: BIN_PROGRAMS is used but not defined.") +} + +func (s *Suite) Test_MkLineChecker_checkVaruseModifiersRange(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--show-autofix", "--source") + t.SetUpVartypes() + mklines := t.NewMkLines("mk/compiler/gcc.mk", MkCvsID, - "PLIST_SUBST+=\tLOCALBASE=${LOCALBASE:Q}") + "CC:=\t${CC:C/^/_asdf_/1:M_asdf_*:S/^_asdf_//}") mklines.Check() t.CheckOutputLines( - "WARN: module.mk:2: Please use PREFIX instead of LOCALBASE.", - "NOTE: module.mk:2: The :Q modifier isn't necessary for ${LOCALBASE} here.") + "NOTE: mk/compiler/gcc.mk:2: "+ + "The modifier \":C/^/_asdf_/1:M_asdf_*:S/^_asdf_//\" can be written as \":[1]\".", + "AUTOFIX: mk/compiler/gcc.mk:2: "+ + "Replacing \":C/^/_asdf_/1:M_asdf_*:S/^_asdf_//\" with \":[1]\".", + "-\tCC:=\t${CC:C/^/_asdf_/1:M_asdf_*:S/^_asdf_//}", + "+\tCC:=\t${CC:[1]}") + + // Now go through all the "almost" cases, to reach full branch coverage. + mklines = t.NewMkLines("gcc.mk", + MkCvsID, + "\t: ${CC:M1:M2:M3}", + "\t: ${CC:C/^begin//:M2:M3}", // M1 pattern not exactly ^ + "\t: ${CC:C/^/_asdf_/g:M2:M3}", // M1 options != "1" + "\t: ${CC:C/^/....../g:M2:M3}", // M1 replacement doesn't match \w+ + "\t: ${CC:C/^/_asdf_/1:O:M3}", // M2 is not a match modifier + "\t: ${CC:C/^/_asdf_/1:N2:M3}", // M2 is :N instead of :M + "\t: ${CC:C/^/_asdf_/1:M_asdf_:M3}", // M2 pattern is missing the * at the end + "\t: ${CC:C/^/_asdf_/1:Mother:M3}", // M2 pattern differs from the M1 pattern + "\t: ${CC:C/^/_asdf_/1:M_asdf_*:M3}", // M3 ist not a substitution modifier + "\t: ${CC:C/^/_asdf_/1:M_asdf_*:S,from,to,}", // M3 pattern differs from the M1 pattern + "\t: ${CC:C/^/_asdf_/1:M_asdf_*:S,^_asdf_,to,}", // M3 replacement is not empty + "\t: ${CC:C/^/_asdf_/1:M_asdf_*:S,^_asdf_,,g}") // M3 modifier has options + + mklines.Check() } func (s *Suite) Test_MkLineChecker_checkVarusePermissions(c *check.C) { @@ -1354,40 +1052,6 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__explain(c *check.C) { "\ttech-pkg@NetBSD.org mailing list.", "") } -func (s *Suite) Test_MkLineChecker_explainPermissions(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("-Wall", "--explain") - t.SetUpVartypes() - - mklines := t.NewMkLines("buildlink3.mk", - MkCvsID, - "AUTO_MKDIRS=\tyes") - - mklines.Check() - - t.CheckOutputLines( - "WARN: buildlink3.mk:2: The variable AUTO_MKDIRS should not be set in this file; "+ - "it would be ok in Makefile, Makefile.* or *.mk, "+ - "but not buildlink3.mk or builtin.mk.", - "", - "\tThe allowed actions for a variable are determined based on the file", - "\tname in which the variable is used or defined. The rules for", - "\tAUTO_MKDIRS are:", - "", - "\t* in buildlink3.mk, it should not be accessed at all", - "\t* in builtin.mk, it should not be accessed at all", - "\t* in Makefile, it may be set, given a default value, or used", - "\t* in Makefile.*, it may be set, given a default value, or used", - "\t* in *.mk, it may be set, given a default value, or used", - // TODO: Add a check for infrastructure permissions - // when the "infra:" prefix is added. - "", - "\tIf these rules seem to be incorrect, please ask on the", - "\ttech-pkg@NetBSD.org mailing list.", - "") -} - func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time(c *check.C) { t := s.Init(c) @@ -1706,43 +1370,6 @@ func (s *Suite) Test_MkLineChecker_warnVarusePermissions__not_directly_and_no_al "WARN: mk-c.mk:7: BUILDLINK_PKGSRCDIR.mk-c should not be used in any file.") } -func (s *Suite) Test_MkLineChecker_checkVarassignDecreasingVersions(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("Makefile", - MkCvsID, - "PYTHON_VERSIONS_ACCEPTED=\t36 __future__ # rationale", - "PYTHON_VERSIONS_ACCEPTED=\t36 -13 # rationale", - "PYTHON_VERSIONS_ACCEPTED=\t36 ${PKGVERSION_NOREV} # rationale", - "PYTHON_VERSIONS_ACCEPTED=\t36 37 # rationale", - "PYTHON_VERSIONS_ACCEPTED=\t37 36 27 25 # rationale") - - // TODO: All but the last of the above assignments should be flagged as - // redundant by RedundantScope; as of March 2019, that check is only - // implemented for package Makefiles, not for individual files. - - mklines.Check() - - // Half of these warnings are from VartypeCheck.Version, the - // other half are from checkVarassignDecreasingVersions. - // Strictly speaking some of them are redundant, but that would - // mean to reject only variable references in checkVarassignDecreasingVersions. - // This is probably ok. - // TODO: Fix this. - t.CheckOutputLines( - "WARN: Makefile:2: Invalid version number \"__future__\".", - "ERROR: Makefile:2: Value \"__future__\" for "+ - "PYTHON_VERSIONS_ACCEPTED must be a positive integer.", - "WARN: Makefile:3: Invalid version number \"-13\".", - "ERROR: Makefile:3: Value \"-13\" for "+ - "PYTHON_VERSIONS_ACCEPTED must be a positive integer.", - "ERROR: Makefile:4: Value \"${PKGVERSION_NOREV}\" for "+ - "PYTHON_VERSIONS_ACCEPTED must be a positive integer.", - "WARN: Makefile:5: The values for PYTHON_VERSIONS_ACCEPTED "+ - "should be in decreasing order (37 before 36).") -} - func (s *Suite) Test_MkLineChecker_warnVaruseToolLoadTime(c *check.C) { t := s.Init(c) @@ -1800,524 +1427,6 @@ func (s *Suite) Test_MkLineChecker_warnVaruseToolLoadTime__local_tool(c *check.C "WARN: ~/category/package/Makefile:7: The tool ${MK_TOOL} cannot be used at load time.") } -func (s *Suite) Test_MkLineChecker_Check__warn_varuse_LOCALBASE(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("options.mk", - MkCvsID, - "PKGNAME=\t${LOCALBASE}") - - mklines.Check() - - t.CheckOutputLines( - "WARN: options.mk:2: Please use PREFIX instead of LOCALBASE.") -} - -func (s *Suite) Test_MkLineChecker_CheckRelativePkgdir(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("other/package/Makefile") - - test := func(relativePkgdir string, diagnostics ...string) { - // Must be in the filesystem because of directory references. - mklines := t.SetUpFileMkLines("category/package/Makefile", - "# dummy") - - checkRelativePkgdir := func(mkline *MkLine) { - MkLineChecker{mklines, mkline}.CheckRelativePkgdir(relativePkgdir) - } - - mklines.ForEach(checkRelativePkgdir) - - t.CheckOutput(diagnostics) - } - - test("../pkgbase", - "ERROR: ~/category/package/Makefile:1: Relative path \"../pkgbase/Makefile\" does not exist.", - "WARN: ~/category/package/Makefile:1: \"../pkgbase\" is not a valid relative package directory.") - - test("../../other/package", - nil...) - - test("../../other/does-not-exist", - "ERROR: ~/category/package/Makefile:1: Relative path \"../../other/does-not-exist/Makefile\" does not exist.") - - test("${OTHER_PACKAGE}", - nil...) -} - -// PR pkg/46570, item 2 -func (s *Suite) Test_MkLineChecker__unclosed_varuse(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("Makefile", - MkCvsID, - "EGDIRS=\t${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d") - - mklines.Check() - - t.CheckOutputLines( - "WARN: Makefile:2: Missing closing \"}\" for \"EGDIR/pam.d\".", - "WARN: Makefile:2: Invalid part \"/pam.d\" after variable name \"EGDIR\".", - "WARN: Makefile:2: Missing closing \"}\" for \"EGDIR/dbus-1/system.d ${EGDIR/pam.d\".", - "WARN: Makefile:2: Invalid part \"/dbus-1/system.d ${EGDIR/pam.d\" after variable name \"EGDIR\".", - "WARN: Makefile:2: Missing closing \"}\" for \"EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".", - "WARN: Makefile:2: Invalid part \"/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\" after variable name \"EGDIR\".", - "WARN: Makefile:2: EGDIRS is defined but not used.", - "WARN: Makefile:2: EGDIR/pam.d is used but not defined.") -} - -func (s *Suite) Test_MkLineChecker_Check__varuse_modifier_L(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("x11/xkeyboard-config/Makefile", - MkCvsID, - "FILES_SUBST+=\tXKBCOMP_SYMLINK=${${XKBBASE}/xkbcomp:L:Q}", - "FILES_SUBST+=\tXKBCOMP_SYMLINK=${${XKBBASE}/xkbcomp:Q}") - - mklines.Check() - - // In line 2, don't warn that ${XKBBASE}/xkbcomp is used but not defined. - // This is because the :L modifier interprets everything before as an expression - // instead of a variable name. - // - // In line 3 the :L modifier is missing, therefore ${XKBBASE}/xkbcomp is the - // name of another variable, and that variable is not known. Only XKBBASE is known. - // - // In line 3, warn about the invalid "/" as part of the variable name. - t.CheckOutputLines( - "WARN: x11/xkeyboard-config/Makefile:3: "+ - "Invalid part \"/xkbcomp\" after variable name \"${XKBBASE}\".", - "WARN: x11/xkeyboard-config/Makefile:3: XKBBASE is used but not defined.") -} - -func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparison_with_shell_command(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("security/openssl/Makefile", - MkCvsID, - ".if ${PKGSRC_COMPILER} == \"gcc\" && ${CC} == \"cc\"", - ".endif") - - mklines.Check() - - // Don't warn about unknown shell command "cc". - t.CheckOutputLines( - "WARN: security/openssl/Makefile:2: Use ${PKGSRC_COMPILER:Mgcc} instead of the == operator.") -} - -// The :N modifier filters unwanted values. After this filter, any variable value -// may be compared with the empty string, regardless of the variable type. -// Effectively, the :N modifier changes the type from T to Option(T). -func (s *Suite) Test_MkLineChecker_checkDirectiveCond__compare_pattern_with_empty(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("filename.mk", - MkCvsID, - ".if ${X11BASE:Npattern} == \"\"", - ".endif", - "", - ".if ${X11BASE:N<>} == \"*\"", - ".endif", - "", - ".if !(${OPSYS:M*BSD} != \"\")", - ".endif") - - mklines.Check() - - // TODO: There should be a warning about "<>" containing invalid - // characters for a path. See VartypeCheck.Pathname - t.CheckOutputLines( - "WARN: filename.mk:5: The pathname pattern \"<>\" contains the invalid characters \"<>\".", - "WARN: filename.mk:5: The pathname \"*\" contains the invalid character \"*\".") -} - -func (s *Suite) Test_MkLineChecker_checkDirectiveCondEmpty(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - t.Chdir(".") - - test := func(before string, diagnosticsAndAfter ...string) { - - mklines := t.SetUpFileMkLines("module.mk", - MkCvsID, - before, - ".endif") - ck := MkLineChecker{mklines, mklines.mklines[1]} - - t.SetUpCommandLine("-Wall") - mklines.ForEach(func(mkline *MkLine) { - if mkline == mklines.mklines[1] { - ck.checkDirectiveCond() - } - }) - - t.SetUpCommandLine("-Wall", "--autofix") - mklines.ForEach(func(mkline *MkLine) { - if mkline == mklines.mklines[1] { - ck.checkDirectiveCond() - } - }) - - mklines.SaveAutofixChanges() - afterMklines := t.LoadMkInclude("module.mk") - - if len(diagnosticsAndAfter) > 0 { - diagLen := len(diagnosticsAndAfter) - diagnostics := diagnosticsAndAfter[:diagLen-1] - after := diagnosticsAndAfter[diagLen-1] - - t.CheckOutput(diagnostics) - t.CheckEquals(afterMklines.mklines[1].Text, after) - } else { - t.CheckOutputEmpty() - } - } - - test( - ".if ${PKGPATH:Mpattern}", - - "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpattern\".", - "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Mpattern}\" with \"${PKGPATH} == pattern\".", - - ".if ${PKGPATH} == pattern") - - // When the pattern contains placeholders, it cannot be converted to == or !=. - test( - ".if ${PKGPATH:Mpa*n}", - nil...) - - // The :tl modifier prevents the autofix. - test( - ".if ${PKGPATH:tl:Mpattern}", - - "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpattern\".", - - ".if ${PKGPATH:tl:Mpattern}") - - test( - ".if ${PKGPATH:Ncategory/package}", - - "NOTE: module.mk:2: PKGPATH should be compared using != instead of matching against \":Ncategory/package\".", - "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Ncategory/package}\" with \"${PKGPATH} != category/package\".", - - ".if ${PKGPATH} != category/package") - - // ${PKGPATH:None:Ntwo} is a short variant of ${PKGPATH} != "one" && - // ${PKGPATH} != "two". Applying the transformation would make the - // condition longer than before, therefore nothing is done here. - test( - ".if ${PKGPATH:None:Ntwo}", - nil...) - - // Note: this combination doesn't make sense since the patterns "one" and "two" don't overlap. - test(".if ${PKGPATH:Mone:Mtwo}", - - "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mone\".", - "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mtwo\".", - - ".if ${PKGPATH:Mone:Mtwo}") - - test(".if !empty(PKGPATH:Mpattern)", - - "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpattern\".", - "AUTOFIX: module.mk:2: Replacing \"!empty(PKGPATH:Mpattern)\" with \"${PKGPATH} == pattern\".", - - ".if ${PKGPATH} == pattern") - - test(".if empty(PKGPATH:Mpattern)", - - "NOTE: module.mk:2: PKGPATH should be compared using != instead of matching against \":Mpattern\".", - "AUTOFIX: module.mk:2: Replacing \"empty(PKGPATH:Mpattern)\" with \"${PKGPATH} != pattern\".", - - ".if ${PKGPATH} != pattern") - - test(".if !!empty(PKGPATH:Mpattern)", - - // TODO: When taking all the ! into account, this is actually a - // test for emptiness, therefore the diagnostics should suggest - // the != operator instead of ==. - "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpattern\".", - "AUTOFIX: module.mk:2: Replacing \"!empty(PKGPATH:Mpattern)\" with \"${PKGPATH} == pattern\".", - - // TODO: The ! and == could be combined into a !=. - // Luckily the !! pattern doesn't occur in practice. - ".if !${PKGPATH} == pattern") - - test(".if empty(PKGPATH:Mpattern) || 0", - - "NOTE: module.mk:2: PKGPATH should be compared using != instead of matching against \":Mpattern\".", - "AUTOFIX: module.mk:2: Replacing \"empty(PKGPATH:Mpattern)\" with \"${PKGPATH} != pattern\".", - - ".if ${PKGPATH} != pattern || 0") - - // No note in this case since there is no implicit !empty around the varUse. - test(".if ${PKGPATH:Mpattern} != ${OTHER}", - - "WARN: module.mk:2: OTHER is used but not defined.", - - ".if ${PKGPATH:Mpattern} != ${OTHER}") - - test( - ".if ${PKGPATH:Mpattern}", - - "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpattern\".", - "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Mpattern}\" with \"${PKGPATH} == pattern\".", - - ".if ${PKGPATH} == pattern") - - test( - ".if !${PKGPATH:Mpattern}", - - "NOTE: module.mk:2: PKGPATH should be compared using != instead of matching against \":Mpattern\".", - "AUTOFIX: module.mk:2: Replacing \"!${PKGPATH:Mpattern}\" with \"${PKGPATH} != pattern\".", - - ".if ${PKGPATH} != pattern") - - test( - ".if !!${PKGPATH:Mpattern}", - - "NOTE: module.mk:2: PKGPATH should be compared using != instead of matching against \":Mpattern\".", - "AUTOFIX: module.mk:2: Replacing \"!${PKGPATH:Mpattern}\" with \"${PKGPATH} != pattern\".", - - ".if !${PKGPATH} != pattern") - - // This pattern with spaces doesn't make sense at all in the :M - // modifier since it can never match. - // Or can it, if the PKGPATH contains quotes? - // How exactly does bmake apply the matching here, are both values unquoted? - test( - ".if ${PKGPATH:Mpattern with spaces}", - - "WARN: module.mk:2: The pathname pattern \"pattern with spaces\" "+ - "contains the invalid characters \" \".", - - ".if ${PKGPATH:Mpattern with spaces}") - // TODO: ".if ${PKGPATH} == \"pattern with spaces\"") - - test( - ".if ${PKGPATH:M'pattern with spaces'}", - - "WARN: module.mk:2: The pathname pattern \"'pattern with spaces'\" "+ - "contains the invalid characters \"' '\".", - - ".if ${PKGPATH:M'pattern with spaces'}") - // TODO: ".if ${PKGPATH} == 'pattern with spaces'") - - test( - ".if ${PKGPATH:M&&}", - - "WARN: module.mk:2: The pathname pattern \"&&\" "+ - "contains the invalid characters \"&&\".", - - ".if ${PKGPATH:M&&}") - // TODO: ".if ${PKGPATH} == '&&'") - - // If PKGPATH is "", the condition is false. - // If PKGPATH is "negative-pattern", the condition is false. - // In all other cases, the condition is true. - // - // Therefore this condition cannot simply be transformed into - // ${PKGPATH} != negative-pattern, since that would produce a - // different result in the case where PKGPATH is empty. - // - // For system-provided variables that are guaranteed to be non-empty, - // such as OPSYS or PKGPATH, this replacement is valid. - // These variables are only guaranteed to be defined after bsd.prefs.mk - // has been included, like everywhere else. - test( - ".if ${PKGPATH:Nnegative-pattern}", - - "NOTE: module.mk:2: PKGPATH should be compared using != instead of matching against \":Nnegative-pattern\".", - "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Nnegative-pattern}\" with \"${PKGPATH} != negative-pattern\".", - - ".if ${PKGPATH} != negative-pattern") - - // Since UNKNOWN is not a well-known system-provided variable that is - // guaranteed to be non-empty (see the previous example), it is not - // transformed at all. - test( - ".if ${UNKNOWN:Nnegative-pattern}", - - "WARN: module.mk:2: UNKNOWN is used but not defined.", - - ".if ${UNKNOWN:Nnegative-pattern}") - - test( - ".if ${PKGPATH:Mpath1} || ${PKGPATH:Mpath2}", - - "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpath1\".", - "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpath2\".", - "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Mpath1}\" with \"${PKGPATH} == path1\".", - "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Mpath2}\" with \"${PKGPATH} == path2\".", - - ".if ${PKGPATH} == path1 || ${PKGPATH} == path2") - - test( - ".if (((((${PKGPATH:Mpath})))))", - - "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpath\".", - "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Mpath}\" with \"${PKGPATH} == path\".", - - ".if (((((${PKGPATH} == path)))))") - - // Note: this combination doesn't make sense since the patterns "one" and "two" don't overlap. - test( - ".if ${PKGPATH:Mone:Mtwo}", - - "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mone\".", - "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mtwo\".", - - ".if ${PKGPATH:Mone:Mtwo}") - - test( - ".if ${MACHINE_ARCH:Mx86_64}", - - "NOTE: module.mk:2: MACHINE_ARCH should be compared using == instead of matching against \":Mx86_64\".", - "AUTOFIX: module.mk:2: Replacing \"${MACHINE_ARCH:Mx86_64}\" with \"${MACHINE_ARCH} == x86_64\".", - - ".if ${MACHINE_ARCH} == x86_64") -} - -func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparing_PKGSRC_COMPILER_with_eqeq(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("Makefile", - MkCvsID, - ".if ${PKGSRC_COMPILER} == \"clang\"", - ".elif ${PKGSRC_COMPILER} != \"gcc\"", - ".endif") - - mklines.Check() - - t.CheckOutputLines( - "WARN: Makefile:2: Use ${PKGSRC_COMPILER:Mclang} instead of the == operator.", - "WARN: Makefile:3: Use ${PKGSRC_COMPILER:Ngcc} instead of the != operator.") -} - -func (s *Suite) Test_MkLineChecker_checkDirectiveCondCompareVarStr__no_tracing(c *check.C) { - t := s.Init(c) - b := NewMkTokenBuilder() - - t.SetUpVartypes() - mklines := t.NewMkLines("filename.mk", - ".if ${DISTFILES:Mpattern:O:u} == NetBSD") - t.DisableTracing() - - ck := MkLineChecker{mklines, mklines.mklines[0]} - varUse := b.VarUse("DISTFILES", "Mpattern", "O", "u") - ck.checkDirectiveCondCompareVarStr(varUse, "==", "distfile-1.0.tar.gz") - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS_with_backticks(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("chat/pidgin-icb/Makefile", - MkCvsID, - "CFLAGS+=\t`pkg-config pidgin --cflags`") - mkline := mklines.mklines[1] - - words := mkline.Fields() - - // bmake handles backticks in the same way, treating them as ordinary characters - t.CheckDeepEquals(words, []string{"`pkg-config", "pidgin", "--cflags`"}) - - ck := MkLineChecker{mklines, mklines.mklines[1]} - ck.checkVartype("CFLAGS", opAssignAppend, "`pkg-config pidgin --cflags`", "") - - // No warning about "`pkg-config" being an unknown CFlag. - // As of September 2019, there is no such check anymore in pkglint. - t.CheckOutputEmpty() -} - -// See PR 46570, Ctrl+F "4. Shell quoting". -// Pkglint is correct, since the shell sees this definition for -// CPPFLAGS as three words, not one word. -func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("Makefile", - MkCvsID, - "CPPFLAGS.SunOS+=\t-DPIPECOMMAND=\\\"/usr/sbin/sendmail -bs %s\\\"") - - mklines.Check() - - t.CheckOutputLines( - "WARN: Makefile:2: Compiler flag \"-DPIPECOMMAND=\\\\\\\"/usr/sbin/sendmail\" has unbalanced double quotes.", - "WARN: Makefile:2: Compiler flag \"%s\\\\\\\"\" has unbalanced double quotes.") -} - -func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--autofix", "-Wspace") - lines := t.SetUpFileLines("filename.mk", - MkCvsID, - ".if defined(A)", - ".for a in ${A}", - ".if defined(C)", - ".endif", - ".endfor", - ".endif") - mklines := NewMkLines(lines) - - mklines.Check() - - t.CheckOutputLines( - "AUTOFIX: ~/filename.mk:3: Replacing \".\" with \". \".", - "AUTOFIX: ~/filename.mk:4: Replacing \".\" with \". \".", - "AUTOFIX: ~/filename.mk:5: Replacing \".\" with \". \".", - "AUTOFIX: ~/filename.mk:6: Replacing \".\" with \". \".") - t.CheckFileLines("filename.mk", - "# $"+"NetBSD$", - ".if defined(A)", - ". for a in ${A}", - ". if defined(C)", - ". endif", - ". endfor", - ".endif") -} - -// Up to 2018-01-28, pkglint applied the autofix also to the continuation -// lines, which is incorrect. It replaced the dot in "4.*" with spaces. -func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix_multiline(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("-Wall", "--autofix") - t.SetUpVartypes() - mklines := t.SetUpFileMkLines("options.mk", - MkCvsID, - ".if ${PKGNAME} == pkgname", - ".if \\", - " ${PLATFORM:MNetBSD-4.*}", - ".endif", - ".endif") - - mklines.Check() - - t.CheckOutputLines( - "AUTOFIX: ~/options.mk:3: Replacing \".\" with \". \".", - "AUTOFIX: ~/options.mk:5: Replacing \".\" with \". \".") - - t.CheckFileLines("options.mk", - MkCvsID, - ".if ${PKGNAME} == pkgname", - ". if \\", - " ${PLATFORM:MNetBSD-4.*}", - ". endif", - ".endif") -} - func (s *Suite) Test_MkLineChecker_checkVarUseQuoting(c *check.C) { t := s.Init(c) @@ -2472,310 +1581,397 @@ func (s *Suite) Test_MkLineChecker_checkVarUseQuoting__list_variable_with_two_co "The list variable BUILD_DIRS should not be embedded in a word.") } -// The ${VARNAME:=suffix} expression should only be used with lists. -// It typically appears in MASTER_SITE definitions. -func (s *Suite) Test_MkLineChecker_CheckVaruse__eq_nonlist(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVarassignOpShell(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") - mklines := t.SetUpFileMkLines("options.mk", + t.SetUpTool("uname", "UNAME", AfterPrefsMk) + t.SetUpTool("echo", "", AtRunTime) + t.SetUpPkgsrc() + t.SetUpPackage("category/package", + ".include \"standalone.mk\"") + t.CreateFileLines("category/package/standalone.mk", MkCvsID, - "WRKSRC=\t\t${WRKDIR:=/subdir}", - "MASTER_SITES=\t${MASTER_SITE_GITHUB:=organization/}") + "", + ".include \"../../mk/bsd.prefs.mk\"", + "", + "OPSYS_NAME!=\t${UNAME}", + ".if ${OPSYS_NAME} == \"NetBSD\"", + ".endif", + "", + "OS_NAME!=\t${UNAME}", + "", + "MUST_BE_EARLY!=\techo 123 # must be evaluated early", + "", + "show-package-vars: .PHONY", + "\techo OS_NAME=${OS_NAME:Q}", + "\techo MUST_BE_EARLY=${MUST_BE_EARLY:Q}") + t.FinishSetUp() - mklines.Check() + G.Check(t.File("category/package/standalone.mk")) + // There is no warning about any variable since no package is currently + // being checked, therefore pkglint cannot decide whether the variable + // is used a load time. t.CheckOutputLines( - "WARN: ~/options.mk:2: The :from=to modifier should only be used with lists, not with WRKDIR.") -} - -func (s *Suite) Test_MkLineChecker_CheckVaruse__for(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") - mklines := t.SetUpFileMkLines("options.mk", - MkCvsID, - ".for var in a b c", - "\t: ${var}", - ".endfor") + "WARN: ~/category/package/standalone.mk:14: Please use \"${ECHO}\" instead of \"echo\".", + "WARN: ~/category/package/standalone.mk:15: Please use \"${ECHO}\" instead of \"echo\".") - mklines.Check() + t.SetUpCommandLine("-Wall", "--explain") + G.Check(t.File("category/package")) - t.CheckOutputEmpty() + // There is no warning for OPSYS_NAME since that variable is used at + // load time. In such a case the command has to be executed anyway, + // and executing it exactly once is the best thing to do. + // + // There is no warning for MUST_BE_EARLY since the comment provides the + // reason that this command really has to be executed at load time. + t.CheckOutputLines( + "NOTE: ~/category/package/standalone.mk:9: Consider the :sh modifier instead of != for \"${UNAME}\".", + "", + "\tFor variable assignments using the != operator, the shell command is", + "\trun every time the file is parsed. In some cases this is too early,", + "\tand the command may not yet be installed. In other cases the command", + "\tis executed more often than necessary. Most commands don't need to", + "\tbe executed for \"make clean\", for example.", + "", + "\tThe :sh modifier defers execution until the variable value is", + "\tactually needed. On the other hand, this means the command is", + "\texecuted each time the variable is evaluated.", + "", + "\tExample:", + "", + "\t\tEARLY_YEAR!= date +%Y", + "", + "\t\tLATE_YEAR_CMD= date +%Y", + "\t\tLATE_YEAR= ${LATE_YEAR_CMD:sh}", + "", + "\t\t# or, in a single line:", + "\t\tLATE_YEAR= ${date +%Y:L:sh}", + "", + "\tTo suppress this note, provide an explanation in a comment at the", + "\tend of the line, or force the variable to be evaluated at load time,", + "\tby using it at the right-hand side of the := operator, or in an .if", + "\tor .for directive.", + "", + "WARN: ~/category/package/standalone.mk:14: Please use \"${ECHO}\" instead of \"echo\".", + "WARN: ~/category/package/standalone.mk:15: Please use \"${ECHO}\" instead of \"echo\".") } -// When a parameterized variable is defined in the pkgsrc infrastructure, -// it does not generate a warning about being "used but not defined". -// Even if the variable parameter differs, like .Linux and .SunOS in this -// case. This pattern is typical for pkgsrc, therefore pkglint doesn't -// check that the variable names match exactly. -func (s *Suite) Test_MkLineChecker_CheckVaruse__varcanon(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkText(c *check.C) { t := s.Init(c) - b := NewMkTokenBuilder() t.SetUpPkgsrc() - t.CreateFileLines("mk/sys-vars.mk", - MkCvsID, - "CPPPATH.Linux=\t/usr/bin/cpp") - t.FinishSetUp() - mklines := t.NewMkLines("module.mk", + t.SetUpCommandLine("-Wall,no-space") + mklines := t.SetUpFileMkLines("module.mk", MkCvsID, - "COMMENT=\t${CPPPATH.SunOS}") - - ck := MkLineChecker{mklines, mklines.mklines[1]} + "CFLAGS+= -Wl,--rpath,${PREFIX}/lib", + "PKG_FAIL_REASON+= \"Group ${GAMEGRP} doesn't exist.\"") + t.FinishSetUp() - ck.CheckVaruse(b.VarUse("CPPPATH.SunOS"), &VarUseContext{ - vartype: &Vartype{ - basicType: BtPathname, - options: Guessed, - aclEntries: nil, - }, - time: VucRunTime, - quoting: VucQuotPlain, - IsWordPart: false, - }) + mklines.Check() - t.CheckOutputEmpty() + t.CheckOutputLines( + "WARN: ~/module.mk:2: Please use ${COMPILER_RPATH_FLAG} instead of \"-Wl,--rpath,\".", + "WARN: ~/module.mk:3: Use of \"GAMEGRP\" is deprecated. Use GAMES_GROUP instead.") } -// Any variable that is defined in the pkgsrc infrastructure in mk/**/*.mk is -// considered defined, and no "used but not defined" warning is logged for it. -// -// See Pkgsrc.loadUntypedVars. -func (s *Suite) Test_MkLineChecker_CheckVaruse__defined_in_infrastructure(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkText__WRKSRC(c *check.C) { t := s.Init(c) - t.SetUpPkgsrc() - t.CreateFileLines("mk/deeply/nested/infra.mk", - MkCvsID, - "INFRA_VAR?=\tvalue") - t.FinishSetUp() - mklines := t.SetUpFileMkLines("category/package/module.mk", + t.SetUpCommandLine("-Wall", "--explain") + mklines := t.SetUpFileMkLines("module.mk", MkCvsID, - "do-fetch:", - "\t: ${INFRA_VAR} ${UNDEFINED}") + "pre-configure:", + "\tcd ${WRKSRC}/..") mklines.Check() t.CheckOutputLines( - "WARN: ~/category/package/module.mk:3: UNDEFINED is used but not defined.") + "WARN: ~/module.mk:3: Building the package should take place entirely inside ${WRKSRC}, not \"${WRKSRC}/..\".", + "", + "\tWRKSRC should be defined so that there is no need to do anything", + "\toutside of this directory.", + "", + "\tExample:", + "", + "\t\tWRKSRC=\t${WRKDIR}", + "\t\tCONFIGURE_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src", + "\t\tBUILD_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src ${WRKSRC}/cmd", + "", + "\tSee the pkgsrc guide, section \"Directories used during the build", + "\tprocess\":", + "\thttps://www.NetBSD.org/docs/pkgsrc/pkgsrc.html#build.builddirs", + "", + "WARN: ~/module.mk:3: WRKSRC is used but not defined.") } -func (s *Suite) Test_MkLineChecker_CheckVaruse__build_defs(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVartype__simple_type(c *check.C) { t := s.Init(c) - // XXX: This paragraph should not be necessary since VARBASE and X11_TYPE - // are also defined in vardefs.go. - t.SetUpPkgsrc() - t.CreateFileLines("mk/defaults/mk.conf", - "VARBASE?= /usr/pkg/var") - t.SetUpCommandLine("-Wall,no-space") - t.FinishSetUp() + t.SetUpVartypes() - mklines := t.SetUpFileMkLines("options.mk", - MkCvsID, - "COMMENT= ${VARBASE} ${X11_TYPE}", - "PKG_FAIL_REASON+= ${VARBASE} ${X11_TYPE}", - "BUILD_DEFS+= X11_TYPE") + // Since COMMENT is defined in vardefs.go its type is certain instead of guessed. + vartype := G.Pkgsrc.VariableType(nil, "COMMENT") + + c.Assert(vartype, check.NotNil) + t.CheckEquals(vartype.basicType.name, "Comment") + t.CheckEquals(vartype.IsGuessed(), false) + t.CheckEquals(vartype.IsList(), false) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "COMMENT=\tA nice package") mklines.Check() t.CheckOutputLines( - "WARN: ~/options.mk:2: The user-defined variable VARBASE is used but not added to BUILD_DEFS.", - "WARN: ~/options.mk:3: PKG_FAIL_REASON should only get one item per line.") + "WARN: Makefile:2: COMMENT should not begin with \"A\".") } -// The LOCALBASE variable may be defined and used in the infrastructure. -// It is always equivalent to PREFIX and only exists for historic reasons. -func (s *Suite) Test_MkLineChecker_CheckVaruse__LOCALBASE_in_infrastructure(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVartype(c *check.C) { t := s.Init(c) - t.SetUpPkgsrc() - t.CreateFileLines("mk/infra.mk", + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", MkCvsID, - "LOCALBASE?=\t${PREFIX}", - "DEFAULT_PREFIX=\t${LOCALBASE}") - t.FinishSetUp() - - G.Check(t.File("mk/infra.mk")) + "DISTNAME=\tgcc-${GCC_VERSION}") - // No warnings about LOCALBASE being used; the infrastructure files may - // do this. In packages though, LOCALBASE is deprecated. + mklines.vars.Define("GCC_VERSION", mklines.mklines[1]) + mklines.Check() - // There is no warning about DEFAULT_PREFIX being "defined but not used" - // since Pkgsrc.loadUntypedVars calls Pkgsrc.vartypes.DefineType, which - // registers that variable globally. t.CheckOutputEmpty() } -func (s *Suite) Test_MkLineChecker_CheckVaruse__user_defined_variable_and_BUILD_DEFS(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVartype__append_to_non_list(c *check.C) { t := s.Init(c) - t.SetUpPkgsrc() - t.CreateFileLines("mk/defaults/mk.conf", - "VARBASE?=\t${PREFIX}/var", - "PYTHON_VER?=\t36") - mklines := t.NewMkLines("file.mk", + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", MkCvsID, - "BUILD_DEFS+=\tPYTHON_VER", - "\t: ${VARBASE}", - "\t: ${VARBASE}", - "\t: ${PYTHON_VER}") - t.FinishSetUp() + "DISTNAME+=\tsuffix", + "COMMENT=\tComment for", + "COMMENT+=\tthe package") mklines.Check() t.CheckOutputLines( - "WARN: file.mk:3: The user-defined variable VARBASE is used but not added to BUILD_DEFS.") + "WARN: filename.mk:2: The variable DISTNAME should not be appended to "+ + "(only set, or given a default value) in this file.", + "WARN: filename.mk:2: The \"+=\" operator should only be used with lists, not with DISTNAME.") } -func (s *Suite) Test_MkLineChecker_checkVaruseModifiersSuffix(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVartype__no_tracing(c *check.C) { t := s.Init(c) t.SetUpVartypes() - mklines := t.NewMkLines("file.mk", + mklines := t.NewMkLines("filename.mk", MkCvsID, - "\t: ${HOMEPAGE:=subdir/:Q}", // wrong - "\t: ${BUILD_DIRS:=subdir/}", // correct - "\t: ${BIN_PROGRAMS:=.exe}") // unknown since BIN_PROGRAMS doesn't have a type + "UNKNOWN=\tvalue", + "CUR_DIR!=\tpwd") + t.DisableTracing() mklines.Check() t.CheckOutputLines( - "WARN: file.mk:2: The :from=to modifier should only be used with lists, not with HOMEPAGE.", - "WARN: file.mk:4: BIN_PROGRAMS is used but not defined.") + "WARN: filename.mk:2: UNKNOWN is defined but not used.", + "WARN: filename.mk:3: CUR_DIR is defined but not used.") } -func (s *Suite) Test_MkLineChecker_checkVaruseModifiersRange(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVartype__one_per_line(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--show-autofix", "--source") t.SetUpVartypes() - mklines := t.NewMkLines("mk/compiler/gcc.mk", + mklines := t.NewMkLines("filename.mk", MkCvsID, - "CC:=\t${CC:C/^/_asdf_/1:M_asdf_*:S/^_asdf_//}") + "PKG_FAIL_REASON+=\tSeveral words are wrong.", + "PKG_FAIL_REASON+=\t\"Properly quoted\"", + "PKG_FAIL_REASON+=\t# none") + t.DisableTracing() mklines.Check() t.CheckOutputLines( - "NOTE: mk/compiler/gcc.mk:2: "+ - "The modifier \":C/^/_asdf_/1:M_asdf_*:S/^_asdf_//\" can be written as \":[1]\".", - "AUTOFIX: mk/compiler/gcc.mk:2: "+ - "Replacing \":C/^/_asdf_/1:M_asdf_*:S/^_asdf_//\" with \":[1]\".", - "-\tCC:=\t${CC:C/^/_asdf_/1:M_asdf_*:S/^_asdf_//}", - "+\tCC:=\t${CC:[1]}") + "WARN: filename.mk:2: PKG_FAIL_REASON should only get one item per line.") +} - // Now go through all the "almost" cases, to reach full branch coverage. - mklines = t.NewMkLines("gcc.mk", +func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS_with_backticks(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("chat/pidgin-icb/Makefile", MkCvsID, - "\t: ${CC:M1:M2:M3}", - "\t: ${CC:C/^begin//:M2:M3}", // M1 pattern not exactly ^ - "\t: ${CC:C/^/_asdf_/g:M2:M3}", // M1 options != "1" - "\t: ${CC:C/^/....../g:M2:M3}", // M1 replacement doesn't match \w+ - "\t: ${CC:C/^/_asdf_/1:O:M3}", // M2 is not a match modifier - "\t: ${CC:C/^/_asdf_/1:N2:M3}", // M2 is :N instead of :M - "\t: ${CC:C/^/_asdf_/1:M_asdf_:M3}", // M2 pattern is missing the * at the end - "\t: ${CC:C/^/_asdf_/1:Mother:M3}", // M2 pattern differs from the M1 pattern - "\t: ${CC:C/^/_asdf_/1:M_asdf_*:M3}", // M3 ist not a substitution modifier - "\t: ${CC:C/^/_asdf_/1:M_asdf_*:S,from,to,}", // M3 pattern differs from the M1 pattern - "\t: ${CC:C/^/_asdf_/1:M_asdf_*:S,^_asdf_,to,}", // M3 replacement is not empty - "\t: ${CC:C/^/_asdf_/1:M_asdf_*:S,^_asdf_,,g}") // M3 modifier has options + "CFLAGS+=\t`pkg-config pidgin --cflags`") + mkline := mklines.mklines[1] - mklines.Check() + words := mkline.Fields() + + // bmake handles backticks in the same way, treating them as ordinary characters + t.CheckDeepEquals(words, []string{"`pkg-config", "pidgin", "--cflags`"}) + + ck := MkLineChecker{mklines, mklines.mklines[1]} + ck.checkVartype("CFLAGS", opAssignAppend, "`pkg-config pidgin --cflags`", "") + + // No warning about "`pkg-config" being an unknown CFlag. + // As of September 2019, there is no such check anymore in pkglint. + t.CheckOutputEmpty() } -func (s *Suite) Test_MkLineChecker_CheckVaruse__deprecated_PKG_DEBUG(c *check.C) { +// See PR 46570, Ctrl+F "4. Shell quoting". +// Pkglint is correct, since the shell sees this definition for +// CPPFLAGS as three words, not one word. +func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS(c *check.C) { t := s.Init(c) t.SetUpVartypes() - G.Pkgsrc.initDeprecatedVars() - - mklines := t.NewMkLines("module.mk", + mklines := t.NewMkLines("Makefile", MkCvsID, - "\t${_PKG_SILENT}${_PKG_DEBUG} :") + "CPPFLAGS.SunOS+=\t-DPIPECOMMAND=\\\"/usr/sbin/sendmail -bs %s\\\"") mklines.Check() t.CheckOutputLines( - "WARN: module.mk:2: Use of _PKG_SILENT and _PKG_DEBUG is deprecated. Use ${RUN} instead.") + "WARN: Makefile:2: Compiler flag \"-DPIPECOMMAND=\\\\\\\"/usr/sbin/sendmail\" has unbalanced double quotes.", + "WARN: Makefile:2: Compiler flag \"%s\\\\\\\"\" has unbalanced double quotes.") } -func (s *Suite) Test_MkLineChecker_checkVaruseUndefined(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVarassignRightCategory__none(c *check.C) { t := s.Init(c) - t.SetUpPkgsrc() - t.CreateFileLines("mk/infra.mk", - MkCvsID, - "#", - "# User-settable variables:", - "#", - "# DOCUMENTED", - "", - "ASSIGNED=\tassigned", - "#COMMENTED=\tcommented") + t.SetUpPackage("obscure/package", + "CATEGORIES=\t# none") t.FinishSetUp() - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "", - "do-build:", - "\t: ${ASSIGNED} ${COMMENTED} ${DOCUMENTED} ${UNKNOWN}") + G.Check(t.File("obscure/package")) - mklines.Check() + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLineChecker_checkVarassignRightCategory__indirect(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("obscure/package", + "CATEGORIES=\t${PKGPATH:C,/.*,,}") + t.FinishSetUp() + + G.Check(t.File("obscure/package")) + // This case does not occur in practice, + // therefore it's ok to have these warnings. t.CheckOutputLines( - "WARN: filename.mk:4: UNKNOWN is used but not defined.") + "WARN: ~/obscure/package/Makefile:5: "+ + "The primary category should be \"obscure\", not \"${PKGPATH:C,/.*,,}\".", + "ERROR: ~/obscure/package/Makefile:5: "+ + "Invalid category \"${PKGPATH:C,/.*,,}\".") } -// PR 46570, item "15. net/uucp/Makefile has a make loop" -func (s *Suite) Test_MkLineChecker_checkVaruseUndefined__indirect_variables(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVarassignRightCategory__wrong(c *check.C) { t := s.Init(c) - t.SetUpTool("echo", "ECHO", AfterPrefsMk) - mklines := t.NewMkLines("net/uucp/Makefile", - MkCvsID, - "\techo ${UUCP_${var}}") + t.SetUpPackage("obscure/package", + "CATEGORIES=\tperl5") + t.FinishSetUp() - mklines.Check() + G.Check(t.File("obscure/package")) - // No warning about UUCP_${var} being used but not defined. - // - // Normally, parameterized variables use a dot instead of an underscore as separator. - // This is one of the few other cases. Pkglint doesn't warn about dynamic variable - // names like UUCP_${var} or SITES_${distfile}. - // - // It does warn about simple variable names though, like ${var} in this example. t.CheckOutputLines( - "WARN: net/uucp/Makefile:2: var is used but not defined.") + "WARN: ~/obscure/package/Makefile:5: The primary category should be \"obscure\", not \"perl5\".") } -// Documented variables are declared as both defined and used since, as -// of April 2019, pkglint doesn't yet interpret the "Package-settable -// variables" comment. -func (s *Suite) Test_MkLineChecker_checkVaruseUndefined__documented(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVarassignRightCategory__wrong_in_package_directory(c *check.C) { t := s.Init(c) - t.SetUpVartypes() + t.SetUpPackage("obscure/package", + "CATEGORIES=\tperl5") + t.FinishSetUp() + t.Chdir("obscure/package") - mklines := t.NewMkLines("interpreter.mk", + G.Check(".") + + t.CheckOutputLines( + "WARN: Makefile:5: The primary category should be \"obscure\", not \"perl5\".") +} + +func (s *Suite) Test_MkLineChecker_checkVarassignRightCategory__append(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("obscure/package", + "CATEGORIES+=\tperl5") + t.FinishSetUp() + + G.Check(t.File("obscure/package")) + + // Appending is ok. + // In this particular case, appending has the same effect as assigning, + // but that can be checked somewhere else. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLineChecker_checkVarassignRightCategory__default(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("obscure/package", + "CATEGORIES?=\tperl5") + t.FinishSetUp() + + G.Check(t.File("obscure/package")) + + // Default assignments set the primary category, just like simple assignments. + t.CheckOutputLines( + "WARN: ~/obscure/package/Makefile:5: The primary category should be \"obscure\", not \"perl5\".") +} + +func (s *Suite) Test_MkLineChecker_checkVarassignRightCategory__autofix(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "--autofix") + t.SetUpPackage("obscure/package", + "CATEGORIES=\tperl5 obscure python") + t.FinishSetUp() + + G.Check(t.File("obscure/package")) + + t.CheckOutputLines( + "AUTOFIX: ~/obscure/package/Makefile:5: " + + "Replacing \"perl5 obscure\" with \"obscure perl5\".") +} + +func (s *Suite) Test_MkLineChecker_checkVarassignRightCategory__third(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("obscure/package", + "CATEGORIES=\tperl5 python obscure") + t.FinishSetUp() + + G.Check(t.File("obscure/package")) + + t.CheckOutputLines( + "WARN: ~/obscure/package/Makefile:5: " + + "The primary category should be \"obscure\", not \"perl5\".") + + t.SetUpCommandLine("-Wall", "--show-autofix") + + G.Check(t.File("obscure/package")) + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLineChecker_checkVarassignRightCategory__other_file(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("obscure/package", + "CATEGORIES=\tperl5 obscure python") + mklines := t.SetUpFileMkLines("obscure/package/module.mk", MkCvsID, - "#", - "# Package-settable variables:", - "#", - "# REPLACE_INTERP", - "#\tThe list of files whose interpreter will be corrected.", "", - "REPLACE_INTERPRETER+=\tinterp", - "REPLACE.interp.old=\t.*/interp", - "REPLACE.interp.new=\t${PREFIX}/bin/interp", - "REPLACE_FILES.interp=\t${REPLACE_INTERP}") + "CATEGORIES=\tperl5") + t.FinishSetUp() mklines.Check() - t.CheckOutputEmpty() + // It doesn't matter in which file the CATEGORIES= line appears. + // If it's a plain assignment, it will end up as the primary category. + t.CheckOutputLines( + "WARN: ~/obscure/package/module.mk:3: " + + "The primary category should be \"obscure\", not \"perl5\".") } func (s *Suite) Test_MkLineChecker_checkVarassignMisc(c *check.C) { @@ -2842,53 +2038,284 @@ func (s *Suite) Test_MkLineChecker_checkVarassignMisc__multiple_inclusion_guards "instead of \"# defined\".") } -func (s *Suite) Test_MkLineChecker_checkText(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVarassignDecreasingVersions(c *check.C) { t := s.Init(c) - t.SetUpPkgsrc() - - t.SetUpCommandLine("-Wall,no-space") - mklines := t.SetUpFileMkLines("module.mk", + t.SetUpVartypes() + mklines := t.NewMkLines("Makefile", MkCvsID, - "CFLAGS+= -Wl,--rpath,${PREFIX}/lib", - "PKG_FAIL_REASON+= \"Group ${GAMEGRP} doesn't exist.\"") - t.FinishSetUp() + "PYTHON_VERSIONS_ACCEPTED=\t36 __future__ # rationale", + "PYTHON_VERSIONS_ACCEPTED=\t36 -13 # rationale", + "PYTHON_VERSIONS_ACCEPTED=\t36 ${PKGVERSION_NOREV} # rationale", + "PYTHON_VERSIONS_ACCEPTED=\t36 37 # rationale", + "PYTHON_VERSIONS_ACCEPTED=\t37 36 27 25 # rationale") + + // TODO: All but the last of the above assignments should be flagged as + // redundant by RedundantScope; as of March 2019, that check is only + // implemented for package Makefiles, not for individual files. mklines.Check() + // Half of these warnings are from VartypeCheck.Version, the + // other half are from checkVarassignDecreasingVersions. + // Strictly speaking some of them are redundant, but that would + // mean to reject only variable references in checkVarassignDecreasingVersions. + // This is probably ok. + // TODO: Fix this. t.CheckOutputLines( - "WARN: ~/module.mk:2: Please use ${COMPILER_RPATH_FLAG} instead of \"-Wl,--rpath,\".", - "WARN: ~/module.mk:3: Use of \"GAMEGRP\" is deprecated. Use GAMES_GROUP instead.") + "WARN: Makefile:2: Invalid version number \"__future__\".", + "ERROR: Makefile:2: Value \"__future__\" for "+ + "PYTHON_VERSIONS_ACCEPTED must be a positive integer.", + "WARN: Makefile:3: Invalid version number \"-13\".", + "ERROR: Makefile:3: Value \"-13\" for "+ + "PYTHON_VERSIONS_ACCEPTED must be a positive integer.", + "ERROR: Makefile:4: Value \"${PKGVERSION_NOREV}\" for "+ + "PYTHON_VERSIONS_ACCEPTED must be a positive integer.", + "WARN: Makefile:5: The values for PYTHON_VERSIONS_ACCEPTED "+ + "should be in decreasing order (37 before 36).") } -func (s *Suite) Test_MkLineChecker_checkText__WRKSRC(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkVarassignRightVaruse(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall", "--explain") - mklines := t.SetUpFileMkLines("module.mk", + t.SetUpVartypes() + + mklines := t.NewMkLines("module.mk", MkCvsID, - "pre-configure:", - "\tcd ${WRKSRC}/..") + "PLIST_SUBST+=\tLOCALBASE=${LOCALBASE:Q}") mklines.Check() t.CheckOutputLines( - "WARN: ~/module.mk:3: Building the package should take place entirely inside ${WRKSRC}, not \"${WRKSRC}/..\".", + "WARN: module.mk:2: Please use PREFIX instead of LOCALBASE.", + "NOTE: module.mk:2: The :Q modifier isn't necessary for ${LOCALBASE} here.") +} + +func (s *Suite) Test_MkLineChecker_checkShellCommand__indentation(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("filename.mk", + MkCvsID, "", - "\tWRKSRC should be defined so that there is no need to do anything", - "\toutside of this directory.", + "do-install:", + "\t\techo 'unnecessarily indented'", + "\t\tfor var in 1 2 3; do \\", + "\t\t\techo \"$$var\"; \\", + "\t echo \"spaces\"; \\", + "\t\tdone", "", - "\tExample:", + "\t\t\t\t\t# comment, not a shell command") + + mklines.Check() + t.SetUpCommandLine("-Wall", "--autofix") + mklines.Check() + + t.CheckOutputLines( + "NOTE: ~/filename.mk:4: Shell programs should be indented with a single tab.", + "WARN: ~/filename.mk:4: Unknown shell command \"echo\".", + "NOTE: ~/filename.mk:5--8: Shell programs should be indented with a single tab.", + "WARN: ~/filename.mk:5--8: Unknown shell command \"echo\".", + "WARN: ~/filename.mk:5--8: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"echo \\\"$$var\\\"\") to separate commands.", + "WARN: ~/filename.mk:5--8: Unknown shell command \"echo\".", + + "AUTOFIX: ~/filename.mk:4: Replacing \"\\t\\t\" with \"\\t\".", + "AUTOFIX: ~/filename.mk:5: Replacing \"\\t\\t\" with \"\\t\".", + "AUTOFIX: ~/filename.mk:6: Replacing \"\\t\\t\" with \"\\t\".", + "AUTOFIX: ~/filename.mk:8: Replacing \"\\t\\t\" with \"\\t\".") + t.CheckFileLinesDetab("filename.mk", + MkCvsID, "", - "\t\tWRKSRC=\t${WRKDIR}", - "\t\tCONFIGURE_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src", - "\t\tBUILD_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src ${WRKSRC}/cmd", + "do-install:", + " echo 'unnecessarily indented'", + " for var in 1 2 3; do \\", + " echo \"$$var\"; \\", + " echo \"spaces\"; \\", // not changed + " done", "", - "\tSee the pkgsrc guide, section \"Directories used during the build", - "\tprocess\":", - "\thttps://www.NetBSD.org/docs/pkgsrc/pkgsrc.html#build.builddirs", + " # comment, not a shell command") +} + +func (s *Suite) Test_MkLineChecker_checkInclude(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + t.CreateFileLines("pkgtools/x11-links/buildlink3.mk") + t.CreateFileLines("graphics/jpeg/buildlink3.mk") + t.CreateFileLines("devel/intltool/buildlink3.mk") + t.CreateFileLines("devel/intltool/builtin.mk") + mklines := t.SetUpFileMkLines("category/package/filename.mk", + MkCvsID, "", - "WARN: ~/module.mk:3: WRKSRC is used but not defined.") + ".include \"../../pkgtools/x11-links/buildlink3.mk\"", + ".include \"../../graphics/jpeg/buildlink3.mk\"", + ".include \"../../devel/intltool/buildlink3.mk\"", + ".include \"../../devel/intltool/builtin.mk\"") + + mklines.Check() + + t.CheckOutputLines( + "ERROR: ~/category/package/filename.mk:3: "+ + "../../pkgtools/x11-links/buildlink3.mk must not be included directly. "+ + "Include \"../../mk/x11.buildlink3.mk\" instead.", + "ERROR: ~/category/package/filename.mk:4: "+ + "../../graphics/jpeg/buildlink3.mk must not be included directly. "+ + "Include \"../../mk/jpeg.buildlink3.mk\" instead.", + "WARN: ~/category/package/filename.mk:5: "+ + "Please write \"USE_TOOLS+= intltool\" instead of this line.", + "ERROR: ~/category/package/filename.mk:6: "+ + "../../devel/intltool/builtin.mk must not be included directly. "+ + "Include \"../../devel/intltool/buildlink3.mk\" instead.") +} + +func (s *Suite) Test_MkLineChecker_checkInclude__Makefile(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines(t.File("Makefile"), + MkCvsID, + ".include \"../../other/package/Makefile\"") + + mklines.Check() + + t.CheckOutputLines( + "ERROR: ~/Makefile:2: Relative path \"../../other/package/Makefile\" does not exist.", + "ERROR: ~/Makefile:2: Other Makefiles must not be included directly.") +} + +func (s *Suite) Test_MkLineChecker_checkInclude__Makefile_exists(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("other/existing/Makefile") + t.SetUpPackage("category/package", + ".include \"../../other/existing/Makefile\"", + ".include \"../../other/not-found/Makefile\"") + t.FinishSetUp() + + G.checkdirPackage(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/Makefile:21: Cannot read \"../../other/not-found/Makefile\".") +} + +func (s *Suite) Test_MkLineChecker_checkInclude__hacks(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/hacks.mk", + MkCvsID, + ".include \"../../category/package/nonexistent.mk\"", + ".include \"../../category/package/builtin.mk\"") + t.CreateFileLines("category/package/builtin.mk", + MkCvsID) + t.FinishSetUp() + + G.checkdirPackage(t.File("category/package")) + + // The purpose of this "nonexistent" diagnostic is only to show that + // hacks.mk is indeed parsed and checked. + t.CheckOutputLines( + "ERROR: ~/category/package/hacks.mk:2: " + + "Relative path \"../../category/package/nonexistent.mk\" does not exist.") +} + +func (s *Suite) Test_MkLineChecker_checkInclude__builtin_mk(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../category/package/builtin.mk\"", + ".include \"../../category/package/builtin.mk\" # ok") + t.CreateFileLines("category/package/builtin.mk", + MkCvsID) + t.FinishSetUp() + + G.checkdirPackage(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/Makefile:20: " + + "../../category/package/builtin.mk must not be included directly. " + + "Include \"../../category/package/buildlink3.mk\" instead.") +} + +func (s *Suite) Test_MkLineChecker_checkInclude__builtin_mk_rationale(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "# I have good reasons for including this file directly.", + ".include \"../../category/package/builtin.mk\"", + "", + ".include \"../../category/package/builtin.mk\"") + t.CreateFileLines("category/package/builtin.mk", + MkCvsID) + t.FinishSetUp() + + G.checkdirPackage(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/Makefile:23: " + + "../../category/package/builtin.mk must not be included directly. " + + "Include \"../../category/package/buildlink3.mk\" instead.") +} + +func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--autofix", "-Wspace") + lines := t.SetUpFileLines("filename.mk", + MkCvsID, + ".if defined(A)", + ".for a in ${A}", + ".if defined(C)", + ".endif", + ".endfor", + ".endif") + mklines := NewMkLines(lines) + + mklines.Check() + + t.CheckOutputLines( + "AUTOFIX: ~/filename.mk:3: Replacing \".\" with \". \".", + "AUTOFIX: ~/filename.mk:4: Replacing \".\" with \". \".", + "AUTOFIX: ~/filename.mk:5: Replacing \".\" with \". \".", + "AUTOFIX: ~/filename.mk:6: Replacing \".\" with \". \".") + t.CheckFileLines("filename.mk", + "# $"+"NetBSD$", + ".if defined(A)", + ". for a in ${A}", + ". if defined(C)", + ". endif", + ". endfor", + ".endif") +} + +// Up to 2018-01-28, pkglint applied the autofix also to the continuation +// lines, which is incorrect. It replaced the dot in "4.*" with spaces. +func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix_multiline(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "--autofix") + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("options.mk", + MkCvsID, + ".if ${PKGNAME} == pkgname", + ".if \\", + " ${PLATFORM:MNetBSD-4.*}", + ".endif", + ".endif") + + mklines.Check() + + t.CheckOutputLines( + "AUTOFIX: ~/options.mk:3: Replacing \".\" with \". \".", + "AUTOFIX: ~/options.mk:5: Replacing \".\" with \". \".") + + t.CheckFileLines("options.mk", + MkCvsID, + ".if ${PKGNAME} == pkgname", + ". if \\", + " ${PLATFORM:MNetBSD-4.*}", + ". endif", + ".endif") } func (s *Suite) Test_MkLineChecker_CheckRelativePath(c *check.C) { @@ -2980,3 +2407,729 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePath__wip_mk(c *check.C) { "WARN: ~/wip/package/Makefile:21: References to other packages "+ "should look like \"../../category/package\", not \"../package\".") } + +func (s *Suite) Test_MkLineChecker_CheckRelativePkgdir(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("other/package/Makefile") + + test := func(relativePkgdir string, diagnostics ...string) { + // Must be in the filesystem because of directory references. + mklines := t.SetUpFileMkLines("category/package/Makefile", + "# dummy") + + checkRelativePkgdir := func(mkline *MkLine) { + MkLineChecker{mklines, mkline}.CheckRelativePkgdir(relativePkgdir) + } + + mklines.ForEach(checkRelativePkgdir) + + t.CheckOutput(diagnostics) + } + + test("../pkgbase", + "ERROR: ~/category/package/Makefile:1: Relative path \"../pkgbase/Makefile\" does not exist.", + "WARN: ~/category/package/Makefile:1: \"../pkgbase\" is not a valid relative package directory.") + + test("../../other/package", + nil...) + + test("../../other/does-not-exist", + "ERROR: ~/category/package/Makefile:1: Relative path \"../../other/does-not-exist/Makefile\" does not exist.") + + test("${OTHER_PACKAGE}", + nil...) +} + +func (s *Suite) Test_MkLineChecker_checkDirective(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + mklines := t.NewMkLines("category/package/filename.mk", + MkCvsID, + "", + ".for", + ".endfor", + "", + ".if", + ".else don't", + ".endif invalid-arg", + "", + ".ifdef FNAME_MK", + ".endif", + ".ifndef FNAME_MK", + ".endif", + "", + ".for var in a b c", + ".endfor", + ".undef var unrelated") + + mklines.Check() + + t.CheckOutputLines( + "ERROR: category/package/filename.mk:3: \".for\" requires arguments.", + "ERROR: category/package/filename.mk:6: \".if\" requires arguments.", + "ERROR: category/package/filename.mk:7: \".else\" does not take arguments. "+ + "If you meant \"else if\", use \".elif\".", + "ERROR: category/package/filename.mk:8: \".endif\" does not take arguments.", + "WARN: category/package/filename.mk:10: The \".ifdef\" directive is deprecated. "+ + "Please use \".if defined(FNAME_MK)\" instead.", + "WARN: category/package/filename.mk:12: The \".ifndef\" directive is deprecated. "+ + "Please use \".if !defined(FNAME_MK)\" instead.", + "NOTE: category/package/filename.mk:17: Using \".undef\" after a \".for\" loop is unnecessary.") +} + +func (s *Suite) Test_MkLineChecker_checkDirective__for_loop_varname(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + ".for VAR in a b c", // Should be lowercase. + ".endfor", + "", + ".for _var_ in a b c", // Should be written without underscores. + ".endfor", + "", + ".for .var. in a b c", // Should be written without dots. + ".endfor", + "", + ".for ${VAR} in a b c", // The variable name really must be an identifier. + ".endfor") + + mklines.Check() + + t.CheckOutputLines( + "WARN: filename.mk:3: The variable name \"VAR\" in the .for loop should not contain uppercase letters.", + "WARN: filename.mk:6: Variable names starting with an underscore (_var_) are reserved for internal pkgsrc use.", + "ERROR: filename.mk:9: Invalid variable name \".var.\".", + "ERROR: filename.mk:12: Invalid variable name \"${VAR}\".") +} + +func (s *Suite) Test_MkLineChecker_checkDirectiveEnd__ending_comments(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("opsys.mk", + MkCvsID, + "", + ".for i in 1 2 3 4 5", + ". if ${OPSYS} == NetBSD", + ". if ${MACHINE_ARCH} == x86_64", + ". if ${OS_VERSION:M8.*}", + ". endif # MACHINE_ARCH", // Wrong, should be OS_VERSION. + ". endif # OS_VERSION", // Wrong, should be MACHINE_ARCH. + ". endif # OPSYS", // Correct. + ".endfor # j", // Wrong, should be i. + "", + ".if ${PKG_OPTIONS:Moption}", + ".endif # option", // Correct. + "", + ".if ${PKG_OPTIONS:Moption}", + ".endif # opti", // This typo goes unnoticed since "opti" is a substring of the condition. + "", + ".if ${OPSYS} == NetBSD", + ".elif ${OPSYS} == FreeBSD", + ".endif # NetBSD", // Wrong, should be FreeBSD from the .elif. + "", + ".for ii in 1 2", + ". for jj in 1 2", + ". endfor # ii", // Note: a simple "i" would not generate a warning because it is found in the word "in". + ".endfor # ii") + + // See MkLineChecker.checkDirective + mklines.Check() + + t.CheckOutputLines( + "WARN: opsys.mk:7: Comment \"MACHINE_ARCH\" does not match condition \"${OS_VERSION:M8.*}\".", + "WARN: opsys.mk:8: Comment \"OS_VERSION\" does not match condition \"${MACHINE_ARCH} == x86_64\".", + "WARN: opsys.mk:10: Comment \"j\" does not match loop \"i in 1 2 3 4 5\".", + "WARN: opsys.mk:12: Unknown option \"option\".", + "WARN: opsys.mk:20: Comment \"NetBSD\" does not match condition \"${OPSYS} == FreeBSD\".", + "WARN: opsys.mk:24: Comment \"ii\" does not match loop \"jj in 1 2\".") +} + +// After removing the dummy indentation in commit d5a926af, +// there was a panic: runtime error: index out of range, +// in wip/jacorb-lib/buildlink3.mk. +func (s *Suite) Test_MkLineChecker_checkDirectiveEnd__unbalanced(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + ".endfor # comment", + ".endif # comment") + + mklines.Check() + + t.CheckOutputLines( + "ERROR: filename.mk:3: Unmatched .endfor.", + "ERROR: filename.mk:4: Unmatched .endif.") +} + +func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + test := func(cond string, output ...string) { + mklines := t.NewMkLines("filename.mk", + cond) + mklines.ForEach(func(mkline *MkLine) { + MkLineChecker{mklines, mkline}.checkDirectiveCond() + }) + t.CheckOutput(output) + } + + test( + ".if !empty(PKGSRC_COMPILER:Mmycc)", + "WARN: filename.mk:1: The pattern \"mycc\" cannot match any of "+ + "{ ccache ccc clang distcc f2c gcc hp icc ido "+ + "mipspro mipspro-ucode pcc sunpro xlc } for PKGSRC_COMPILER.") + + test( + ".elif ${A} != ${B}", + "WARN: filename.mk:1: A is used but not defined.", + "WARN: filename.mk:1: B is used but not defined.") + + test(".if ${HOMEPAGE} == \"mailto:someone@example.org\"", + "WARN: filename.mk:1: \"mailto:someone@example.org\" is not a valid URL.", + "WARN: filename.mk:1: HOMEPAGE should not be used at load time in any file.") + + test(".if !empty(PKGSRC_RUN_TEST:M[Y][eE][sS])", + "WARN: filename.mk:1: PKGSRC_RUN_TEST should be matched "+ + "against \"[yY][eE][sS]\" or \"[nN][oO]\", not \"[Y][eE][sS]\".") + + test(".if !empty(IS_BUILTIN.Xfixes:M[yY][eE][sS])") + + test(".if !empty(${IS_BUILTIN.Xfixes:M[yY][eE][sS]})", + "WARN: filename.mk:1: The empty() function takes a variable name as parameter, "+ + "not a variable expression.") + + test(".if ${PKGSRC_COMPILER} == \"msvc\"", + "WARN: filename.mk:1: \"msvc\" is not valid for PKGSRC_COMPILER. "+ + "Use one of { ccache ccc clang distcc f2c gcc hp icc ido mipspro mipspro-ucode pcc sunpro xlc } instead.", + "WARN: filename.mk:1: Use ${PKGSRC_COMPILER:Mmsvc} instead of the == operator.") + + test(".if ${PKG_LIBTOOL:Mlibtool}", + "NOTE: filename.mk:1: PKG_LIBTOOL should be compared using == instead of matching against \":Mlibtool\".", + "WARN: filename.mk:1: PKG_LIBTOOL should not be used at load time in any file.") + + test(".if ${MACHINE_PLATFORM:MUnknownOS-*-*} || ${MACHINE_ARCH:Mx86}", + "WARN: filename.mk:1: "+ + "The pattern \"UnknownOS\" cannot match any of "+ + "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ + "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ + "} for the operating system part of MACHINE_PLATFORM.", + "WARN: filename.mk:1: "+ + "The pattern \"x86\" cannot match any of "+ + "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm "+ + "earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb "+ + "earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 "+ + "m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax "+ + "powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ + "} for MACHINE_ARCH.", + "NOTE: filename.mk:1: MACHINE_ARCH should be compared using == instead of matching against \":Mx86\".") + + // Doesn't occur in practice since it is surprising that the ! applies + // to the comparison operator, and not to one of its arguments. + test(".if !${VAR} == value", + "WARN: filename.mk:1: VAR is used but not defined.") + + // Doesn't occur in practice since this string can never be empty. + test(".if !\"${VAR}str\"", + "WARN: filename.mk:1: VAR is used but not defined.") + + // Doesn't occur in practice since !${VAR} && !${VAR2} is more idiomatic. + test(".if !\"${VAR}${VAR2}\"", + "WARN: filename.mk:1: VAR is used but not defined.", + "WARN: filename.mk:1: VAR2 is used but not defined.") + + // Just for code coverage; always evaluates to true. + test(".if \"string\"", + nil...) + + // Code coverage for checkVar. + test(".if ${OPSYS} || ${MACHINE_ARCH}", + nil...) + + test(".if ${VAR}", + "WARN: filename.mk:1: VAR is used but not defined.") + + test(".if ${VAR} == 3", + "WARN: filename.mk:1: VAR is used but not defined.") + + test(".if \"value\" == ${VAR}", + "WARN: filename.mk:1: VAR is used but not defined.") + + test(".if ${MASTER_SITES:Mftp://*} == \"ftp://netbsd.org/\"", + "WARN: filename.mk:1: Invalid variable modifier \"//*\" for \"MASTER_SITES\".", + "WARN: filename.mk:1: \"ftp\" is not a valid URL.", + "WARN: filename.mk:1: MASTER_SITES should not be used at load time in any file.", + "WARN: filename.mk:1: Invalid variable modifier \"//*\" for \"MASTER_SITES\".") +} + +func (s *Suite) Test_MkLineChecker_checkDirectiveCond__tracing(c *check.C) { + t := s.Init(c) + + t.EnableTracingToLog() + mklines := t.NewMkLines("filename.mk", + ".if ${VAR:Mpattern1:Mpattern2} == comparison") + + mklines.ForEach(func(mkline *MkLine) { + MkLineChecker{mklines, mkline}.checkDirectiveCond() + }) + + t.CheckOutputLinesMatching(`^WARN|checkCompare`, + "TRACE: 1 checkCompareVarStr ${VAR:Mpattern1:Mpattern2} == comparison", + "WARN: filename.mk:1: VAR is used but not defined.") +} + +func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparison_with_shell_command(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("security/openssl/Makefile", + MkCvsID, + ".if ${PKGSRC_COMPILER} == \"gcc\" && ${CC} == \"cc\"", + ".endif") + + mklines.Check() + + // Don't warn about unknown shell command "cc". + t.CheckOutputLines( + "WARN: security/openssl/Makefile:2: Use ${PKGSRC_COMPILER:Mgcc} instead of the == operator.") +} + +// The :N modifier filters unwanted values. After this filter, any variable value +// may be compared with the empty string, regardless of the variable type. +// Effectively, the :N modifier changes the type from T to Option(T). +func (s *Suite) Test_MkLineChecker_checkDirectiveCond__compare_pattern_with_empty(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", + MkCvsID, + ".if ${X11BASE:Npattern} == \"\"", + ".endif", + "", + ".if ${X11BASE:N<>} == \"*\"", + ".endif", + "", + ".if !(${OPSYS:M*BSD} != \"\")", + ".endif") + + mklines.Check() + + // TODO: There should be a warning about "<>" containing invalid + // characters for a path. See VartypeCheck.Pathname + t.CheckOutputLines( + "WARN: filename.mk:5: The pathname pattern \"<>\" contains the invalid characters \"<>\".", + "WARN: filename.mk:5: The pathname \"*\" contains the invalid character \"*\".") +} + +func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparing_PKGSRC_COMPILER_with_eqeq(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("Makefile", + MkCvsID, + ".if ${PKGSRC_COMPILER} == \"clang\"", + ".elif ${PKGSRC_COMPILER} != \"gcc\"", + ".endif") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:2: Use ${PKGSRC_COMPILER:Mclang} instead of the == operator.", + "WARN: Makefile:3: Use ${PKGSRC_COMPILER:Ngcc} instead of the != operator.") +} + +func (s *Suite) Test_MkLineChecker_checkDirectiveCondEmpty(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.Chdir(".") + + test := func(before string, diagnosticsAndAfter ...string) { + + mklines := t.SetUpFileMkLines("module.mk", + MkCvsID, + before, + ".endif") + ck := MkLineChecker{mklines, mklines.mklines[1]} + + t.SetUpCommandLine("-Wall") + mklines.ForEach(func(mkline *MkLine) { + if mkline == mklines.mklines[1] { + ck.checkDirectiveCond() + } + }) + + t.SetUpCommandLine("-Wall", "--autofix") + mklines.ForEach(func(mkline *MkLine) { + if mkline == mklines.mklines[1] { + ck.checkDirectiveCond() + } + }) + + mklines.SaveAutofixChanges() + afterMklines := t.LoadMkInclude("module.mk") + + if len(diagnosticsAndAfter) > 0 { + diagLen := len(diagnosticsAndAfter) + diagnostics := diagnosticsAndAfter[:diagLen-1] + after := diagnosticsAndAfter[diagLen-1] + + t.CheckOutput(diagnostics) + t.CheckEquals(afterMklines.mklines[1].Text, after) + } else { + t.CheckOutputEmpty() + } + } + + test( + ".if ${PKGPATH:Mpattern}", + + "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpattern\".", + "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Mpattern}\" with \"${PKGPATH} == pattern\".", + + ".if ${PKGPATH} == pattern") + + // When the pattern contains placeholders, it cannot be converted to == or !=. + test( + ".if ${PKGPATH:Mpa*n}", + nil...) + + // The :tl modifier prevents the autofix. + test( + ".if ${PKGPATH:tl:Mpattern}", + + "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpattern\".", + + ".if ${PKGPATH:tl:Mpattern}") + + test( + ".if ${PKGPATH:Ncategory/package}", + + "NOTE: module.mk:2: PKGPATH should be compared using != instead of matching against \":Ncategory/package\".", + "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Ncategory/package}\" with \"${PKGPATH} != category/package\".", + + ".if ${PKGPATH} != category/package") + + // ${PKGPATH:None:Ntwo} is a short variant of ${PKGPATH} != "one" && + // ${PKGPATH} != "two". Applying the transformation would make the + // condition longer than before, therefore nothing is done here. + test( + ".if ${PKGPATH:None:Ntwo}", + nil...) + + // Note: this combination doesn't make sense since the patterns "one" and "two" don't overlap. + test(".if ${PKGPATH:Mone:Mtwo}", + + "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mone\".", + "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mtwo\".", + + ".if ${PKGPATH:Mone:Mtwo}") + + test(".if !empty(PKGPATH:Mpattern)", + + "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpattern\".", + "AUTOFIX: module.mk:2: Replacing \"!empty(PKGPATH:Mpattern)\" with \"${PKGPATH} == pattern\".", + + ".if ${PKGPATH} == pattern") + + test(".if empty(PKGPATH:Mpattern)", + + "NOTE: module.mk:2: PKGPATH should be compared using != instead of matching against \":Mpattern\".", + "AUTOFIX: module.mk:2: Replacing \"empty(PKGPATH:Mpattern)\" with \"${PKGPATH} != pattern\".", + + ".if ${PKGPATH} != pattern") + + test(".if !!empty(PKGPATH:Mpattern)", + + // TODO: When taking all the ! into account, this is actually a + // test for emptiness, therefore the diagnostics should suggest + // the != operator instead of ==. + "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpattern\".", + "AUTOFIX: module.mk:2: Replacing \"!empty(PKGPATH:Mpattern)\" with \"${PKGPATH} == pattern\".", + + // TODO: The ! and == could be combined into a !=. + // Luckily the !! pattern doesn't occur in practice. + ".if !${PKGPATH} == pattern") + + test(".if empty(PKGPATH:Mpattern) || 0", + + "NOTE: module.mk:2: PKGPATH should be compared using != instead of matching against \":Mpattern\".", + "AUTOFIX: module.mk:2: Replacing \"empty(PKGPATH:Mpattern)\" with \"${PKGPATH} != pattern\".", + + ".if ${PKGPATH} != pattern || 0") + + // No note in this case since there is no implicit !empty around the varUse. + test(".if ${PKGPATH:Mpattern} != ${OTHER}", + + "WARN: module.mk:2: OTHER is used but not defined.", + + ".if ${PKGPATH:Mpattern} != ${OTHER}") + + test( + ".if ${PKGPATH:Mpattern}", + + "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpattern\".", + "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Mpattern}\" with \"${PKGPATH} == pattern\".", + + ".if ${PKGPATH} == pattern") + + test( + ".if !${PKGPATH:Mpattern}", + + "NOTE: module.mk:2: PKGPATH should be compared using != instead of matching against \":Mpattern\".", + "AUTOFIX: module.mk:2: Replacing \"!${PKGPATH:Mpattern}\" with \"${PKGPATH} != pattern\".", + + ".if ${PKGPATH} != pattern") + + test( + ".if !!${PKGPATH:Mpattern}", + + "NOTE: module.mk:2: PKGPATH should be compared using != instead of matching against \":Mpattern\".", + "AUTOFIX: module.mk:2: Replacing \"!${PKGPATH:Mpattern}\" with \"${PKGPATH} != pattern\".", + + ".if !${PKGPATH} != pattern") + + // This pattern with spaces doesn't make sense at all in the :M + // modifier since it can never match. + // Or can it, if the PKGPATH contains quotes? + // How exactly does bmake apply the matching here, are both values unquoted? + test( + ".if ${PKGPATH:Mpattern with spaces}", + + "WARN: module.mk:2: The pathname pattern \"pattern with spaces\" "+ + "contains the invalid characters \" \".", + + ".if ${PKGPATH:Mpattern with spaces}") + // TODO: ".if ${PKGPATH} == \"pattern with spaces\"") + + test( + ".if ${PKGPATH:M'pattern with spaces'}", + + "WARN: module.mk:2: The pathname pattern \"'pattern with spaces'\" "+ + "contains the invalid characters \"' '\".", + + ".if ${PKGPATH:M'pattern with spaces'}") + // TODO: ".if ${PKGPATH} == 'pattern with spaces'") + + test( + ".if ${PKGPATH:M&&}", + + "WARN: module.mk:2: The pathname pattern \"&&\" "+ + "contains the invalid characters \"&&\".", + + ".if ${PKGPATH:M&&}") + // TODO: ".if ${PKGPATH} == '&&'") + + // If PKGPATH is "", the condition is false. + // If PKGPATH is "negative-pattern", the condition is false. + // In all other cases, the condition is true. + // + // Therefore this condition cannot simply be transformed into + // ${PKGPATH} != negative-pattern, since that would produce a + // different result in the case where PKGPATH is empty. + // + // For system-provided variables that are guaranteed to be non-empty, + // such as OPSYS or PKGPATH, this replacement is valid. + // These variables are only guaranteed to be defined after bsd.prefs.mk + // has been included, like everywhere else. + test( + ".if ${PKGPATH:Nnegative-pattern}", + + "NOTE: module.mk:2: PKGPATH should be compared using != instead of matching against \":Nnegative-pattern\".", + "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Nnegative-pattern}\" with \"${PKGPATH} != negative-pattern\".", + + ".if ${PKGPATH} != negative-pattern") + + // Since UNKNOWN is not a well-known system-provided variable that is + // guaranteed to be non-empty (see the previous example), it is not + // transformed at all. + test( + ".if ${UNKNOWN:Nnegative-pattern}", + + "WARN: module.mk:2: UNKNOWN is used but not defined.", + + ".if ${UNKNOWN:Nnegative-pattern}") + + test( + ".if ${PKGPATH:Mpath1} || ${PKGPATH:Mpath2}", + + "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpath1\".", + "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpath2\".", + "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Mpath1}\" with \"${PKGPATH} == path1\".", + "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Mpath2}\" with \"${PKGPATH} == path2\".", + + ".if ${PKGPATH} == path1 || ${PKGPATH} == path2") + + test( + ".if (((((${PKGPATH:Mpath})))))", + + "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mpath\".", + "AUTOFIX: module.mk:2: Replacing \"${PKGPATH:Mpath}\" with \"${PKGPATH} == path\".", + + ".if (((((${PKGPATH} == path)))))") + + // Note: this combination doesn't make sense since the patterns "one" and "two" don't overlap. + test( + ".if ${PKGPATH:Mone:Mtwo}", + + "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mone\".", + "NOTE: module.mk:2: PKGPATH should be compared using == instead of matching against \":Mtwo\".", + + ".if ${PKGPATH:Mone:Mtwo}") + + test( + ".if ${MACHINE_ARCH:Mx86_64}", + + "NOTE: module.mk:2: MACHINE_ARCH should be compared using == instead of matching against \":Mx86_64\".", + "AUTOFIX: module.mk:2: Replacing \"${MACHINE_ARCH:Mx86_64}\" with \"${MACHINE_ARCH} == x86_64\".", + + ".if ${MACHINE_ARCH} == x86_64") +} + +func (s *Suite) Test_MkLineChecker_checkDirectiveCondCompare(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + test := func(cond string, output ...string) { + mklines := t.NewMkLines("filename.mk", + cond) + mklines.ForEach(func(mkline *MkLine) { + MkLineChecker{mklines, mkline}.checkDirectiveCond() + }) + t.CheckOutput(output) + } + + // As of July 2019, pkglint doesn't have specific checks for comparing + // variables to numbers. + test(".if ${VAR} > 0", + "WARN: filename.mk:1: VAR is used but not defined.") + + // For string comparisons, the checks from vartypecheck.go are + // performed. + test(".if ${DISTNAME} == \"<>\"", + "WARN: filename.mk:1: The filename \"<>\" contains the invalid characters \"<>\".", + "WARN: filename.mk:1: DISTNAME should not be used at load time in any file.") + + // This type of comparison doesn't occur in practice since it is + // overly verbose. + test(".if \"${BUILD_DIRS}str\" == \"str\"", + // TODO: why should it not be used? In a .for loop it sounds pretty normal. + "WARN: filename.mk:1: BUILD_DIRS should not be used at load time in any file.") + + // This is a shorthand for defined(VAR), but it is not used in practice. + test(".if VAR", + "WARN: filename.mk:1: Invalid condition, unrecognized part: \"VAR\".") + + // Calling a function with braces instead of parentheses is syntactically + // invalid. Pkglint is stricter than bmake in this situation. + // + // Bmake reads the "empty{VAR}" as a variable name. It then checks whether + // this variable is defined. It is not, of course, therefore the expression + // is false. The ! in front of it negates this false, which makes the whole + // condition true. + // + // See https://mail-index.netbsd.org/tech-pkg/2019/07/07/msg021539.html + test(".if !empty{VAR}", + "WARN: filename.mk:1: Invalid condition, unrecognized part: \"empty{VAR}\".") +} + +func (s *Suite) Test_MkLineChecker_checkDirectiveCondCompareVarStr__no_tracing(c *check.C) { + t := s.Init(c) + b := NewMkTokenBuilder() + + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", + ".if ${DISTFILES:Mpattern:O:u} == NetBSD") + t.DisableTracing() + + ck := MkLineChecker{mklines, mklines.mklines[0]} + varUse := b.VarUse("DISTFILES", "Mpattern", "O", "u") + ck.checkDirectiveCondCompareVarStr(varUse, "==", "distfile-1.0.tar.gz") + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLineChecker_checkDirectiveFor(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("for.mk", + MkCvsID, + ".for dir in ${PATH:C,:, ,g}", + ".endfor", + "", + ".for dir in ${PATH}", + ".endfor", + "", + ".for dir in ${PATH:M*/bin}", + ".endfor") + + mklines.Check() + + t.CheckOutputLines( + // No warning about a missing :Q in line 2 since the :C modifier + // converts the colon-separated list into a space-separated list, + // as required by the .for loop. + + // This warning is correct since PATH is separated by colons, not by spaces. + "WARN: for.mk:5: Please use ${PATH:Q} instead of ${PATH}.", + + // This warning is also correct since the :M modifier doesn't change the + // word boundaries. + "WARN: for.mk:8: Please use ${PATH:M*/bin:Q} instead of ${PATH:M*/bin}.") +} + +func (s *Suite) Test_MkLineChecker_checkDirectiveFor__infrastructure(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("mk/file.mk", + MkCvsID, + ".for i = 1 2 3", // The "=" should rather be "in". + ".endfor", + "", + ".for _i_ in 1 2 3", // Underscores are only allowed in infrastructure files. + ".endfor") + t.FinishSetUp() + + G.Check(t.File("mk/file.mk")) + + // Pkglint doesn't care about trivial syntax errors like the "=" instead + // of "in" above; bmake will already catch these. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLineChecker_checkDependencyRule(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + mklines := t.NewMkLines("category/package/filename.mk", + MkCvsID, + "", + ".PHONY: target-1", + "target-2: .PHONY", + ".ORDER: target-1 target-2", + "target-1:", + "target-2:", + "target-3:", + "${_COOKIE.test}:") + + mklines.Check() + + t.CheckOutputLines( + "WARN: category/package/filename.mk:8: Undeclared target \"target-3\".") +} diff --git a/pkgtools/pkglint/files/mklineparser.go b/pkgtools/pkglint/files/mklineparser.go index 1a1d2da49f5..4d638f9c3b3 100644 --- a/pkgtools/pkglint/files/mklineparser.go +++ b/pkgtools/pkglint/files/mklineparser.go @@ -28,10 +28,12 @@ func (p MkLineParser) Parse(line *Line) *MkLine { // at the end of the line. if hasPrefix(text, "\t") { lex := textproc.NewLexer(text) - for lex.SkipByte('\t') { - } + lex.SkipHspace() splitResult := p.split(line, lex.Rest(), false) + if lex.PeekByte() == '#' { + return p.parseCommentOrEmpty(line, p.split(line, lex.Rest(), true)) + } return p.parseShellcmd(line, splitResult) } diff --git a/pkgtools/pkglint/files/mklineparser_test.go b/pkgtools/pkglint/files/mklineparser_test.go index ab36d9499ea..b3b3b183817 100644 --- a/pkgtools/pkglint/files/mklineparser_test.go +++ b/pkgtools/pkglint/files/mklineparser_test.go @@ -60,7 +60,8 @@ func (s *Suite) Test_MkLineParser_Parse__comment_or_not(c *check.C) { // From shells/zsh/Makefile.common, rev. 1.78 mklineCommandUnescaped := t.NewMkLine("filename.mk", 1, "\t# $ sha1 patches/patch-ac") - t.CheckEquals(mklineCommandUnescaped.ShellCommand(), "# $ sha1 patches/patch-ac") + t.CheckEquals(mklineCommandUnescaped.IsComment(), true) + t.CheckEquals(mklineCommandUnescaped.Comment(), " $ sha1 patches/patch-ac") t.CheckOutputEmpty() // No warning about parsing the lonely dollar sign. mklineVarassignUnescaped := t.NewMkLine("filename.mk", 1, "SED_CMD=\t's,#,hash,'") @@ -85,6 +86,7 @@ func (s *Suite) Test_MkLineParser_Parse__commented_lines(c *check.C) { test(".include \"other.mk\" # the comment") test(".include <other.mk> # the comment") test("target: source # the comment") + test("\t\t# the comment") } func (s *Suite) Test_MkLineParser_parseVarassign(c *check.C) { diff --git a/pkgtools/pkglint/files/mklines.go b/pkgtools/pkglint/files/mklines.go index eca17c26969..486eae68661 100644 --- a/pkgtools/pkglint/files/mklines.go +++ b/pkgtools/pkglint/files/mklines.go @@ -1,8 +1,6 @@ package pkglint -import ( - "strings" -) +import "strings" // MkLines contains data for the Makefile (or *.mk) that is currently checked. type MkLines struct { @@ -70,19 +68,6 @@ func NewMkLines(lines *Lines) *MkLines { // ck.AfterLine // ck.Finish -// Whole returns a virtual line that can be used for issuing diagnostics -// and explanations, but not for text replacements. -func (mklines *MkLines) Whole() *Line { return mklines.lines.Whole() } - -// UseVar remembers that the given variable is used in the given line. -// This controls the "defined but not used" warning. -func (mklines *MkLines) UseVar(mkline *MkLine, varname string, time VucTime) { - mklines.vars.Use(varname, mkline, time) - if G.Pkg != nil { - G.Pkg.vars.Use(varname, mkline, time) - } -} - func (mklines *MkLines) Check() { if trace.Tracing { defer trace.Call1(mklines.lines.Filename)() @@ -105,203 +90,112 @@ func (mklines *MkLines) Check() { SaveAutofixChanges(mklines.lines) } -func (mklines *MkLines) checkAll() { - allowedTargets := map[string]bool{ - "pre-fetch": true, "do-fetch": true, "post-fetch": true, - "pre-extract": true, "do-extract": true, "post-extract": true, - "pre-patch": true, "do-patch": true, "post-patch": true, - "pre-tools": true, "do-tools": true, "post-tools": true, - "pre-wrapper": true, "do-wrapper": true, "post-wrapper": true, - "pre-configure": true, "do-configure": true, "post-configure": true, - "pre-build": true, "do-build": true, "post-build": true, - "pre-test": true, "do-test": true, "post-test": true, - "pre-install": true, "do-install": true, "post-install": true, - "pre-package": true, "do-package": true, "post-package": true, - "pre-clean": true, "do-clean": true, "post-clean": true} - - mklines.lines.CheckCvsID(0, `#[\t ]+`, "# ") - - substContext := NewSubstContext() - var varalign VaralignBlock - vargroupsChecker := NewVargroupsChecker(mklines) - isHacksMk := mklines.lines.BaseName == "hacks.mk" - - lineAction := func(mkline *MkLine) bool { - if isHacksMk { - // Needs to be set here because it is reset in MkLines.ForEach. - mklines.Tools.SeenPrefs = true - } - - ck := MkLineChecker{mklines, mkline} - ck.Check() - vargroupsChecker.Check(mkline) - - varalign.Process(mkline) - mklines.Tools.ParseToolLine(mklines, mkline, false, false) - substContext.Process(mkline) - - switch { - - case mkline.IsVarassign(): - mklines.target = "" - mkline.Tokenize(mkline.Value(), true) // Just for the side-effect of the warnings. - - mklines.checkVarassignPlist(mkline) - - case mkline.IsInclude(): - mklines.target = "" - if G.Pkg != nil { - G.Pkg.checkIncludeConditionally(mkline, mklines.indentation) - } - - case mkline.IsDirective(): - ck.checkDirective(mklines.forVars, mklines.indentation) - - case mkline.IsDependency(): - ck.checkDependencyRule(allowedTargets) - mklines.target = mkline.Targets() - - case mkline.IsShellCommand(): - mkline.Tokenize(mkline.ShellCommand(), true) // Just for the side-effect of the warnings. - } +func (mklines *MkLines) collectRationale() { - return true + isUseful := func(mkline *MkLine) bool { + comment := trimHspace(mkline.Comment()) + return comment != "" && !hasPrefix(comment, "$NetBSD") } - atEnd := func(mkline *MkLine) { - mklines.indentation.CheckFinish(mklines.lines.Filename) - vargroupsChecker.Finish(mkline) + isRealComment := func(mkline *MkLine) bool { + return mkline.IsComment() && !mkline.IsCommentedVarassign() } - if trace.Tracing { - trace.Stepf("Starting main checking loop") + rationale := false + for _, mkline := range mklines.mklines { + rationale = rationale || isRealComment(mkline) && isUseful(mkline) + mkline.splitResult.hasRationale = rationale || isUseful(mkline) + rationale = rationale && !mkline.IsEmpty() } - mklines.ForEachEnd(lineAction, atEnd) +} - substContext.Finish(mklines.EOFLine()) - varalign.Finish() +func (mklines *MkLines) collectUsedVariables() { + for _, mkline := range mklines.mklines { + mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { + mklines.UseVar(mkline, varUse.varname, time) + }) + } - CheckLinesTrailingEmptyLines(mklines.lines) + mklines.collectDocumentedVariables() } -func (mklines *MkLines) checkVarassignPlist(mkline *MkLine) { - switch mkline.Varcanon() { - case "PLIST_VARS": - for _, id := range mkline.ValueFields(resolveVariableRefs(mklines, mkline.Value())) { - if !mklines.plistVarSkip && mklines.plistVarSet[id] == nil { - mkline.Warnf("%q is added to PLIST_VARS, but PLIST.%s is not defined in this file.", id, id) - } - } - - case "PLIST.*": - id := mkline.Varparam() - if !mklines.plistVarSkip && mklines.plistVarAdded[id] == nil { - mkline.Warnf("PLIST.%s is defined, but %q is not added to PLIST_VARS in this file.", id, id) - } +// UseVar remembers that the given variable is used in the given line. +// This controls the "defined but not used" warning. +func (mklines *MkLines) UseVar(mkline *MkLine, varname string, time VucTime) { + mklines.vars.Use(varname, mkline, time) + if G.Pkg != nil { + G.Pkg.vars.Use(varname, mkline, time) } } -func (mklines *MkLines) SplitToParagraphs() []*Paragraph { - var paras []*Paragraph - - lines := mklines.mklines - isEmpty := func(i int) bool { - if lines[i].IsEmpty() { - return true - } - return lines[i].IsComment() && - lines[i].Text == "#" && - (i == 0 || lines[i-1].IsComment()) && - (i == len(lines)-1 || lines[i+1].IsComment()) - } +// collectDocumentedVariables collects the variables that are mentioned in the human-readable +// documentation of the Makefile fragments from the pkgsrc infrastructure. +// +// Loosely based on mk/help/help.awk, revision 1.28, but much simpler. +func (mklines *MkLines) collectDocumentedVariables() { + scope := NewScope() + commentLines := 0 + relevant := true - i := 0 - for i < len(lines) { - from := i - for from < len(lines) && isEmpty(from) { - from++ - } + // TODO: Correctly interpret declarations like "package-settable variables:" and + // "user-settable variables", as well as "default: ...", "allowed: ...", + // "list of" and other types. - to := from - for to < len(lines) && !isEmpty(to) { - to++ + finish := func() { + // The commentLines include the the line containing the variable name, + // leaving 2 of these 3 lines for the actual documentation. + if commentLines >= 3 && relevant { + for varname, mkline := range scope.used { + mklines.vars.Define(varname, mkline) + mklines.vars.Use(varname, mkline, VucRunTime) + } } - if from != to { - paras = append(paras, NewParagraph(mklines, from, to)) - } - i = to + scope = NewScope() + commentLines = 0 + relevant = true } - return paras -} - -// ForEach calls the action for each line, until the action returns false. -// It keeps track of the indentation (see MkLines.indentation) -// and all conditional variables (see Indentation.IsConditional). -func (mklines *MkLines) ForEach(action func(mkline *MkLine)) { - mklines.ForEachEnd( - func(mkline *MkLine) bool { action(mkline); return true }, - func(mkline *MkLine) {}) -} - -// ForEachEnd calls the action for each line, until the action returns false. -// It keeps track of the indentation and all conditional variables. -// At the end, atEnd is called with the last line as its argument. -func (mklines *MkLines) ForEachEnd(action func(mkline *MkLine) bool, atEnd func(lastMkline *MkLine)) bool { - - // XXX: To avoid looping over the lines multiple times, it would - // be nice to have an interface LinesChecker that checks a single topic. - // Multiple of these line checkers could be run in parallel, so that - // the diagnostics appear in the correct order, from top to bottom. - - mklines.indentation = NewIndentation() - mklines.Tools.SeenPrefs = false - - result := true for _, mkline := range mklines.mklines { - mklines.indentation.TrackBefore(mkline) - if !action(mkline) { - result = false - break - } - mklines.indentation.TrackAfter(mkline) - } - - if len(mklines.mklines) > 0 { - atEnd(mklines.mklines[len(mklines.mklines)-1]) - } - mklines.indentation = nil + text := mkline.Text + switch { + case hasPrefix(text, "#"): + words := strings.Fields(text) + if len(words) <= 1 { + break + } - return result -} + commentLines++ -// ExpandLoopVar searches the surrounding .for loops for the given -// variable and returns a slice containing all its values, fully -// expanded. -// -// It can only be used during an active ForEach call. -func (mklines *MkLines) ExpandLoopVar(varname string) []string { + parser := NewMkParser(nil, words[1]) + varname := parser.Varname() + if len(varname) < 3 { + break + } + if hasSuffix(varname, ".") { + if !parser.lexer.SkipRegexp(regcomp(`^<\w+>`)) { + break + } + varname += "*" + } + parser.lexer.SkipByte(':') - // From the inner loop to the outer loop, just in case - // that two loops should ever use the same variable. - for i := len(mklines.indentation.levels) - 1; i >= 0; i-- { - ind := mklines.indentation.levels[i] + varcanon := varnameCanon(varname) + if varcanon == strings.ToUpper(varcanon) && matches(varcanon, `[A-Z]`) && parser.EOF() { + scope.Define(varcanon, mkline) + scope.Use(varcanon, mkline, VucRunTime) + } - mkline := ind.mkline - if mkline.Directive() != "for" { - continue - } + if words[1] == "Copyright" { + relevant = false + } - // TODO: If needed, add support for multi-variable .for loops. - resolved := resolveVariableRefs(mklines, mkline.Args()) - words := mkline.ValueFields(resolved) - if len(words) >= 3 && words[0] == varname && words[1] == "in" { - return words[2:] + case mkline.IsEmpty(): + finish() } } - return nil + finish() } func (mklines *MkLines) collectVariables() { @@ -369,6 +263,46 @@ func (mklines *MkLines) collectVariables() { }) } +// ForEach calls the action for each line, until the action returns false. +// It keeps track of the indentation (see MkLines.indentation) +// and all conditional variables (see Indentation.IsConditional). +func (mklines *MkLines) ForEach(action func(mkline *MkLine)) { + mklines.ForEachEnd( + func(mkline *MkLine) bool { action(mkline); return true }, + func(mkline *MkLine) {}) +} + +// ForEachEnd calls the action for each line, until the action returns false. +// It keeps track of the indentation and all conditional variables. +// At the end, atEnd is called with the last line as its argument. +func (mklines *MkLines) ForEachEnd(action func(mkline *MkLine) bool, atEnd func(lastMkline *MkLine)) bool { + + // XXX: To avoid looping over the lines multiple times, it would + // be nice to have an interface LinesChecker that checks a single topic. + // Multiple of these line checkers could be run in parallel, so that + // the diagnostics appear in the correct order, from top to bottom. + + mklines.indentation = NewIndentation() + mklines.Tools.SeenPrefs = false + + result := true + for _, mkline := range mklines.mklines { + mklines.indentation.TrackBefore(mkline) + if !action(mkline) { + result = false + break + } + mklines.indentation.TrackAfter(mkline) + } + + if len(mklines.mklines) > 0 { + atEnd(mklines.mklines[len(mklines.mklines)-1]) + } + mklines.indentation = nil + + return result +} + // defineVar marks a variable as defined in both the current package and the current file. func (mklines *MkLines) defineVar(pkg *Package, mkline *MkLine, varname string) { mklines.vars.Define(varname, mkline) @@ -408,103 +342,100 @@ func (mklines *MkLines) collectElse() { // TODO: Check whether this ForEach is redundant because it is already run somewhere else. } -func (mklines *MkLines) collectRationale() { +func (mklines *MkLines) checkAll() { + allowedTargets := map[string]bool{ + "pre-fetch": true, "do-fetch": true, "post-fetch": true, + "pre-extract": true, "do-extract": true, "post-extract": true, + "pre-patch": true, "do-patch": true, "post-patch": true, + "pre-tools": true, "do-tools": true, "post-tools": true, + "pre-wrapper": true, "do-wrapper": true, "post-wrapper": true, + "pre-configure": true, "do-configure": true, "post-configure": true, + "pre-build": true, "do-build": true, "post-build": true, + "pre-test": true, "do-test": true, "post-test": true, + "pre-install": true, "do-install": true, "post-install": true, + "pre-package": true, "do-package": true, "post-package": true, + "pre-clean": true, "do-clean": true, "post-clean": true} - useful := func(mkline *MkLine) bool { - comment := trimHspace(mkline.Comment()) - return comment != "" && !hasPrefix(comment, "$") - } + mklines.lines.CheckCvsID(0, `#[\t ]+`, "# ") - realComment := func(mkline *MkLine) bool { - return mkline.IsComment() && !mkline.IsCommentedVarassign() - } + substContext := NewSubstContext() + var varalign VaralignBlock + vargroupsChecker := NewVargroupsChecker(mklines) + isHacksMk := mklines.lines.BaseName == "hacks.mk" - rationale := false - for _, mkline := range mklines.mklines { - rationale = rationale || realComment(mkline) && useful(mkline) - mkline.splitResult.hasRationale = rationale || useful(mkline) - rationale = rationale && !mkline.IsEmpty() - } -} + lineAction := func(mkline *MkLine) bool { + if isHacksMk { + // Needs to be set here because it is reset in MkLines.ForEach. + mklines.Tools.SeenPrefs = true + } -func (mklines *MkLines) collectUsedVariables() { - for _, mkline := range mklines.mklines { - mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { - mklines.UseVar(mkline, varUse.varname, time) - }) - } + ck := MkLineChecker{mklines, mkline} + ck.Check() + vargroupsChecker.Check(mkline) - mklines.collectDocumentedVariables() -} + varalign.Process(mkline) + mklines.Tools.ParseToolLine(mklines, mkline, false, false) + substContext.Process(mkline) -// collectDocumentedVariables collects the variables that are mentioned in the human-readable -// documentation of the Makefile fragments from the pkgsrc infrastructure. -// -// Loosely based on mk/help/help.awk, revision 1.28, but much simpler. -func (mklines *MkLines) collectDocumentedVariables() { - scope := NewScope() - commentLines := 0 - relevant := true + switch { - // TODO: Correctly interpret declarations like "package-settable variables:" and - // "user-settable variables", as well as "default: ...", "allowed: ...", - // "list of" and other types. + case mkline.IsVarassign(): + mklines.target = "" + mkline.Tokenize(mkline.Value(), true) // Just for the side-effect of the warnings. - finish := func() { - // The commentLines include the the line containing the variable name, - // leaving 2 of these 3 lines for the actual documentation. - if commentLines >= 3 && relevant { - for varname, mkline := range scope.used { - mklines.vars.Define(varname, mkline) - mklines.vars.Use(varname, mkline, VucRunTime) + mklines.checkVarassignPlist(mkline) + + case mkline.IsInclude(): + mklines.target = "" + if G.Pkg != nil { + G.Pkg.checkIncludeConditionally(mkline, mklines.indentation) } + + case mkline.IsDirective(): + ck.checkDirective(mklines.forVars, mklines.indentation) + + case mkline.IsDependency(): + ck.checkDependencyRule(allowedTargets) + mklines.target = mkline.Targets() + + case mkline.IsShellCommand(): + mkline.Tokenize(mkline.ShellCommand(), true) // Just for the side-effect of the warnings. } - scope = NewScope() - commentLines = 0 - relevant = true + return true } - for _, mkline := range mklines.mklines { - text := mkline.Text - switch { - case hasPrefix(text, "#"): - words := strings.Fields(text) - if len(words) <= 1 { - break - } + atEnd := func(mkline *MkLine) { + mklines.indentation.CheckFinish(mklines.lines.Filename) + vargroupsChecker.Finish(mkline) + } - commentLines++ + if trace.Tracing { + trace.Stepf("Starting main checking loop") + } + mklines.ForEachEnd(lineAction, atEnd) - parser := NewMkParser(nil, words[1]) - varname := parser.Varname() - if len(varname) < 3 { - break - } - if hasSuffix(varname, ".") { - if !parser.lexer.SkipRegexp(regcomp(`^<\w+>`)) { - break - } - varname += "*" - } - parser.lexer.SkipByte(':') + substContext.Finish(mklines.EOFLine()) + varalign.Finish() - varcanon := varnameCanon(varname) - if varcanon == strings.ToUpper(varcanon) && matches(varcanon, `[A-Z]`) && parser.EOF() { - scope.Define(varcanon, mkline) - scope.Use(varcanon, mkline, VucRunTime) - } + CheckLinesTrailingEmptyLines(mklines.lines) +} - if words[1] == "Copyright" { - relevant = false +func (mklines *MkLines) checkVarassignPlist(mkline *MkLine) { + switch mkline.Varcanon() { + case "PLIST_VARS": + for _, id := range mkline.ValueFields(resolveVariableRefs(mklines, mkline.Value())) { + if !mklines.plistVarSkip && mklines.plistVarSet[id] == nil { + mkline.Warnf("%q is added to PLIST_VARS, but PLIST.%s is not defined in this file.", id, id) } + } - case mkline.IsEmpty(): - finish() + case "PLIST.*": + id := mkline.Varparam() + if !mklines.plistVarSkip && mklines.plistVarAdded[id] == nil { + mkline.Warnf("PLIST.%s is defined, but %q is not added to PLIST_VARS in this file.", id, id) } } - - finish() } // CheckUsedBy checks that this file (a Makefile.common) has the given @@ -597,6 +528,69 @@ func (mklines *MkLines) CheckUsedBy(relativeName string) { SaveAutofixChanges(lines) } +func (mklines *MkLines) SplitToParagraphs() []*Paragraph { + var paras []*Paragraph + + lines := mklines.mklines + isEmpty := func(i int) bool { + if lines[i].IsEmpty() { + return true + } + return lines[i].IsComment() && + lines[i].Text == "#" && + (i == 0 || lines[i-1].IsComment()) && + (i == len(lines)-1 || lines[i+1].IsComment()) + } + + i := 0 + for i < len(lines) { + from := i + for from < len(lines) && isEmpty(from) { + from++ + } + + to := from + for to < len(lines) && !isEmpty(to) { + to++ + } + + if from != to { + paras = append(paras, NewParagraph(mklines, from, to)) + } + i = to + } + + return paras +} + +// ExpandLoopVar searches the surrounding .for loops for the given +// variable and returns a slice containing all its values, fully +// expanded. +// +// It can only be used during an active ForEach call. +func (mklines *MkLines) ExpandLoopVar(varname string) []string { + + // From the inner loop to the outer loop, just in case + // that two loops should ever use the same variable. + for i := len(mklines.indentation.levels) - 1; i >= 0; i-- { + ind := mklines.indentation.levels[i] + + mkline := ind.mkline + if mkline.Directive() != "for" { + continue + } + + // TODO: If needed, add support for multi-variable .for loops. + resolved := resolveVariableRefs(mklines, mkline.Args()) + words := mkline.ValueFields(resolved) + if len(words) >= 3 && words[0] == varname && words[1] == "in" { + return words[2:] + } + } + + return nil +} + func (mklines *MkLines) SaveAutofixChanges() { mklines.lines.SaveAutofixChanges() } @@ -604,3 +598,7 @@ func (mklines *MkLines) SaveAutofixChanges() { func (mklines *MkLines) EOFLine() *MkLine { return NewMkLineParser().Parse(mklines.lines.EOFLine()) } + +// Whole returns a virtual line that can be used for issuing diagnostics +// and explanations, but not for text replacements. +func (mklines *MkLines) Whole() *Line { return mklines.lines.Whole() } diff --git a/pkgtools/pkglint/files/mklines_test.go b/pkgtools/pkglint/files/mklines_test.go index a8b01925fe4..04bd725426f 100644 --- a/pkgtools/pkglint/files/mklines_test.go +++ b/pkgtools/pkglint/files/mklines_test.go @@ -6,23 +6,6 @@ import ( "strings" ) -func (s *Suite) Test_MkLines_Check__unusual_target(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - t.SetUpTool("cc", "CC", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "echo: echo.c", - "\tcc -o ${.TARGET} ${.IMPSRC}") - - mklines.Check() - - t.CheckOutputLines( - "WARN: Makefile:3: Undeclared target \"echo\".") -} - func (s *Suite) Test_MkLines__quoting_LDFLAGS_for_GNU_configure(c *check.C) { t := s.Init(c) @@ -116,12 +99,48 @@ func (s *Suite) Test_MkLines__varuse_sh_modifier(c *check.C) { t.CheckOutputEmpty() } +func (s *Suite) Test_MkLines_Check__unusual_target(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpTool("cc", "CC", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "echo: echo.c", + "\tcc -o ${.TARGET} ${.IMPSRC}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:3: Undeclared target \"echo\".") +} + +func (s *Suite) Test_MkLines_Check__loop_variable_used_outside_loop(c *check.C) { + t := s.Init(c) + + t.SetUpTool("echo", "", AtRunTime) + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + "do-install:", + "\techo ${msg}", + ".for msg in message", + "\techo ${msg}", + ".endfor") + + mklines.Check() + + t.CheckOutputLines( + "WARN: filename.mk:4: msg is used but not defined.") +} + // For parameterized variables, the "defined but not used" and // the "used but not defined" checks are loosened a bit. // When VAR.param1 is defined or used, VAR.param2 is also regarded // as defined or used since often in pkgsrc, parameterized variables // are not referred to by their exact names but by VAR.${param}. -func (s *Suite) Test_MkLines__varuse_parameterized(c *check.C) { +func (s *Suite) Test_MkLines_Check__varuse_parameterized(c *check.C) { t := s.Init(c) t.SetUpVartypes() @@ -158,7 +177,7 @@ func (s *Suite) Test_MkLines__varuse_parameterized(c *check.C) { // Pkglint could offer to either add the missing semicolon. // Or, if it knows what INSTALL_DATA does, it could simply say that INSTALL_DATA // can handle multiple files in a single invocation. -func (s *Suite) Test_MkLines__loop_modifier(c *check.C) { +func (s *Suite) Test_MkLines_Check__loop_modifier(c *check.C) { t := s.Init(c) t.SetUpVartypes() @@ -175,7 +194,7 @@ func (s *Suite) Test_MkLines__loop_modifier(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_MkLines__PKG_SKIP_REASON_depending_on_OPSYS(c *check.C) { +func (s *Suite) Test_MkLines_Check__PKG_SKIP_REASON_depending_on_OPSYS(c *check.C) { t := s.Init(c) t.SetUpVartypes() @@ -230,275 +249,138 @@ func (s *Suite) Test_MkLines_Check__absolute_pathname_depending_on_OPSYS(c *chec "WARN: games/heretic2-demo/Makefile:5: Unknown shell command \"/usr/bin/bsdtar\".") } -func (s *Suite) Test_MkLines_CheckUsedBy__show_autofix(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--show-autofix") - - test := func(pkgpath string, lines []string, diagnostics []string) { - mklines := t.NewMkLines("Makefile.common", lines...) - - mklines.CheckUsedBy(pkgpath) - - t.CheckOutput(diagnostics) - } - - lines := func(lines ...string) []string { return lines } - diagnostics := func(diagnostics ...string) []string { return diagnostics } - - // This file is too short to be checked. - test( - "category/package", - lines(), - diagnostics()) - - // Still too short. - test( - "category/package", - lines( - MkCvsID), - diagnostics()) - - // Still too short. - test( - "category/package", - lines( - MkCvsID, - ""), - diagnostics()) - - // This file is correctly mentioned. - test( - "sysutils/mc", - lines( - MkCvsID, - "", - "# used by sysutils/mc"), - diagnostics()) - - // This file is not correctly mentioned, therefore the line is inserted. - test( - "category/package", - lines( - MkCvsID, - "", - "VARNAME=\tvalue"), - diagnostics( - "WARN: Makefile.common:1: Please add a line \"# used by category/package\" here.", - "AUTOFIX: Makefile.common:1: Inserting a line \"# used by category/package\" after this line.")) - - // The "used by" comments may either start in line 2 or in line 3. - test( - "category/package", - lines( - MkCvsID, - "#", - "#"), - diagnostics( - "WARN: Makefile.common:1: Please add a line \"# used by category/package\" here.", - "AUTOFIX: Makefile.common:1: Inserting a line \"# used by category/package\" after this line.")) - - // TODO: What if there is an introductory comment first? That should stay at the top of the file. - // TODO: What if the "used by" comments appear in the second paragraph, preceded by only comments and empty lines? - - // Since the first paragraph already has some comments, the "used by" - // comments need their separate paragraph, which is inserted after - // the first paragraph. - test("category/package", - lines( - MkCvsID, - "# A normal comment", - "# that spans", - "# several lines"), - diagnostics( - "AUTOFIX: Makefile.common:4: Inserting a line \"\" after this line.", - "WARN: Makefile.common:4: Please add a line \"# used by category/package\" here.", - "AUTOFIX: Makefile.common:4: Inserting a line \"# used by category/package\" after this line.")) - - t.CheckEquals(G.Logger.autofixAvailable, true) -} - -func (s *Suite) Test_MkLines_CheckUsedBy(c *check.C) { +func (s *Suite) Test_MkLines_Check__indentation(c *check.C) { t := s.Init(c) - test := func(pkgpath string, lines []string, diagnostics []string) { - mklines := t.NewMkLines("Makefile.common", lines...) - - mklines.CheckUsedBy(pkgpath) - - t.CheckOutput(diagnostics) - } - - lines := func(lines ...string) []string { return lines } - diagnostics := func(diagnostics ...string) []string { return diagnostics } - - // The including package is already mentioned in the single "used by" - // paragraph. Nothing needs to be changed. - test("category/package2/Makefile", - lines( - MkCvsID, - "# This Makefile fragment is", - "# used by category/package1/Makefile, as well as", // looks similar to the formal "used by". - "# some others.", - "", - "# used by category/package2/Makefile"), - diagnostics()) - - // The including file is not yet mentioned. There is a single "used by" - // paragraph, and the including file needs to be added to that paragraph. - // It is added in the correct sorting order. The entries are simply - // sorted alphabetically. - test("category/package/Makefile", - lines( - MkCvsID, - "# This Makefile fragment is", - "# used by category/package1/Makefile, as well as", // looks similar to the formal "used by". - "# some others.", - "", - "# used by category/package2/Makefile"), - diagnostics( - "WARN: Makefile.common:6: Please add a line \"# used by category/package/Makefile\" here.")) - - // There are two separate paragraphs with "used by" lines. The first of - // them is the interesting one. The new line is added to the first paragraph. - test("category/package", - lines( - MkCvsID, - "# used by category/package1", - "", - "# used by category/package2"), - diagnostics( - "WARN: Makefile.common:4: There should only be a single \"used by\" paragraph per file.", - "WARN: Makefile.common:2: Please add a line \"# used by category/package\" here.")) - - // The empty comment also separates the two paragraphs, like in the - // previous test case. - test("category/package", - lines( - MkCvsID, - "# used by category/package1", - "#", - "# used by category/package2"), - diagnostics( - "WARN: Makefile.common:4: There should only be a single \"used by\" paragraph per file.", - "WARN: Makefile.common:2: Please add a line \"# used by category/package\" here.")) - - // Code coverage for hasOther being true and conflict being non-nil. - // Ensures that the warning is printed in the first wrong line. - test("category/package", - lines( - MkCvsID, - "", - "# Unrelated comment.", - "# used by category/package1", - "# used by category/package2"), - diagnostics( - "WARN: Makefile.common:4: The \"used by\" lines should be in a separate paragraph.", - "WARN: Makefile.common:1: Please add a line \"# used by category/package\" here.")) + t.SetUpVartypes() + mklines := t.NewMkLines("options.mk", + MkCvsID, + ". if !defined(GUARD_MK)", + ". if ${OPSYS} == ${OPSYS}", + ". for i in ${FILES}", + ". if !defined(GUARD2_MK)", + ". else", + ". endif", + ". endfor", + ". if ${COND1}", + ". elif ${COND2}", + ". else ${COND3}", + ". endif", + ". endif", + ". endif", + ". endif") - // Code coverage for hasUsedBy being true and conflict being non-nil. - // Ensures that the warning is printed in the first wrong line. - test("category/package", - lines( - MkCvsID, - "", - "# used by category/package1", - "# Unrelated comment.", - "# Unrelated comment 2."), - diagnostics( - "WARN: Makefile.common:4: The \"used by\" lines should be in a separate paragraph.", - "WARN: Makefile.common:1: Please add a line \"# used by category/package\" here.")) + mklines.Check() - t.CheckEquals(G.Logger.autofixAvailable, true) + t.CheckOutputLines( + "NOTE: options.mk:2: This directive should be indented by 0 spaces.", + "WARN: options.mk:2: GUARD_MK is used but not defined.", + "NOTE: options.mk:3: This directive should be indented by 0 spaces.", + "NOTE: options.mk:4: This directive should be indented by 2 spaces.", + "WARN: options.mk:4: FILES is used but not defined.", + "NOTE: options.mk:5: This directive should be indented by 4 spaces.", + "WARN: options.mk:5: GUARD2_MK is used but not defined.", + "NOTE: options.mk:6: This directive should be indented by 4 spaces.", + "NOTE: options.mk:7: This directive should be indented by 4 spaces.", + "NOTE: options.mk:8: This directive should be indented by 2 spaces.", + "NOTE: options.mk:9: This directive should be indented by 2 spaces.", + "WARN: options.mk:9: COND1 is used but not defined.", + "NOTE: options.mk:10: This directive should be indented by 2 spaces.", + "WARN: options.mk:10: COND2 is used but not defined.", + "NOTE: options.mk:11: This directive should be indented by 2 spaces.", + "ERROR: options.mk:11: \".else\" does not take arguments. If you meant \"else if\", use \".elif\".", + "NOTE: options.mk:12: This directive should be indented by 2 spaces.", + "NOTE: options.mk:13: This directive should be indented by 0 spaces.", + "NOTE: options.mk:14: This directive should be indented by 0 spaces.", + "NOTE: options.mk:15: This directive should be indented by 0 spaces.", + "ERROR: options.mk:15: Unmatched .endif.") } -func (s *Suite) Test_MkLines_CheckUsedBy__separate_paragraph(c *check.C) { +// The .include directives do not need to be indented. They have the +// syntactical form of directives but cannot be nested in a single file. +// Therefore they may be either indented at the correct indentation depth +// or not indented at all. +func (s *Suite) Test_MkLines_Check__indentation_include(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("Makefile.common", + t.SetUpVartypes() + t.CreateFileLines("included.mk") + mklines := t.SetUpFileMkLines("module.mk", MkCvsID, - "# a comment", - "# used by category/package", - "# a comment") + "", + ".if ${PKGPATH} == \"category/package\"", + ".include \"included.mk\"", + ". include \"included.mk\"", + ". include \"included.mk\"", + ". include \"included.mk\"", + ".endif") - mklines.CheckUsedBy("category/package") + mklines.Check() t.CheckOutputLines( - "WARN: Makefile.common:3: The \"used by\" lines should be in a separate paragraph.") + "ERROR: ~/module.mk:3: There is no package in \"category/package\".", + "NOTE: ~/module.mk:5: This directive should be indented by 2 spaces.", + "NOTE: ~/module.mk:7: This directive should be indented by 2 spaces.") } -func (s *Suite) Test_MkLines_ExpandLoopVar(c *check.C) { +func (s *Suite) Test_MkLines_Check__unfinished_directives(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("filename.mk", + t.SetUpVartypes() + mklines := t.NewMkLines("opsys.mk", MkCvsID, "", - ".for file in a b c d e f g h", - ". for rank in 1 2 3 4 5 6 7 8", - "CHESS_BOARD+=\t${file}${rank}", - ". endfor", - ".endfor") + ".for i in 1 2 3 4 5", + ". if ${OPSYS} == NetBSD", + ". if ${MACHINE_ARCH} == x86_64", + ". if ${OS_VERSION:M8.*}") - var files []string - var ranks []string - var diagonals []string - mklines.ForEach(func(mkline *MkLine) { - if mkline.IsVarassign() { - ranks = mklines.ExpandLoopVar("rank") - files = mklines.ExpandLoopVar("file") - diagonals = mklines.ExpandLoopVar("diagonals") - } - }) + mklines.Check() - t.CheckDeepEquals(files, strings.Split("abcdefgh", "")) - t.CheckDeepEquals(ranks, strings.Split("12345678", "")) - t.Check(diagonals, check.HasLen, 0) + t.CheckOutputLines( + "ERROR: opsys.mk:EOF: .if from line 6 must be closed.", + "ERROR: opsys.mk:EOF: .if from line 5 must be closed.", + "ERROR: opsys.mk:EOF: .if from line 4 must be closed.", + "ERROR: opsys.mk:EOF: .for from line 3 must be closed.") } -func (s *Suite) Test_MkLines_ExpandLoopVar__multi(c *check.C) { +func (s *Suite) Test_MkLines_Check__unbalanced_directives(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("filename.mk", + t.SetUpVartypes() + mklines := t.NewMkLines("opsys.mk", MkCvsID, "", - ".if 1", - ". for key value in 1 one 2 two 3 three", - "VAR.${key}=\t${value}", + ".for i in 1 2 3 4 5", + ". if ${OPSYS} == NetBSD", ". endfor", ".endif") - var keys []string - var values []string - mklines.ForEach(func(mkline *MkLine) { - if mkline.IsVarassign() { - keys = mklines.ExpandLoopVar("key") - values = mklines.ExpandLoopVar("value") - } - }) + mklines.Check() - // As of June 2019, multi-variable .for loops are not yet implemented. - t.Check(keys, check.HasLen, 0) - t.Check(values, check.HasLen, 0) + // As of November 2018 pkglint doesn't find that the inner .if is closed by an .endfor. + // This is checked by bmake, though. + // + // As soon as pkglint starts to analyze .if/.for as regular statements + // like in most programming languages, it will find this inconsistency, too. + t.CheckOutputEmpty() } -func (s *Suite) Test_MkLines_ExpandLoopVar__malformed_for(c *check.C) { +func (s *Suite) Test_MkLines_Check__incomplete_subst_at_end(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("filename.mk", + t.SetUpVartypes() + mklines := t.NewMkLines("subst.mk", MkCvsID, "", - ".for var in", - "VAR=\t${var}", - ".endfor") + "SUBST_CLASSES+=\tclass") - var values = []string{"uninitialized"} - mklines.ForEach(func(mkline *MkLine) { - if mkline.IsVarassign() { - values = mklines.ExpandLoopVar("key") - } - }) + mklines.Check() - t.Check(values, check.HasLen, 0) + t.CheckOutputLines( + "WARN: subst.mk:EOF: Incomplete SUBST block: SUBST_STAGE.class missing.", + "WARN: subst.mk:EOF: Incomplete SUBST block: SUBST_FILES.class missing.", + "WARN: subst.mk:EOF: Incomplete SUBST block: SUBST_SED.class, SUBST_VARS.class or SUBST_FILTER_CMD.class missing.") } func (s *Suite) Test_MkLines_collectRationale(c *check.C) { @@ -572,6 +454,106 @@ func (s *Suite) Test_MkLines_collectRationale(c *check.C) { "- VAR=\tvalue") } +func (s *Suite) Test_MkLines_collectUsedVariables__simple(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("filename.mk", + "\t${VAR}") + mkline := mklines.mklines[0] + + mklines.collectUsedVariables() + + t.CheckDeepEquals(mklines.vars.used, map[string]*MkLine{"VAR": mkline}) + t.CheckEquals(mklines.vars.FirstUse("VAR"), mkline) +} + +func (s *Suite) Test_MkLines_collectUsedVariables__nested(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + "LHS.${lparam}=\tRHS.${rparam}", + "", + "target:", + "\t${outer.${inner}}") + assignMkline := mklines.mklines[2] + shellMkline := mklines.mklines[5] + + mklines.collectUsedVariables() + + t.CheckEquals(len(mklines.vars.used), 5) + t.CheckEquals(mklines.vars.FirstUse("lparam"), assignMkline) + t.CheckEquals(mklines.vars.FirstUse("rparam"), assignMkline) + t.CheckEquals(mklines.vars.FirstUse("inner"), shellMkline) + t.CheckEquals(mklines.vars.FirstUse("outer.*"), shellMkline) + t.CheckEquals(mklines.vars.FirstUse("outer.${inner}"), shellMkline) +} + +func (s *Suite) Test_MkLines_collectDocumentedVariables(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpTool("rm", "RM", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "#", + "# Copyright 2000-2018", + "#", + "# This whole comment is ignored, until the next empty line.", + "# Since it contains the word \"copyright\", it's probably legalese", + "# instead of documentation.", + "", + "# User-settable variables:", + "#", + "# PKG_DEBUG_LEVEL", + "#\tHow verbose should pkgsrc be when running shell commands?", + "#", + "#\t* 0:\tdon't show most shell ...", + "", + "# PKG_VERBOSE", + "#\tWhen this variable is defined, pkgsrc gets a bit more verbose", + "#\t(i.e. \"-v\" option is passed to some commands ...", + "", + "# VARIABLE", + "#\tA paragraph of a single line is not enough to be recognized as \"relevant\".", + "", + "# PARAGRAPH", + "#\tA paragraph may end in a", + "#\tPARA_END_VARNAME.", + "", + "# VARBASE1.<param1>", + "# VARBASE2.*", + "# VARBASE3.${id}", + "", + "# NETBSD/amd64", + "#\tThis is not a variable name.", + "#\tThe slash must not appear in a variable name.", + "", + "# _____", + "#\tThis is not a variable name.", + "#\tVariable names must have at least one letter.") + + // The variables that appear in the documentation are marked as + // both used and defined, to prevent the "defined but not used" warnings. + mklines.collectDocumentedVariables() + + var varnames []string + for varname, mkline := range mklines.vars.used { + varnames = append(varnames, sprintf("%s (line %s)", varname, mkline.Linenos())) + } + sort.Strings(varnames) + + expected := []string{ + "PARAGRAPH (line 23)", + "PKG_DEBUG_LEVEL (line 11)", + "PKG_VERBOSE (line 16)", + "VARBASE1.* (line 27)", + "VARBASE2.* (line 28)", + "VARBASE3.* (line 29)"} + t.CheckDeepEquals(varnames, expected) +} + func (s *Suite) Test_MkLines_collectVariables(c *check.C) { t := s.Init(c) @@ -650,212 +632,75 @@ func (s *Suite) Test_MkLines_collectVariables__no_tracing(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_MkLines_collectUsedVariables__simple(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("filename.mk", - "\t${VAR}") - mkline := mklines.mklines[0] - - mklines.collectUsedVariables() - - t.CheckDeepEquals(mklines.vars.used, map[string]*MkLine{"VAR": mkline}) - t.CheckEquals(mklines.vars.FirstUse("VAR"), mkline) -} - -func (s *Suite) Test_MkLines_collectUsedVariables__nested(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "", - "LHS.${lparam}=\tRHS.${rparam}", - "", - "target:", - "\t${outer.${inner}}") - assignMkline := mklines.mklines[2] - shellMkline := mklines.mklines[5] - - mklines.collectUsedVariables() - - t.CheckEquals(len(mklines.vars.used), 5) - t.CheckEquals(mklines.vars.FirstUse("lparam"), assignMkline) - t.CheckEquals(mklines.vars.FirstUse("rparam"), assignMkline) - t.CheckEquals(mklines.vars.FirstUse("inner"), shellMkline) - t.CheckEquals(mklines.vars.FirstUse("outer.*"), shellMkline) - t.CheckEquals(mklines.vars.FirstUse("outer.${inner}"), shellMkline) -} - -func (s *Suite) Test_MkLines__private_tool_undefined(c *check.C) { +// Ensures that during MkLines.ForEach, the conditional variables in +// MkLines.Indentation are correctly updated for each line. +func (s *Suite) Test_MkLines_ForEach__conditional_variables(c *check.C) { t := s.Init(c) + t.SetUpCommandLine("-Wall,no-space") t.SetUpVartypes() - mklines := t.NewMkLines("filename.mk", + mklines := t.NewMkLines("conditional.mk", MkCvsID, "", - "\tmd5sum filename") - - mklines.Check() - - t.CheckOutputLines( - "WARN: filename.mk:3: Unknown shell command \"md5sum\".") -} - -// Tools that are defined by a package by adding to TOOLS_CREATE can -// be used without adding them to USE_TOOLS again. -func (s *Suite) Test_MkLines__private_tool_defined(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "TOOLS_CREATE+=\tmd5sum", + ".if defined(PKG_DEVELOPER)", + "DEVELOPER=\tyes", + ".endif", "", - "\tmd5sum filename") - - mklines.Check() - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_Check__indentation(c *check.C) { - t := s.Init(c) + ".if ${USE_TOOLS:Mgettext}", + "USES_GETTEXT=\tyes", + ".endif") - t.SetUpVartypes() - mklines := t.NewMkLines("options.mk", - MkCvsID, - ". if !defined(GUARD_MK)", - ". if ${OPSYS} == ${OPSYS}", - ". for i in ${FILES}", - ". if !defined(GUARD2_MK)", - ". else", - ". endif", - ". endfor", - ". if ${COND1}", - ". elif ${COND2}", - ". else ${COND3}", - ". endif", - ". endif", - ". endif", - ". endif") + seenDeveloper := false + seenUsesGettext := false - mklines.Check() + mklines.ForEach(func(mkline *MkLine) { + if mkline.IsVarassign() { + switch mkline.Varname() { + case "DEVELOPER": + t.CheckEquals(mklines.indentation.IsConditional(), true) + seenDeveloper = true + case "USES_GETTEXT": + t.CheckEquals(mklines.indentation.IsConditional(), true) + seenUsesGettext = true + } + } + }) - t.CheckOutputLines( - "NOTE: options.mk:2: This directive should be indented by 0 spaces.", - "WARN: options.mk:2: GUARD_MK is used but not defined.", - "NOTE: options.mk:3: This directive should be indented by 0 spaces.", - "NOTE: options.mk:4: This directive should be indented by 2 spaces.", - "WARN: options.mk:4: FILES is used but not defined.", - "NOTE: options.mk:5: This directive should be indented by 4 spaces.", - "WARN: options.mk:5: GUARD2_MK is used but not defined.", - "NOTE: options.mk:6: This directive should be indented by 4 spaces.", - "NOTE: options.mk:7: This directive should be indented by 4 spaces.", - "NOTE: options.mk:8: This directive should be indented by 2 spaces.", - "NOTE: options.mk:9: This directive should be indented by 2 spaces.", - "WARN: options.mk:9: COND1 is used but not defined.", - "NOTE: options.mk:10: This directive should be indented by 2 spaces.", - "WARN: options.mk:10: COND2 is used but not defined.", - "NOTE: options.mk:11: This directive should be indented by 2 spaces.", - "ERROR: options.mk:11: \".else\" does not take arguments. If you meant \"else if\", use \".elif\".", - "NOTE: options.mk:12: This directive should be indented by 2 spaces.", - "NOTE: options.mk:13: This directive should be indented by 0 spaces.", - "NOTE: options.mk:14: This directive should be indented by 0 spaces.", - "NOTE: options.mk:15: This directive should be indented by 0 spaces.", - "ERROR: options.mk:15: Unmatched .endif.") + t.CheckEquals(seenDeveloper, true) + t.CheckEquals(seenUsesGettext, true) } -// The .include directives do not need to be indented. They have the -// syntactical form of directives but cannot be nested in a single file. -// Therefore they may be either indented at the correct indentation depth -// or not indented at all. -func (s *Suite) Test_MkLines_Check__indentation_include(c *check.C) { +func (s *Suite) Test_MkLines_collectElse(c *check.C) { t := s.Init(c) + t.SetUpCommandLine("-Wno-space") t.SetUpVartypes() - t.CreateFileLines("included.mk") - mklines := t.SetUpFileMkLines("module.mk", - MkCvsID, - "", - ".if ${PKGPATH} == \"category/package\"", - ".include \"included.mk\"", - ". include \"included.mk\"", - ". include \"included.mk\"", - ". include \"included.mk\"", - ".endif") - - mklines.Check() - t.CheckOutputLines( - "ERROR: ~/module.mk:3: There is no package in \"category/package\".", - "NOTE: ~/module.mk:5: This directive should be indented by 2 spaces.", - "NOTE: ~/module.mk:7: This directive should be indented by 2 spaces.") -} - -func (s *Suite) Test_MkLines_Check__unfinished_directives(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("opsys.mk", + mklines := t.NewMkLines("module.mk", MkCvsID, "", - ".for i in 1 2 3 4 5", - ". if ${OPSYS} == NetBSD", - ". if ${MACHINE_ARCH} == x86_64", - ". if ${OS_VERSION:M8.*}") - - mklines.Check() - - t.CheckOutputLines( - "ERROR: opsys.mk:EOF: .if from line 6 must be closed.", - "ERROR: opsys.mk:EOF: .if from line 5 must be closed.", - "ERROR: opsys.mk:EOF: .if from line 4 must be closed.", - "ERROR: opsys.mk:EOF: .for from line 3 must be closed.") -} - -func (s *Suite) Test_MkLines_Check__unbalanced_directives(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("opsys.mk", - MkCvsID, + ".if 0", + ".endif", "", - ".for i in 1 2 3 4 5", - ". if ${OPSYS} == NetBSD", - ". endfor", - ".endif") - - mklines.Check() - - // As of November 2018 pkglint doesn't find that the inner .if is closed by an .endfor. - // This is checked by bmake, though. - // - // As soon as pkglint starts to analyze .if/.for as regular statements - // like in most programming languages, it will find this inconsistency, too. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_Check__incomplete_subst_at_end(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("subst.mk", - MkCvsID, + ".if 0", + ".else", + ".endif", "", - "SUBST_CLASSES+=\tclass") + ".if 0", + ".elif 0", + ".endif") - mklines.Check() + mklines.collectElse() - t.CheckOutputLines( - "WARN: subst.mk:EOF: Incomplete SUBST block: SUBST_STAGE.class missing.", - "WARN: subst.mk:EOF: Incomplete SUBST block: SUBST_FILES.class missing.", - "WARN: subst.mk:EOF: Incomplete SUBST block: SUBST_SED.class, SUBST_VARS.class or SUBST_FILTER_CMD.class missing.") + t.CheckEquals(mklines.mklines[2].HasElseBranch(), false) + t.CheckEquals(mklines.mklines[5].HasElseBranch(), true) + t.CheckEquals(mklines.mklines[9].HasElseBranch(), false) } // Demonstrates how to define your own make(1) targets for creating // files in the current directory. The pkgsrc-wip category Makefile // does this, while all other categories don't need any custom code. -func (s *Suite) Test_MkLines__wip_category_Makefile(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__wip_category_Makefile(c *check.C) { t := s.Init(c) t.SetUpCommandLine("-Wall", "--explain") @@ -897,90 +742,7 @@ func (s *Suite) Test_MkLines__wip_category_Makefile(c *check.C) { "") } -func (s *Suite) Test_MkLines_collectDocumentedVariables(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - t.SetUpTool("rm", "RM", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "#", - "# Copyright 2000-2018", - "#", - "# This whole comment is ignored, until the next empty line.", - "# Since it contains the word \"copyright\", it's probably legalese", - "# instead of documentation.", - "", - "# User-settable variables:", - "#", - "# PKG_DEBUG_LEVEL", - "#\tHow verbose should pkgsrc be when running shell commands?", - "#", - "#\t* 0:\tdon't show most shell ...", - "", - "# PKG_VERBOSE", - "#\tWhen this variable is defined, pkgsrc gets a bit more verbose", - "#\t(i.e. \"-v\" option is passed to some commands ...", - "", - "# VARIABLE", - "#\tA paragraph of a single line is not enough to be recognized as \"relevant\".", - "", - "# PARAGRAPH", - "#\tA paragraph may end in a", - "#\tPARA_END_VARNAME.", - "", - "# VARBASE1.<param1>", - "# VARBASE2.*", - "# VARBASE3.${id}", - "", - "# NETBSD/amd64", - "#\tThis is not a variable name.", - "#\tThe slash must not appear in a variable name.", - "", - "# _____", - "#\tThis is not a variable name.", - "#\tVariable names must have at least one letter.") - - // The variables that appear in the documentation are marked as - // both used and defined, to prevent the "defined but not used" warnings. - mklines.collectDocumentedVariables() - - var varnames []string - for varname, mkline := range mklines.vars.used { - varnames = append(varnames, sprintf("%s (line %s)", varname, mkline.Linenos())) - } - sort.Strings(varnames) - - expected := []string{ - "PARAGRAPH (line 23)", - "PKG_DEBUG_LEVEL (line 11)", - "PKG_VERBOSE (line 16)", - "VARBASE1.* (line 27)", - "VARBASE2.* (line 28)", - "VARBASE3.* (line 29)"} - t.CheckDeepEquals(varnames, expected) -} - -func (s *Suite) Test_MkLines__shell_command_indentation(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("Makefile", - MkCvsID, - "#", - "pre-configure:", - "\tcd 'indented correctly'", - "\t\tcd 'indented needlessly'", - "\tcd 'indented correctly' \\", - "\t\t&& cd 'with indented continuation'") - - mklines.Check() - - t.CheckOutputLines( - "NOTE: Makefile:5: Shell programs should be indented with a single tab.") -} - -func (s *Suite) Test_MkLines__unknown_options(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__unknown_options(c *check.C) { t := s.Init(c) t.SetUpVartypes() @@ -998,7 +760,7 @@ func (s *Suite) Test_MkLines__unknown_options(c *check.C) { "WARN: options.mk:4: Unknown option \"unknown\".") } -func (s *Suite) Test_MkLines_Check__PLIST_VARS(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__PLIST_VARS(c *check.C) { t := s.Init(c) t.SetUpCommandLine("-Wno-space") @@ -1034,7 +796,7 @@ func (s *Suite) Test_MkLines_Check__PLIST_VARS(c *check.C) { "WARN: ~/category/package/options.mk:16: PLIST.only-defined is defined, but \"only-defined\" is not added to PLIST_VARS in this file.") } -func (s *Suite) Test_MkLines_Check__PLIST_VARS_indirect(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__PLIST_VARS_indirect(c *check.C) { t := s.Init(c) t.SetUpCommandLine("-Wno-space") @@ -1070,7 +832,7 @@ func (s *Suite) Test_MkLines_Check__PLIST_VARS_indirect(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_MkLines_Check__PLIST_VARS_indirect_2(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__PLIST_VARS_indirect_2(c *check.C) { t := s.Init(c) t.SetUpCommandLine("-Wno-space") @@ -1097,34 +859,7 @@ func (s *Suite) Test_MkLines_Check__PLIST_VARS_indirect_2(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_MkLines_collectElse(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("-Wno-space") - t.SetUpVartypes() - - mklines := t.NewMkLines("module.mk", - MkCvsID, - "", - ".if 0", - ".endif", - "", - ".if 0", - ".else", - ".endif", - "", - ".if 0", - ".elif 0", - ".endif") - - mklines.collectElse() - - t.CheckEquals(mklines.mklines[2].HasElseBranch(), false) - t.CheckEquals(mklines.mklines[5].HasElseBranch(), true) - t.CheckEquals(mklines.mklines[9].HasElseBranch(), false) -} - -func (s *Suite) Test_MkLines_Check__defined_and_used_variables(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__defined_and_used_variables(c *check.C) { t := s.Init(c) t.SetUpCommandLine("-Wno-space") @@ -1151,7 +886,7 @@ func (s *Suite) Test_MkLines_Check__defined_and_used_variables(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_MkLines_Check__hacks_mk(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__hacks_mk(c *check.C) { t := s.Init(c) t.SetUpCommandLine("-Wall,no-space") @@ -1169,7 +904,7 @@ func (s *Suite) Test_MkLines_Check__hacks_mk(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_MkLines_Check__MASTER_SITE_in_HOMEPAGE(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__MASTER_SITE_in_HOMEPAGE(c *check.C) { t := s.Init(c) t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") @@ -1194,11 +929,11 @@ func (s *Suite) Test_MkLines_Check__MASTER_SITE_in_HOMEPAGE(c *check.C) { // Up to June 2019, pkglint wrongly replaced the HOMEPAGE // with an empty string. -func (s *Suite) Test_MkLines_Check__autofix_MASTER_SITE_in_HOMEPAGE(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__autofix_MASTER_SITE_in_HOMEPAGE(c *check.C) { t := s.Init(c) test := func(diagnostics ...string) { - mklines := t.NewMkLines("Makefile", + mklines := t.SetUpFileMkLines("Makefile", MkCvsID, "", "MASTER_SITES= \\", @@ -1213,6 +948,7 @@ func (s *Suite) Test_MkLines_Check__autofix_MASTER_SITE_in_HOMEPAGE(c *check.C) } t.SetUpVartypes() + t.Chdir(".") t.SetUpCommandLine("-Wall") test( @@ -1224,7 +960,7 @@ func (s *Suite) Test_MkLines_Check__autofix_MASTER_SITE_in_HOMEPAGE(c *check.C) } -func (s *Suite) Test_MkLines_Check__autofix_MASTER_SITE_in_HOMEPAGE_in_package(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__autofix_MASTER_SITE_in_HOMEPAGE_in_package(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package", @@ -1248,7 +984,7 @@ func (s *Suite) Test_MkLines_Check__autofix_MASTER_SITE_in_HOMEPAGE_in_package(c "with \"https://cdn1.example.org/\".") } -func (s *Suite) Test_MkLines_Check__VERSION_as_word_part_in_MASTER_SITES(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__VERSION_as_word_part_in_MASTER_SITES(c *check.C) { t := s.Init(c) t.SetUpVartypes() @@ -1265,7 +1001,7 @@ func (s *Suite) Test_MkLines_Check__VERSION_as_word_part_in_MASTER_SITES(c *chec "WARN: geography/viking/Makefile:2: VERSION is used but not defined.") } -func (s *Suite) Test_MkLines_Check__shell_command_as_word_part_in_ENV_list(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__shell_command_as_word_part_in_ENV_list(c *check.C) { t := s.Init(c) t.SetUpVartypes() @@ -1279,7 +1015,7 @@ func (s *Suite) Test_MkLines_Check__shell_command_as_word_part_in_ENV_list(c *ch "WARN: x11/lablgtk1/Makefile:2: Please use ${CC:Q} instead of ${CC}.") } -func (s *Suite) Test_MkLines_Check__extra_warnings(c *check.C) { +func (s *Suite) Test_MkLines_checkAll__extra_warnings(c *check.C) { t := s.Init(c) t.SetUpCommandLine("-Wextra") @@ -1307,6 +1043,224 @@ func (s *Suite) Test_MkLines_Check__extra_warnings(c *check.C) { "NOTE: options.mk:11: You can use \"../build\" instead of \"${WRKSRC}/../build\".") } +// At 2018-12-02, pkglint had resolved ${MY_PLIST_VARS} into a single word, +// whereas the correct behavior is to resolve it into two words. +// It had produced warnings about mismatched PLIST_VARS IDs. +func (s *Suite) Test_MkLines_checkVarassignPlist__indirect(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("plist.mk", + MkCvsID, + "", + "MY_PLIST_VARS=\tvar1 var2", + "PLIST_VARS+=\t${MY_PLIST_VARS}", + "", + "PLIST.var1=\tyes", + "PLIST.var2=\tyes") + + mklines.Check() + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLines_CheckUsedBy__show_autofix(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--show-autofix") + + test := func(pkgpath string, lines []string, diagnostics []string) { + mklines := t.NewMkLines("Makefile.common", lines...) + + mklines.CheckUsedBy(pkgpath) + + t.CheckOutput(diagnostics) + } + + lines := func(lines ...string) []string { return lines } + diagnostics := func(diagnostics ...string) []string { return diagnostics } + + // This file is too short to be checked. + test( + "category/package", + lines(), + diagnostics()) + + // Still too short. + test( + "category/package", + lines( + MkCvsID), + diagnostics()) + + // Still too short. + test( + "category/package", + lines( + MkCvsID, + ""), + diagnostics()) + + // This file is correctly mentioned. + test( + "sysutils/mc", + lines( + MkCvsID, + "", + "# used by sysutils/mc"), + diagnostics()) + + // This file is not correctly mentioned, therefore the line is inserted. + test( + "category/package", + lines( + MkCvsID, + "", + "VARNAME=\tvalue"), + diagnostics( + "WARN: Makefile.common:1: Please add a line \"# used by category/package\" here.", + "AUTOFIX: Makefile.common:1: Inserting a line \"# used by category/package\" after this line.")) + + // The "used by" comments may either start in line 2 or in line 3. + test( + "category/package", + lines( + MkCvsID, + "#", + "#"), + diagnostics( + "WARN: Makefile.common:1: Please add a line \"# used by category/package\" here.", + "AUTOFIX: Makefile.common:1: Inserting a line \"# used by category/package\" after this line.")) + + // TODO: What if there is an introductory comment first? That should stay at the top of the file. + // TODO: What if the "used by" comments appear in the second paragraph, preceded by only comments and empty lines? + + // Since the first paragraph already has some comments, the "used by" + // comments need their separate paragraph, which is inserted after + // the first paragraph. + test("category/package", + lines( + MkCvsID, + "# A normal comment", + "# that spans", + "# several lines"), + diagnostics( + "AUTOFIX: Makefile.common:4: Inserting a line \"\" after this line.", + "WARN: Makefile.common:4: Please add a line \"# used by category/package\" here.", + "AUTOFIX: Makefile.common:4: Inserting a line \"# used by category/package\" after this line.")) + + t.CheckEquals(G.Logger.autofixAvailable, true) +} + +func (s *Suite) Test_MkLines_CheckUsedBy(c *check.C) { + t := s.Init(c) + + test := func(pkgpath string, lines []string, diagnostics []string) { + mklines := t.NewMkLines("Makefile.common", lines...) + + mklines.CheckUsedBy(pkgpath) + + t.CheckOutput(diagnostics) + } + + lines := func(lines ...string) []string { return lines } + diagnostics := func(diagnostics ...string) []string { return diagnostics } + + // The including package is already mentioned in the single "used by" + // paragraph. Nothing needs to be changed. + test("category/package2/Makefile", + lines( + MkCvsID, + "# This Makefile fragment is", + "# used by category/package1/Makefile, as well as", // looks similar to the formal "used by". + "# some others.", + "", + "# used by category/package2/Makefile"), + diagnostics()) + + // The including file is not yet mentioned. There is a single "used by" + // paragraph, and the including file needs to be added to that paragraph. + // It is added in the correct sorting order. The entries are simply + // sorted alphabetically. + test("category/package/Makefile", + lines( + MkCvsID, + "# This Makefile fragment is", + "# used by category/package1/Makefile, as well as", // looks similar to the formal "used by". + "# some others.", + "", + "# used by category/package2/Makefile"), + diagnostics( + "WARN: Makefile.common:6: Please add a line \"# used by category/package/Makefile\" here.")) + + // There are two separate paragraphs with "used by" lines. The first of + // them is the interesting one. The new line is added to the first paragraph. + test("category/package", + lines( + MkCvsID, + "# used by category/package1", + "", + "# used by category/package2"), + diagnostics( + "WARN: Makefile.common:4: There should only be a single \"used by\" paragraph per file.", + "WARN: Makefile.common:2: Please add a line \"# used by category/package\" here.")) + + // The empty comment also separates the two paragraphs, like in the + // previous test case. + test("category/package", + lines( + MkCvsID, + "# used by category/package1", + "#", + "# used by category/package2"), + diagnostics( + "WARN: Makefile.common:4: There should only be a single \"used by\" paragraph per file.", + "WARN: Makefile.common:2: Please add a line \"# used by category/package\" here.")) + + // Code coverage for hasOther being true and conflict being non-nil. + // Ensures that the warning is printed in the first wrong line. + test("category/package", + lines( + MkCvsID, + "", + "# Unrelated comment.", + "# used by category/package1", + "# used by category/package2"), + diagnostics( + "WARN: Makefile.common:4: The \"used by\" lines should be in a separate paragraph.", + "WARN: Makefile.common:1: Please add a line \"# used by category/package\" here.")) + + // Code coverage for hasUsedBy being true and conflict being non-nil. + // Ensures that the warning is printed in the first wrong line. + test("category/package", + lines( + MkCvsID, + "", + "# used by category/package1", + "# Unrelated comment.", + "# Unrelated comment 2."), + diagnostics( + "WARN: Makefile.common:4: The \"used by\" lines should be in a separate paragraph.", + "WARN: Makefile.common:1: Please add a line \"# used by category/package\" here.")) + + t.CheckEquals(G.Logger.autofixAvailable, true) +} + +func (s *Suite) Test_MkLines_CheckUsedBy__separate_paragraph(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile.common", + MkCvsID, + "# a comment", + "# used by category/package", + "# a comment") + + mklines.CheckUsedBy("category/package") + + t.CheckOutputLines( + "WARN: Makefile.common:3: The \"used by\" lines should be in a separate paragraph.") +} + func (s *Suite) Test_MkLines_SplitToParagraphs(c *check.C) { t := s.Init(c) @@ -1374,61 +1328,76 @@ func (s *Suite) Test_MkLines_SplitToParagraphs(c *check.C) { para(0, 4)) } -// Ensures that during MkLines.ForEach, the conditional variables in -// MkLines.Indentation are correctly updated for each line. -func (s *Suite) Test_MkLines_ForEach__conditional_variables(c *check.C) { +func (s *Suite) Test_MkLines_ExpandLoopVar(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall,no-space") - t.SetUpVartypes() - mklines := t.NewMkLines("conditional.mk", + mklines := t.NewMkLines("filename.mk", MkCvsID, "", - ".if defined(PKG_DEVELOPER)", - "DEVELOPER=\tyes", - ".endif", + ".for file in a b c d e f g h", + ". for rank in 1 2 3 4 5 6 7 8", + "CHESS_BOARD+=\t${file}${rank}", + ". endfor", + ".endfor") + + var files []string + var ranks []string + var diagonals []string + mklines.ForEach(func(mkline *MkLine) { + if mkline.IsVarassign() { + ranks = mklines.ExpandLoopVar("rank") + files = mklines.ExpandLoopVar("file") + diagonals = mklines.ExpandLoopVar("diagonals") + } + }) + + t.CheckDeepEquals(files, strings.Split("abcdefgh", "")) + t.CheckDeepEquals(ranks, strings.Split("12345678", "")) + t.Check(diagonals, check.HasLen, 0) +} + +func (s *Suite) Test_MkLines_ExpandLoopVar__multi(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("filename.mk", + MkCvsID, "", - ".if ${USE_TOOLS:Mgettext}", - "USES_GETTEXT=\tyes", + ".if 1", + ". for key value in 1 one 2 two 3 three", + "VAR.${key}=\t${value}", + ". endfor", ".endif") - seenDeveloper := false - seenUsesGettext := false - + var keys []string + var values []string mklines.ForEach(func(mkline *MkLine) { if mkline.IsVarassign() { - switch mkline.Varname() { - case "DEVELOPER": - t.CheckEquals(mklines.indentation.IsConditional(), true) - seenDeveloper = true - case "USES_GETTEXT": - t.CheckEquals(mklines.indentation.IsConditional(), true) - seenUsesGettext = true - } + keys = mklines.ExpandLoopVar("key") + values = mklines.ExpandLoopVar("value") } }) - t.CheckEquals(seenDeveloper, true) - t.CheckEquals(seenUsesGettext, true) + // As of June 2019, multi-variable .for loops are not yet implemented. + t.Check(keys, check.HasLen, 0) + t.Check(values, check.HasLen, 0) } -// At 2018-12-02, pkglint had resolved ${MY_PLIST_VARS} into a single word, -// whereas the correct behavior is to resolve it into two words. -// It had produced warnings about mismatched PLIST_VARS IDs. -func (s *Suite) Test_MkLines_checkVarassignPlist__indirect(c *check.C) { +func (s *Suite) Test_MkLines_ExpandLoopVar__malformed_for(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - mklines := t.SetUpFileMkLines("plist.mk", + mklines := t.NewMkLines("filename.mk", MkCvsID, "", - "MY_PLIST_VARS=\tvar1 var2", - "PLIST_VARS+=\t${MY_PLIST_VARS}", - "", - "PLIST.var1=\tyes", - "PLIST.var2=\tyes") + ".for var in", + "VAR=\t${var}", + ".endfor") - mklines.Check() + var values = []string{"uninitialized"} + mklines.ForEach(func(mkline *MkLine) { + if mkline.IsVarassign() { + values = mklines.ExpandLoopVar("key") + } + }) - t.CheckOutputEmpty() + t.Check(values, check.HasLen, 0) } diff --git a/pkgtools/pkglint/files/mkparser.go b/pkgtools/pkglint/files/mkparser.go index c8bef3e1fa1..19414ba22f8 100644 --- a/pkgtools/pkglint/files/mkparser.go +++ b/pkgtools/pkglint/files/mkparser.go @@ -673,26 +673,6 @@ func (p *MkParser) Op() (bool, MkOperator) { return false, 0 } -// isPkgbasePart returns whether str, when following a hyphen, -// continues the package base (as in "mysql-client"), or whether it -// starts the version (as in "mysql-1.0"). -func (*MkParser) isPkgbasePart(str string) bool { - lexer := textproc.NewLexer(str) - - lexer.SkipByte('{') - lexer.SkipByte('[') - if lexer.NextByteSet(textproc.Alpha) != -1 { - return true - } - - varUse := NewMkParser(nil, lexer.Rest()).VarUse() - if varUse != nil { - return !contains(varUse.varname, "VER") && len(varUse.modifiers) == 0 - } - - return false -} - func (p *MkParser) PkgbasePattern() string { lexer := p.lexer @@ -722,6 +702,26 @@ func (p *MkParser) PkgbasePattern() string { return "" } +// isPkgbasePart returns whether str, when following a hyphen, +// continues the package base (as in "mysql-client"), or whether it +// starts the version (as in "mysql-1.0"). +func (*MkParser) isPkgbasePart(str string) bool { + lexer := textproc.NewLexer(str) + + lexer.SkipByte('{') + lexer.SkipByte('[') + if lexer.NextByteSet(textproc.Alpha) != -1 { + return true + } + + varUse := NewMkParser(nil, lexer.Rest()).VarUse() + if varUse != nil { + return !contains(varUse.varname, "VER") && len(varUse.modifiers) == 0 + } + + return false +} + type DependencyPattern struct { Pkgbase string // "freeciv-client", "{gcc48,gcc48-libs}", "${EMACS_REQD}" LowerOp string // ">=", ">" diff --git a/pkgtools/pkglint/files/mkparser_test.go b/pkgtools/pkglint/files/mkparser_test.go index 635b7ae1e6a..f103fae406f 100644 --- a/pkgtools/pkglint/files/mkparser_test.go +++ b/pkgtools/pkglint/files/mkparser_test.go @@ -382,6 +382,101 @@ func (s *Suite) Test_MkParser_VarUse(c *check.C) { "WARN: Test_MkParser_VarUse.mk:1: Invalid part \" text\" after variable name \"arbitrary\".") } +func (s *Suite) Test_MkParser_VarUse__ambiguous(c *check.C) { + t := s.Init(c) + b := NewMkTokenBuilder() + + t.SetUpCommandLine("--explain") + + line := t.NewLine("module.mk", 123, "\t$Varname $X") + p := NewMkParser(line, line.Text[1:]) + + tokens := p.MkTokens() + t.CheckDeepEquals(tokens, b.Tokens( + b.VaruseTextToken("$V", "V"), + b.TextToken("arname "), + b.VaruseTextToken("$X", "X"))) + + t.CheckOutputLines( + "ERROR: module.mk:123: $Varname is ambiguous. Use ${Varname} if you mean a Make variable or $$Varname if you mean a shell variable.", + "", + "\tOnly the first letter after the dollar is the variable name.", + "\tEverything following it is normal text, even if it looks like a", + "\tvariable name to human readers.", + "", + "WARN: module.mk:123: $X is ambiguous. Use ${X} if you mean a Make variable or $$X if you mean a shell variable.", + "", + "\tIn its current form, this variable is parsed as a Make variable. For", + "\thuman readers though, $x looks more like a shell variable than a", + "\tMake variable, since Make variables are usually written using braces", + "\t(BSD-style) or parentheses (GNU-style).", + "") +} + +// Pkglint can replace $(VAR) with ${VAR}. It doesn't look at all components +// of nested variables though because this case is not important enough to +// invest much development time. It occurs so seldom that it is acceptable +// to run pkglint multiple times in such a case. +func (s *Suite) Test_MkParser_VarUse__parentheses_autofix(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--autofix") + t.SetUpVartypes() + lines := t.SetUpFileLines("Makefile", + MkCvsID, + "COMMENT=$(P1) $(P2)) $(P3:Q) ${BRACES} $(A.$(B.$(C)))") + mklines := NewMkLines(lines) + + mklines.Check() + + t.CheckOutputLines( + "AUTOFIX: ~/Makefile:2: Replacing \"$(P1)\" with \"${P1}\".", + "AUTOFIX: ~/Makefile:2: Replacing \"$(P2)\" with \"${P2}\".", + "AUTOFIX: ~/Makefile:2: Replacing \"$(P3:Q)\" with \"${P3:Q}\".", + "AUTOFIX: ~/Makefile:2: Replacing \"$(C)\" with \"${C}\".") + t.CheckFileLines("Makefile", + MkCvsID, + "COMMENT=${P1} ${P2}) ${P3:Q} ${BRACES} $(A.$(B.${C}))") +} + +func (s *Suite) Test_MkParser_VarUseModifiers(c *check.C) { + t := s.Init(c) + + varUse := NewMkTokenBuilder().VarUse + test := func(text string, varUse *MkVarUse, diagnostics ...string) { + line := t.NewLine("Makefile", 20, "\t"+text) + p := NewMkParser(line, text) + + actual := p.VarUse() + + t.CheckDeepEquals(actual, varUse) + t.CheckEquals(p.Rest(), "") + t.CheckOutput(diagnostics) + } + + // The !command! modifier is used so seldom that pkglint does not + // check whether the command is actually valid. + // At least not while parsing the modifier since at this point it might + // be still unknown which of the commands can be used and which cannot. + test("${VAR:!command!}", varUse("VAR", "!command!")) + + test("${VAR:!command}", varUse("VAR"), + "WARN: Makefile:20: Invalid variable modifier \"!command\" for \"VAR\".") + + test("${VAR:command!}", varUse("VAR"), + "WARN: Makefile:20: Invalid variable modifier \"command!\" for \"VAR\".") + + // The :L modifier makes the variable value "echo hello", and the :[1] + // modifier extracts the "echo". + test("${echo hello:L:[1]}", varUse("echo hello", "L", "[1]")) + + // bmake ignores the :[3] modifier, and the :L modifier just returns the + // variable name, in this case BUILD_DIRS. + test("${BUILD_DIRS:[3]:L}", varUse("BUILD_DIRS", "[3]", "L")) + + test("${PATH:ts::Q}", varUse("PATH", "ts:", "Q")) +} + func (s *Suite) Test_MkParser_varUseModifier__invalid_ts_modifier_with_warning(c *check.C) { t := s.Init(c) @@ -479,6 +574,64 @@ func (s *Suite) Test_MkParser_varUseModifier__varuse_in_malformed_modifier(c *ch "WARN: filename.mk:123: Invalid variable modifier \"?yes${INNER}\" for \"${VAR}\".") } +func (s *Suite) Test_MkParser_varUseModifierSubst(c *check.C) { + t := s.Init(c) + + varUse := NewMkTokenBuilder().VarUse + test := func(text string, varUse *MkVarUse, rest string, diagnostics ...string) { + line := t.NewLine("Makefile", 20, "\t"+text) + p := NewMkParser(line, text) + + actual := p.VarUse() + + t.CheckDeepEquals(actual, varUse) + t.CheckEquals(p.Rest(), rest) + t.CheckOutput(diagnostics) + } + + test("${VAR:S", varUse("VAR"), "", + "WARN: Makefile:20: Invalid variable modifier \"S\" for \"VAR\".", + "WARN: Makefile:20: Missing closing \"}\" for \"VAR\".") + + test("${VAR:S}", varUse("VAR"), "", + "WARN: Makefile:20: Invalid variable modifier \"S\" for \"VAR\".") + + test("${VAR:S,}", varUse("VAR"), "", + "WARN: Makefile:20: Invalid variable modifier \"S,\" for \"VAR\".") + + test("${VAR:S,from,to}", varUse("VAR"), "", + "WARN: Makefile:20: Invalid variable modifier \"S,from,to\" for \"VAR\".") + + test("${VAR:S,from,to,}", varUse("VAR", "S,from,to,"), "") + + test("${VAR:S,^from$,to,}", varUse("VAR", "S,^from$,to,"), "") + + test("${VAR:S,@F@,${F},}", varUse("VAR", "S,@F@,${F},"), "") + + test("${VAR:S,from,to,1}", varUse("VAR", "S,from,to,1"), "") + test("${VAR:S,from,to,g}", varUse("VAR", "S,from,to,g"), "") + test("${VAR:S,from,to,W}", varUse("VAR", "S,from,to,W"), "") + + test("${VAR:S,from,to,1gW}", varUse("VAR", "S,from,to,1gW"), "") + + // Inside the :S or :C modifiers, neither a colon nor the closing + // brace need to be escaped. Otherwise these patterns would become + // too difficult to read and write. + test("${VAR:C/[[:alnum:]]{2}/**/g}", + varUse("VAR", "C/[[:alnum:]]{2}/**/g"), + "") + + // Some pkgsrc users really explore the darkest corners of bmake by using + // the backslash as the separator in the :S modifier. Sure, it works, it + // just looks totally unexpected to the average pkgsrc reader. + // + // Using the backslash as separator means that it cannot be used for anything + // else, not even for escaping other characters. + test("${VAR:S\\.post1\\\\1}", + varUse("VAR", "S\\.post1\\\\1"), + "") +} + func (s *Suite) Test_MkParser_varUseModifierAt__missing_at_after_variable_name(c *check.C) { t := s.Init(c) b := NewMkTokenBuilder() @@ -521,34 +674,35 @@ func (s *Suite) Test_MkParser_varUseModifierAt__incomplete_without_warning(c *ch t.CheckOutputEmpty() } -func (s *Suite) Test_MkParser_VarUse__ambiguous(c *check.C) { +func (s *Suite) Test_MkParser_varUseModifierAt(c *check.C) { t := s.Init(c) - b := NewMkTokenBuilder() - t.SetUpCommandLine("--explain") + varUse := NewMkTokenBuilder().VarUse + test := func(text string, varUse *MkVarUse, rest string, diagnostics ...string) { + line := t.NewLine("Makefile", 20, "\t"+text) + p := NewMkParser(line, text) - line := t.NewLine("module.mk", 123, "\t$Varname $X") - p := NewMkParser(line, line.Text[1:]) + actual := p.VarUse() - tokens := p.MkTokens() - t.CheckDeepEquals(tokens, b.Tokens( - b.VaruseTextToken("$V", "V"), - b.TextToken("arname "), - b.VaruseTextToken("$X", "X"))) + t.CheckDeepEquals(actual, varUse) + t.CheckEquals(p.Rest(), rest) + t.CheckOutput(diagnostics) + } - t.CheckOutputLines( - "ERROR: module.mk:123: $Varname is ambiguous. Use ${Varname} if you mean a Make variable or $$Varname if you mean a shell variable.", - "", - "\tOnly the first letter after the dollar is the variable name.", - "\tEverything following it is normal text, even if it looks like a", - "\tvariable name to human readers.", - "", - "WARN: module.mk:123: $X is ambiguous. Use ${X} if you mean a Make variable or $$X if you mean a shell variable.", + test("${VAR:@", + varUse("VAR"), "", - "\tIn its current form, this variable is parsed as a Make variable. For", - "\thuman readers though, $x looks more like a shell variable than a", - "\tMake variable, since Make variables are usually written using braces", - "\t(BSD-style) or parentheses (GNU-style).", + "WARN: Makefile:20: Invalid variable modifier \"@\" for \"VAR\".", + "WARN: Makefile:20: Missing closing \"}\" for \"VAR\".") + + test("${VAR:@i@${i}}", varUse("VAR", "@i@${i}}"), "", + "WARN: Makefile:20: Modifier ${VAR:@i@...@} is missing the final \"@\".", + "WARN: Makefile:20: Missing closing \"}\" for \"VAR\".") + + test("${VAR:@i@${i}@}", varUse("VAR", "@i@${i}@"), "") + + test("${PKG_GROUPS:@g@${g:Q}:${PKG_GID.${g}:Q}@:C/:*$//g}", + varUse("PKG_GROUPS", "@g@${g:Q}:${PKG_GID.${g}:Q}@", "C/:*$//g"), "") } @@ -851,184 +1005,6 @@ func (s *Suite) Test_MkParser_Varname(c *check.C) { testRest("VARNAME/rest", "VARNAME", "/rest") } -// Pkglint can replace $(VAR) with ${VAR}. It doesn't look at all components -// of nested variables though because this case is not important enough to -// invest much development time. It occurs so seldom that it is acceptable -// to run pkglint multiple times in such a case. -func (s *Suite) Test_MkParser_VarUse__parentheses_autofix(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--autofix") - t.SetUpVartypes() - lines := t.SetUpFileLines("Makefile", - MkCvsID, - "COMMENT=$(P1) $(P2)) $(P3:Q) ${BRACES} $(A.$(B.$(C)))") - mklines := NewMkLines(lines) - - mklines.Check() - - t.CheckOutputLines( - "AUTOFIX: ~/Makefile:2: Replacing \"$(P1)\" with \"${P1}\".", - "AUTOFIX: ~/Makefile:2: Replacing \"$(P2)\" with \"${P2}\".", - "AUTOFIX: ~/Makefile:2: Replacing \"$(P3:Q)\" with \"${P3:Q}\".", - "AUTOFIX: ~/Makefile:2: Replacing \"$(C)\" with \"${C}\".") - t.CheckFileLines("Makefile", - MkCvsID, - "COMMENT=${P1} ${P2}) ${P3:Q} ${BRACES} $(A.$(B.${C}))") -} - -func (s *Suite) Test_MkParser_VarUseModifiers(c *check.C) { - t := s.Init(c) - - varUse := NewMkTokenBuilder().VarUse - test := func(text string, varUse *MkVarUse, diagnostics ...string) { - line := t.NewLine("Makefile", 20, "\t"+text) - p := NewMkParser(line, text) - - actual := p.VarUse() - - t.CheckDeepEquals(actual, varUse) - t.CheckEquals(p.Rest(), "") - t.CheckOutput(diagnostics) - } - - // The !command! modifier is used so seldom that pkglint does not - // check whether the command is actually valid. - // At least not while parsing the modifier since at this point it might - // be still unknown which of the commands can be used and which cannot. - test("${VAR:!command!}", varUse("VAR", "!command!")) - - test("${VAR:!command}", varUse("VAR"), - "WARN: Makefile:20: Invalid variable modifier \"!command\" for \"VAR\".") - - test("${VAR:command!}", varUse("VAR"), - "WARN: Makefile:20: Invalid variable modifier \"command!\" for \"VAR\".") - - // The :L modifier makes the variable value "echo hello", and the :[1] - // modifier extracts the "echo". - test("${echo hello:L:[1]}", varUse("echo hello", "L", "[1]")) - - // bmake ignores the :[3] modifier, and the :L modifier just returns the - // variable name, in this case BUILD_DIRS. - test("${BUILD_DIRS:[3]:L}", varUse("BUILD_DIRS", "[3]", "L")) - - test("${PATH:ts::Q}", varUse("PATH", "ts:", "Q")) -} - -func (s *Suite) Test_MkParser_varUseModifierSubst(c *check.C) { - t := s.Init(c) - - varUse := NewMkTokenBuilder().VarUse - test := func(text string, varUse *MkVarUse, rest string, diagnostics ...string) { - line := t.NewLine("Makefile", 20, "\t"+text) - p := NewMkParser(line, text) - - actual := p.VarUse() - - t.CheckDeepEquals(actual, varUse) - t.CheckEquals(p.Rest(), rest) - t.CheckOutput(diagnostics) - } - - test("${VAR:S", varUse("VAR"), "", - "WARN: Makefile:20: Invalid variable modifier \"S\" for \"VAR\".", - "WARN: Makefile:20: Missing closing \"}\" for \"VAR\".") - - test("${VAR:S}", varUse("VAR"), "", - "WARN: Makefile:20: Invalid variable modifier \"S\" for \"VAR\".") - - test("${VAR:S,}", varUse("VAR"), "", - "WARN: Makefile:20: Invalid variable modifier \"S,\" for \"VAR\".") - - test("${VAR:S,from,to}", varUse("VAR"), "", - "WARN: Makefile:20: Invalid variable modifier \"S,from,to\" for \"VAR\".") - - test("${VAR:S,from,to,}", varUse("VAR", "S,from,to,"), "") - - test("${VAR:S,^from$,to,}", varUse("VAR", "S,^from$,to,"), "") - - test("${VAR:S,@F@,${F},}", varUse("VAR", "S,@F@,${F},"), "") - - test("${VAR:S,from,to,1}", varUse("VAR", "S,from,to,1"), "") - test("${VAR:S,from,to,g}", varUse("VAR", "S,from,to,g"), "") - test("${VAR:S,from,to,W}", varUse("VAR", "S,from,to,W"), "") - - test("${VAR:S,from,to,1gW}", varUse("VAR", "S,from,to,1gW"), "") - - // Inside the :S or :C modifiers, neither a colon nor the closing - // brace need to be escaped. Otherwise these patterns would become - // too difficult to read and write. - test("${VAR:C/[[:alnum:]]{2}/**/g}", - varUse("VAR", "C/[[:alnum:]]{2}/**/g"), - "") - - // Some pkgsrc users really explore the darkest corners of bmake by using - // the backslash as the separator in the :S modifier. Sure, it works, it - // just looks totally unexpected to the average pkgsrc reader. - // - // Using the backslash as separator means that it cannot be used for anything - // else, not even for escaping other characters. - test("${VAR:S\\.post1\\\\1}", - varUse("VAR", "S\\.post1\\\\1"), - "") -} - -func (s *Suite) Test_MkParser_varUseModifierAt(c *check.C) { - t := s.Init(c) - - varUse := NewMkTokenBuilder().VarUse - test := func(text string, varUse *MkVarUse, rest string, diagnostics ...string) { - line := t.NewLine("Makefile", 20, "\t"+text) - p := NewMkParser(line, text) - - actual := p.VarUse() - - t.CheckDeepEquals(actual, varUse) - t.CheckEquals(p.Rest(), rest) - t.CheckOutput(diagnostics) - } - - test("${VAR:@", - varUse("VAR"), - "", - "WARN: Makefile:20: Invalid variable modifier \"@\" for \"VAR\".", - "WARN: Makefile:20: Missing closing \"}\" for \"VAR\".") - - test("${VAR:@i@${i}}", varUse("VAR", "@i@${i}}"), "", - "WARN: Makefile:20: Modifier ${VAR:@i@...@} is missing the final \"@\".", - "WARN: Makefile:20: Missing closing \"}\" for \"VAR\".") - - test("${VAR:@i@${i}@}", varUse("VAR", "@i@${i}@"), "") - - test("${PKG_GROUPS:@g@${g:Q}:${PKG_GID.${g}:Q}@:C/:*$//g}", - varUse("PKG_GROUPS", "@g@${g:Q}:${PKG_GID.${g}:Q}@", "C/:*$//g"), - "") -} - -func (s *Suite) Test_MkParser_isPkgbasePart(c *check.C) { - t := s.Init(c) - - test := func(str string, expected bool) { - actual := (*MkParser)(nil).isPkgbasePart(str) - - t.CheckEquals(actual, expected) - } - - test("X11", true) - test("client", true) - test("${PKGNAME}", true) - test("[a-z]", true) - test("{client,server}", true) - - test("1.2", false) - test("[0-9]*", false) - test("{5.[1-7].*,6.[0-9]*}", false) - test("${PKGVERSION}", false) - test("${PKGNAME:C/^.*-//}", false) - test(">=1.0", false) - test("_client", false) // The combination foo-_client looks strange. -} - func (s *Suite) Test_MkParser_PkgbasePattern(c *check.C) { t := s.Init(c) @@ -1075,6 +1051,30 @@ func (s *Suite) Test_MkParser_PkgbasePattern(c *check.C) { test("{ssh{,6}-[0-9]*,openssh-[0-9]*}", "", "{ssh{,6}-[0-9]*,openssh-[0-9]*}") } +func (s *Suite) Test_MkParser_isPkgbasePart(c *check.C) { + t := s.Init(c) + + test := func(str string, expected bool) { + actual := (*MkParser)(nil).isPkgbasePart(str) + + t.CheckEquals(actual, expected) + } + + test("X11", true) + test("client", true) + test("${PKGNAME}", true) + test("[a-z]", true) + test("{client,server}", true) + + test("1.2", false) + test("[0-9]*", false) + test("{5.[1-7].*,6.[0-9]*}", false) + test("${PKGVERSION}", false) + test("${PKGNAME:C/^.*-//}", false) + test(">=1.0", false) + test("_client", false) // The combination foo-_client looks strange. +} + func (s *Suite) Test_MkParser_Dependency(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/mkshparser.go b/pkgtools/pkglint/files/mkshparser.go index 98547f5822a..1fd0af412a5 100644 --- a/pkgtools/pkglint/files/mkshparser.go +++ b/pkgtools/pkglint/files/mkshparser.go @@ -66,8 +66,6 @@ type ShellLexer struct { func NewShellLexer(tokens []string, rest string) *ShellLexer { return &ShellLexer{ - current: "", - ioRedirect: "", remaining: tokens, atCommandStart: true, error: rest} diff --git a/pkgtools/pkglint/files/mkshparser_test.go b/pkgtools/pkglint/files/mkshparser_test.go index 3c36762250a..0c828fdb8a4 100644 --- a/pkgtools/pkglint/files/mkshparser_test.go +++ b/pkgtools/pkglint/files/mkshparser_test.go @@ -64,15 +64,15 @@ type ShSuite struct { var _ = check.Suite(&ShSuite{}) -func (s *ShSuite) SetUpTest(c *check.C) { - G = NewPkglint() +func (s *ShSuite) SetUpTest(*check.C) { + G = NewPkglint(nil, nil) } -func (s *ShSuite) TearDownTest(c *check.C) { +func (s *ShSuite) TearDownTest(*check.C) { G = unusablePkglint() } -func (s *ShSuite) Test_ShellParser__program(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__program(c *check.C) { b := s.init(c) s.test("", @@ -149,7 +149,7 @@ func (s *ShSuite) Test_ShellParser__program(c *check.C) { b.List().AddCommand(b.SimpleCommand("action2")).AddSemicolon()))) } -func (s *ShSuite) Test_ShellParser__list(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__list(c *check.C) { b := s.init(c) s.test("echo1 && echo2", @@ -171,7 +171,7 @@ func (s *ShSuite) Test_ShellParser__list(c *check.C) { AddBackground()) } -func (s *ShSuite) Test_ShellParser__and_or(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__and_or(c *check.C) { b := s.init(c) s.test("echo1 | echo2", @@ -200,7 +200,7 @@ func (s *ShSuite) Test_ShellParser__and_or(c *check.C) { b.SimpleCommand("echo4"))))) } -func (s *ShSuite) Test_ShellParser__pipeline(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__pipeline(c *check.C) { b := s.init(c) s.test("command1 | command2", @@ -214,7 +214,7 @@ func (s *ShSuite) Test_ShellParser__pipeline(c *check.C) { b.SimpleCommand("command2"))))) } -func (s *ShSuite) Test_ShellParser__pipe_sequence(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__pipe_sequence(c *check.C) { b := s.init(c) s.test("command1 | if true ; then : ; fi", @@ -225,7 +225,7 @@ func (s *ShSuite) Test_ShellParser__pipe_sequence(c *check.C) { b.List().AddCommand(b.SimpleCommand(":")).AddSemicolon()))))) } -func (s *ShSuite) Test_ShellParser__command(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__command(c *check.C) { b := s.init(c) s.test("simple_command", @@ -251,7 +251,7 @@ func (s *ShSuite) Test_ShellParser__command(c *check.C) { b.Redirection(2, ">&", "1")))) } -func (s *ShSuite) Test_ShellParser__compound_command(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__compound_command(c *check.C) { b := s.init(c) s.test("{ brace ; }", @@ -274,7 +274,7 @@ func (s *ShSuite) Test_ShellParser__compound_command(c *check.C) { } -func (s *ShSuite) Test_ShellParser__subshell(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__subshell(c *check.C) { b := s.init(c) sub3 := b.Subshell(b.List().AddCommand(b.SimpleCommand("sub3"))) @@ -283,7 +283,7 @@ func (s *ShSuite) Test_ShellParser__subshell(c *check.C) { s.test("( ( ( sub3 ) ; sub2 ) ; sub1 )", b.List().AddCommand(sub1)) } -func (s *ShSuite) Test_ShellParser__compound_list(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__compound_list(c *check.C) { b := s.init(c) s.test("( \n echo )", @@ -291,7 +291,7 @@ func (s *ShSuite) Test_ShellParser__compound_list(c *check.C) { b.List().AddCommand(b.SimpleCommand("echo"))))) } -func (s *ShSuite) Test_ShellParser__term(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__term(c *check.C) { b := s.init(c) s.test("echo1 ; echo2 ;", @@ -302,7 +302,7 @@ func (s *ShSuite) Test_ShellParser__term(c *check.C) { AddSemicolon()) } -func (s *ShSuite) Test_ShellParser__for_clause(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__for_clause(c *check.C) { b := s.init(c) // If this test fails, the cause might be in shell.y, in the for_clause rule. @@ -360,7 +360,7 @@ func (s *ShSuite) Test_ShellParser__for_clause(c *check.C) { b.List().AddCommand(b.SimpleCommand("echo", "$$i$$j")).AddSemicolon()))))) } -func (s *ShSuite) Test_ShellParser__case_clause(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__case_clause(c *check.C) { b := s.init(c) s.test("case $var in esac", @@ -442,7 +442,7 @@ func (s *ShSuite) Test_ShellParser__case_clause(c *check.C) { []string{}...) } -func (s *ShSuite) Test_ShellParser__if_clause(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__if_clause(c *check.C) { b := s.init(c) s.test( @@ -469,7 +469,7 @@ func (s *ShSuite) Test_ShellParser__if_clause(c *check.C) { b.List().AddCommand(b.SimpleCommand("action3")).AddSemicolon()))) } -func (s *ShSuite) Test_ShellParser__while_clause(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__while_clause(c *check.C) { b := s.init(c) s.test("while condition ; do action ; done", @@ -478,7 +478,7 @@ func (s *ShSuite) Test_ShellParser__while_clause(c *check.C) { b.List().AddCommand(b.SimpleCommand("action")).AddSemicolon()))) } -func (s *ShSuite) Test_ShellParser__until_clause(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__until_clause(c *check.C) { b := s.init(c) s.test("until condition ; do action ; done", @@ -487,7 +487,7 @@ func (s *ShSuite) Test_ShellParser__until_clause(c *check.C) { b.List().AddCommand(b.SimpleCommand("action")).AddSemicolon()))) } -func (s *ShSuite) Test_ShellParser__function_definition(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__function_definition(c *check.C) { b := s.init(c) s.test("fn() { simple-command; }", @@ -502,7 +502,7 @@ func (s *ShSuite) Test_ShellParser__function_definition(c *check.C) { // a single command without braces or parentheses. } -func (s *ShSuite) Test_ShellParser__brace_group(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__brace_group(c *check.C) { b := s.init(c) // No semicolon necessary after the closing brace. @@ -513,7 +513,7 @@ func (s *ShSuite) Test_ShellParser__brace_group(c *check.C) { b.List().AddCommand(b.SimpleCommand("echo", "yes")).AddSemicolon()))))) } -func (s *ShSuite) Test_ShellParser__simple_command(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__simple_command(c *check.C) { b := s.init(c) s.test( @@ -554,7 +554,7 @@ func (s *ShSuite) Test_ShellParser__simple_command(c *check.C) { b.List().AddCommand(b.SimpleCommand("{OpenGrok", "args"))) } -func (s *ShSuite) Test_ShellParser__io_redirect(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__io_redirect(c *check.C) { b := s.init(c) s.test("echo >> ${PLIST_SRC}", @@ -615,7 +615,7 @@ func (s *ShSuite) Test_ShellParser__io_redirect(c *check.C) { {1, ">", b.Token("output")}}}})) } -func (s *ShSuite) Test_ShellParser__redirect_list(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__redirect_list(c *check.C) { b := s.init(c) s.test("(:) 1>out", @@ -632,7 +632,7 @@ func (s *ShSuite) Test_ShellParser__redirect_list(c *check.C) { b.Redirection(2, ">", "out")))) } -func (s *ShSuite) Test_ShellParser__io_here(c *check.C) { +func (s *ShSuite) Test_parseShellProgram__io_here(c *check.C) { // In pkgsrc Makefiles, the IO here-documents cannot be used since // all the text is joined into a single line. Therefore these test // cases only show that pkglint can indeed not parse <<EOF @@ -665,16 +665,13 @@ func (s *ShSuite) init(c *check.C) *MkShBuilder { func (s *ShSuite) test(program string, expected *MkShList) { t := s.t + // See parseShellProgram tokens, rest := splitIntoShellTokens(dummyLine, program) t.CheckEquals(rest, "") - lexer := ShellLexer{ - current: "", - remaining: tokens, - atCommandStart: true, - error: ""} + lexer := NewShellLexer(tokens, rest) parser := shyyParserImpl{} - zeroMeansSuccess := parser.Parse(&lexer) + zeroMeansSuccess := parser.Parse(lexer) c := s.c diff --git a/pkgtools/pkglint/files/mktypes_test.go b/pkgtools/pkglint/files/mktypes_test.go index fd89c22f40b..ad2cc690eda 100644 --- a/pkgtools/pkglint/files/mktypes_test.go +++ b/pkgtools/pkglint/files/mktypes_test.go @@ -49,85 +49,6 @@ func (list *MkShList) AddCommand(command *MkShCommand) *MkShList { return list.AddAndOr(andOr) } -func (s *Suite) Test_MkVarUse_Mod(c *check.C) { - t := s.Init(c) - - test := func(varUseText string, mod string) { - line := t.NewLine("filename.mk", 123, "") - varUse := NewMkParser(line, varUseText).VarUse() - t.CheckOutputEmpty() - t.CheckEquals(varUse.Mod(), mod) - } - - test("${varname:Q}", ":Q") - test("${PATH:ts::Q}", ":ts::Q") -} - -func (s *Suite) Test_MkVarUseModifier_MatchMatch(c *check.C) { - t := s.Init(c) - - testFail := func(modifier string) { - mod := MkVarUseModifier{modifier} - ok, _, _, _ := mod.MatchMatch() - t.CheckEquals(ok, false) - } - test := func(modifier string, positive bool, pattern string, exact bool) { - mod := MkVarUseModifier{modifier} - actualOk, actualPositive, actualPattern, actualExact := mod.MatchMatch() - t.CheckDeepEquals( - []interface{}{actualOk, actualPositive, actualPattern, actualExact}, - []interface{}{true, positive, pattern, exact}) - } - - testFail("") - testFail("X") - - test("Mpattern", true, "pattern", true) - test("M*", true, "*", false) - test("M${VAR}", true, "${VAR}", false) - test("Npattern", false, "pattern", true) -} - -func (s *Suite) Test_MkVarUseModifier_ChangesWords(c *check.C) { - t := s.Init(c) - - test := func(modifier string, changes bool) { - mod := MkVarUseModifier{modifier} - t.CheckEquals(mod.ChangesWords(), changes) - } - - test("E", false) - test("R", false) - test("Mpattern", false) - test("Npattern", false) - test("S,from,to,", true) - test("C,from,to,", true) - test("tl", false) - test("tu", false) - test("sh", true) - - test("unknown", true) -} - -// Ensures that ChangesWords cannot be called with an empty string as modifier. -func (s *Suite) Test_MkVarUseModifier_ChangesWords__empty(c *check.C) { - t := s.Init(c) - - mkline := t.NewMkLine("filename.mk", 123, "\t${VAR:}") - - n := 0 - mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { - n += 100 - for _, mod := range varUse.modifiers { - mod.ChangesWords() - n++ - } - }) - - t.CheckOutputEmpty() - t.CheckEquals(n, 100) -} - func (s *Suite) Test_MkVarUseModifier_MatchSubst(c *check.C) { t := s.Init(c) @@ -228,3 +149,82 @@ func (s *Suite) Test_MkVarUseModifier_Subst__C_with_complex_replacement(c *check t.CheckEquals(ok, false) t.CheckEquals(result, "") } + +func (s *Suite) Test_MkVarUseModifier_MatchMatch(c *check.C) { + t := s.Init(c) + + testFail := func(modifier string) { + mod := MkVarUseModifier{modifier} + ok, _, _, _ := mod.MatchMatch() + t.CheckEquals(ok, false) + } + test := func(modifier string, positive bool, pattern string, exact bool) { + mod := MkVarUseModifier{modifier} + actualOk, actualPositive, actualPattern, actualExact := mod.MatchMatch() + t.CheckDeepEquals( + []interface{}{actualOk, actualPositive, actualPattern, actualExact}, + []interface{}{true, positive, pattern, exact}) + } + + testFail("") + testFail("X") + + test("Mpattern", true, "pattern", true) + test("M*", true, "*", false) + test("M${VAR}", true, "${VAR}", false) + test("Npattern", false, "pattern", true) +} + +func (s *Suite) Test_MkVarUseModifier_ChangesWords(c *check.C) { + t := s.Init(c) + + test := func(modifier string, changes bool) { + mod := MkVarUseModifier{modifier} + t.CheckEquals(mod.ChangesWords(), changes) + } + + test("E", false) + test("R", false) + test("Mpattern", false) + test("Npattern", false) + test("S,from,to,", true) + test("C,from,to,", true) + test("tl", false) + test("tu", false) + test("sh", true) + + test("unknown", true) +} + +// Ensures that ChangesWords cannot be called with an empty string as modifier. +func (s *Suite) Test_MkVarUseModifier_ChangesWords__empty(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("filename.mk", 123, "\t${VAR:}") + + n := 0 + mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) { + n += 100 + for _, mod := range varUse.modifiers { + mod.ChangesWords() + n++ + } + }) + + t.CheckOutputEmpty() + t.CheckEquals(n, 100) +} + +func (s *Suite) Test_MkVarUse_Mod(c *check.C) { + t := s.Init(c) + + test := func(varUseText string, mod string) { + line := t.NewLine("filename.mk", 123, "") + varUse := NewMkParser(line, varUseText).VarUse() + t.CheckOutputEmpty() + t.CheckEquals(varUse.Mod(), mod) + } + + test("${varname:Q}", ":Q") + test("${PATH:ts::Q}", ":ts::Q") +} diff --git a/pkgtools/pkglint/files/options_test.go b/pkgtools/pkglint/files/options_test.go index 662ee576802..36468712b85 100755 --- a/pkgtools/pkglint/files/options_test.go +++ b/pkgtools/pkglint/files/options_test.go @@ -2,6 +2,66 @@ package pkglint import "gopkg.in/check.v1" +func (s *Suite) Test_CheckLinesOptionsMk__autofix(c *check.C) { + t := s.Init(c) + + t.SetUpOption("opt", "") + t.CreateFileLines("mk/bsd.options.mk") + t.SetUpPackage("category/package", + ".include \"options.mk\"") + t.CreateFileLines("category/package/options.mk", + MkCvsID, + "", + "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", + "PKG_SUPPORTED_OPTIONS=\t# none", + "", + ".include \"../../mk/bsd.options.mk\"", + "", + ".if 0", + ".if 0", + ".endif", + ".endif") + t.FinishSetUp() + t.Chdir("category/package") + + G.Check(".") + + t.CheckOutputLines( + "NOTE: options.mk:9: This directive should be indented by 2 spaces.", + "NOTE: options.mk:10: This directive should be indented by 2 spaces.") + + t.SetUpCommandLine("-Wall", "--show-autofix") + + G.Check(".") + + t.CheckOutputLines( + "NOTE: options.mk:9: This directive should be indented by 2 spaces.", + "AUTOFIX: options.mk:9: Replacing \".\" with \". \".", + "NOTE: options.mk:10: This directive should be indented by 2 spaces.", + "AUTOFIX: options.mk:10: Replacing \".\" with \". \".") + + t.SetUpCommandLine("-Wall", "--autofix") + + G.Check(".") + + t.CheckOutputLines( + "AUTOFIX: options.mk:9: Replacing \".\" with \". \".", + "AUTOFIX: options.mk:10: Replacing \".\" with \". \".") + + t.CheckFileLinesDetab("options.mk", + MkCvsID, + "", + "PKG_OPTIONS_VAR= PKG_OPTIONS.package", + "PKG_SUPPORTED_OPTIONS= # none", + "", + ".include \"../../mk/bsd.options.mk\"", + "", + ".if 0", + ". if 0", + ". endif", + ".endif") +} + func (s *Suite) Test_CheckLinesOptionsMk__literal(c *check.C) { t := s.Init(c) @@ -361,96 +421,6 @@ func (s *Suite) Test_CheckLinesOptionsMk__PLIST_VARS_based_on_PKG_SUPPORTED_OPTI "WARN: options.mk:5: Option \"two\" should be handled below in an .if block.") } -// Up to April 2019, pkglint logged a wrong note saying that OTHER_VARIABLE -// should have the positive branch first. That note was only ever intended -// for PKG_OPTIONS. -func (s *Suite) Test_OptionsLinesChecker_handleLowerCondition__foreign_variable(c *check.C) { - t := s.Init(c) - - t.SetUpOption("opt", "") - t.CreateFileLines("mk/bsd.options.mk") - t.SetUpPackage("category/package", - ".include \"options.mk\"") - t.CreateFileLines("category/package/options.mk", - MkCvsID, - "", - "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", - "PKG_SUPPORTED_OPTIONS=\topt", - "", - ".include \"../../mk/bsd.options.mk\"", - "", - ".if empty(OTHER_VARIABLE)", - ".else", - ".endif") - t.FinishSetUp() - - G.Check(t.File("category/package")) - - t.CheckOutputLines( - "WARN: ~/category/package/options.mk:8: OTHER_VARIABLE is used but not defined.", - "WARN: ~/category/package/options.mk:4: Option \"opt\" should be handled below in an .if block.") -} - -func (s *Suite) Test_CheckLinesOptionsMk__autofix(c *check.C) { - t := s.Init(c) - - t.SetUpOption("opt", "") - t.CreateFileLines("mk/bsd.options.mk") - t.SetUpPackage("category/package", - ".include \"options.mk\"") - t.CreateFileLines("category/package/options.mk", - MkCvsID, - "", - "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", - "PKG_SUPPORTED_OPTIONS=\t# none", - "", - ".include \"../../mk/bsd.options.mk\"", - "", - ".if 0", - ".if 0", - ".endif", - ".endif") - t.FinishSetUp() - t.Chdir("category/package") - - G.Check(".") - - t.CheckOutputLines( - "NOTE: options.mk:9: This directive should be indented by 2 spaces.", - "NOTE: options.mk:10: This directive should be indented by 2 spaces.") - - t.SetUpCommandLine("-Wall", "--show-autofix") - - G.Check(".") - - t.CheckOutputLines( - "NOTE: options.mk:9: This directive should be indented by 2 spaces.", - "AUTOFIX: options.mk:9: Replacing \".\" with \". \".", - "NOTE: options.mk:10: This directive should be indented by 2 spaces.", - "AUTOFIX: options.mk:10: Replacing \".\" with \". \".") - - t.SetUpCommandLine("-Wall", "--autofix") - - G.Check(".") - - t.CheckOutputLines( - "AUTOFIX: options.mk:9: Replacing \".\" with \". \".", - "AUTOFIX: options.mk:10: Replacing \".\" with \". \".") - - t.CheckFileLinesDetab("options.mk", - MkCvsID, - "", - "PKG_OPTIONS_VAR= PKG_OPTIONS.package", - "PKG_SUPPORTED_OPTIONS= # none", - "", - ".include \"../../mk/bsd.options.mk\"", - "", - ".if 0", - ". if 0", - ". endif", - ".endif") -} - // A few packages (such as www/w3m) define several options that are // handled by a single .if block in the lower part. func (s *Suite) Test_CheckLinesOptionsMk__combined_option_handling(c *check.C) { @@ -700,3 +670,33 @@ func (s *Suite) Test_CheckLinesOptionsMk__supported_but_not_checked(c *check.C) // does not issue any warnings about possibly unhandled options at all. t.CheckOutputEmpty() } + +// Up to April 2019, pkglint logged a wrong note saying that OTHER_VARIABLE +// should have the positive branch first. That note was only ever intended +// for PKG_OPTIONS. +func (s *Suite) Test_OptionsLinesChecker_handleLowerCondition__foreign_variable(c *check.C) { + t := s.Init(c) + + t.SetUpOption("opt", "") + t.CreateFileLines("mk/bsd.options.mk") + t.SetUpPackage("category/package", + ".include \"options.mk\"") + t.CreateFileLines("category/package/options.mk", + MkCvsID, + "", + "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", + "PKG_SUPPORTED_OPTIONS=\topt", + "", + ".include \"../../mk/bsd.options.mk\"", + "", + ".if empty(OTHER_VARIABLE)", + ".else", + ".endif") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/package/options.mk:8: OTHER_VARIABLE is used but not defined.", + "WARN: ~/category/package/options.mk:4: Option \"opt\" should be handled below in an .if block.") +} diff --git a/pkgtools/pkglint/files/package.go b/pkgtools/pkglint/files/package.go index 8e50d3e6ca8..7029931e883 100644 --- a/pkgtools/pkglint/files/package.go +++ b/pkgtools/pkglint/files/package.go @@ -97,113 +97,6 @@ func NewPackage(dir string) *Package { return &pkg } -// File returns the (possibly absolute) path to relativeFileName, -// as resolved from the package's directory. -// Variables that are known in the package are resolved, e.g. ${PKGDIR}. -func (pkg *Package) File(relativeFileName string) string { - return cleanpath(resolveVariableRefs(nil /* XXX: or maybe some mklines? */, joinPath(pkg.dir, relativeFileName))) -} - -// Rel returns the path by which the given filename (as seen from the -// current working directory) can be reached as a relative path from -// the package directory. -// -// Example: -// NewPackage("category/package").Rel("other/package") == "../../other/package" -func (pkg *Package) Rel(filename string) string { - return relpath(pkg.dir, filename) -} - -// Returns whether the given file (relative to the package directory) -// is included somewhere in the package, either directly or indirectly. -func (pkg *Package) Includes(filename string) bool { - return pkg.unconditionalIncludes[filename] != nil || - pkg.conditionalIncludes[filename] != nil -} - -func (pkg *Package) checkPossibleDowngrade() { - if trace.Tracing { - defer trace.Call0()() - } - - m, _, pkgversion := match2(pkg.EffectivePkgname, rePkgname) - if !m { - return - } - - mkline := pkg.EffectivePkgnameLine - - change := G.Pkgsrc.LastChange[pkg.Pkgpath] - if change == nil { - if trace.Tracing { - trace.Step1("No change log for package %q", pkg.Pkgpath) - } - return - } - - if change.Action == Updated { - pkgversionNorev := replaceAll(pkgversion, `nb\d+$`, "") - changeNorev := replaceAll(change.Version(), `nb\d+$`, "") - cmp := pkgver.Compare(pkgversionNorev, changeNorev) - switch { - case cmp < 0: - mkline.Warnf("The package is being downgraded from %s (see %s) to %s.", - change.Version(), mkline.Line.RefToLocation(change.Location), pkgversion) - mkline.Explain( - "The files in doc/CHANGES-*, in which all version changes are", - "recorded, have a higher version number than what the package says.", - "This is unusual, since packages are typically upgraded instead of", - "downgraded.") - - case cmp > 0 && !isLocallyModified(mkline.Filename): - mkline.Notef("Package version %q is greater than the latest %q from %s.", - pkgversion, change.Version(), mkline.Line.RefToLocation(change.Location)) - mkline.Explain( - "Each update to a package should be mentioned in the doc/CHANGES file.", - "That file is used for the quarterly statistics of updated packages.", - "", - "To do this after updating a package, run", - sprintf("%q,", bmake("cce")), - "which is the abbreviation for commit-changes-entry.") - } - } -} - -// checkLinesBuildlink3Inclusion checks whether the package Makefile includes -// at least those buildlink3.mk files that are included by the buildlink3.mk -// file of the package. -// -// The other direction is not checked since it is perfectly fine for a package -// to have more dependencies than are needed for buildlink the package. -// (This might be worth re-checking though.) -func (pkg *Package) checkLinesBuildlink3Inclusion(mklines *MkLines) { - if trace.Tracing { - defer trace.Call0()() - } - - // Collect all the included buildlink3.mk files from the file. - includedFiles := make(map[string]*MkLine) - for _, mkline := range mklines.mklines { - if mkline.IsInclude() { - includedFile := mkline.IncludedFile() - if hasSuffix(includedFile, "/buildlink3.mk") { - includedFiles[includedFile] = mkline - if pkg.bl3[includedFile] == nil { - mkline.Warnf("%s is included by this file but not by the package.", includedFile) - } - } - } - } - - if trace.Tracing { - for packageBl3 := range pkg.bl3 { - if includedFiles[packageBl3] == nil { - trace.Step1("%s is included by the package but not by the buildlink3.mk file.", packageBl3) - } - } - } -} - func (pkg *Package) load() ([]string, *MkLines, *MkLines) { // Load the package Makefile and all included files, // to collect all used and defined variables and similar data. @@ -242,96 +135,6 @@ func (pkg *Package) load() ([]string, *MkLines, *MkLines) { return files, mklines, allLines } -func (pkg *Package) check(filenames []string, mklines, allLines *MkLines) { - haveDistinfo := false - havePatches := false - - for _, filename := range filenames { - if containsVarRef(filename) { - if trace.Tracing { - trace.Stepf("Skipping file %q because the name contains an unresolved variable.", filename) - } - continue - } - - st, err := os.Lstat(filename) - switch { - case err != nil: - // For a missing custom distinfo file, an error message is already generated - // for the line where DISTINFO_FILE is defined. - // - // For all other cases it is next to impossible to reach this branch - // since all those files come from calls to dirglob. - break - - case path.Base(filename) == "Makefile" && strings.Count(G.Pkgsrc.ToRel(filename), "/") == 2: - G.checkExecutable(filename, st.Mode()) - pkg.checkfilePackageMakefile(filename, mklines, allLines) - - default: - pkg.checkDirent(filename, st.Mode()) - } - - if contains(filename, "/patches/patch-") { - havePatches = true - } else if hasSuffix(filename, "/distinfo") { - haveDistinfo = true - } - pkg.checkOwnerMaintainer(filename) - pkg.checkFreeze(filename) - } - - if pkg.Pkgdir == "." { - if havePatches && !haveDistinfo { - line := NewLineWhole(pkg.File(pkg.DistinfoFile)) - line.Warnf("A package with patches should have a distinfo file.") - line.Explain( - "To generate a distinfo file for the existing patches, run", - sprintf("%q.", bmake("makepatchsum"))) - } - } -} - -// checkDirent checks a directory entry based on its filename and its mode -// (regular file, directory, symlink). -func (pkg *Package) checkDirent(dirent string, mode os.FileMode) { - // TODO: merge duplicate code in Pkglint.checkMode - - basename := path.Base(dirent) - - switch { - - case mode.IsRegular(): - pkgsrcRel := G.Pkgsrc.ToRel(dirent) - depth := strings.Count(pkgsrcRel, "/") - G.checkReg(dirent, basename, depth) - - case hasPrefix(basename, "work"): - if G.Opts.Import { - NewLineWhole(dirent).Errorf("Must be cleaned up before committing the package.") - } - return - - case mode.IsDir(): - switch { - case basename == "files", - basename == "patches", - matches(dirent, `(?:^|/)files/[^/]*$`), - isEmptyDir(dirent): - break - - default: - NewLineWhole(dirent).Warnf("Unknown directory name.") - } - - case mode&os.ModeSymlink != 0: - NewLineWhole(dirent).Warnf("Invalid symlink name.") - - default: - NewLineWhole(dirent).Errorf("Only files and directories are allowed in pkgsrc.") - } -} - func (pkg *Package) loadPackageMakefile() (*MkLines, *MkLines) { filename := pkg.File("Makefile") if trace.Tracing { @@ -360,7 +163,7 @@ func (pkg *Package) loadPackageMakefile() (*MkLines, *MkLines) { } // See mk/tools/cmake.mk - if pkg.vars.Defined("USE_CMAKE") { + if pkg.vars.IsDefined("USE_CMAKE") { allLines.Tools.def("cmake", "", false, AtRunTime, nil) allLines.Tools.def("cpack", "", false, AtRunTime, nil) } @@ -373,11 +176,11 @@ func (pkg *Package) loadPackageMakefile() (*MkLines, *MkLines) { pkg.Patchdir = pkg.vars.LastValue("PATCHDIR") // See lang/php/ext.mk - if pkg.vars.DefinedSimilar("PHPEXT_MK") { - if !pkg.vars.DefinedSimilar("USE_PHP_EXT_PATCHES") { + if pkg.vars.IsDefinedSimilar("PHPEXT_MK") { + if !pkg.vars.IsDefinedSimilar("USE_PHP_EXT_PATCHES") { pkg.Patchdir = "patches" } - if pkg.vars.DefinedSimilar("PECL_VERSION") { + if pkg.vars.IsDefinedSimilar("PECL_VERSION") { pkg.DistinfoFile = "distinfo" } else { pkg.IgnoreMissingPatches = true @@ -397,21 +200,6 @@ func (pkg *Package) loadPackageMakefile() (*MkLines, *MkLines) { return mainLines, allLines } -func (pkg *Package) collectConditionalIncludes(mklines *MkLines) { - mklines.ForEach(func(mkline *MkLine) { - if mkline.IsInclude() { - mkline.SetConditionalVars(mklines.indentation.Varnames()) - - key := pkg.Rel(mkline.IncludedFileFull()) - if mklines.indentation.IsConditional() { - pkg.conditionalIncludes[key] = mkline - } else { - pkg.unconditionalIncludes[key] = mkline - } - } - }) -} - // TODO: What is allLines used for, is it still necessary? Would it be better as a field in Package? func (pkg *Package) parse(mklines *MkLines, allLines *MkLines, includingFileForUsedCheck string) bool { if trace.Tracing { @@ -471,7 +259,7 @@ func (pkg *Package) parseLine(mklines *MkLines, mkline *MkLine, allLines *MkLine if mkline.IsVarassign() { varname, op, value := mkline.Varname(), mkline.Op(), mkline.Value() - if op != opAssignDefault || !pkg.vars.Defined(varname) { + if op != opAssignDefault || !pkg.vars.IsDefined(varname) { if trace.Tracing { trace.Stepf("varassign(%q, %q, %q)", varname, op, value) } @@ -500,7 +288,7 @@ func (pkg *Package) loadIncluded(mkline *MkLine, includingFile string) (included fullIncluded := joinPath(dirname, includedFile) relIncludedFile := relpath(pkg.dir, fullIncluded) - if !pkg.diveInto(includingFile, includedFile) { + if !pkg.shouldDiveInto(includingFile, includedFile) { return nil, true } @@ -554,11 +342,38 @@ func (pkg *Package) loadIncluded(mkline *MkLine, includingFile string) (included return includedMklines, false } -// diveInto decides whether to load the includedFile. +// resolveIncludedFile resolves Makefile variables such as ${PKGPATH} to +// their actual values. +func (pkg *Package) resolveIncludedFile(mkline *MkLine, includingFilename string) string { + + // TODO: resolveVariableRefs uses G.Pkg implicitly. It should be made explicit. + // TODO: Try to combine resolveVariableRefs and ResolveVarsInRelativePath. + includedFile := resolveVariableRefs(nil /* XXX: or maybe some mklines? */, mkline.ResolveVarsInRelativePath(mkline.IncludedFile())) + if containsVarRef(includedFile) { + if trace.Tracing && !contains(includingFilename, "/mk/") { + trace.Stepf("%s:%s: Skipping unresolvable include file %q.", + mkline.Filename, mkline.Linenos(), includedFile) + } + return "" + } + + if mkline.Basename != "buildlink3.mk" { + if hasSuffix(includedFile, "/buildlink3.mk") { + pkg.bl3[includedFile] = mkline + if trace.Tracing { + trace.Step1("Buildlink3 file in package: %q", includedFile) + } + } + } + + return includedFile +} + +// shouldDiveInto decides whether to load the includedFile. // // The includingFile is relative to the current working directory, // the includedFile is taken directly from the .include directive. -func (*Package) diveInto(includingFile string, includedFile string) bool { +func (*Package) shouldDiveInto(includingFile string, includedFile string) bool { if hasSuffix(includedFile, "/bsd.pkg.mk") || IsPrefs(includedFile) { return false @@ -592,31 +407,88 @@ func (pkg *Package) collectSeenInclude(mkline *MkLine, includedFile string) { pkg.seenInclude = true } -// resolveIncludedFile resolves Makefile variables such as ${PKGPATH} to -// their actual values. -func (pkg *Package) resolveIncludedFile(mkline *MkLine, includingFilename string) string { +func (pkg *Package) collectConditionalIncludes(mklines *MkLines) { + mklines.ForEach(func(mkline *MkLine) { + if mkline.IsInclude() { + mkline.SetConditionalVars(mklines.indentation.Varnames()) - // TODO: resolveVariableRefs uses G.Pkg implicitly. It should be made explicit. - // TODO: Try to combine resolveVariableRefs and ResolveVarsInRelativePath. - includedFile := resolveVariableRefs(nil /* XXX: or maybe some mklines? */, mkline.ResolveVarsInRelativePath(mkline.IncludedFile())) - if containsVarRef(includedFile) { - if trace.Tracing && !contains(includingFilename, "/mk/") { - trace.Stepf("%s:%s: Skipping unresolvable include file %q.", - mkline.Filename, mkline.Linenos(), includedFile) + key := pkg.Rel(mkline.IncludedFileFull()) + if mklines.indentation.IsConditional() { + pkg.conditionalIncludes[key] = mkline + } else { + pkg.unconditionalIncludes[key] = mkline + } } - return "" + }) +} + +func (pkg *Package) loadPlistDirs(plistFilename string) { + lines := Load(plistFilename, MustSucceed) + ck := PlistChecker{ + pkg, + make(map[string]*PlistLine), + make(map[string]*PlistLine), + "", + Once{}, + false} + ck.Load(lines) + + for filename, pline := range ck.allFiles { + pkg.Plist.Files[filename] = pline + } + for dirname, pline := range ck.allDirs { + pkg.Plist.Dirs[dirname] = pline } +} - if mkline.Basename != "buildlink3.mk" { - if hasSuffix(includedFile, "/buildlink3.mk") { - pkg.bl3[includedFile] = mkline +func (pkg *Package) check(filenames []string, mklines, allLines *MkLines) { + haveDistinfo := false + havePatches := false + + for _, filename := range filenames { + if containsVarRef(filename) { if trace.Tracing { - trace.Step1("Buildlink3 file in package: %q", includedFile) + trace.Stepf("Skipping file %q because the name contains an unresolved variable.", filename) } + continue + } + + st, err := os.Lstat(filename) + switch { + case err != nil: + // For a missing custom distinfo file, an error message is already generated + // for the line where DISTINFO_FILE is defined. + // + // For all other cases it is next to impossible to reach this branch + // since all those files come from calls to dirglob. + break + + case path.Base(filename) == "Makefile" && strings.Count(G.Pkgsrc.ToRel(filename), "/") == 2: + G.checkExecutable(filename, st.Mode()) + pkg.checkfilePackageMakefile(filename, mklines, allLines) + + default: + pkg.checkDirent(filename, st.Mode()) + } + + if contains(filename, "/patches/patch-") { + havePatches = true + } else if hasSuffix(filename, "/distinfo") { + haveDistinfo = true } + pkg.checkOwnerMaintainer(filename) + pkg.checkFreeze(filename) } - return includedFile + if pkg.Pkgdir == "." { + if havePatches && !haveDistinfo { + line := NewLineWhole(pkg.File(pkg.DistinfoFile)) + line.Warnf("A package with patches should have a distinfo file.") + line.Explain( + "To generate a distinfo file for the existing patches, run", + sprintf("%q.", bmake("makepatchsum"))) + } + } } func (pkg *Package) checkfilePackageMakefile(filename string, mklines *MkLines, allLines *MkLines) { @@ -627,9 +499,9 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines *MkLines, vars := pkg.vars pkg.checkPlist() - want := !vars.Defined("NO_CHECKSUM") - want = want && !vars.Defined("META_PACKAGE") - want = want && !(vars.Defined("DISTFILES") && vars.LastValue("DISTFILES") == "") + want := !vars.IsDefined("NO_CHECKSUM") + want = want && !vars.IsDefined("META_PACKAGE") + want = want && !(vars.IsDefined("DISTFILES") && vars.LastValue("DISTFILES") == "") want = want || !isEmptyDir(pkg.File(pkg.Patchdir)) if !want { @@ -658,7 +530,7 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines *MkLines, } } - if !vars.Defined("LICENSE") && !vars.Defined("META_PACKAGE") { + if !vars.IsDefined("LICENSE") && !vars.IsDefined("META_PACKAGE") { line := NewLineWhole(filename) line.Errorf("Each package must define its LICENSE.") // TODO: Explain why the LICENSE is necessary. @@ -669,13 +541,14 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines *MkLines, pkg.redundant = NewRedundantScope() pkg.redundant.Check(allLines) // Updates the variables in the scope + pkg.checkCategories() pkg.checkGnuConfigureUseLanguages() pkg.checkUseLanguagesCompilerMk(allLines) pkg.determineEffectivePkgVars() pkg.checkPossibleDowngrade() - if !vars.Defined("COMMENT") { + if !vars.IsDefined("COMMENT") { NewLineWhole(filename).Warnf("Each package should define a COMMENT.") } @@ -714,7 +587,7 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines *MkLines, // or whether that file should be omitted since it is autogenerated. func (pkg *Package) checkPlist() { vars := pkg.vars - if vars.Defined("PLIST_SRC") || vars.Defined("GENERATE_PLIST") { + if vars.IsDefined("PLIST_SRC") || vars.IsDefined("GENERATE_PLIST") { return } @@ -744,210 +617,22 @@ func (pkg *Package) needsPlist() (bool, *Line) { // TODO: In the below code, it shouldn't be necessary to mention // each variable name twice. - if vars.Defined("PERL5_PACKLIST") { + if vars.IsDefined("PERL5_PACKLIST") { return false, vars.LastDefinition("PERL5_PACKLIST").Line } - if vars.Defined("PERL5_USE_PACKLIST") { + if vars.IsDefined("PERL5_USE_PACKLIST") { needed := strings.ToLower(vars.LastValue("PERL5_USE_PACKLIST")) == "no" return needed, vars.LastDefinition("PERL5_USE_PACKLIST").Line } - if vars.Defined("META_PACKAGE") { + if vars.IsDefined("META_PACKAGE") { return false, vars.LastDefinition("META_PACKAGE").Line } return true, NewLineWhole(pkg.File("Makefile")) } -func (pkg *Package) checkGnuConfigureUseLanguages() { - s := pkg.redundant - - gnuConfigure := s.vars["GNU_CONFIGURE"] - if gnuConfigure == nil || !gnuConfigure.vari.Constant() { - return - } - - useLanguages := s.vars["USE_LANGUAGES"] - if useLanguages == nil || !useLanguages.vari.Constant() { - return - } - - var wrongLines []*MkLine - for _, mkline := range useLanguages.vari.WriteLocations() { - - if G.Pkgsrc.IsInfra(mkline.Line.Filename) { - continue - } - - if matches(mkline.Comment(), `(?-i)\b(?:c|empty|none)\b`) { - // Don't emit a warning since the comment probably contains a - // statement that C is really not needed. - return - } - - languages := mkline.Value() - if matches(languages, `(?:^|[\t ]+)(?:c|c99|objc)(?:[\t ]+|$)`) { - return - } - - wrongLines = append(wrongLines, mkline) - } - - gnuLine := gnuConfigure.vari.WriteLocations()[0] - for _, useLine := range wrongLines { - gnuLine.Warnf( - "GNU_CONFIGURE almost always needs a C compiler, "+ - "but \"c\" is not added to USE_LANGUAGES in %s.", - gnuLine.RefTo(useLine)) - } -} - -// nbPart determines the smallest part of the package version number, -// typically "nb13" or an empty string. -// -// It is only used inside pkgsrc to mark changes that are -// independent from the upstream package. -func (pkg *Package) nbPart() string { - pkgrevision := pkg.vars.LastValue("PKGREVISION") - if rev, err := strconv.Atoi(pkgrevision); err == nil { - return "nb" + strconv.Itoa(rev) - } - return "" -} - -func (pkg *Package) determineEffectivePkgVars() { - distnameLine := pkg.vars.FirstDefinition("DISTNAME") - pkgnameLine := pkg.vars.FirstDefinition("PKGNAME") - - distname := "" - if distnameLine != nil { - distname = distnameLine.Value() - } - - pkgname := "" - if pkgnameLine != nil { - pkgname = pkgnameLine.Value() - } - - effname := pkgname - if distname != "" && effname != "" { - merged, ok := pkg.pkgnameFromDistname(effname, distname) - if ok { - effname = merged - } - } - - if pkgnameLine != nil && (pkgname == distname || pkgname == "${DISTNAME}") { - if !pkgnameLine.HasComment() { - pkgnameLine.Notef("This assignment is probably redundant " + - "since PKGNAME is ${DISTNAME} by default.") - pkgnameLine.Explain( - "To mark this assignment as necessary, add a comment to the end of this line.") - } - } - - if pkgname == "" && distnameLine != nil && !containsVarRef(distname) && !matches(distname, rePkgname) { - distnameLine.Warnf("As DISTNAME is not a valid package name, please define the PKGNAME explicitly.") - } - - if pkgname != "" { - distname = "" - } - - if effname != "" && !containsVarRef(effname) { - if m, m1, m2 := match2(effname, rePkgname); m { - pkg.EffectivePkgname = effname + pkg.nbPart() - pkg.EffectivePkgnameLine = pkgnameLine - pkg.EffectivePkgbase = m1 - pkg.EffectivePkgversion = m2 - } - } - - if pkg.EffectivePkgnameLine == nil && distname != "" && !containsVarRef(distname) { - if m, m1, m2 := match2(distname, rePkgname); m { - pkg.EffectivePkgname = distname + pkg.nbPart() - pkg.EffectivePkgnameLine = distnameLine - pkg.EffectivePkgbase = m1 - pkg.EffectivePkgversion = m2 - } - } - - if pkg.EffectivePkgnameLine != nil { - if trace.Tracing { - trace.Stepf("Effective name=%q base=%q version=%q", - pkg.EffectivePkgname, pkg.EffectivePkgbase, pkg.EffectivePkgversion) - } - } -} - -func (pkg *Package) pkgnameFromDistname(pkgname, distname string) (string, bool) { - tokens := NewMkParser(nil, pkgname).MkTokens() - - // TODO: Make this resolving of variable references available to all other variables as well. - - var result strings.Builder - for _, token := range tokens { - if token.Varuse != nil { - if token.Varuse.varname != "DISTNAME" { - return "", false - } - - newDistname := distname - for _, mod := range token.Varuse.modifiers { - if mod.IsToLower() { - newDistname = strings.ToLower(newDistname) - } else if subst, ok := mod.Subst(newDistname); ok { - newDistname = subst - } else { - return "", false - } - } - result.WriteString(newDistname) - } else { - result.WriteString(token.Text) - } - } - return result.String(), true -} - -func (pkg *Package) checkUpdate() { - if pkg.EffectivePkgbase == "" { - return - } - - for _, sugg := range G.Pkgsrc.SuggestedUpdates() { - if pkg.EffectivePkgbase != sugg.Pkgname { - continue - } - - suggver, comment := sugg.Version, sugg.Comment - if comment != "" { - comment = " (" + comment + ")" - } - - pkgnameLine := pkg.EffectivePkgnameLine - cmp := pkgver.Compare(pkg.EffectivePkgversion, suggver) - switch { - - case cmp < 0: - pkgnameLine.Warnf("This package should be updated to %s%s.", - sugg.Version, comment) - pkgnameLine.Explain( - "The wishlist for package updates in doc/TODO mentions that a newer", - "version of this package is available.") - - case cmp > 0: - pkgnameLine.Notef("This package is newer than the update request to %s%s.", - suggver, comment) - - default: - pkgnameLine.Notef("The update request to %s from doc/TODO%s has been done.", - suggver, comment) - } - } -} - // CheckVarorder checks that in simple package Makefiles, // the most common variables appear in a fixed order. // The order itself is a little arbitrary but provides @@ -1181,26 +866,351 @@ func (pkg *Package) CheckVarorder(mklines *MkLines) { seeGuide("Package components, Makefile", "components.Makefile")) } -func (pkg *Package) checkFileMakefileExt(filename string) { - base := path.Base(filename) - if !hasPrefix(base, "Makefile.") || base == "Makefile.common" { +func (pkg *Package) checkCategories() { + categories := pkg.redundant.vars["CATEGORIES"] + if categories == nil || !categories.vari.IsConstant() { return } - ext := strings.TrimPrefix(base, "Makefile.") - line := NewLineWhole(filename) - line.Notef("Consider renaming %q to %q.", base, ext+".mk") - line.Explain( - "The main definition of a pkgsrc package should be in the Makefile.", - "Common definitions for a few very closely related packages can be", - "placed in a Makefile.common, these may cover various topics.", - "", - "All other definitions should be grouped by topics and implemented", - "in separate files named *.mk after their topics. Typical examples", - "are extension.mk, module.mk, version.mk.", - "", - "These topic files should be documented properly so that their", - sprintf("content can be queried using %q.", bmakeHelp("help"))) + seen := map[string]*MkLine{} + for _, mkline := range categories.vari.WriteLocations() { + switch mkline.Op() { + case opAssignDefault: + for _, category := range mkline.ValueFields(mkline.Value()) { + if seen[category] == nil { + seen[category] = mkline + } + } + case opAssign, opAssignAppend: + for _, category := range mkline.ValueFields(mkline.Value()) { + if seen[category] != nil { + mkline.Notef("Category %q is already added in %s.", + category, mkline.RefTo(seen[category])) + } + if seen[category] == nil { + seen[category] = mkline + } + } + } + } +} + +func (pkg *Package) checkGnuConfigureUseLanguages() { + s := pkg.redundant + + gnuConfigure := s.vars["GNU_CONFIGURE"] + if gnuConfigure == nil || !gnuConfigure.vari.IsConstant() { + return + } + + useLanguages := s.vars["USE_LANGUAGES"] + if useLanguages == nil || !useLanguages.vari.IsConstant() { + return + } + + var wrongLines []*MkLine + for _, mkline := range useLanguages.vari.WriteLocations() { + + if G.Pkgsrc.IsInfra(mkline.Line.Filename) { + continue + } + + if matches(mkline.Comment(), `(?-i)\b(?:c|empty|none)\b`) { + // Don't emit a warning since the comment probably contains a + // statement that C is really not needed. + return + } + + languages := mkline.Value() + if matches(languages, `(?:^|[\t ]+)(?:c|c99|objc)(?:[\t ]+|$)`) { + return + } + + wrongLines = append(wrongLines, mkline) + } + + gnuLine := gnuConfigure.vari.WriteLocations()[0] + for _, useLine := range wrongLines { + gnuLine.Warnf( + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in %s.", + gnuLine.RefTo(useLine)) + } +} + +// checkUseLanguagesCompilerMk checks that after including mk/compiler.mk +// or mk/endian.mk for the first time, there are no more changes to +// USE_LANGUAGES, as these would be ignored by the pkgsrc infrastructure. +func (pkg *Package) checkUseLanguagesCompilerMk(mklines *MkLines) { + + var seen Once + + handleVarassign := func(mkline *MkLine) { + if mkline.Varname() != "USE_LANGUAGES" { + return + } + + if !seen.Seen("../../mk/compiler.mk") && !seen.Seen("../../mk/endian.mk") { + return + } + + if mkline.Basename == "compiler.mk" { + if relpath(pkg.dir, mkline.Filename) == "../../mk/compiler.mk" { + return + } + } + + mkline.Warnf("Modifying USE_LANGUAGES after including ../../mk/compiler.mk has no effect.") + mkline.Explain( + "The file compiler.mk guards itself against multiple inclusion.") + } + + handleInclude := func(mkline *MkLine) { + _ = seen.FirstTime(pkg.Rel(mkline.IncludedFileFull())) + } + + mklines.ForEach(func(mkline *MkLine) { + switch { + case mkline.IsVarassign(): + handleVarassign(mkline) + + case mkline.IsInclude(): + handleInclude(mkline) + } + }) +} + +func (pkg *Package) determineEffectivePkgVars() { + distnameLine := pkg.vars.FirstDefinition("DISTNAME") + pkgnameLine := pkg.vars.FirstDefinition("PKGNAME") + + distname := "" + if distnameLine != nil { + distname = distnameLine.Value() + } + + pkgname := "" + if pkgnameLine != nil { + pkgname = pkgnameLine.Value() + } + + effname := pkgname + if distname != "" && effname != "" { + merged, ok := pkg.pkgnameFromDistname(effname, distname) + if ok { + effname = merged + } + } + + if pkgnameLine != nil && (pkgname == distname || pkgname == "${DISTNAME}") { + if !pkgnameLine.HasComment() { + pkgnameLine.Notef("This assignment is probably redundant " + + "since PKGNAME is ${DISTNAME} by default.") + pkgnameLine.Explain( + "To mark this assignment as necessary, add a comment to the end of this line.") + } + } + + if pkgname == "" && distnameLine != nil && !containsVarRef(distname) && !matches(distname, rePkgname) { + distnameLine.Warnf("As DISTNAME is not a valid package name, please define the PKGNAME explicitly.") + } + + if pkgname != "" { + distname = "" + } + + if effname != "" && !containsVarRef(effname) { + if m, m1, m2 := match2(effname, rePkgname); m { + pkg.EffectivePkgname = effname + pkg.nbPart() + pkg.EffectivePkgnameLine = pkgnameLine + pkg.EffectivePkgbase = m1 + pkg.EffectivePkgversion = m2 + } + } + + if pkg.EffectivePkgnameLine == nil && distname != "" && !containsVarRef(distname) { + if m, m1, m2 := match2(distname, rePkgname); m { + pkg.EffectivePkgname = distname + pkg.nbPart() + pkg.EffectivePkgnameLine = distnameLine + pkg.EffectivePkgbase = m1 + pkg.EffectivePkgversion = m2 + } + } + + if pkg.EffectivePkgnameLine != nil { + if trace.Tracing { + trace.Stepf("Effective name=%q base=%q version=%q", + pkg.EffectivePkgname, pkg.EffectivePkgbase, pkg.EffectivePkgversion) + } + } +} + +// nbPart determines the smallest part of the package version number, +// typically "nb13" or an empty string. +// +// It is only used inside pkgsrc to mark changes that are +// independent from the upstream package. +func (pkg *Package) nbPart() string { + pkgrevision := pkg.vars.LastValue("PKGREVISION") + if rev, err := strconv.Atoi(pkgrevision); err == nil { + return "nb" + strconv.Itoa(rev) + } + return "" +} + +func (pkg *Package) pkgnameFromDistname(pkgname, distname string) (string, bool) { + tokens := NewMkParser(nil, pkgname).MkTokens() + + // TODO: Make this resolving of variable references available to all other variables as well. + + var result strings.Builder + for _, token := range tokens { + if token.Varuse != nil { + if token.Varuse.varname != "DISTNAME" { + return "", false + } + + newDistname := distname + for _, mod := range token.Varuse.modifiers { + if mod.IsToLower() { + newDistname = strings.ToLower(newDistname) + } else if subst, ok := mod.Subst(newDistname); ok { + newDistname = subst + } else { + return "", false + } + } + result.WriteString(newDistname) + } else { + result.WriteString(token.Text) + } + } + return result.String(), true +} + +func (pkg *Package) checkPossibleDowngrade() { + if trace.Tracing { + defer trace.Call0()() + } + + m, _, pkgversion := match2(pkg.EffectivePkgname, rePkgname) + if !m { + return + } + + mkline := pkg.EffectivePkgnameLine + + change := G.Pkgsrc.LastChange[pkg.Pkgpath] + if change == nil { + if trace.Tracing { + trace.Step1("No change log for package %q", pkg.Pkgpath) + } + return + } + + if change.Action == Updated { + pkgversionNorev := replaceAll(pkgversion, `nb\d+$`, "") + changeNorev := replaceAll(change.Version(), `nb\d+$`, "") + cmp := pkgver.Compare(pkgversionNorev, changeNorev) + switch { + case cmp < 0: + mkline.Warnf("The package is being downgraded from %s (see %s) to %s.", + change.Version(), mkline.Line.RefToLocation(change.Location), pkgversion) + mkline.Explain( + "The files in doc/CHANGES-*, in which all version changes are", + "recorded, have a higher version number than what the package says.", + "This is unusual, since packages are typically upgraded instead of", + "downgraded.") + + case cmp > 0 && !isLocallyModified(mkline.Filename): + mkline.Notef("Package version %q is greater than the latest %q from %s.", + pkgversion, change.Version(), mkline.Line.RefToLocation(change.Location)) + mkline.Explain( + "Each update to a package should be mentioned in the doc/CHANGES file.", + "That file is used for the quarterly statistics of updated packages.", + "", + "To do this after updating a package, run", + sprintf("%q,", bmake("cce")), + "which is the abbreviation for commit-changes-entry.") + } + } +} + +func (pkg *Package) checkUpdate() { + if pkg.EffectivePkgbase == "" { + return + } + + for _, sugg := range G.Pkgsrc.SuggestedUpdates() { + if pkg.EffectivePkgbase != sugg.Pkgname { + continue + } + + suggver, comment := sugg.Version, sugg.Comment + if comment != "" { + comment = " (" + comment + ")" + } + + pkgnameLine := pkg.EffectivePkgnameLine + cmp := pkgver.Compare(pkg.EffectivePkgversion, suggver) + switch { + + case cmp < 0: + pkgnameLine.Warnf("This package should be updated to %s%s.", + sugg.Version, comment) + pkgnameLine.Explain( + "The wishlist for package updates in doc/TODO mentions that a newer", + "version of this package is available.") + + case cmp > 0: + pkgnameLine.Notef("This package is newer than the update request to %s%s.", + suggver, comment) + + default: + pkgnameLine.Notef("The update request to %s from doc/TODO%s has been done.", + suggver, comment) + } + } +} + +// checkDirent checks a directory entry based on its filename and its mode +// (regular file, directory, symlink). +func (pkg *Package) checkDirent(dirent string, mode os.FileMode) { + // TODO: merge duplicate code in Pkglint.checkMode + + basename := path.Base(dirent) + + switch { + + case mode.IsRegular(): + pkgsrcRel := G.Pkgsrc.ToRel(dirent) + depth := strings.Count(pkgsrcRel, "/") + G.checkReg(dirent, basename, depth) + + case hasPrefix(basename, "work"): + if G.Opts.Import { + NewLineWhole(dirent).Errorf("Must be cleaned up before committing the package.") + } + return + + case mode.IsDir(): + switch { + case basename == "files", + basename == "patches", + matches(dirent, `(?:^|/)files/[^/]*$`), + isEmptyDir(dirent): + break + + default: + NewLineWhole(dirent).Warnf("Unknown directory name.") + } + + case mode&os.ModeSymlink != 0: + NewLineWhole(dirent).Warnf("Invalid symlink name.") + + default: + NewLineWhole(dirent).Errorf("Only files and directories are allowed in pkgsrc.") + } } // checkOwnerMaintainer checks files that are about to be committed. @@ -1268,6 +1278,63 @@ func (pkg *Package) checkFreeze(filename string) { "See https://www.NetBSD.org/developers/pkgsrc/ for the exact rules.") } +func (pkg *Package) checkFileMakefileExt(filename string) { + base := path.Base(filename) + if !hasPrefix(base, "Makefile.") || base == "Makefile.common" { + return + } + ext := strings.TrimPrefix(base, "Makefile.") + + line := NewLineWhole(filename) + line.Notef("Consider renaming %q to %q.", base, ext+".mk") + line.Explain( + "The main definition of a pkgsrc package should be in the Makefile.", + "Common definitions for a few very closely related packages can be", + "placed in a Makefile.common, these may cover various topics.", + "", + "All other definitions should be grouped by topics and implemented", + "in separate files named *.mk after their topics. Typical examples", + "are extension.mk, module.mk, version.mk.", + "", + "These topic files should be documented properly so that their", + sprintf("content can be queried using %q.", bmakeHelp("help"))) +} + +// checkLinesBuildlink3Inclusion checks whether the package Makefile includes +// at least those buildlink3.mk files that are included by the buildlink3.mk +// file of the package. +// +// The other direction is not checked since it is perfectly fine for a package +// to have more dependencies than are needed for buildlink the package. +// (This might be worth re-checking though.) +func (pkg *Package) checkLinesBuildlink3Inclusion(mklines *MkLines) { + if trace.Tracing { + defer trace.Call0()() + } + + // Collect all the included buildlink3.mk files from the file. + includedFiles := make(map[string]*MkLine) + for _, mkline := range mklines.mklines { + if mkline.IsInclude() { + includedFile := mkline.IncludedFile() + if hasSuffix(includedFile, "/buildlink3.mk") { + includedFiles[includedFile] = mkline + if pkg.bl3[includedFile] == nil { + mkline.Warnf("%s is included by this file but not by the package.", includedFile) + } + } + } + } + + if trace.Tracing { + for packageBl3 := range pkg.bl3 { + if includedFiles[packageBl3] == nil { + trace.Step1("%s is included by the package but not by the buildlink3.mk file.", packageBl3) + } + } + } +} + func (pkg *Package) checkIncludeConditionally(mkline *MkLine, indentation *Indentation) { if IsPrefs(mkline.IncludedFile()) { return @@ -1322,25 +1389,6 @@ func (pkg *Package) checkIncludeConditionally(mkline *MkLine, indentation *Inden // already done with *_MK variables. } -func (pkg *Package) loadPlistDirs(plistFilename string) { - lines := Load(plistFilename, MustSucceed) - ck := PlistChecker{ - pkg, - make(map[string]*PlistLine), - make(map[string]*PlistLine), - "", - Once{}, - false} - ck.Load(lines) - - for filename, pline := range ck.allFiles { - pkg.Plist.Files[filename] = pline - } - for dirname, pline := range ck.allDirs { - pkg.Plist.Dirs[dirname] = pline - } -} - func (pkg *Package) AutofixDistinfo(oldSha1, newSha1 string) { distinfoFilename := pkg.File(pkg.DistinfoFile) if lines := Load(distinfoFilename, NotEmpty|LogErrors); lines != nil { @@ -1354,46 +1402,28 @@ func (pkg *Package) AutofixDistinfo(oldSha1, newSha1 string) { } } -// checkUseLanguagesCompilerMk checks that after including mk/compiler.mk -// or mk/endian.mk for the first time, there are no more changes to -// USE_LANGUAGES, as these would be ignored by the pkgsrc infrastructure. -func (pkg *Package) checkUseLanguagesCompilerMk(mklines *MkLines) { - - var seen Once - - handleVarassign := func(mkline *MkLine) { - if mkline.Varname() != "USE_LANGUAGES" { - return - } - - if !seen.Seen("../../mk/compiler.mk") && !seen.Seen("../../mk/endian.mk") { - return - } - - if mkline.Basename == "compiler.mk" { - if relpath(pkg.dir, mkline.Filename) == "../../mk/compiler.mk" { - return - } - } - - mkline.Warnf("Modifying USE_LANGUAGES after including ../../mk/compiler.mk has no effect.") - mkline.Explain( - "The file compiler.mk guards itself against multiple inclusion.") - } - - handleInclude := func(mkline *MkLine) { - _ = seen.FirstTime(pkg.Rel(mkline.IncludedFileFull())) - } +// File returns the (possibly absolute) path to relativeFileName, +// as resolved from the package's directory. +// Variables that are known in the package are resolved, e.g. ${PKGDIR}. +func (pkg *Package) File(relativeFileName string) string { + return cleanpath(resolveVariableRefs(nil /* XXX: or maybe some mklines? */, joinPath(pkg.dir, relativeFileName))) +} - mklines.ForEach(func(mkline *MkLine) { - switch { - case mkline.IsVarassign(): - handleVarassign(mkline) +// Rel returns the path by which the given filename (as seen from the +// current working directory) can be reached as a relative path from +// the package directory. +// +// Example: +// NewPackage("category/package").Rel("other/package") == "../../other/package" +func (pkg *Package) Rel(filename string) string { + return relpath(pkg.dir, filename) +} - case mkline.IsInclude(): - handleInclude(mkline) - } - }) +// Returns whether the given file (relative to the package directory) +// is included somewhere in the package, either directly or indirectly. +func (pkg *Package) Includes(filename string) bool { + return pkg.unconditionalIncludes[filename] != nil || + pkg.conditionalIncludes[filename] != nil } // PlistContent lists the directories and files that appear in the diff --git a/pkgtools/pkglint/files/package_test.go b/pkgtools/pkglint/files/package_test.go index abb70719ea0..5285b635d9b 100644 --- a/pkgtools/pkglint/files/package_test.go +++ b/pkgtools/pkglint/files/package_test.go @@ -7,981 +7,329 @@ import ( "strings" ) -func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__file_but_not_package(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("category/dependency/buildlink3.mk") - t.CreateFileLines("category/dependency/module.mk") - G.Pkg = NewPackage(t.File("category/package")) - mklines := t.NewMkLines("category/package/buildlink3.mk", - MkCvsID, - "", - ".include \"../../category/dependency/buildlink3.mk\"", - ".include \"../../category/dependency/module.mk\"") - - G.Pkg.checkLinesBuildlink3Inclusion(mklines) - - t.CheckOutputLines( - "WARN: category/package/buildlink3.mk:3: " + - "../../category/dependency/buildlink3.mk is included by this file " + - "but not by the package.") -} - -// Several files from the pkgsrc infrastructure are named *.buildlink3.mk, -// even though they don't follow the typical file format for buildlink3.mk -// files. Therefore they are ignored by this check. -func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__infra_buildlink_file(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package", - ".include \"../../mk/motif.buildlink3.mk\"") - t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", - ".include \"../../mk/motif.buildlink3.mk\"") - t.CreateFileLines("mk/motif.buildlink3.mk", - MkCvsID) - - t.Main("--quiet", "-Wall", "category/package") - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__package_but_not_file(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("category/dependency/buildlink3.mk") - G.Pkg = NewPackage(t.File("category/package")) - G.Pkg.bl3["../../category/dependency/buildlink3.mk"] = - t.NewMkLine("../../category/dependency/buildlink3.mk", 1, "") - mklines := t.NewMkLines("category/package/buildlink3.mk", - MkCvsID) - - t.EnableTracingToLog() - G.Pkg.checkLinesBuildlink3Inclusion(mklines) - - // This is only traced but not logged as a regular warning since - // several packages have build dependencies that are not needed - // for building other packages. These cannot be flagged as warnings. - t.CheckOutputLines( - "TRACE: + (*Package).checkLinesBuildlink3Inclusion()", - "TRACE: 1 ../../category/dependency/buildlink3.mk "+ - "is included by the package but not by the buildlink3.mk file.", - "TRACE: - (*Package).checkLinesBuildlink3Inclusion()") -} - -// Just for code coverage. -func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__no_tracing(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package") - t.CreateFileDummyBuildlink3("category/package/buildlink3.mk") - t.FinishSetUp() - - t.DisableTracing() - G.Check(t.File("category/package")) - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Package_pkgnameFromDistname(c *check.C) { +func (s *Suite) Test_Package__varuse_at_load_time(c *check.C) { t := s.Init(c) - var once Once - test := func(pkgname, distname, expectedPkgname string, diagnostics ...string) { - t.SetUpPackage("category/package", - "PKGNAME=\t"+pkgname, - "DISTNAME=\t"+distname) - if once.FirstTime("called") { - t.FinishSetUp() - } - - pkg := NewPackage(t.File("category/package")) - pkg.loadPackageMakefile() - pkg.determineEffectivePkgVars() - t.CheckEquals(pkg.EffectivePkgname, expectedPkgname) - t.CheckOutput(diagnostics) - } - - test("pkgname-1.0", "whatever", "pkgname-1.0") - - test("${DISTNAME}", "distname-1.0", "distname-1.0", - "NOTE: ~/category/package/Makefile:4: This assignment is probably redundant since PKGNAME is ${DISTNAME} by default.") - - test("${DISTNAME:S/dist/pkg/}", "distname-1.0", "pkgname-1.0") - - test("${DISTNAME:S|a|b|g}", "panama-0.13", "pbnbmb-0.13") - - // The substitution succeeds, but the substituted value is missing - // the package version. Therefore it is discarded completely. - test("${DISTNAME:S|^lib||}", "libncurses", "") - - // The substitution succeeds, but the substituted value is missing - // the package version. Therefore it is discarded completely. - test("${DISTNAME:S|^lib||}", "mylib", "") - - test("${DISTNAME:tl:S/-/./g:S/he/-/1}", "SaxonHE9-5-0-1J", "saxon-9.5.0.1j") - - test("${DISTNAME:C/beta/.0./}", "fspanel-0.8beta1", "fspanel-0.8.0.1") - - test("${DISTNAME:C/Gtk2/p5-gtk2/}", "Gtk2-1.0", "p5-gtk2-1.0") - - test("${DISTNAME:S/-0$/.0/1}", "aspell-af-0.50-0", "aspell-af-0.50.0") - - test("${DISTNAME:M*.tar.gz:C,\\..*,,}", "aspell-af-0.50-0", "") - - test("${DISTNAME:S,a,b,c,d}", "aspell-af-0.50-0", "bspell-af-0.50-0", - "WARN: ~/category/package/Makefile:4: Invalid variable modifier \"c,d\" for \"DISTNAME\".") - - test("${DISTFILE:C,\\..*,,}", "aspell-af-0.50-0", "") -} - -func (s *Suite) Test_Package_CheckVarorder__only_required_variables(c *check.C) { - t := s.Init(c) + t.SetUpPkgsrc() + t.SetUpTool("printf", "", AtRunTime) + t.CreateFileLines("licenses/2-clause-bsd", + "# dummy") + t.CreateFileLines("category/Makefile") + t.CreateFileLines("mk/tools/defaults.mk", + "TOOLS_CREATE+=false", + "TOOLS_CREATE+=nice", + "TOOLS_CREATE+=true", + "_TOOLS_VARNAME.nice=NICE") - pkg := NewPackage(t.File("x11/9term")) - mklines := t.NewMkLines("Makefile", + t.CreateFileLines("category/pkgbase/Makefile", MkCvsID, "", - "DISTNAME=9term", - "CATEGORIES=x11", + "PKGNAME= loadtime-vartest-1.0", + "CATEGORIES= category", "", - ".include \"../../mk/bsd.pkg.mk\"") - - pkg.CheckVarorder(mklines) - - t.CheckOutputLines( - "WARN: Makefile:3: The canonical order of the variables is " + - "DISTNAME, CATEGORIES, empty line, COMMENT, LICENSE.") -} - -func (s *Suite) Test_Package_CheckVarorder__with_optional_variables(c *check.C) { - t := s.Init(c) - - pkg := NewPackage(t.File("x11/9term")) - mklines := t.NewMkLines("Makefile", - MkCvsID, + "COMMENT= Demonstrate variable values during parsing", + "LICENSE= 2-clause-bsd", "", - "GITHUB_PROJECT=project", - "DISTNAME=9term", - "CATEGORIES=x11") - - pkg.CheckVarorder(mklines) - - // TODO: Make this warning more specific to the actual situation. - t.CheckOutputLines( - "WARN: Makefile:3: The canonical order of the variables is " + - "GITHUB_PROJECT, DISTNAME, CATEGORIES, GITHUB_PROJECT, empty line, " + - "COMMENT, LICENSE.") -} - -// Just for code coverage. -func (s *Suite) Test_Package_CheckVarorder__no_tracing(c *check.C) { - t := s.Init(c) - - pkg := NewPackage(t.File("x11/9term")) - mklines := t.NewMkLines("Makefile", - MkCvsID, + "PLIST_SRC= # none", + "NO_CHECKSUM= yes", + "NO_CONFIGURE= yes", "", - "DISTNAME=9term", - "CATEGORIES=x11", + "USE_TOOLS+= echo false", + "FALSE_BEFORE!= echo false=${FALSE:Q}", // false= + "NICE_BEFORE!= echo nice=${NICE:Q}", // nice= + "TRUE_BEFORE!= echo true=${TRUE:Q}", // true= + // + // All three variables above are empty since the tool + // variables are initialized by bsd.prefs.mk. The variables + // from share/mk/sys.mk are available, though. + // "", - ".include \"../../mk/bsd.pkg.mk\"") - t.DisableTracing() - - pkg.CheckVarorder(mklines) - - t.CheckOutputLines( - "WARN: Makefile:3: The canonical order of the variables is " + - "DISTNAME, CATEGORIES, empty line, COMMENT, LICENSE.") -} - -// Ensure that comments and empty lines do not lead to panics. -// This would be when accessing fields from the MkLine without checking the line type before. -func (s *Suite) Test_Package_CheckVarorder__comments_do_not_crash(c *check.C) { - t := s.Init(c) - - pkg := NewPackage(t.File("x11/9term")) - mklines := t.NewMkLines("Makefile", - MkCvsID, + ".include \"../../mk/bsd.prefs.mk\"", + // + // At this point, all tools from USE_TOOLS are defined with their variables. + // ${FALSE} works, but a plain "false" might call the wrong tool. + // That's because the tool wrappers are not set up yet. This + // happens between the post-depends and pre-fetch stages. Even + // then, the plain tool names may only be used in the + // {pre,do,post}-* targets, since a recursive make(1) needs to be + // run to set up the correct PATH. + // "", - "GITHUB_PROJECT=project", + "USE_TOOLS+= nice", + // + // The "nice" tool will only be available as ${NICE} after bsd.pkg.mk + // has been included. Even including bsd.prefs.mk another time does + // not have any effect since it is guarded against multiple inclusion. + // "", - "# comment", + ".include \"../../mk/bsd.prefs.mk\"", // Has no effect. "", - "DISTNAME=9term", - "# comment", - "CATEGORIES=x11") - - pkg.CheckVarorder(mklines) - - t.CheckOutputLines( - "WARN: Makefile:3: The canonical order of the variables is " + - "GITHUB_PROJECT, DISTNAME, CATEGORIES, GITHUB_PROJECT, empty line, " + - "COMMENT, LICENSE.") -} - -func (s *Suite) Test_Package_CheckVarorder__comments_are_ignored(c *check.C) { - t := s.Init(c) - - pkg := NewPackage(t.File("x11/9term")) - mklines := t.NewMkLines("Makefile", - MkCvsID, + "FALSE_AFTER!= echo false=${FALSE:Q}", // false=false + "NICE_AFTER!= echo nice=${NICE:Q}", // nice= + "TRUE_AFTER!= echo true=${TRUE:Q}", // true=true "", - "DISTNAME=\tdistname-1.0", - "CATEGORIES=\tsysutils", + "do-build:", + "\t${RUN} printf 'before: %-20s %-20s %-20s\\n' ${FALSE_BEFORE} ${NICE_BEFORE} ${TRUE_BEFORE}", + "\t${RUN} printf 'after: %-20s %-20s %-20s\\n' ${FALSE_AFTER} ${NICE_AFTER} ${TRUE_AFTER}", + "\t${RUN} printf 'runtime: %-20s %-20s %-20s\\n' false=${FALSE:Q} nice=${NICE:Q} true=${TRUE:Q}", "", - "MAINTAINER=\tpkgsrc-users@NetBSD.org", - "# comment", - "COMMENT=\tComment", - "LICENSE=\tgnu-gpl-v2") - - pkg.CheckVarorder(mklines) + ".include \"../../mk/bsd.pkg.mk\"") - t.CheckOutputEmpty() -} + t.SetUpCommandLine("-q", "-Wall,no-space") + t.FinishSetUp() -func (s *Suite) Test_Package_CheckVarorder__commented_variable_assignment(c *check.C) { - t := s.Init(c) + G.Check(t.File("category/pkgbase")) - pkg := NewPackage(t.File("x11/9term")) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "DISTNAME=\tdistname-1.0", - "CATEGORIES=\tsysutils", - "", - "MAINTAINER=\tpkgsrc-users@NetBSD.org", - "#HOMEPAGE=\thttps://example.org/", - "COMMENT=\tComment", - "LICENSE=\tgnu-gpl-v2") + t.CheckOutputLines( + "NOTE: ~/category/pkgbase/Makefile:14: Consider the :sh modifier instead of != for \"echo false=${FALSE:Q}\".", + "WARN: ~/category/pkgbase/Makefile:14: To use the tool ${FALSE} at load time, bsd.prefs.mk has to be included before.", + "NOTE: ~/category/pkgbase/Makefile:15: Consider the :sh modifier instead of != for \"echo nice=${NICE:Q}\".", - pkg.CheckVarorder(mklines) + // TODO: replace "at load time" with "before including bsd.prefs.mk in line ###". + // TODO: ${NICE} could be used at load time if it were added to USE_TOOLS earlier. + "WARN: ~/category/pkgbase/Makefile:15: The tool ${NICE} cannot be used at load time.", - t.CheckOutputEmpty() + "NOTE: ~/category/pkgbase/Makefile:16: Consider the :sh modifier instead of != for \"echo true=${TRUE:Q}\".", + "WARN: ~/category/pkgbase/Makefile:16: To use the tool ${TRUE} at load time, bsd.prefs.mk has to be included before.", + "NOTE: ~/category/pkgbase/Makefile:24: Consider the :sh modifier instead of != for \"echo false=${FALSE:Q}\".", + "NOTE: ~/category/pkgbase/Makefile:25: Consider the :sh modifier instead of != for \"echo nice=${NICE:Q}\".", + "WARN: ~/category/pkgbase/Makefile:25: The tool ${NICE} cannot be used at load time.", + "NOTE: ~/category/pkgbase/Makefile:26: Consider the :sh modifier instead of != for \"echo true=${TRUE:Q}\".") } -func (s *Suite) Test_Package_CheckVarorder__skip_because_of_foreign_variable(c *check.C) { +func (s *Suite) Test_Package__relative_included_filenames_in_same_directory(c *check.C) { t := s.Init(c) - pkg := NewPackage(t.File("x11/9term")) - mklines := t.NewMkLines("Makefile", + t.SetUpPackage("category/package", + "PKGNAME=\tpkgname-1.67", + "DISTNAME=\tdistfile_1_67", + ".include \"../../category/package/other.mk\"") + t.CreateFileLines("category/package/other.mk", MkCvsID, - "", - "DISTNAME=\tdistname-1.0", - "USE_TOOLS+=gmake", - "CATEGORIES=\tsysutils", - "", - "MAINTAINER=\tpkgsrc-users@NetBSD.org", - "#HOMEPAGE=\thttps://example.org/", - "COMMENT=\tComment", - "LICENSE=\tgnu-gpl-v2") + "PKGNAME=\tpkgname-1.67", + "DISTNAME=\tdistfile_1_67", + ".include \"../../category/package/other.mk\"") + t.FinishSetUp() - t.EnableTracingToLog() - pkg.CheckVarorder(mklines) + G.Check(t.File("category/package")) - t.CheckOutputLinesMatching(`.*varorder.*`, - "TRACE: 1 Skipping varorder because of line 4.") + // TODO: Since other.mk is referenced via "../../category/package", + // it would be nice if this relative path would be reflected in the output + // instead of referring just to "other.mk". + // This needs to be fixed somewhere near relpath. + // + // The notes are in reverse order because they are produced when checking + // other.mk, and there the relative order is correct (line 2 before line 3). + t.CheckOutputLines( + "NOTE: ~/category/package/Makefile:4: "+ + "Definition of PKGNAME is redundant because of other.mk:2.", + "NOTE: ~/category/package/Makefile:3: "+ + "Definition of DISTNAME is redundant because of other.mk:3.") } -func (s *Suite) Test_Package_CheckVarorder__skip_if_there_are_directives(c *check.C) { +func (s *Suite) Test_Package__using_common_Makefile_overriding_DISTINFO_FILE(c *check.C) { t := s.Init(c) - pkg := NewPackage(t.File("category/package")) - mklines := t.NewMkLines("Makefile", + t.SetUpPackage("security/pinentry") + t.CreateFileLines("security/pinentry/Makefile.common", MkCvsID, + "DISTINFO_FILE=\t${.CURDIR}/../../security/pinentry/distinfo") + t.SetUpPackage("security/pinentry-fltk", + ".include \"../../security/pinentry/Makefile.common\"", + "DISTINFO_FILE=\t${.CURDIR}/distinfo") + t.CreateFileDummyPatch("security/pinentry-fltk/patches/patch-aa") + t.CreateFileLines("security/pinentry-fltk/distinfo", + CvsID, "", - "DISTNAME=\tdistname-1.0", - "CATEGORIES=\tsysutils", - "", - ".if ${DISTNAME:Mdistname-*}", - "MAINTAINER=\tpkgsrc-users@NetBSD.org", - ".endif", - "LICENSE=\tgnu-gpl-v2") - - pkg.CheckVarorder(mklines) + "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") + t.FinishSetUp() - // No warning about the missing COMMENT since the .if directive - // causes the whole check to be skipped. - t.CheckOutputEmpty() + G.Check(t.File("security/pinentry")) - // Just for code coverage. - t.DisableTracing() - pkg.CheckVarorder(mklines) t.CheckOutputEmpty() -} -// TODO: Add more tests like skip_if_there_are_directives for other line types. - -func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_top(c *check.C) { - t := s.Init(c) - - pkg := NewPackage(t.File("x11/9term")) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "GITHUB_PROJECT=\t\tautocutsel", - "DISTNAME=\t\tautocutsel-0.10.0", - "CATEGORIES=\t\tx11", - "MASTER_SITES=\t\t${MASTER_SITE_GITHUB:=sigmike/}", - "GITHUB_TAG=\t\t${PKGVERSION_NOREV}", - "", - "COMMENT=\tComment", - "LICENSE=\tgnu-gpl-v2") - - pkg.CheckVarorder(mklines) + G.Check(t.File("security/pinentry-fltk")) + // The DISTINFO_FILE definition from pinentry-fltk overrides + // the one from pinentry since it appears later. + // Therefore the patch is searched for at the right location. t.CheckOutputEmpty() } -func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_bottom(c *check.C) { +func (s *Suite) Test_Package__redundant_variable_in_unrelated_files(c *check.C) { t := s.Init(c) - pkg := NewPackage(t.File("x11/9term")) - mklines := t.NewMkLines("Makefile", + t.SetUpPackage("databases/py-trytond-ldap-authentication", + ".include \"../../devel/py-trytond/Makefile.common\"", + ".include \"../../lang/python/egg.mk\"") + t.CreateFileLines("devel/py-trytond/Makefile.common", MkCvsID, - "", - "DISTNAME=\t\tautocutsel-0.10.0", - "CATEGORIES=\t\tx11", - "MASTER_SITES=\t\t${MASTER_SITE_GITHUB:=sigmike/}", - "GITHUB_PROJECT=\t\tautocutsel", - "GITHUB_TAG=\t\t${PKGVERSION_NOREV}", - "", - "COMMENT=\tComment", - "LICENSE=\tgnu-gpl-v2") + "PY_PATCHPLIST=\tyes") + t.CreateFileLines("lang/python/egg.mk", + MkCvsID, + "PY_PATCHPLIST=\tyes") + t.FinishSetUp() - pkg.CheckVarorder(mklines) + G.Check(t.File("databases/py-trytond-ldap-authentication")) + // Since egg.mk and Makefile.common are unrelated, the definition of + // PY_PATCHPLIST is not redundant in these files. t.CheckOutputEmpty() } -func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { +// As of April 2019, there are only a few files in the whole pkgsrc tree +// that are called Makefile.*, except Makefile.common, which occurs more +// often. +// +// Using the file extension for variants of that Makefile is confusing, +// therefore they should be renamed to *.mk. +func (s *Suite) Test_Package__Makefile_files(c *check.C) { t := s.Init(c) - t.CreateFileLines("mk/bsd.pkg.mk", "# dummy") - t.CreateFileLines("x11/Makefile", MkCvsID) - t.CreateFileLines("x11/9term/PLIST", PlistCvsID, "bin/9term") - t.CreateFileLines("x11/9term/Makefile", - MkCvsID, - "", - "DISTNAME=\t9term-1.0", - "CATEGORIES=\tx11", - "", - "COMMENT=\tTerminal", - "", - "NO_CHECKSUM=\tyes", - "", - ".include \"../../mk/bsd.pkg.mk\"") - - t.SetUpVartypes() + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/Makefile.common", + MkCvsID) + t.CreateFileLines("category/package/Makefile.orig", + MkCvsID) + t.CreateFileLines("category/package/Makefile.php", + MkCvsID) + t.CreateFileLines("category/package/ext.mk", + MkCvsID) + t.FinishSetUp() - G.Check(t.File("x11/9term")) + G.Check(t.File("category/package")) - // Since the error is grave enough, the warning about the correct position is suppressed. - // TODO: Knowing the correct position helps, though. + // No warning for the Makefile.orig since the package is not + // being imported at the moment; see Pkglint.checkReg. t.CheckOutputLines( - "ERROR: ~/x11/9term/Makefile: Each package must define its LICENSE.") + "NOTE: ~/category/package/Makefile.php: " + + "Consider renaming \"Makefile.php\" to \"php.mk\".") } -// https://mail-index.netbsd.org/tech-pkg/2017/01/18/msg017698.html -func (s *Suite) Test_Package_CheckVarorder__MASTER_SITES(c *check.C) { +func (s *Suite) Test_Package__patch_in_FILESDIR(c *check.C) { t := s.Init(c) - pkg := NewPackage(t.File("category/package")) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "PKGNAME=\tpackage-1.0", - "CATEGORIES=\tcategory", - "MASTER_SITES=\thttp://example.org/", - "MASTER_SITES+=\thttp://mirror.example.org/", - "", - "COMMENT=\tComment", - "LICENSE=\tgnu-gpl-v2") + t.SetUpCommandLine("-Wall", "-Call") + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/files/patch-aa", + "This file can contain anything, no matter what the filename says.") + t.FinishSetUp() - pkg.CheckVarorder(mklines) + G.Check(t.File("category/package")) - // No warning that "MASTER_SITES appears too late" + // No warnings. The files in FILESDIR are independent of pkgsrc + // and may contain anything. There are no naming conventions or + // anything else. t.CheckOutputEmpty() } -func (s *Suite) Test_Package_CheckVarorder__diagnostics(c *check.C) { +// See https://mail-index.netbsd.org/tech-pkg/2018/07/22/msg020092.html +func (s *Suite) Test_Package__redundant_master_sites(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - pkg := NewPackage(t.File("category/package")) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "CATEGORIES= net", - "", - "COMMENT= Comment", - "LICENSE= gnu-gpl-v3", - "", - "GITHUB_PROJECT= pkgbase", - "DISTNAME= v1.0", - "PKGNAME= ${GITHUB_PROJECT}-${DISTNAME}", - "MASTER_SITES= ${MASTER_SITE_GITHUB:=project/}", - "DIST_SUBDIR= ${GITHUB_PROJECT}", - "", - "MAINTAINER= maintainer@example.org", - "HOMEPAGE= https://github.com/project/pkgbase/", - "", - ".include \"../../mk/bsd.pkg.mk\"") - - pkg.CheckVarorder(mklines) - - t.CheckOutputLines( - "WARN: Makefile:3: The canonical order of the variables is " + - "GITHUB_PROJECT, DISTNAME, PKGNAME, CATEGORIES, " + - "MASTER_SITES, GITHUB_PROJECT, DIST_SUBDIR, empty line, " + - "MAINTAINER, HOMEPAGE, COMMENT, LICENSE.") - - // After moving the variables according to the warning: - mklines = t.NewMkLines("Makefile", + t.SetUpPkgsrc() + t.SetUpMasterSite("MASTER_SITE_R_CRAN", "http://cran.r-project.org/src/") + t.CreateFileLines("math/R/Makefile.extension", MkCvsID, "", - "GITHUB_PROJECT= pkgbase", - "DISTNAME= v1.0", - "PKGNAME= ${GITHUB_PROJECT}-${DISTNAME}", - "CATEGORIES= net", - "MASTER_SITES= ${MASTER_SITE_GITHUB:=project/}", - "DIST_SUBDIR= ${GITHUB_PROJECT}", - "", - "MAINTAINER= maintainer@example.org", - "HOMEPAGE= https://github.com/project/pkgbase/", - "COMMENT= Comment", - "LICENSE= gnu-gpl-v3", - "", - ".include \"../../mk/bsd.pkg.mk\"") - - pkg.CheckVarorder(mklines) - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Package_CheckVarorder__comment_at_end_of_section(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - pkg := NewPackage(t.File("category/package")) - mklines := t.NewMkLines("Makefile", + "PKGNAME?=\tR-${R_PKGNAME}-${R_PKGVER}", + "MASTER_SITES?=\t${MASTER_SITE_R_CRAN:=contrib/}", + "GENERATE_PLIST+=\techo \"bin/r-package\";", + "NO_CHECKSUM=\tyes", + "LICENSE?=\tgnu-gpl-v2") + t.CreateFileLines("math/R-date/Makefile", MkCvsID, "", - "CATEGORIES= net", - "SITES.*= # none", - "# comment after the last variable of a section", - "", - "MAINTAINER= maintainer@example.org", - "HOMEPAGE= https://github.com/project/pkgbase/", - "COMMENT= Comment", - "LICENSE= gnu-gpl-v3", + "R_PKGNAME=\tdate", + "R_PKGVER=\t1.2.3", + "COMMENT=\tR package for handling date arithmetic", + "MASTER_SITES=\t${MASTER_SITE_R_CRAN:=contrib/}", // Redundant; see math/R/Makefile.extension. "", + ".include \"../../math/R/Makefile.extension\"", ".include \"../../mk/bsd.pkg.mk\"") + t.FinishSetUp() - t.EnableTracingToLog() - pkg.CheckVarorder(mklines) + // See Package.checkfilePackageMakefile + G.checkdirPackage(t.File("math/R-date")) - // The varorder code is not skipped, not even because of the comment - // after SITES.*. - t.CheckOutputLinesMatching(`.*varorder.*`, - nil...) + // The definition in Makefile:6 is redundant because the same definition + // occurs later in Makefile.extension:4. + // + // When a file includes another file, it's always the including file that + // is marked as redundant since the included file typically provides the + // generally useful value for several packages; + // see RedundantScope.handleVarassign, keyword includePath. + t.CheckOutputLines( + "NOTE: ~/math/R-date/Makefile:6: " + + "Definition of MASTER_SITES is redundant " + + "because of ../../math/R/Makefile.extension:4.") } -func (s *Suite) Test_Package_CheckVarorder__comments_between_sections(c *check.C) { +// Before 2018-09-09, the .CURDIR variable did not have a fallback value. +// When resolving the relative path x11/gst-x11/${.CURDIR}/../../multimedia/gst-base/distinfo, +// "gst-x11/${.CURDIR}" was interpreted as "category/package", and the whole +// path was resolved to "x11/multimedia/gst-base/distinfo, which of course +// could not be found. +func (s *Suite) Test_Package__distinfo_from_other_package(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - pkg := NewPackage(t.File("category/package")) - mklines := t.NewMkLines("Makefile", + t.SetUpCommandLine("-Wall,no-space") + t.SetUpPkgsrc() + t.Chdir(".") + t.CreateFileLines("x11/gst-x11/Makefile", MkCvsID, - "", - "CATEGORIES= net", - "", - "# comment 1", - "", - "# comment 2", - "", - "MAINTAINER= maintainer@example.org", - "HOMEPAGE= https://github.com/project/pkgbase/", - "COMMENT= Comment", - "LICENSE= gnu-gpl-v3", - "", + ".include \"../../multimedia/gst-base/Makefile.common\"", ".include \"../../mk/bsd.pkg.mk\"") - - pkg.CheckVarorder(mklines) - - // The empty line between the comments is not treated as a section separator. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Package_CheckVarorder__commented_varassign(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - pkg := NewPackage(t.File("category/package")) - mklines := t.NewMkLines("Makefile", + t.CreateFileLines("multimedia/gst-base/Makefile.common", MkCvsID, - "", - "CATEGORIES= net", - "#MASTER_SITES= # none", - "", - "HOMEPAGE= https://github.com/project/pkgbase/", - "#HOMEPAGE= https://github.com/project/pkgbase/", - "#HOMEPAGE= https://github.com/project/pkgbase/", - "#HOMEPAGE= https://github.com/project/pkgbase/", - "#HOMEPAGE= https://github.com/project/pkgbase/", - "LICENSE= gnu-gpl-v3", - "COMMENT= Comment", - "", - ".include \"../../mk/bsd.pkg.mk\"") - - pkg.CheckVarorder(mklines) - - // The order of the variables LICENSE and COMMENT is intentionally - // wrong to force the warning. - // - // Up to June 2019 (308099138a62) pkglint mentioned in the warning - // each commented variable assignment, even repeatedly for the same - // variable name. - // - // These variable assignments should be in the correct order, even - // if they are commented out. It's not necessary though to list a - // variable more than once. - t.CheckOutputLines( - "WARN: Makefile:3: The canonical order of the variables is " + - "CATEGORIES, MASTER_SITES, empty line, HOMEPAGE, COMMENT, LICENSE.") -} - -func (s *Suite) Test_Package_CheckVarorder__DEPENDS(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - pkg := NewPackage(t.File("category/package")) - mklines := t.NewMkLines("Makefile", + ".include \"plugins.mk\"") + t.CreateFileLines("multimedia/gst-base/plugins.mk", MkCvsID, + "DISTINFO_FILE=\t${.CURDIR}/../../multimedia/gst-base/distinfo") + t.CreateFileLines("multimedia/gst-base/distinfo", + CvsID, "", - "CATEGORIES= net", - "", - "COMMENT= Comment", - "LICENSE= license", - "MAINTAINER= maintainer@example.org", // In wrong order - "", - "DEPENDS+= dependency>=1.0:../../category/dependency", - "", - ".include \"../../mk/bsd.pkg.mk\"") - - pkg.CheckVarorder(mklines) - - t.CheckOutputLines( - "WARN: Makefile:3: The canonical order of the variables is " + - "CATEGORIES, empty line, MAINTAINER, COMMENT, LICENSE, empty line, DEPENDS.") -} - -func (s *Suite) Test_Package_nbPart(c *check.C) { - t := s.Init(c) - - pkg := NewPackage(t.File("category/pkgbase")) - pkg.vars.Define("PKGREVISION", t.NewMkLine("Makefile", 1, "PKGREVISION=14")) - - t.CheckEquals(pkg.nbPart(), "nb14") - - pkg.vars = NewScope() - pkg.vars.Define("PKGREVISION", t.NewMkLine("Makefile", 1, "PKGREVISION=asdf")) - - t.CheckEquals(pkg.nbPart(), "") -} - -// PKGNAME is stronger than DISTNAME. -func (s *Suite) Test_Package_determineEffectivePkgVars__precedence(c *check.C) { - t := s.Init(c) - - pkg := NewPackage(t.File("category/pkgbase")) - pkgnameLine := t.NewMkLine("Makefile", 3, "PKGNAME=pkgname-1.0") - distnameLine := t.NewMkLine("Makefile", 4, "DISTNAME=distname-1.0") - pkgrevisionLine := t.NewMkLine("Makefile", 5, "PKGREVISION=13") - - pkg.vars.Define(pkgnameLine.Varname(), pkgnameLine) - pkg.vars.Define(distnameLine.Varname(), distnameLine) - pkg.vars.Define(pkgrevisionLine.Varname(), pkgrevisionLine) - - pkg.determineEffectivePkgVars() - - t.CheckEquals(pkg.EffectivePkgbase, "pkgname") - t.CheckEquals(pkg.EffectivePkgname, "pkgname-1.0nb13") - t.CheckEquals(pkg.EffectivePkgversion, "1.0") -} - -func (s *Suite) Test_Package_determineEffectivePkgVars__same(c *check.C) { - t := s.Init(c) - - pkg := t.SetUpPackage("category/package", - "DISTNAME=\tdistname-1.0", - "PKGNAME=\tdistname-1.0") - t.FinishSetUp() - - G.Check(pkg) - - t.CheckOutputLines( - "NOTE: ~/category/package/Makefile:4: " + - "This assignment is probably redundant since PKGNAME is ${DISTNAME} by default.") -} - -func (s *Suite) Test_Package_determineEffectivePkgVars__simple_reference(c *check.C) { - t := s.Init(c) - - pkg := t.SetUpPackage("category/package", - "DISTNAME=\tdistname-1.0", - "PKGNAME=\t${DISTNAME}") - t.FinishSetUp() - - G.Check(pkg) - - t.CheckOutputLines( - "NOTE: ~/category/package/Makefile:4: " + - "This assignment is probably redundant since PKGNAME is ${DISTNAME} by default.") -} - -func (s *Suite) Test_Package_determineEffectivePkgVars__commented(c *check.C) { - t := s.Init(c) - - pkg := t.SetUpPackage("category/package", - "DISTNAME=\tdistname-1.0", - "PKGNAME=\t${DISTNAME} # intentionally") - t.FinishSetUp() - - G.Check(pkg) - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Package_determineEffectivePkgVars__invalid_DISTNAME(c *check.C) { - t := s.Init(c) - - pkg := t.SetUpPackage("category/package", - "DISTNAME=\tpkgname-version") - t.FinishSetUp() - - G.Check(pkg) - - t.CheckOutputLines( - "WARN: ~/category/package/Makefile:3: " + - "As DISTNAME is not a valid package name, please define the PKGNAME explicitly.") -} - -func (s *Suite) Test_Package_determineEffectivePkgVars__indirect_DISTNAME(c *check.C) { - t := s.Init(c) - - pkg := t.SetUpPackage("category/package", - "DISTNAME=\t${DISTFILES:[1]:C,\\..*,,}") - t.FinishSetUp() - - G.Check(pkg) - - // No warning since the case of DISTNAME being dependent on another - // variable is too difficult to analyze. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Package_determineEffectivePkgVars__C_modifier(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("x11/p5-gtk2", - "DISTNAME=\tGtk2-1.0", - "PKGNAME=\t${DISTNAME:C:Gtk2:p5-gtk2:}") - t.FinishSetUp() - pkg := NewPackage(t.File("x11/p5-gtk2")) - files, mklines, allLines := pkg.load() - - pkg.check(files, mklines, allLines) - - t.CheckEquals(pkg.EffectivePkgname, "p5-gtk2-1.0") -} - -// In some cases the PKGNAME is derived from DISTNAME, and it seems as -// if the :C modifier would not affect anything. This may nevertheless -// be on purpose since the modifier may apply to future versions and -// do things like replacing a "-1" with a ".1". -func (s *Suite) Test_Package_determineEffectivePkgVars__ineffective_C_modifier(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package", - "DISTNAME=\tdistname-1.0", - "PKGNAME=\t${DISTNAME:C:does_not_match:replacement:}") + "SHA1 (patch-aa) = 1234") t.FinishSetUp() - pkg := NewPackage(t.File("category/package")) - files, mklines, allLines := pkg.load() - - pkg.check(files, mklines, allLines) - t.CheckEquals(pkg.EffectivePkgname, "distname-1.0") - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Package_determineEffectivePkgVars__Python_prefix(c *check.C) { - t := s.Init(c) - - G.Experimental = true - t.SetUpPackage("category/package", - "PKGNAME=\tpackage-2.0", - ".include \"../../lang/python/extension.mk\"") - t.CreateFileLines("lang/python/extension.mk", - MkCvsID) - - t.Main("-Wall", "category/package") - - t.CheckOutputLines( - "WARN: ~/category/package/Makefile:4: The PKGNAME of Python extensions should start with ${PYPKGPREFIX}.", - "1 warning found.") -} - -func (s *Suite) Test_Package_determineEffectivePkgVars__Python_prefix_PKGNAME_variable(c *check.C) { - t := s.Init(c) - - G.Experimental = true - t.SetUpPackage("category/package", - "PKGNAME=\t${VAR}-package-2.0", - ".include \"../../lang/python/extension.mk\"") - t.CreateFileLines("lang/python/extension.mk", - MkCvsID, - "VAR=\tvalue") - - t.Main("-Wall", "category/package") + G.Check("x11/gst-x11") - // Since PKGNAME starts with a variable, pkglint doesn't investigate - // further what the possible value of this variable could be. If it - // did, it would see that the prefix is not PYPKGPREFIX and would - // complain. t.CheckOutputLines( - "Looks fine.") + "WARN: x11/gst-x11/Makefile: This package should have a PLIST file.", + "ERROR: x11/gst-x11/Makefile: Each package must define its LICENSE.", + "WARN: x11/gst-x11/Makefile: Each package should define a COMMENT.", + "WARN: x11/gst-x11/../../multimedia/gst-base/distinfo:3: Patch file \"patch-aa\" does not exist in directory \"../../x11/gst-x11/patches\".") } -// As of August 2019, pkglint loads the package files in alphabetical order. -// This means that the package Makefile is loaded early, and includes by -// other files may be invisible yet. This applies to both Makefile.* and to -// *.mk since both of these appear later. -// -// The effects of these files are nevertheless visible at the right time -// because the package Makefile is loaded including all its included files. -func (s *Suite) Test_Package_determineEffectivePkgVars__Python_prefix_late(c *check.C) { +func (s *Suite) Test_Package__case_insensitive(c *check.C) { t := s.Init(c) - G.Experimental = true + t.SetUpPkgsrc() + t.SetUpPackage("net/p5-Net-DNS") t.SetUpPackage("category/package", - "PKGNAME=\tpackage-2.0", - ".include \"common.mk\"") - t.CreateFileLines("category/package/common.mk", - MkCvsID, - ".include \"../../lang/python/extension.mk\"") - t.CreateFileLines("lang/python/extension.mk", - MkCvsID) - - t.Main("-Wall", "category/package") - - t.CheckOutputLines( - "WARN: ~/category/package/Makefile:4: "+ - "The PKGNAME of Python extensions should start with ${PYPKGPREFIX}.", - "1 warning found.") -} - -func (s *Suite) Test_Package_checkPossibleDowngrade(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("doc/CHANGES-2018", - "\tUpdated category/pkgbase to 1.8 [committer 2018-01-05]") - G.Pkgsrc.loadDocChanges() - - t.Chdir("category/pkgbase") - G.Pkg = NewPackage(".") - G.Pkg.EffectivePkgname = "package-1.0nb15" - G.Pkg.EffectivePkgnameLine = t.NewMkLine("Makefile", 5, "PKGNAME=dummy") - - G.Pkg.checkPossibleDowngrade() - - t.CheckOutputLines( - "WARN: Makefile:5: The package is being downgraded from 1.8 (see ../../doc/CHANGES-2018:1) to 1.0nb15.") - - G.Pkgsrc.LastChange["category/pkgbase"].target = "1.0nb22" - - G.Pkg.checkPossibleDowngrade() - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Package_checkPossibleDowngrade__moved(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/pkgbase", - "PKGNAME=\tpackage-1.0") - t.CreateFileLines("doc/CHANGES-2018", - "\tUpdated category/old-package to 1.8 [committer 2018-01-05]", - "\tMoved category/old-package to category/pkgbase [committer 2018-01-05]") + "DEPENDS+=\tp5-Net-DNS>=0:../../net/p5-net-dns") t.FinishSetUp() - pkg := NewPackage(t.File("category/pkgbase")) - pkg.load() - pkg.determineEffectivePkgVars() - pkg.checkPossibleDowngrade() - - t.CheckEquals(G.Pkgsrc.LastChange["category/pkgbase"].Action, Moved) - // No warning because the latest action is not Updated. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Package_checkPossibleDowngrade__locally_modified_update(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package", - "PKGNAME=\tpackage-1.8") - t.CreateFileLines("doc/CHANGES-2018", - "\tUpdated category/package to 1.0 [committer 2018-01-05]") - t.CreateFileLines("category/package/CVS/Entries", - "/Makefile//modified//") - t.FinishSetUp() + // this test is only interesting on a case-insensitive filesystem + if !fileExists(t.File("mk/BSD.PKG.MK")) { + return + } G.Check(t.File("category/package")) - // Since the Makefile is locally modified, pkglint doesn't issue - // any warning since it assumes the package is being upgraded. + // FIXME: On a case-sensitive filesystem, p5-net-dns would not be found. t.CheckOutputEmpty() - - // When the Makefile is no longer locally modified, the warning - // is activated again. - t.Remove("category/package/CVS/Entries") - G.cvsEntriesDir = "" - - G.Check(t.File("category/package")) - - t.CheckOutputLines( - "NOTE: ~/category/package/Makefile:4: Package version \"1.8\" " + - "is greater than the latest \"1.0\" from ../../doc/CHANGES-2018:1.") -} - -func (s *Suite) Test_Package_loadPackageMakefile__dump(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--dumpmakefile") - t.SetUpPkgsrc() - t.CreateFileLines("category/Makefile") - t.CreateFileLines("category/package/PLIST", - PlistCvsID, - "bin/program") - t.CreateFileLines("category/package/distinfo", - CvsID, - "", - "SHA1 (distfile-1.0.tar.gz) = 12341234...", - "RMD160 (distfile-1.0.tar.gz) = 12341234...", - "SHA512 (distfile-1.0.tar.gz) = 12341234...", - "Size (distfile-1.0.tar.gz) = 12341234...") - t.CreateFileLines("category/package/Makefile", - MkCvsID, - "", - "CATEGORIES=category", - "", - "COMMENT=\tComment", - "LICENSE=\t2-clause-bsd") - // TODO: There is no .include line at the end of the Makefile. - // This should always be checked though. - t.FinishSetUp() - - G.checkdirPackage(t.File("category/package")) - - t.CheckOutputLines( - "Whole Makefile (with all included files) follows:", - "~/category/package/Makefile:1: "+MkCvsID, - "~/category/package/Makefile:2: ", - "~/category/package/Makefile:3: CATEGORIES=category", - "~/category/package/Makefile:4: ", - "~/category/package/Makefile:5: COMMENT=\tComment", - "~/category/package/Makefile:6: LICENSE=\t2-clause-bsd") } -func (s *Suite) Test_Package__varuse_at_load_time(c *check.C) { +func (s *Suite) Test_NewPackage(c *check.C) { t := s.Init(c) t.SetUpPkgsrc() - t.SetUpTool("printf", "", AtRunTime) - t.CreateFileLines("licenses/2-clause-bsd", - "# dummy") - t.CreateFileLines("misc/Makefile") - t.CreateFileLines("mk/tools/defaults.mk", - "TOOLS_CREATE+=false", - "TOOLS_CREATE+=nice", - "TOOLS_CREATE+=true", - "_TOOLS_VARNAME.nice=NICE") - - t.CreateFileLines("category/pkgbase/Makefile", - MkCvsID, - "", - "PKGNAME= loadtime-vartest-1.0", - "CATEGORIES= misc", - "", - "COMMENT= Demonstrate variable values during parsing", - "LICENSE= 2-clause-bsd", - "", - "PLIST_SRC= # none", - "NO_CHECKSUM= yes", - "NO_CONFIGURE= yes", - "", - "USE_TOOLS+= echo false", - "FALSE_BEFORE!= echo false=${FALSE:Q}", // false= - "NICE_BEFORE!= echo nice=${NICE:Q}", // nice= - "TRUE_BEFORE!= echo true=${TRUE:Q}", // true= - // - // All three variables above are empty since the tool - // variables are initialized by bsd.prefs.mk. The variables - // from share/mk/sys.mk are available, though. - // - "", - ".include \"../../mk/bsd.prefs.mk\"", - // - // At this point, all tools from USE_TOOLS are defined with their variables. - // ${FALSE} works, but a plain "false" might call the wrong tool. - // That's because the tool wrappers are not set up yet. This - // happens between the post-depends and pre-fetch stages. Even - // then, the plain tool names may only be used in the - // {pre,do,post}-* targets, since a recursive make(1) needs to be - // run to set up the correct PATH. - // - "", - "USE_TOOLS+= nice", - // - // The "nice" tool will only be available as ${NICE} after bsd.pkg.mk - // has been included. Even including bsd.prefs.mk another time does - // not have any effect since it is guarded against multiple inclusion. - // - "", - ".include \"../../mk/bsd.prefs.mk\"", // Has no effect. - "", - "FALSE_AFTER!= echo false=${FALSE:Q}", // false=false - "NICE_AFTER!= echo nice=${NICE:Q}", // nice= - "TRUE_AFTER!= echo true=${TRUE:Q}", // true=true - "", - "do-build:", - "\t${RUN} printf 'before: %-20s %-20s %-20s\\n' ${FALSE_BEFORE} ${NICE_BEFORE} ${TRUE_BEFORE}", - "\t${RUN} printf 'after: %-20s %-20s %-20s\\n' ${FALSE_AFTER} ${NICE_AFTER} ${TRUE_AFTER}", - "\t${RUN} printf 'runtime: %-20s %-20s %-20s\\n' false=${FALSE:Q} nice=${NICE:Q} true=${TRUE:Q}", - "", - ".include \"../../mk/bsd.pkg.mk\"") - - t.SetUpCommandLine("-q", "-Wall,no-space") + t.CreateFileLines("category/Makefile", + MkCvsID) t.FinishSetUp() - G.Check(t.File("category/pkgbase")) - - t.CheckOutputLines( - "NOTE: ~/category/pkgbase/Makefile:14: Consider the :sh modifier instead of != for \"echo false=${FALSE:Q}\".", - "WARN: ~/category/pkgbase/Makefile:14: To use the tool ${FALSE} at load time, bsd.prefs.mk has to be included before.", - "NOTE: ~/category/pkgbase/Makefile:15: Consider the :sh modifier instead of != for \"echo nice=${NICE:Q}\".", - - // TODO: replace "at load time" with "before including bsd.prefs.mk in line ###". - // TODO: ${NICE} could be used at load time if it were added to USE_TOOLS earlier. - "WARN: ~/category/pkgbase/Makefile:15: The tool ${NICE} cannot be used at load time.", - - "NOTE: ~/category/pkgbase/Makefile:16: Consider the :sh modifier instead of != for \"echo true=${TRUE:Q}\".", - "WARN: ~/category/pkgbase/Makefile:16: To use the tool ${TRUE} at load time, bsd.prefs.mk has to be included before.", - "NOTE: ~/category/pkgbase/Makefile:24: Consider the :sh modifier instead of != for \"echo false=${FALSE:Q}\".", - "NOTE: ~/category/pkgbase/Makefile:25: Consider the :sh modifier instead of != for \"echo nice=${NICE:Q}\".", - "WARN: ~/category/pkgbase/Makefile:25: The tool ${NICE} cannot be used at load time.", - "NOTE: ~/category/pkgbase/Makefile:26: Consider the :sh modifier instead of != for \"echo true=${TRUE:Q}\".") + t.ExpectAssert(func() { NewPackage("category") }) } // Demonstrates that Makefile fragments are handled differently, @@ -1075,6 +423,45 @@ func (s *Suite) Test_Package_load__extra_files(c *check.C) { "WARN: patches/readme.mk: Patch files should be named \"patch-\", followed by letters, '-', '_', '.', and digits only.") } +func (s *Suite) Test_Package_loadPackageMakefile__dump(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--dumpmakefile") + t.SetUpPkgsrc() + t.CreateFileLines("category/Makefile") + t.CreateFileLines("category/package/PLIST", + PlistCvsID, + "bin/program") + t.CreateFileLines("category/package/distinfo", + CvsID, + "", + "SHA1 (distfile-1.0.tar.gz) = 12341234...", + "RMD160 (distfile-1.0.tar.gz) = 12341234...", + "SHA512 (distfile-1.0.tar.gz) = 12341234...", + "Size (distfile-1.0.tar.gz) = 12341234...") + t.CreateFileLines("category/package/Makefile", + MkCvsID, + "", + "CATEGORIES=category", + "", + "COMMENT=\tComment", + "LICENSE=\t2-clause-bsd") + // TODO: There is no .include line at the end of the Makefile. + // This should always be checked though. + t.FinishSetUp() + + G.checkdirPackage(t.File("category/package")) + + t.CheckOutputLines( + "Whole Makefile (with all included files) follows:", + "~/category/package/Makefile:1: "+MkCvsID, + "~/category/package/Makefile:2: ", + "~/category/package/Makefile:3: CATEGORIES=category", + "~/category/package/Makefile:4: ", + "~/category/package/Makefile:5: COMMENT=\tComment", + "~/category/package/Makefile:6: LICENSE=\t2-clause-bsd") +} + func (s *Suite) Test_Package_loadPackageMakefile(c *check.C) { t := s.Init(c) @@ -1095,36 +482,6 @@ func (s *Suite) Test_Package_loadPackageMakefile(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_Package__relative_included_filenames_in_same_directory(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package", - "PKGNAME=\tpkgname-1.67", - "DISTNAME=\tdistfile_1_67", - ".include \"../../category/package/other.mk\"") - t.CreateFileLines("category/package/other.mk", - MkCvsID, - "PKGNAME=\tpkgname-1.67", - "DISTNAME=\tdistfile_1_67", - ".include \"../../category/package/other.mk\"") - t.FinishSetUp() - - G.Check(t.File("category/package")) - - // TODO: Since other.mk is referenced via "../../category/package", - // it would be nice if this relative path would be reflected in the output - // instead of referring just to "other.mk". - // This needs to be fixed somewhere near relpath. - // - // The notes are in reverse order because they are produced when checking - // other.mk, and there the relative order is correct (line 2 before line 3). - t.CheckOutputLines( - "NOTE: ~/category/package/Makefile:4: "+ - "Definition of PKGNAME is redundant because of other.mk:2.", - "NOTE: ~/category/package/Makefile:3: "+ - "Definition of DISTNAME is redundant because of other.mk:3.") -} - func (s *Suite) Test_Package_loadPackageMakefile__PECL_VERSION(c *check.C) { t := s.Init(c) @@ -1151,564 +508,671 @@ func (s *Suite) Test_Package_loadPackageMakefile__PECL_VERSION(c *check.C) { G.Check(pkg) } -func (s *Suite) Test_Package_check__files_Makefile(c *check.C) { +// Pkglint loads some files from the pkgsrc infrastructure and skips others. +// +// When a buildlink3.mk file from the infrastructure is included, it should +// be allowed to include its corresponding builtin.mk file in turn. +// +// This is necessary to load the correct variable assignments for the +// redundancy check, in particular variable assignments that serve as +// arguments to "procedure calls", such as mk/find-files.mk. +func (s *Suite) Test_Package_parse__include_infrastructure(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") - t.CreateFileLines("category/package/files/Makefile", - "This file may contain anything.") + t.SetUpCommandLine("--dumpmakefile") + t.SetUpPackage("category/package", + ".include \"../../mk/dlopen.buildlink3.mk\"", + ".include \"../../mk/pthread.buildlink3.mk\"") + t.CreateFileLines("mk/dlopen.buildlink3.mk", + ".include \"dlopen.builtin.mk\"") + t.CreateFileLines("mk/dlopen.builtin.mk", + ".include \"pthread.builtin.mk\"") + t.CreateFileLines("mk/pthread.buildlink3.mk", + ".include \"pthread.builtin.mk\"") + t.CreateFileLines("mk/pthread.builtin.mk", + "# This should be included by pthread.buildlink3.mk") + t.FinishSetUp() - t.Main("category/package/files/Makefile") + G.Check(t.File("category/package")) - // Since there is nothing to check in files/*, pkglint could - // as well report this as a usage error. - // - // Until June 2019, checking individual files in FILESDIR had - // been enabled by the -Call command line option. t.CheckOutputLines( - "Looks fine.") + "Whole Makefile (with all included files) follows:", + "~/category/package/Makefile:1: "+MkCvsID, + "~/category/package/Makefile:2: ", + "~/category/package/Makefile:3: DISTNAME=\tpackage-1.0", + "~/category/package/Makefile:4: #PKGNAME=\tpackage-1.0", + "~/category/package/Makefile:5: CATEGORIES=\tcategory", + "~/category/package/Makefile:6: MASTER_SITES=\t# none", + "~/category/package/Makefile:7: ", + "~/category/package/Makefile:8: MAINTAINER=\tpkgsrc-users@NetBSD.org", + "~/category/package/Makefile:9: HOMEPAGE=\t# none", + "~/category/package/Makefile:10: COMMENT=\tDummy package", + "~/category/package/Makefile:11: LICENSE=\t2-clause-bsd", + "~/category/package/Makefile:12: ", + "~/category/package/Makefile:13: .include \"suppress-varorder.mk\"", + "~/category/package/suppress-varorder.mk:1: "+MkCvsID, + "~/category/package/Makefile:14: ", + "~/category/package/Makefile:15: # filler", + "~/category/package/Makefile:16: # filler", + "~/category/package/Makefile:17: # filler", + "~/category/package/Makefile:18: # filler", + "~/category/package/Makefile:19: ", + "~/category/package/Makefile:20: .include \"../../mk/dlopen.buildlink3.mk\"", + "~/category/package/../../mk/dlopen.buildlink3.mk:1: .include \"dlopen.builtin.mk\"", + "~/mk/dlopen.builtin.mk:1: .include \"pthread.builtin.mk\"", + "~/category/package/Makefile:21: .include \"../../mk/pthread.buildlink3.mk\"", + "~/category/package/../../mk/pthread.buildlink3.mk:1: .include \"pthread.builtin.mk\"", + "~/mk/pthread.builtin.mk:1: # This should be included by pthread.buildlink3.mk", + "~/category/package/Makefile:22: ", + "~/category/package/Makefile:23: .include \"../../mk/bsd.pkg.mk\"") +} - t.Main("category/package") +// See https://github.com/rillig/pkglint/issues/1 +func (s *Suite) Test_Package_parse__include_without_exists(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"options.mk\"") + t.FinishSetUp() + + G.checkdirPackage(t.File("category/package")) t.CheckOutputLines( - "Looks fine.") + "ERROR: ~/category/package/Makefile:20: Cannot read \"options.mk\".") } -func (s *Suite) Test_Package_check__patches_Makefile(c *check.C) { +// See https://github.com/rillig/pkglint/issues/1 +func (s *Suite) Test_Package_parse__include_after_exists(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") - t.CreateFileLines("category/package/patches/Makefile", - "This file may contain anything.") + t.SetUpPackage("category/package", + ".if exists(options.mk)", + ". include \"options.mk\"", + ".endif") + t.FinishSetUp() - t.Main("category/package") + G.checkdirPackage(t.File("category/package")) - t.CheckOutputLines( - "WARN: ~/category/package/patches/Makefile: Patch files should be "+ - "named \"patch-\", followed by letters, '-', '_', '.', and digits only.", - "1 warning found.") + // No error message at all because of the .if exists before. + t.CheckOutputEmpty() } -func (s *Suite) Test_Package_checkDirent__errors(c *check.C) { +// See https://github.com/rillig/pkglint/issues/1 +func (s *Suite) Test_Package_parse__include_other_after_exists(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Call", "-Wall,no-space") - t.SetUpPkgsrc() - t.CreateFileLines("category/package/files/subdir/file") - t.CreateFileLines("category/package/files/subdir/subsub/file") + t.SetUpPackage("category/package", + ".if exists(options.mk)", + ". include \"another.mk\"", + ".endif") t.FinishSetUp() - pkg := NewPackage(t.File("category/package")) - pkg.checkDirent(t.File("category/package/options.mk"), 0444) - pkg.checkDirent(t.File("category/package/files/subdir"), 0555|os.ModeDir) - pkg.checkDirent(t.File("category/package/files/subdir/subsub"), 0555|os.ModeDir) - pkg.checkDirent(t.File("category/package/files"), 0555|os.ModeDir) + G.checkdirPackage(t.File("category/package")) t.CheckOutputLines( - "ERROR: ~/category/package/options.mk: Cannot be read.", - "WARN: ~/category/package/files/subdir/subsub: Unknown directory name.") + "ERROR: ~/category/package/Makefile:21: Cannot read \"another.mk\".") } -func (s *Suite) Test_Package_checkDirent__file_selection(c *check.C) { +func (s *Suite) Test_Package_parse__simple(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Call", "-Wall,no-space") - t.SetUpPkgsrc() - t.CreateFileLines("doc/CHANGES-2018", - CvsID) - t.CreateFileLines("category/package/buildlink3.mk", - MkCvsID) - t.CreateFileLines("category/package/unexpected.txt", - CvsID) + t.SetUpPackage("category/package") + t.Chdir("category/package") t.FinishSetUp() - pkg := NewPackage(t.File("category/package")) - pkg.checkDirent(t.File("doc/CHANGES-2018"), 0444) - pkg.checkDirent(t.File("category/package/buildlink3.mk"), 0444) - pkg.checkDirent(t.File("category/package/unexpected.txt"), 0444) + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() t.CheckOutputLines( - "WARN: ~/category/package/buildlink3.mk:EOF: Expected a BUILDLINK_TREE line.", - "WARN: ~/category/package/unexpected.txt: Unexpected file found.") + "FirstTime: suppress-varorder.mk") } -// Since all required information is passed to G.checkDirent via parameters, -// this test produces the expected results even though none of these files actually exists. -func (s *Suite) Test_Package_checkDirent__skipped(c *check.C) { +func (s *Suite) Test_Package_parse__nonexistent_Makefile(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package") - t.FinishSetUp() t.Chdir("category/package") - pkg := NewPackage(".") - - pkg.checkDirent("work", os.ModeSymlink) - pkg.checkDirent("work.i386", os.ModeSymlink) - pkg.checkDirent("work.hostname", os.ModeSymlink) - pkg.checkDirent("other", os.ModeSymlink) + t.Remove("Makefile") + t.FinishSetUp() - pkg.checkDirent("device", os.ModeDevice) + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() t.CheckOutputLines( - "WARN: other: Invalid symlink name.", - "ERROR: device: Only files and directories are allowed in pkgsrc.") + "ERROR: Makefile: Cannot be read.") } -func (s *Suite) Test_Package_checkIncludeConditionally__conditional_and_unconditional_include(c *check.C) { +func (s *Suite) Test_Package_parse__include_in_same_directory(c *check.C) { t := s.Init(c) - t.SetUpOption("zlib", "") t.SetUpPackage("category/package", - ".include \"../../devel/zlib/buildlink3.mk\"", - ".if ${OPSYS} == \"Linux\"", - ".include \"../../sysutils/coreutils/buildlink3.mk\"", - ".endif") - t.CreateFileLines("mk/bsd.options.mk", "") - t.CreateFileLines("devel/zlib/buildlink3.mk", "") - t.CreateFileLines("sysutils/coreutils/buildlink3.mk", "") - - t.CreateFileLines("category/package/options.mk", - MkCvsID, - "", - "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", - "PKG_SUPPORTED_OPTIONS=\tzlib", - "", - ".include \"../../mk/bsd.options.mk\"", - "", - ".if !empty(PKG_OPTIONS:Mzlib)", - ". include \"../../devel/zlib/buildlink3.mk\"", - ".endif", - ".include \"../../sysutils/coreutils/buildlink3.mk\"") + ".include \"version.mk\"") t.Chdir("category/package") + t.CreateFileLines("version.mk", + MkCvsID) t.FinishSetUp() - G.checkdirPackage(".") + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() t.CheckOutputLines( - "WARN: Makefile:20: \"../../devel/zlib/buildlink3.mk\" is included "+ - "unconditionally here "+ - "and conditionally in options.mk:9 (depending on PKG_OPTIONS).", - "WARN: Makefile:22: \"../../sysutils/coreutils/buildlink3.mk\" is included "+ - "conditionally here (depending on OPSYS) and "+ - "unconditionally in options.mk:11.") + "FirstTime: suppress-varorder.mk", + "FirstTime: version.mk") } -func (s *Suite) Test_Package_checkIncludeConditionally__explain_PKG_OPTIONS_in_Makefile(c *check.C) { +func (s *Suite) Test_Package_parse__nonexistent_include(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall", "--explain") - t.SetUpOption("zlib", "use zlib compression") - - t.CreateFileLines("mk/bsd.options.mk", - MkCvsID) - t.CreateFileLines("devel/zlib/buildlink3.mk", - MkCvsID) t.SetUpPackage("category/package", - "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", - "PKG_SUPPORTED_OPTIONS=\tzlib", - "", - ".include \"../../mk/bsd.options.mk\"", - "", - ".if ${PKG_OPTIONS:Mzlib}", - ".include \"../../devel/zlib/buildlink3.mk\"", - ".endif") - t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", - ".include \"../../devel/zlib/buildlink3.mk\"") + ".include \"version.mk\"") t.Chdir("category/package") t.FinishSetUp() - G.checkdirPackage(".") + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() t.CheckOutputLines( - "WARN: Makefile:26: "+ - "\"../../devel/zlib/buildlink3.mk\" is included conditionally here "+ - "(depending on PKG_OPTIONS) and unconditionally in buildlink3.mk:12.", - "", - "\tWhen including a dependent file, the conditions in the buildlink3.mk", - "\tfile should be the same as in options.mk or the Makefile.", - "", - "\tTo find out the PKG_OPTIONS of this package at build time, have a", - "\tlook at mk/pkg-build-options.mk.", - "") + "FirstTime: suppress-varorder.mk", + "FirstTime: version.mk", + "ERROR: Makefile:20: Cannot read \"version.mk\".") } -func (s *Suite) Test_Package_checkIncludeConditionally__no_explanation(c *check.C) { +// When reading the package Makefile, pkglint loads and interprets each +// file only once. This is especially important for packages with a large +// dependency graph containing many common subdependencies. +func (s *Suite) Test_Package_parse__include_twice(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall", "--explain") - t.CreateFileLines("devel/zlib/buildlink3.mk", - MkCvsID) t.SetUpPackage("category/package", - ".if ${OPSYS} == Linux", - ".include \"../../devel/zlib/buildlink3.mk\"", - ".endif") - t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", - ".include \"../../devel/zlib/buildlink3.mk\"") + ".include \"version.mk\"", + ".include \"version.mk\"") t.Chdir("category/package") + t.CreateFileLines("version.mk", + MkCvsID) t.FinishSetUp() - G.checkdirPackage(".") + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() t.CheckOutputLines( - "WARN: Makefile:21: " + - "\"../../devel/zlib/buildlink3.mk\" is included conditionally here " + - "(depending on OPSYS) and unconditionally in buildlink3.mk:12.") + "FirstTime: suppress-varorder.mk", + "FirstTime: version.mk") } -func (s *Suite) Test_Package_checkIncludeConditionally__explain_PKG_OPTIONS_in_options_mk(c *check.C) { +func (s *Suite) Test_Package_parse__include_in_other_directory(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall", "--explain") - t.SetUpOption("zlib", "use zlib compression") - - t.CreateFileLines("mk/bsd.options.mk", - MkCvsID) - t.CreateFileLines("devel/zlib/buildlink3.mk", - MkCvsID) t.SetUpPackage("category/package", - ".include \"options.mk\"") - t.CreateFileLines("category/package/options.mk", - MkCvsID, - "", - "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", - "PKG_SUPPORTED_OPTIONS=\tzlib", - "", - ".include \"../../mk/bsd.options.mk\"", - "", - ".if ${PKG_OPTIONS:Mzlib}", - ".include \"../../devel/zlib/buildlink3.mk\"", - ".endif") - t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", - ".include \"../../devel/zlib/buildlink3.mk\"") + ".include \"../../category/other/version.mk\"") t.Chdir("category/package") + t.CreateFileLines("../../category/other/version.mk", + MkCvsID) t.FinishSetUp() - G.checkdirPackage(".") + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() t.CheckOutputLines( - "WARN: buildlink3.mk:12: "+ - "\"../../devel/zlib/buildlink3.mk\" is included unconditionally here "+ - "and conditionally in options.mk:9 (depending on PKG_OPTIONS).", - "", - "\tWhen including a dependent file, the conditions in the buildlink3.mk", - "\tfile should be the same as in options.mk or the Makefile.", - "", - "\tTo find out the PKG_OPTIONS of this package at build time, have a", - "\tlook at mk/pkg-build-options.mk.", - "") + "FirstTime: suppress-varorder.mk", + "FirstTime: ../../category/other/version.mk") } -func (s *Suite) Test_Package_checkIncludeConditionally__unconditionally_first(c *check.C) { +// Demonstrates that Package.included contains the file paths of the +// included files, relative to the package directory. +func (s *Suite) Test_Package_parse__includes_in_other_directory(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") + t.SetUpPackage("category/package", + ".include \"../../category/other/module.mk\"") t.Chdir("category/package") - t.CreateFileLines("including.mk", + t.CreateFileLines("../../category/other/module.mk", MkCvsID, - "", - ".include \"included.mk\"", - ".if ${OPSYS} == \"Linux\"", - ".include \"included.mk\"", - ".endif") - t.CreateFileLines("included.mk", + ".include \"version.mk\"") + t.CreateFileLines("../../category/other/version.mk", MkCvsID) t.FinishSetUp() - G.Check(".") + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() t.CheckOutputLines( - "WARN: including.mk:3: \"included.mk\" is included " + - "unconditionally here and conditionally in line 5 (depending on OPSYS).") + "FirstTime: suppress-varorder.mk", + "FirstTime: ../../category/other/module.mk", + "FirstTime: ../../category/other/version.mk") } -func (s *Suite) Test_Package_checkIncludeConditionally__only_conditionally(c *check.C) { +func (s *Suite) Test_Package_parse__nonexistent_in_other_directory(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package", - ".if ${OPSYS} == \"Linux\"", - ".include \"included.mk\"", - ".endif") + ".include \"../../category/other/module.mk\"") t.Chdir("category/package") - t.CreateFileLines("included.mk", - MkCvsID) + t.CreateFileLines("../../category/other/module.mk", + MkCvsID, + ".include \"version.mk\"") t.FinishSetUp() - G.Check(".") + G.Pkg = NewPackage(".") + G.Pkg.included.Trace = true + G.Pkg.load() - t.CheckOutputEmpty() + t.CheckOutputLines( + "FirstTime: suppress-varorder.mk", + "FirstTime: ../../category/other/module.mk", + "FirstTime: ../../category/other/version.mk", + "ERROR: ../../category/other/module.mk:2: Cannot read \"version.mk\".") } -func (s *Suite) Test_Package_checkIncludeConditionally__conditionally_first(c *check.C) { +func (s *Suite) Test_Package_parse__skipping(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") - t.Chdir("category/package") - t.CreateFileLines("including.mk", - MkCvsID, - "", - ".if ${OPSYS} == \"Linux\"", - ".include \"included.mk\"", - ".endif", - ".include \"included.mk\"") - t.CreateFileLines("included.mk", - MkCvsID) + t.SetUpCommandLine("-Wall,no-space") + pkg := t.SetUpPackage("category/package", + ".include \"${MYSQL_PKGSRCDIR:S/-client$/-server/}/buildlink3.mk\"") t.FinishSetUp() - G.Check(".") + t.EnableTracingToLog() + G.Check(pkg) + t.EnableSilentTracing() + + // Since 2018-12-16 there is no warning or note anymore for the + // buildlink3.mk file being skipped since it didn't help the average + // pkglint user. + + // The information is still available in the trace log though. + + output := t.Output() + var relevant []string + for _, line := range strings.Split(output, "\n") { + if contains(line, "Skipping") { + relevant = append(relevant, line) + } + } + + t.CheckDeepEquals(relevant, []string{ + "TRACE: 1 2 3 4 ~/category/package/Makefile:20: " + + "Skipping unresolvable include file \"${MYSQL_PKGSRCDIR:S/-client$/-server/}/buildlink3.mk\"."}) +} + +func (s *Suite) Test_Package_parse__not_found(c *check.C) { + t := s.Init(c) + + pkg := t.SetUpPackage("category/package", + ".include \"../../devel/zlib/buildlink3.mk\"") + t.CreateFileLines("devel/zlib/buildlink3.mk", + ".include \"../../enoent/enoent/buildlink3.mk\"") + t.FinishSetUp() + + G.checkdirPackage(pkg) t.CheckOutputLines( - "WARN: including.mk:4: \"included.mk\" is included " + - "conditionally here (depending on OPSYS) and unconditionally in line 6.") + "ERROR: ~/devel/zlib/buildlink3.mk:1: Cannot read \"../../enoent/enoent/buildlink3.mk\".") } -func (s *Suite) Test_Package_checkIncludeConditionally__included_multiple_times(c *check.C) { +func (s *Suite) Test_Package_parse__relative(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") - t.Chdir("category/package") - t.CreateFileLines("including.mk", - MkCvsID, - "", - ".include \"included.mk\"", - ".if ${OPSYS} == \"Linux\"", - ".include \"included.mk\"", - ".endif", - "", - ".include \"included.mk\"", - ".if ${OPSYS} == \"Linux\"", - ".include \"included.mk\"", - ".endif") - t.CreateFileLines("included.mk", + t.CreateFileLines("category/package/extra.mk", MkCvsID) + pkg := t.SetUpPackage("category/package", + ".include \"../package/extra.mk\"") t.FinishSetUp() - G.Check(".") + G.Check(pkg) t.CheckOutputLines( - "WARN: including.mk:3: \"included.mk\" is included "+ - "unconditionally here and conditionally in line 10 (depending on OPSYS).", - "WARN: including.mk:5: \"included.mk\" is included "+ - "conditionally here (depending on OPSYS) and unconditionally in line 8.", - "WARN: including.mk:8: \"included.mk\" is included "+ - "unconditionally here and conditionally in line 10 (depending on OPSYS).") + "WARN: ~/category/package/Makefile:20: " + + "References to other packages should look " + + "like \"../../category/package\", not \"../package\".") } -// For preferences files, it doesn't matter whether they are included -// conditionally or unconditionally since at the end they are included -// anyway by the infrastructure. -func (s *Suite) Test_Package_checkIncludeConditionally__prefs(c *check.C) { +// When a buildlink3.mk file is included, the corresponding builtin.mk +// file is included by the pkgsrc infrastructure. Therefore all variables +// declared in the builtin.mk file become known in the package. +func (s *Suite) Test_Package_parse__builtin_mk(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") - t.Chdir("category/package") - t.CreateFileLines("including.mk", - MkCvsID, + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpPackage("category/package", + ".include \"../../category/lib1/buildlink3.mk\"", "", - ".include \"../../mk/bsd.prefs.mk\"", - ".if ${OPSYS} == \"Linux\"", - ".include \"../../mk/bsd.prefs.mk\"", - ".endif") + "show-var-from-builtin: .PHONY", + "\techo ${VAR_FROM_BUILTIN} ${OTHER_VAR}") + t.CreateFileDummyBuildlink3("category/lib1/buildlink3.mk") + t.CreateFileLines("category/lib1/builtin.mk", + MkCvsID, + "VAR_FROM_BUILTIN=\t# defined") t.FinishSetUp() - G.Check(".") + G.Check(t.File("category/package")) - t.CheckOutputEmpty() + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:23: Please use \"${ECHO}\" instead of \"echo\".", + "WARN: ~/category/package/Makefile:23: OTHER_VAR is used but not defined.") } -func (s *Suite) Test_Package_checkIncludeConditionally__other_directory(c *check.C) { +// Ensures that the paths in Package.included are indeed relative to the +// package directory. This hadn't been the case until March 2019. +func (s *Suite) Test_Package_parse__included(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package", - ".include \"../../category/package-base/including.mk\"") - t.CreateFileLines("category/package-base/including.mk", + ".include \"../../devel/library/buildlink3.mk\"", + ".include \"../../lang/language/module.mk\"") + t.SetUpPackage("devel/library") + t.CreateFileDummyBuildlink3("devel/library/buildlink3.mk") + t.CreateFileLines("devel/library/builtin.mk", + MkCvsID) + t.CreateFileLines("lang/language/module.mk", MkCvsID, - "", - ".include \"included.mk\"", - ".if ${OPSYS} == \"Linux\"", - ".include \"included.mk\"", - ".endif", - "", - ".include \"included.mk\"", - ".if ${OPSYS} == \"Linux\"", - ".include \"included.mk\"", - ".endif") - t.CreateFileLines("category/package-base/included.mk", + ".include \"version.mk\"") + t.CreateFileLines("lang/language/version.mk", MkCvsID) + t.FinishSetUp() + t.Chdir("category/package") + pkg := NewPackage(".") - t.Main("-Wall", "-Call", "category/package") + pkg.included.Trace = true + pkg.loadPackageMakefile() - // TODO: Understand why ../../category/package-base/including.mk is - // not checked for (un)conditional includes. t.CheckOutputLines( - "Looks fine.") + "FirstTime: suppress-varorder.mk", + "FirstTime: ../../devel/library/buildlink3.mk", + "FirstTime: ../../devel/library/builtin.mk", + "FirstTime: ../../lang/language/module.mk", + "FirstTime: ../../lang/language/version.mk") } -// See https://github.com/rillig/pkglint/issues/1 -func (s *Suite) Test_Package__include_without_exists(c *check.C) { +func (s *Suite) Test_Package_parse__include_Makefile_common_same_directory(c *check.C) { t := s.Init(c) + t.SetUpPackage("category/dependency") + t.CreateFileLines("category/dependency/Makefile.common", + MkCvsID, + "#", + "#") t.SetUpPackage("category/package", - ".include \"options.mk\"") + ".include \"../../category/dependency/Makefile.common\"", + ".include \"Makefile.common\"") + t.CreateFileLines("category/package/Makefile.common", + MkCvsID, + "#", + "#") t.FinishSetUp() - G.checkdirPackage(t.File("category/package")) + G.Check(t.File("category/package")) t.CheckOutputLines( - "ERROR: ~/category/package/Makefile:20: Cannot read \"options.mk\".") + "WARN: ~/category/dependency/Makefile.common:1: " + + "Please add a line \"# used by category/package/Makefile\" here.") } -// See https://github.com/rillig/pkglint/issues/1 -func (s *Suite) Test_Package__include_after_exists(c *check.C) { +func (s *Suite) Test_Package_parse__include_Makefile_common_explicit(c *check.C) { t := s.Init(c) + t.SetUpPackage("category/dependency") + t.CreateFileLines("category/dependency/Makefile.common", + MkCvsID, + "#", + "#") t.SetUpPackage("category/package", - ".if exists(options.mk)", - ". include \"options.mk\"", - ".endif") + ".include \"../../category/dependency/Makefile.common\"", + ".include \"../../category/package/Makefile.common\"") + t.CreateFileLines("category/package/Makefile.common", + MkCvsID, + "#", + "#") t.FinishSetUp() - G.checkdirPackage(t.File("category/package")) + G.Check(t.File("category/package")) - // No error message at all because of the .if exists before. - t.CheckOutputEmpty() + t.CheckOutputLines( + "WARN: ~/category/dependency/Makefile.common:1: " + + "Please add a line \"# used by category/package/Makefile\" here.") } -// See https://github.com/rillig/pkglint/issues/1 -func (s *Suite) Test_Package_parse__include_other_after_exists(c *check.C) { +func (s *Suite) Test_Package_parse__fallback_lookup_in_package_directory(c *check.C) { t := s.Init(c) + t.CreateFileLines("mk/pthread.buildlink3.mk", + MkCvsID, + ".include \"../../mk/pthread.builtin.mk\"") + t.CreateFileLines("mk/pthread.builtin.mk", + MkCvsID) t.SetUpPackage("category/package", - ".if exists(options.mk)", - ". include \"another.mk\"", - ".endif") + ".include \"../../mk/pthread.buildlink3.mk\"") t.FinishSetUp() - G.checkdirPackage(t.File("category/package")) + G.Check(t.File("category/package")) t.CheckOutputLines( - "ERROR: ~/category/package/Makefile:21: Cannot read \"another.mk\".") + "NOTE: ~/mk/pthread.buildlink3.mk:2: " + + "The path to the included file should be \"pthread.builtin.mk\".") } -// See https://mail-index.netbsd.org/tech-pkg/2018/07/22/msg020092.html -func (s *Suite) Test_Package__redundant_master_sites(c *check.C) { +// Just for code coverage. +func (s *Suite) Test_Package_resolveIncludedFile__no_tracing(c *check.C) { t := s.Init(c) - t.SetUpPkgsrc() - t.SetUpMasterSite("MASTER_SITE_R_CRAN", "http://cran.r-project.org/src/") - t.CreateFileLines("math/R/Makefile.extension", - MkCvsID, - "", - "PKGNAME?=\tR-${R_PKGNAME}-${R_PKGVER}", - "MASTER_SITES?=\t${MASTER_SITE_R_CRAN:=contrib/}", - "GENERATE_PLIST+=\techo \"bin/r-package\";", - "NO_CHECKSUM=\tyes", - "LICENSE?=\tgnu-gpl-v2") - t.CreateFileLines("math/R-date/Makefile", - MkCvsID, - "", - "R_PKGNAME=\tdate", - "R_PKGVER=\t1.2.3", - "COMMENT=\tR package for handling date arithmetic", - "MASTER_SITES=\t${MASTER_SITE_R_CRAN:=contrib/}", // Redundant; see math/R/Makefile.extension. - "", - ".include \"../../math/R/Makefile.extension\"", - ".include \"../../mk/bsd.pkg.mk\"") + t.SetUpPackage("category/package", + ".include \"../../mk/${UNKNOWN_PKGPATH}.mk\"", + ".include \"../../${UNKNOWN_PKGPATH}/buildlink3.mk\"", + ".include \"../../lang/language/buildlink3.mk\"") + t.CreateFileLines("lang/language/buildlink3.mk", + MkCvsID) t.FinishSetUp() + pkg := NewPackage(t.File("category/package")) + t.DisableTracing() - // See Package.checkfilePackageMakefile - G.checkdirPackage(t.File("math/R-date")) + pkg.included.Trace = true + pkg.loadPackageMakefile() - // The definition in Makefile:6 is redundant because the same definition - // occurs later in Makefile.extension:4. - // - // When a file includes another file, it's always the including file that - // is marked as redundant since the included file typically provides the - // generally useful value for several packages; - // see RedundantScope.handleVarassign, keyword includePath. t.CheckOutputLines( - "NOTE: ~/math/R-date/Makefile:6: " + - "Definition of MASTER_SITES is redundant " + - "because of ../../math/R/Makefile.extension:4.") + "FirstTime: suppress-varorder.mk", + "FirstTime: ../../lang/language/buildlink3.mk", + "FirstTime: ../../lang/language/builtin.mk") } -func (s *Suite) Test_Package_checkUpdate(c *check.C) { +func (s *Suite) Test_Package_resolveIncludedFile__skipping(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/pkg1", - "PKGNAME= package1-1.0") - t.SetUpPackage("category/pkg2", - "PKGNAME= package2-1.0") - t.SetUpPackage("category/pkg3", - "PKGNAME= package3-5.0") - t.CreateFileLines("doc/TODO", - "Suggested package updates", - "", - "", - "\t"+"O wrong bullet", - "\t"+"o package-without-version", - "\t"+"o package1-1.0", - "\t"+"o package2-2.0 [nice new features]", - "\t"+"o package3-3.0 [security update]") + t.SetUpPackage("category/package", + ".include \"../../mk/known.mk\"", + ".include \"../../${UNKNOWN_PKGPATH}/buildlink3.mk\"", + ".include \"../../lang/language/buildlink3.mk\"") + t.CreateFileLines("mk/known.mk", + MkCvsID, + ".include \"${UNKNOWN}.mk\"") + t.CreateFileLines("lang/language/buildlink3.mk", + MkCvsID) + t.FinishSetUp() + pkg := NewPackage(t.File("category/package")) + + t.EnableTracingToLog() + pkg.loadPackageMakefile() + + // The trace log does not contain the message that mk/known.mk includes + // a file that is skipped. This is because most package authors are not + // involved in the pkgsrc infrastructure, therefore there's no point in + // logging anything about these files. + t.CheckOutputLinesMatching(`.*Skipping.*`, + "TRACE: 1 2 ~/category/package/Makefile:21: "+ + "Skipping unresolvable include file \"../../${UNKNOWN_PKGPATH}/buildlink3.mk\".") +} + +func (s *Suite) Test_Package_shouldDiveInto(c *check.C) { + t := s.Init(c) t.Chdir(".") - t.Main("-Wall,no-space", "category/pkg1", "category/pkg2", "category/pkg3") + test := func(including, included string, expected bool) { + actual := (*Package)(nil).shouldDiveInto(including, included) + t.CheckEquals(actual, expected) + } - t.CheckOutputLines( - "WARN: category/pkg1/../../doc/TODO:3: Invalid line format \"\".", - "WARN: category/pkg1/../../doc/TODO:4: Invalid line format \"\\tO wrong bullet\".", - "WARN: category/pkg1/../../doc/TODO:5: Invalid package name \"package-without-version\".", - "NOTE: category/pkg1/Makefile:4: The update request to 1.0 from doc/TODO has been done.", - "WARN: category/pkg2/Makefile:4: This package should be updated to 2.0 ([nice new features]).", - "NOTE: category/pkg3/Makefile:4: This package is newer than the update request to 3.0 ([security update]).", - "4 warnings and 2 notes found.", - "(Run \"pkglint -e -Wall,no-space category/pkg1 category/pkg2 category/pkg3\" to show explanations.)") + // The variables that appear in these files are largely modeled by + // pkglint in the file vardefs.go. Therefore parsing these files again + // doesn't make much sense. + test("Makefile", "../../mk/bsd.pkg.mk", false) + test("Makefile", "../../mk/bsd.prefs.mk", false) + test("Makefile", "../../mk/bsd.fast.prefs.mk", false) + + // All files that are included from outside of the pkgsrc infrastructure + // are relevant. This is typically mk/compiler.mk or the various + // mk/*.buildlink3.mk files. + test("Makefile", "Makefile.common", true) + test("Makefile", "../../mk/compiler.mk", true) + + // The mk/*.buildlink3.mk files often come with a companion file called + // mk/*.builtin.mk, which also defines variables that are visible from + // the package. + // + // This case is needed for getting the redundancy check right. Without it + // there will be warnings about redundant assignments to the + // BUILTIN_CHECK.pthread variable. + test("pthread.buildlink3.mk", "pthread.builtin.mk", true) + test("../../mk/pthread.buildlink3.mk", "pthread.builtin.mk", true) + test("../../mk/pthread.buildlink3.mk", "../../mk/pthread.builtin.mk", true) + + // Files other than the companion builtin.mk are ignored. + test("../../mk/pthread.buildlink3.mk", "pthread.internals.mk", false) + + // Files that are included from within the pkgsrc infrastructure are not + // interesting since their content is largely modeled by pkglint in the + // file vardefs.go. + test("../../mk/one.mk", "two.mk", false) + test("../../mk/one.mk", "../../mk/two.mk", false) + test("../../mk/one.mk", "../lang/go/version.mk", false) + + // wip/mk doesn't count as infrastructure since it is often used as a + // second layer, using the API of the main mk/ infrastructure. + test("wip/mk/cargo-binary.mk", "../../lang/rust/cargo.mk", true) } -func (s *Suite) Test_NewPackage(c *check.C) { +func (s *Suite) Test_Package_collectSeenInclude__builtin_mk(c *check.C) { t := s.Init(c) - t.SetUpPkgsrc() - t.CreateFileLines("category/Makefile", + t.SetUpPackage("category/package", + ".include \"builtin.mk\"") + t.CreateFileLines("category/package/builtin.mk", MkCvsID) t.FinishSetUp() - t.ExpectAssert(func() { NewPackage("category") }) + pkg := NewPackage(t.File("category/package")) + pkg.load() + + t.CheckEquals(pkg.seenInclude, true) } -// Before 2018-09-09, the .CURDIR variable did not have a fallback value. -// When resolving the relative path x11/gst-x11/${.CURDIR}/../../multimedia/gst-base/distinfo, -// "gst-x11/${.CURDIR}" was interpreted as "category/package", and the whole -// path was resolved to "x11/multimedia/gst-base/distinfo, which of course -// could not be found. -func (s *Suite) Test__distinfo_from_other_package(c *check.C) { +func (s *Suite) Test_Package_collectSeenInclude__multiple(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall,no-space") - t.SetUpPkgsrc() - t.Chdir(".") - t.CreateFileLines("x11/gst-x11/Makefile", - MkCvsID, - ".include \"../../multimedia/gst-base/Makefile.common\"", - ".include \"../../mk/bsd.pkg.mk\"") - t.CreateFileLines("multimedia/gst-base/Makefile.common", - MkCvsID, - ".include \"plugins.mk\"") - t.CreateFileLines("multimedia/gst-base/plugins.mk", - MkCvsID, - "DISTINFO_FILE=\t${.CURDIR}/../../multimedia/gst-base/distinfo") - t.CreateFileLines("multimedia/gst-base/distinfo", - CvsID, - "", - "SHA1 (patch-aa) = 1234") + t.SetUpPackage("category/package", + ".include \"001.mk\"", + ".include \"002.mk\"") + t.CreateFileLines("category/package/001.mk", + MkCvsID) + t.CreateFileLines("category/package/002.mk", + MkCvsID) t.FinishSetUp() - G.Check("x11/gst-x11") + t.EnableTracingToLog() + G.Check(t.File("category/package")) + t.EnableSilentTracing() + + // TODO: It's not necessary to trace this message three times. + t.CheckOutputLinesMatching(`^TRACE: .*seenInclude`, + "TRACE: 1 2 3 4 Including \"suppress-varorder.mk\" sets seenInclude.", + "TRACE: 1 2 3 4 Including \"001.mk\" sets seenInclude.", + "TRACE: 1 2 3 4 Including \"002.mk\" sets seenInclude.") +} + +// Just ensure that pkglint doesn't crash. +func (s *Suite) Test_Package_loadPlistDirs__empty(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/PLIST.common", + nil...) + t.FinishSetUp() + + pkg := NewPackage(t.File("category/package")) + pkg.load() + + var dirs []string + for dir := range pkg.Plist.Dirs { + dirs = append(dirs, dir) + } + sort.Strings(dirs) + + t.CheckDeepEquals(dirs, []string{"bin"}) +} + +func (s *Suite) Test_Package_loadPlistDirs(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/PLIST.common", + PlistCvsID, + "@exec echo hello", + "${PLIST.condition}dir/subdir/file", + "@unexec echo bye") + t.FinishSetUp() + + pkg := NewPackage(t.File("category/package")) + pkg.load() + + var dirs []string + for dir := range pkg.Plist.Dirs { + dirs = append(dirs, dir) + } + sort.Strings(dirs) + + t.CheckDeepEquals(dirs, []string{"bin", "dir", "dir/subdir"}) +} + +func (s *Suite) Test_Package_check__files_Makefile(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/files/Makefile", + "This file may contain anything.") + t.Main("category/package/files/Makefile") + + // Since there is nothing to check in files/*, pkglint could + // as well report this as a usage error. + // + // Until June 2019, checking individual files in FILESDIR had + // been enabled by the -Call command line option. t.CheckOutputLines( - "WARN: x11/gst-x11/Makefile: This package should have a PLIST file.", - "ERROR: x11/gst-x11/Makefile: Each package must define its LICENSE.", - "WARN: x11/gst-x11/Makefile: Each package should define a COMMENT.", - "WARN: x11/gst-x11/../../multimedia/gst-base/distinfo:3: Patch file \"patch-aa\" does not exist in directory \"../../x11/gst-x11/patches\".") + "Looks fine.") + + t.Main("category/package") + + t.CheckOutputLines( + "Looks fine.") +} + +func (s *Suite) Test_Package_check__patches_Makefile(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/patches/Makefile", + "This file may contain anything.") + + t.Main("category/package") + + t.CheckOutputLines( + "WARN: ~/category/package/patches/Makefile: Patch files should be "+ + "named \"patch-\", followed by letters, '-', '_', '.', and digits only.", + "1 warning found.") } func (s *Suite) Test_Package_checkfilePackageMakefile__GNU_CONFIGURE(c *check.C) { @@ -1897,6 +1361,563 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__distfiles(c *check.C) { "A package that downloads files should have a distinfo file.") } +// When a package defines PLIST_SRC, it may or may not use the +// PLIST file from the package directory. Therefore the check +// is skipped completely. +func (s *Suite) Test_Package_checkPlist__PLIST_SRC(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "PLIST_SRC=\t${WRKDIR}/PLIST") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkPlist__META_PACKAGE(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "META_PACKAGE=\tyes") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:20: This package should not have a PLIST file.", + "WARN: ~/category/package/distinfo: This file should not exist "+ + "since NO_CHECKSUM or META_PACKAGE is set.") +} + +func (s *Suite) Test_Package_checkPlist__Perl5_packlist(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/p5-Packlist", + "PERL5_PACKLIST=\tauto/Packlist/.packlist") + t.FinishSetUp() + + G.Check(t.File("category/p5-Packlist")) + + t.CheckOutputLines( + "WARN: ~/category/p5-Packlist/Makefile:20: This package should not have a PLIST file.") +} + +func (s *Suite) Test_Package_checkPlist__PERL5_USE_PACKLIST_no(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/p5-NoPacklist", + "PERL5_USE_PACKLIST=\tno") + t.FinishSetUp() + + G.Check(t.File("category/p5-NoPacklist")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkPlist__PERL5_USE_PACKLIST_yes(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/p5-Packlist", + "PERL5_USE_PACKLIST=\tyes") + t.FinishSetUp() + + G.Check(t.File("category/p5-Packlist")) + + t.CheckOutputLines( + "WARN: ~/category/p5-Packlist/Makefile:20: This package should not have a PLIST file.") +} + +func (s *Suite) Test_Package_CheckVarorder__only_required_variables(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("x11/9term")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "DISTNAME=9term", + "CATEGORIES=x11", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + pkg.CheckVarorder(mklines) + + t.CheckOutputLines( + "WARN: Makefile:3: The canonical order of the variables is " + + "DISTNAME, CATEGORIES, empty line, COMMENT, LICENSE.") +} + +func (s *Suite) Test_Package_CheckVarorder__with_optional_variables(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("x11/9term")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "GITHUB_PROJECT=project", + "DISTNAME=9term", + "CATEGORIES=x11") + + pkg.CheckVarorder(mklines) + + // TODO: Make this warning more specific to the actual situation. + t.CheckOutputLines( + "WARN: Makefile:3: The canonical order of the variables is " + + "GITHUB_PROJECT, DISTNAME, CATEGORIES, GITHUB_PROJECT, empty line, " + + "COMMENT, LICENSE.") +} + +// Just for code coverage. +func (s *Suite) Test_Package_CheckVarorder__no_tracing(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("x11/9term")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "DISTNAME=9term", + "CATEGORIES=x11", + "", + ".include \"../../mk/bsd.pkg.mk\"") + t.DisableTracing() + + pkg.CheckVarorder(mklines) + + t.CheckOutputLines( + "WARN: Makefile:3: The canonical order of the variables is " + + "DISTNAME, CATEGORIES, empty line, COMMENT, LICENSE.") +} + +// Ensure that comments and empty lines do not lead to panics. +// This would be when accessing fields from the MkLine without checking the line type before. +func (s *Suite) Test_Package_CheckVarorder__comments_do_not_crash(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("x11/9term")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "GITHUB_PROJECT=project", + "", + "# comment", + "", + "DISTNAME=9term", + "# comment", + "CATEGORIES=x11") + + pkg.CheckVarorder(mklines) + + t.CheckOutputLines( + "WARN: Makefile:3: The canonical order of the variables is " + + "GITHUB_PROJECT, DISTNAME, CATEGORIES, GITHUB_PROJECT, empty line, " + + "COMMENT, LICENSE.") +} + +func (s *Suite) Test_Package_CheckVarorder__comments_are_ignored(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("x11/9term")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "DISTNAME=\tdistname-1.0", + "CATEGORIES=\tsysutils", + "", + "MAINTAINER=\tpkgsrc-users@NetBSD.org", + "# comment", + "COMMENT=\tComment", + "LICENSE=\tgnu-gpl-v2") + + pkg.CheckVarorder(mklines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_CheckVarorder__commented_variable_assignment(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("x11/9term")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "DISTNAME=\tdistname-1.0", + "CATEGORIES=\tsysutils", + "", + "MAINTAINER=\tpkgsrc-users@NetBSD.org", + "#HOMEPAGE=\thttps://example.org/", + "COMMENT=\tComment", + "LICENSE=\tgnu-gpl-v2") + + pkg.CheckVarorder(mklines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_CheckVarorder__skip_because_of_foreign_variable(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("x11/9term")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "DISTNAME=\tdistname-1.0", + "USE_TOOLS+=gmake", + "CATEGORIES=\tsysutils", + "", + "MAINTAINER=\tpkgsrc-users@NetBSD.org", + "#HOMEPAGE=\thttps://example.org/", + "COMMENT=\tComment", + "LICENSE=\tgnu-gpl-v2") + + t.EnableTracingToLog() + pkg.CheckVarorder(mklines) + + t.CheckOutputLinesMatching(`.*varorder.*`, + "TRACE: 1 Skipping varorder because of line 4.") +} + +func (s *Suite) Test_Package_CheckVarorder__skip_if_there_are_directives(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("category/package")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "DISTNAME=\tdistname-1.0", + "CATEGORIES=\tsysutils", + "", + ".if ${DISTNAME:Mdistname-*}", + "MAINTAINER=\tpkgsrc-users@NetBSD.org", + ".endif", + "LICENSE=\tgnu-gpl-v2") + + pkg.CheckVarorder(mklines) + + // No warning about the missing COMMENT since the .if directive + // causes the whole check to be skipped. + t.CheckOutputEmpty() + + // Just for code coverage. + t.DisableTracing() + pkg.CheckVarorder(mklines) + t.CheckOutputEmpty() +} + +// TODO: Add more tests like skip_if_there_are_directives for other line types. + +func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_top(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("x11/9term")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "GITHUB_PROJECT=\t\tautocutsel", + "DISTNAME=\t\tautocutsel-0.10.0", + "CATEGORIES=\t\tx11", + "MASTER_SITES=\t\t${MASTER_SITE_GITHUB:=sigmike/}", + "GITHUB_TAG=\t\t${PKGVERSION_NOREV}", + "", + "COMMENT=\tComment", + "LICENSE=\tgnu-gpl-v2") + + pkg.CheckVarorder(mklines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_bottom(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("x11/9term")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "DISTNAME=\t\tautocutsel-0.10.0", + "CATEGORIES=\t\tx11", + "MASTER_SITES=\t\t${MASTER_SITE_GITHUB:=sigmike/}", + "GITHUB_PROJECT=\t\tautocutsel", + "GITHUB_TAG=\t\t${PKGVERSION_NOREV}", + "", + "COMMENT=\tComment", + "LICENSE=\tgnu-gpl-v2") + + pkg.CheckVarorder(mklines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("mk/bsd.pkg.mk", "# dummy") + t.CreateFileLines("x11/Makefile", MkCvsID) + t.CreateFileLines("x11/9term/PLIST", PlistCvsID, "bin/9term") + t.CreateFileLines("x11/9term/Makefile", + MkCvsID, + "", + "DISTNAME=\t9term-1.0", + "CATEGORIES=\tx11", + "", + "COMMENT=\tTerminal", + "", + "NO_CHECKSUM=\tyes", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + t.SetUpVartypes() + + G.Check(t.File("x11/9term")) + + // Since the error is grave enough, the warning about the correct position is suppressed. + // TODO: Knowing the correct position helps, though. + t.CheckOutputLines( + "ERROR: ~/x11/9term/Makefile: Each package must define its LICENSE.") +} + +// https://mail-index.netbsd.org/tech-pkg/2017/01/18/msg017698.html +func (s *Suite) Test_Package_CheckVarorder__MASTER_SITES(c *check.C) { + t := s.Init(c) + + pkg := NewPackage(t.File("category/package")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "PKGNAME=\tpackage-1.0", + "CATEGORIES=\tcategory", + "MASTER_SITES=\thttp://example.org/", + "MASTER_SITES+=\thttp://mirror.example.org/", + "", + "COMMENT=\tComment", + "LICENSE=\tgnu-gpl-v2") + + pkg.CheckVarorder(mklines) + + // No warning that "MASTER_SITES appears too late" + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_CheckVarorder__diagnostics(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + pkg := NewPackage(t.File("category/package")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "CATEGORIES= net", + "", + "COMMENT= Comment", + "LICENSE= gnu-gpl-v3", + "", + "GITHUB_PROJECT= pkgbase", + "DISTNAME= v1.0", + "PKGNAME= ${GITHUB_PROJECT}-${DISTNAME}", + "MASTER_SITES= ${MASTER_SITE_GITHUB:=project/}", + "DIST_SUBDIR= ${GITHUB_PROJECT}", + "", + "MAINTAINER= maintainer@example.org", + "HOMEPAGE= https://github.com/project/pkgbase/", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + pkg.CheckVarorder(mklines) + + t.CheckOutputLines( + "WARN: Makefile:3: The canonical order of the variables is " + + "GITHUB_PROJECT, DISTNAME, PKGNAME, CATEGORIES, " + + "MASTER_SITES, GITHUB_PROJECT, DIST_SUBDIR, empty line, " + + "MAINTAINER, HOMEPAGE, COMMENT, LICENSE.") + + // After moving the variables according to the warning: + mklines = t.NewMkLines("Makefile", + MkCvsID, + "", + "GITHUB_PROJECT= pkgbase", + "DISTNAME= v1.0", + "PKGNAME= ${GITHUB_PROJECT}-${DISTNAME}", + "CATEGORIES= net", + "MASTER_SITES= ${MASTER_SITE_GITHUB:=project/}", + "DIST_SUBDIR= ${GITHUB_PROJECT}", + "", + "MAINTAINER= maintainer@example.org", + "HOMEPAGE= https://github.com/project/pkgbase/", + "COMMENT= Comment", + "LICENSE= gnu-gpl-v3", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + pkg.CheckVarorder(mklines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_CheckVarorder__comment_at_end_of_section(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + pkg := NewPackage(t.File("category/package")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "CATEGORIES= net", + "SITES.*= # none", + "# comment after the last variable of a section", + "", + "MAINTAINER= maintainer@example.org", + "HOMEPAGE= https://github.com/project/pkgbase/", + "COMMENT= Comment", + "LICENSE= gnu-gpl-v3", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + t.EnableTracingToLog() + pkg.CheckVarorder(mklines) + + // The varorder code is not skipped, not even because of the comment + // after SITES.*. + t.CheckOutputLinesMatching(`.*varorder.*`, + nil...) +} + +func (s *Suite) Test_Package_CheckVarorder__comments_between_sections(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + pkg := NewPackage(t.File("category/package")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "CATEGORIES= net", + "", + "# comment 1", + "", + "# comment 2", + "", + "MAINTAINER= maintainer@example.org", + "HOMEPAGE= https://github.com/project/pkgbase/", + "COMMENT= Comment", + "LICENSE= gnu-gpl-v3", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + pkg.CheckVarorder(mklines) + + // The empty line between the comments is not treated as a section separator. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_CheckVarorder__commented_varassign(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + pkg := NewPackage(t.File("category/package")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "CATEGORIES= net", + "#MASTER_SITES= # none", + "", + "HOMEPAGE= https://github.com/project/pkgbase/", + "#HOMEPAGE= https://github.com/project/pkgbase/", + "#HOMEPAGE= https://github.com/project/pkgbase/", + "#HOMEPAGE= https://github.com/project/pkgbase/", + "#HOMEPAGE= https://github.com/project/pkgbase/", + "LICENSE= gnu-gpl-v3", + "COMMENT= Comment", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + pkg.CheckVarorder(mklines) + + // The order of the variables LICENSE and COMMENT is intentionally + // wrong to force the warning. + // + // Up to June 2019 (308099138a62) pkglint mentioned in the warning + // each commented variable assignment, even repeatedly for the same + // variable name. + // + // These variable assignments should be in the correct order, even + // if they are commented out. It's not necessary though to list a + // variable more than once. + t.CheckOutputLines( + "WARN: Makefile:3: The canonical order of the variables is " + + "CATEGORIES, MASTER_SITES, empty line, HOMEPAGE, COMMENT, LICENSE.") +} + +func (s *Suite) Test_Package_CheckVarorder__DEPENDS(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + pkg := NewPackage(t.File("category/package")) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "CATEGORIES= net", + "", + "COMMENT= Comment", + "LICENSE= license", + "MAINTAINER= maintainer@example.org", // In wrong order + "", + "DEPENDS+= dependency>=1.0:../../category/dependency", + "", + ".include \"../../mk/bsd.pkg.mk\"") + + pkg.CheckVarorder(mklines) + + t.CheckOutputLines( + "WARN: Makefile:3: The canonical order of the variables is " + + "CATEGORIES, empty line, MAINTAINER, COMMENT, LICENSE, empty line, DEPENDS.") +} + +func (s *Suite) Test_Package_checkCategories__redundant(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "CATEGORIES=\tcategory perl5", + ".include \"included.mk\"") + t.CreateFileLines("category/package/included.mk", + MkCvsID, + "CATEGORIES+=\tperl5 python", + "CATEGORIES+=\tpython", + "CATEGORIES?=\tcategory japanese") + t.Chdir("category/package") + t.FinishSetUp() + + G.Check(".") + + t.CheckOutputLines( + // TODO: Warn in the including file, not in the included file, just as in RedundantScope. + "NOTE: included.mk:2: Category \"perl5\" is already added in Makefile:5.", + "NOTE: included.mk:3: Category \"python\" is already added in line 2.") +} + +func (s *Suite) Test_Package_checkCategories__redundant_but_not_constant(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "CATEGORIES=\tcategory", + ".include \"included.mk\"") + t.CreateFileLines("category/package/included.mk", + MkCvsID, + "CATEGORIES+=\tperl5 python", + "CATEGORIES+=\tpython", + "CATEGORIES?=\tcategory japanese", + "", + ".if 1", + "CATEGORIES+=\tchinese", + ".endif") + t.Chdir("category/package") + t.FinishSetUp() + + G.Check(".") + + // No diagnostics at all, because CATEGORIES is not constant, + // as "chinese" may or may not be added. + t.CheckOutputEmpty() +} + func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__no_C(c *check.C) { t := s.Init(c) @@ -2039,50 +2060,6 @@ func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__not_constant_2(c *ch t.CheckOutputEmpty() } -func (s *Suite) Test_Package_loadPlistDirs(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package") - t.CreateFileLines("category/package/PLIST.common", - PlistCvsID, - "@exec echo hello", - "${PLIST.condition}dir/subdir/file", - "@unexec echo bye") - t.FinishSetUp() - - pkg := NewPackage(t.File("category/package")) - pkg.load() - - var dirs []string - for dir := range pkg.Plist.Dirs { - dirs = append(dirs, dir) - } - sort.Strings(dirs) - - t.CheckDeepEquals(dirs, []string{"bin", "dir", "dir/subdir"}) -} - -// Just ensure that pkglint doesn't crash. -func (s *Suite) Test_Package_loadPlistDirs__empty(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package") - t.CreateFileLines("category/package/PLIST.common", - nil...) - t.FinishSetUp() - - pkg := NewPackage(t.File("category/package")) - pkg.load() - - var dirs []string - for dir := range pkg.Plist.Dirs { - dirs = append(dirs, dir) - } - sort.Strings(dirs) - - t.CheckDeepEquals(dirs, []string{"bin"}) -} - func (s *Suite) Test_Package_checkUseLanguagesCompilerMk__too_late(c *check.C) { t := s.Init(c) @@ -2145,484 +2122,433 @@ func (s *Suite) Test_Package_checkUseLanguagesCompilerMk__endian_mk(c *check.C) "Modifying USE_LANGUAGES after including ../../mk/compiler.mk has no effect.") } -func (s *Suite) Test_Package_parse__simple(c *check.C) { +// PKGNAME is stronger than DISTNAME. +func (s *Suite) Test_Package_determineEffectivePkgVars__precedence(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") - t.Chdir("category/package") - t.FinishSetUp() + pkg := NewPackage(t.File("category/pkgbase")) + pkgnameLine := t.NewMkLine("Makefile", 3, "PKGNAME=pkgname-1.0") + distnameLine := t.NewMkLine("Makefile", 4, "DISTNAME=distname-1.0") + pkgrevisionLine := t.NewMkLine("Makefile", 5, "PKGREVISION=13") - G.Pkg = NewPackage(".") - G.Pkg.included.Trace = true - G.Pkg.load() + pkg.vars.Define(pkgnameLine.Varname(), pkgnameLine) + pkg.vars.Define(distnameLine.Varname(), distnameLine) + pkg.vars.Define(pkgrevisionLine.Varname(), pkgrevisionLine) - t.CheckOutputLines( - "FirstTime: suppress-varorder.mk") + pkg.determineEffectivePkgVars() + + t.CheckEquals(pkg.EffectivePkgbase, "pkgname") + t.CheckEquals(pkg.EffectivePkgname, "pkgname-1.0nb13") + t.CheckEquals(pkg.EffectivePkgversion, "1.0") } -func (s *Suite) Test_Package_parse__nonexistent_Makefile(c *check.C) { +func (s *Suite) Test_Package_determineEffectivePkgVars__same(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") - t.Chdir("category/package") - t.Remove("Makefile") + pkg := t.SetUpPackage("category/package", + "DISTNAME=\tdistname-1.0", + "PKGNAME=\tdistname-1.0") t.FinishSetUp() - G.Pkg = NewPackage(".") - G.Pkg.included.Trace = true - G.Pkg.load() + G.Check(pkg) t.CheckOutputLines( - "ERROR: Makefile: Cannot be read.") + "NOTE: ~/category/package/Makefile:4: " + + "This assignment is probably redundant since PKGNAME is ${DISTNAME} by default.") } -func (s *Suite) Test_Package_parse__include_in_same_directory(c *check.C) { +func (s *Suite) Test_Package_determineEffectivePkgVars__simple_reference(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - ".include \"version.mk\"") - t.Chdir("category/package") - t.CreateFileLines("version.mk", - MkCvsID) + pkg := t.SetUpPackage("category/package", + "DISTNAME=\tdistname-1.0", + "PKGNAME=\t${DISTNAME}") t.FinishSetUp() - G.Pkg = NewPackage(".") - G.Pkg.included.Trace = true - G.Pkg.load() + G.Check(pkg) t.CheckOutputLines( - "FirstTime: suppress-varorder.mk", - "FirstTime: version.mk") + "NOTE: ~/category/package/Makefile:4: " + + "This assignment is probably redundant since PKGNAME is ${DISTNAME} by default.") } -func (s *Suite) Test_Package_parse__nonexistent_include(c *check.C) { +func (s *Suite) Test_Package_determineEffectivePkgVars__commented(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - ".include \"version.mk\"") - t.Chdir("category/package") + pkg := t.SetUpPackage("category/package", + "DISTNAME=\tdistname-1.0", + "PKGNAME=\t${DISTNAME} # intentionally") t.FinishSetUp() - G.Pkg = NewPackage(".") - G.Pkg.included.Trace = true - G.Pkg.load() + G.Check(pkg) - t.CheckOutputLines( - "FirstTime: suppress-varorder.mk", - "FirstTime: version.mk", - "ERROR: Makefile:20: Cannot read \"version.mk\".") + t.CheckOutputEmpty() } -// When reading the package Makefile, pkglint loads and interprets each -// file only once. This is especially important for packages with a large -// dependency graph containing many common subdependencies. -func (s *Suite) Test_Package_parse__include_twice(c *check.C) { +func (s *Suite) Test_Package_determineEffectivePkgVars__invalid_DISTNAME(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - ".include \"version.mk\"", - ".include \"version.mk\"") - t.Chdir("category/package") - t.CreateFileLines("version.mk", - MkCvsID) + pkg := t.SetUpPackage("category/package", + "DISTNAME=\tpkgname-version") t.FinishSetUp() - G.Pkg = NewPackage(".") - G.Pkg.included.Trace = true - G.Pkg.load() + G.Check(pkg) t.CheckOutputLines( - "FirstTime: suppress-varorder.mk", - "FirstTime: version.mk") + "WARN: ~/category/package/Makefile:3: " + + "As DISTNAME is not a valid package name, please define the PKGNAME explicitly.") } -func (s *Suite) Test_Package_parse__include_in_other_directory(c *check.C) { +func (s *Suite) Test_Package_determineEffectivePkgVars__indirect_DISTNAME(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - ".include \"../../category/other/version.mk\"") - t.Chdir("category/package") - t.CreateFileLines("../../category/other/version.mk", - MkCvsID) + pkg := t.SetUpPackage("category/package", + "DISTNAME=\t${DISTFILES:[1]:C,\\..*,,}") t.FinishSetUp() - G.Pkg = NewPackage(".") - G.Pkg.included.Trace = true - G.Pkg.load() + G.Check(pkg) - t.CheckOutputLines( - "FirstTime: suppress-varorder.mk", - "FirstTime: ../../category/other/version.mk") + // No warning since the case of DISTNAME being dependent on another + // variable is too difficult to analyze. + t.CheckOutputEmpty() } -// Demonstrates that Package.included contains the file paths of the -// included files, relative to the package directory. -func (s *Suite) Test_Package_parse__includes_in_other_directory(c *check.C) { +func (s *Suite) Test_Package_determineEffectivePkgVars__C_modifier(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - ".include \"../../category/other/module.mk\"") - t.Chdir("category/package") - t.CreateFileLines("../../category/other/module.mk", - MkCvsID, - ".include \"version.mk\"") - t.CreateFileLines("../../category/other/version.mk", - MkCvsID) + t.SetUpPackage("x11/p5-gtk2", + "DISTNAME=\tGtk2-1.0", + "PKGNAME=\t${DISTNAME:C:Gtk2:p5-gtk2:}") t.FinishSetUp() + pkg := NewPackage(t.File("x11/p5-gtk2")) + files, mklines, allLines := pkg.load() - G.Pkg = NewPackage(".") - G.Pkg.included.Trace = true - G.Pkg.load() + pkg.check(files, mklines, allLines) - t.CheckOutputLines( - "FirstTime: suppress-varorder.mk", - "FirstTime: ../../category/other/module.mk", - "FirstTime: ../../category/other/version.mk") + t.CheckEquals(pkg.EffectivePkgname, "p5-gtk2-1.0") } -func (s *Suite) Test_Package_parse__nonexistent_in_other_directory(c *check.C) { +// In some cases the PKGNAME is derived from DISTNAME, and it seems as +// if the :C modifier would not affect anything. This may nevertheless +// be on purpose since the modifier may apply to future versions and +// do things like replacing a "-1" with a ".1". +func (s *Suite) Test_Package_determineEffectivePkgVars__ineffective_C_modifier(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package", - ".include \"../../category/other/module.mk\"") - t.Chdir("category/package") - t.CreateFileLines("../../category/other/module.mk", - MkCvsID, - ".include \"version.mk\"") + "DISTNAME=\tdistname-1.0", + "PKGNAME=\t${DISTNAME:C:does_not_match:replacement:}") t.FinishSetUp() + pkg := NewPackage(t.File("category/package")) + files, mklines, allLines := pkg.load() - G.Pkg = NewPackage(".") - G.Pkg.included.Trace = true - G.Pkg.load() + pkg.check(files, mklines, allLines) - t.CheckOutputLines( - "FirstTime: suppress-varorder.mk", - "FirstTime: ../../category/other/module.mk", - "FirstTime: ../../category/other/version.mk", - "ERROR: ../../category/other/module.mk:2: Cannot read \"version.mk\".") + t.CheckEquals(pkg.EffectivePkgname, "distname-1.0") + t.CheckOutputEmpty() } -func (s *Suite) Test_Package_parse__skipping(c *check.C) { +func (s *Suite) Test_Package_determineEffectivePkgVars__Python_prefix(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall,no-space") - pkg := t.SetUpPackage("category/package", - ".include \"${MYSQL_PKGSRCDIR:S/-client$/-server/}/buildlink3.mk\"") - t.FinishSetUp() - - t.EnableTracingToLog() - G.Check(pkg) - t.EnableSilentTracing() - - // Since 2018-12-16 there is no warning or note anymore for the - // buildlink3.mk file being skipped since it didn't help the average - // pkglint user. - - // The information is still available in the trace log though. + G.Experimental = true + t.SetUpPackage("category/package", + "PKGNAME=\tpackage-2.0", + ".include \"../../lang/python/extension.mk\"") + t.CreateFileLines("lang/python/extension.mk", + MkCvsID) - output := t.Output() - var relevant []string - for _, line := range strings.Split(output, "\n") { - if contains(line, "Skipping") { - relevant = append(relevant, line) - } - } + t.Main("-Wall", "category/package") - t.CheckDeepEquals(relevant, []string{ - "TRACE: 1 2 3 4 ~/category/package/Makefile:20: " + - "Skipping unresolvable include file \"${MYSQL_PKGSRCDIR:S/-client$/-server/}/buildlink3.mk\"."}) + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:4: The PKGNAME of Python extensions should start with ${PYPKGPREFIX}.", + "1 warning found.") } -func (s *Suite) Test_Package_parse__not_found(c *check.C) { +func (s *Suite) Test_Package_determineEffectivePkgVars__Python_prefix_PKGNAME_variable(c *check.C) { t := s.Init(c) - pkg := t.SetUpPackage("category/package", - ".include \"../../devel/zlib/buildlink3.mk\"") - t.CreateFileLines("devel/zlib/buildlink3.mk", - ".include \"../../enoent/enoent/buildlink3.mk\"") - t.FinishSetUp() + G.Experimental = true + t.SetUpPackage("category/package", + "PKGNAME=\t${VAR}-package-2.0", + ".include \"../../lang/python/extension.mk\"") + t.CreateFileLines("lang/python/extension.mk", + MkCvsID, + "VAR=\tvalue") - G.checkdirPackage(pkg) + t.Main("-Wall", "category/package") + // Since PKGNAME starts with a variable, pkglint doesn't investigate + // further what the possible value of this variable could be. If it + // did, it would see that the prefix is not PYPKGPREFIX and would + // complain. t.CheckOutputLines( - "ERROR: ~/devel/zlib/buildlink3.mk:1: Cannot read \"../../enoent/enoent/buildlink3.mk\".") + "Looks fine.") } -func (s *Suite) Test_Package_parse__relative(c *check.C) { +// As of August 2019, pkglint loads the package files in alphabetical order. +// This means that the package Makefile is loaded early, and includes by +// other files may be invisible yet. This applies to both Makefile.* and to +// *.mk since both of these appear later. +// +// The effects of these files are nevertheless visible at the right time +// because the package Makefile is loaded including all its included files. +func (s *Suite) Test_Package_determineEffectivePkgVars__Python_prefix_late(c *check.C) { t := s.Init(c) - t.CreateFileLines("category/package/extra.mk", + G.Experimental = true + t.SetUpPackage("category/package", + "PKGNAME=\tpackage-2.0", + ".include \"common.mk\"") + t.CreateFileLines("category/package/common.mk", + MkCvsID, + ".include \"../../lang/python/extension.mk\"") + t.CreateFileLines("lang/python/extension.mk", MkCvsID) - pkg := t.SetUpPackage("category/package", - ".include \"../package/extra.mk\"") - t.FinishSetUp() - G.Check(pkg) + t.Main("-Wall", "category/package") t.CheckOutputLines( - "WARN: ~/category/package/Makefile:20: " + - "References to other packages should look " + - "like \"../../category/package\", not \"../package\".") + "WARN: ~/category/package/Makefile:4: "+ + "The PKGNAME of Python extensions should start with ${PYPKGPREFIX}.", + "1 warning found.") } -// When a buildlink3.mk file is included, the corresponding builtin.mk -// file is included by the pkgsrc infrastructure. Therefore all variables -// declared in the builtin.mk file become known in the package. -func (s *Suite) Test_Package_parse__builtin_mk(c *check.C) { +func (s *Suite) Test_Package_nbPart(c *check.C) { t := s.Init(c) - t.SetUpTool("echo", "ECHO", AtRunTime) - t.SetUpPackage("category/package", - ".include \"../../category/lib1/buildlink3.mk\"", - "", - "show-var-from-builtin: .PHONY", - "\techo ${VAR_FROM_BUILTIN} ${OTHER_VAR}") - t.CreateFileDummyBuildlink3("category/lib1/buildlink3.mk") - t.CreateFileLines("category/lib1/builtin.mk", - MkCvsID, - "VAR_FROM_BUILTIN=\t# defined") - t.FinishSetUp() + pkg := NewPackage(t.File("category/pkgbase")) + pkg.vars.Define("PKGREVISION", t.NewMkLine("Makefile", 1, "PKGREVISION=14")) - G.Check(t.File("category/package")) + t.CheckEquals(pkg.nbPart(), "nb14") - t.CheckOutputLines( - "WARN: ~/category/package/Makefile:23: Please use \"${ECHO}\" instead of \"echo\".", - "WARN: ~/category/package/Makefile:23: OTHER_VAR is used but not defined.") + pkg.vars = NewScope() + pkg.vars.Define("PKGREVISION", t.NewMkLine("Makefile", 1, "PKGREVISION=asdf")) + + t.CheckEquals(pkg.nbPart(), "") } -// Ensures that the paths in Package.included are indeed relative to the -// package directory. This hadn't been the case until March 2019. -func (s *Suite) Test_Package_parse__included(c *check.C) { +func (s *Suite) Test_Package_pkgnameFromDistname(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - ".include \"../../devel/library/buildlink3.mk\"", - ".include \"../../lang/language/module.mk\"") - t.SetUpPackage("devel/library") - t.CreateFileDummyBuildlink3("devel/library/buildlink3.mk") - t.CreateFileLines("devel/library/builtin.mk", - MkCvsID) - t.CreateFileLines("lang/language/module.mk", - MkCvsID, - ".include \"version.mk\"") - t.CreateFileLines("lang/language/version.mk", - MkCvsID) - t.FinishSetUp() - t.Chdir("category/package") - pkg := NewPackage(".") + var once Once + test := func(pkgname, distname, expectedPkgname string, diagnostics ...string) { + t.SetUpPackage("category/package", + "PKGNAME=\t"+pkgname, + "DISTNAME=\t"+distname) + if once.FirstTime("called") { + t.FinishSetUp() + } - pkg.included.Trace = true - pkg.loadPackageMakefile() + pkg := NewPackage(t.File("category/package")) + pkg.loadPackageMakefile() + pkg.determineEffectivePkgVars() + t.CheckEquals(pkg.EffectivePkgname, expectedPkgname) + t.CheckOutput(diagnostics) + } - t.CheckOutputLines( - "FirstTime: suppress-varorder.mk", - "FirstTime: ../../devel/library/buildlink3.mk", - "FirstTime: ../../devel/library/builtin.mk", - "FirstTime: ../../lang/language/module.mk", - "FirstTime: ../../lang/language/version.mk") -} + test("pkgname-1.0", "whatever", "pkgname-1.0") -func (s *Suite) Test_Package_parse__include_Makefile_common_same_directory(c *check.C) { - t := s.Init(c) + test("${DISTNAME}", "distname-1.0", "distname-1.0", + "NOTE: ~/category/package/Makefile:4: This assignment is probably redundant since PKGNAME is ${DISTNAME} by default.") - t.SetUpPackage("category/dependency") - t.CreateFileLines("category/dependency/Makefile.common", - MkCvsID, - "#", - "#") - t.SetUpPackage("category/package", - ".include \"../../category/dependency/Makefile.common\"", - ".include \"Makefile.common\"") - t.CreateFileLines("category/package/Makefile.common", - MkCvsID, - "#", - "#") - t.FinishSetUp() + test("${DISTNAME:S/dist/pkg/}", "distname-1.0", "pkgname-1.0") - G.Check(t.File("category/package")) + test("${DISTNAME:S|a|b|g}", "panama-0.13", "pbnbmb-0.13") - t.CheckOutputLines( - "WARN: ~/category/dependency/Makefile.common:1: " + - "Please add a line \"# used by category/package/Makefile\" here.") -} + // The substitution succeeds, but the substituted value is missing + // the package version. Therefore it is discarded completely. + test("${DISTNAME:S|^lib||}", "libncurses", "") -func (s *Suite) Test_Package_parse__include_Makefile_common_explicit(c *check.C) { - t := s.Init(c) + // The substitution succeeds, but the substituted value is missing + // the package version. Therefore it is discarded completely. + test("${DISTNAME:S|^lib||}", "mylib", "") - t.SetUpPackage("category/dependency") - t.CreateFileLines("category/dependency/Makefile.common", - MkCvsID, - "#", - "#") - t.SetUpPackage("category/package", - ".include \"../../category/dependency/Makefile.common\"", - ".include \"../../category/package/Makefile.common\"") - t.CreateFileLines("category/package/Makefile.common", - MkCvsID, - "#", - "#") - t.FinishSetUp() + test("${DISTNAME:tl:S/-/./g:S/he/-/1}", "SaxonHE9-5-0-1J", "saxon-9.5.0.1j") - G.Check(t.File("category/package")) + test("${DISTNAME:C/beta/.0./}", "fspanel-0.8beta1", "fspanel-0.8.0.1") - t.CheckOutputLines( - "WARN: ~/category/dependency/Makefile.common:1: " + - "Please add a line \"# used by category/package/Makefile\" here.") + test("${DISTNAME:C/Gtk2/p5-gtk2/}", "Gtk2-1.0", "p5-gtk2-1.0") + + test("${DISTNAME:S/-0$/.0/1}", "aspell-af-0.50-0", "aspell-af-0.50.0") + + test("${DISTNAME:M*.tar.gz:C,\\..*,,}", "aspell-af-0.50-0", "") + + test("${DISTNAME:S,a,b,c,d}", "aspell-af-0.50-0", "bspell-af-0.50-0", + "WARN: ~/category/package/Makefile:4: Invalid variable modifier \"c,d\" for \"DISTNAME\".") + + test("${DISTFILE:C,\\..*,,}", "aspell-af-0.50-0", "") } -func (s *Suite) Test_Package_parse__fallback_lookup_in_package_directory(c *check.C) { +func (s *Suite) Test_Package_checkPossibleDowngrade(c *check.C) { t := s.Init(c) - t.CreateFileLines("mk/pthread.buildlink3.mk", - MkCvsID, - ".include \"../../mk/pthread.builtin.mk\"") - t.CreateFileLines("mk/pthread.builtin.mk", - MkCvsID) - t.SetUpPackage("category/package", - ".include \"../../mk/pthread.buildlink3.mk\"") - t.FinishSetUp() + t.CreateFileLines("doc/CHANGES-2018", + "\tUpdated category/pkgbase to 1.8 [committer 2018-01-05]") + G.Pkgsrc.loadDocChanges() - G.Check(t.File("category/package")) + t.Chdir("category/pkgbase") + G.Pkg = NewPackage(".") + G.Pkg.EffectivePkgname = "package-1.0nb15" + G.Pkg.EffectivePkgnameLine = t.NewMkLine("Makefile", 5, "PKGNAME=dummy") + + G.Pkg.checkPossibleDowngrade() t.CheckOutputLines( - "NOTE: ~/mk/pthread.buildlink3.mk:2: " + - "The path to the included file should be \"pthread.builtin.mk\".") + "WARN: Makefile:5: The package is being downgraded from 1.8 (see ../../doc/CHANGES-2018:1) to 1.0nb15.") + + G.Pkgsrc.LastChange["category/pkgbase"].target = "1.0nb22" + + G.Pkg.checkPossibleDowngrade() + + t.CheckOutputEmpty() } -func (s *Suite) Test_Package_collectSeenInclude__builtin_mk(c *check.C) { +func (s *Suite) Test_Package_checkPossibleDowngrade__moved(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - ".include \"builtin.mk\"") - t.CreateFileLines("category/package/builtin.mk", - MkCvsID) + t.SetUpPackage("category/pkgbase", + "PKGNAME=\tpackage-1.0") + t.CreateFileLines("doc/CHANGES-2018", + "\tUpdated category/old-package to 1.8 [committer 2018-01-05]", + "\tMoved category/old-package to category/pkgbase [committer 2018-01-05]") t.FinishSetUp() - pkg := NewPackage(t.File("category/package")) + pkg := NewPackage(t.File("category/pkgbase")) pkg.load() + pkg.determineEffectivePkgVars() + pkg.checkPossibleDowngrade() - t.CheckEquals(pkg.seenInclude, true) + t.CheckEquals(G.Pkgsrc.LastChange["category/pkgbase"].Action, Moved) + // No warning because the latest action is not Updated. + t.CheckOutputEmpty() } -func (s *Suite) Test_Package_diveInto(c *check.C) { +func (s *Suite) Test_Package_checkPossibleDowngrade__locally_modified_update(c *check.C) { t := s.Init(c) - t.Chdir(".") - test := func(including, included string, expected bool) { - actual := (*Package)(nil).diveInto(including, included) - t.CheckEquals(actual, expected) - } + t.SetUpPackage("category/package", + "PKGNAME=\tpackage-1.8") + t.CreateFileLines("doc/CHANGES-2018", + "\tUpdated category/package to 1.0 [committer 2018-01-05]") + t.CreateFileLines("category/package/CVS/Entries", + "/Makefile//modified//") + t.FinishSetUp() - // The variables that appear in these files are largely modeled by - // pkglint in the file vardefs.go. Therefore parsing these files again - // doesn't make much sense. - test("Makefile", "../../mk/bsd.pkg.mk", false) - test("Makefile", "../../mk/bsd.prefs.mk", false) - test("Makefile", "../../mk/bsd.fast.prefs.mk", false) + G.Check(t.File("category/package")) - // All files that are included from outside of the pkgsrc infrastructure - // are relevant. This is typically mk/compiler.mk or the various - // mk/*.buildlink3.mk files. - test("Makefile", "Makefile.common", true) - test("Makefile", "../../mk/compiler.mk", true) + // Since the Makefile is locally modified, pkglint doesn't issue + // any warning since it assumes the package is being upgraded. + t.CheckOutputEmpty() - // The mk/*.buildlink3.mk files often come with a companion file called - // mk/*.builtin.mk, which also defines variables that are visible from - // the package. - // - // This case is needed for getting the redundancy check right. Without it - // there will be warnings about redundant assignments to the - // BUILTIN_CHECK.pthread variable. - test("pthread.buildlink3.mk", "pthread.builtin.mk", true) - test("../../mk/pthread.buildlink3.mk", "pthread.builtin.mk", true) - test("../../mk/pthread.buildlink3.mk", "../../mk/pthread.builtin.mk", true) + // When the Makefile is no longer locally modified, the warning + // is activated again. + t.Remove("category/package/CVS/Entries") + G.cvsEntriesDir = "" - // Files other than the companion builtin.mk are ignored. - test("../../mk/pthread.buildlink3.mk", "pthread.internals.mk", false) + G.Check(t.File("category/package")) - // Files that are included from within the pkgsrc infrastructure are not - // interesting since their content is largely modeled by pkglint in the - // file vardefs.go. - test("../../mk/one.mk", "two.mk", false) - test("../../mk/one.mk", "../../mk/two.mk", false) - test("../../mk/one.mk", "../lang/go/version.mk", false) + t.CheckOutputLines( + "NOTE: ~/category/package/Makefile:4: Package version \"1.8\" " + + "is greater than the latest \"1.0\" from ../../doc/CHANGES-2018:1.") +} - // wip/mk doesn't count as infrastructure since it is often used as a - // second layer, using the API of the main mk/ infrastructure. - test("wip/mk/cargo-binary.mk", "../../lang/rust/cargo.mk", true) +func (s *Suite) Test_Package_checkUpdate(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/pkg1", + "PKGNAME= package1-1.0") + t.SetUpPackage("category/pkg2", + "PKGNAME= package2-1.0") + t.SetUpPackage("category/pkg3", + "PKGNAME= package3-5.0") + t.CreateFileLines("doc/TODO", + "Suggested package updates", + "", + "", + "\t"+"O wrong bullet", + "\t"+"o package-without-version", + "\t"+"o package1-1.0", + "\t"+"o package2-2.0 [nice new features]", + "\t"+"o package3-3.0 [security update]") + t.Chdir(".") + + t.Main("-Wall,no-space", "category/pkg1", "category/pkg2", "category/pkg3") + + t.CheckOutputLines( + "WARN: category/pkg1/../../doc/TODO:3: Invalid line format \"\".", + "WARN: category/pkg1/../../doc/TODO:4: Invalid line format \"\\tO wrong bullet\".", + "WARN: category/pkg1/../../doc/TODO:5: Invalid package name \"package-without-version\".", + "NOTE: category/pkg1/Makefile:4: The update request to 1.0 from doc/TODO has been done.", + "WARN: category/pkg2/Makefile:4: This package should be updated to 2.0 ([nice new features]).", + "NOTE: category/pkg3/Makefile:4: This package is newer than the update request to 3.0 ([security update]).", + "4 warnings and 2 notes found.", + "(Run \"pkglint -e -Wall,no-space category/pkg1 category/pkg2 category/pkg3\" to show explanations.)") } -func (s *Suite) Test_Package_collectSeenInclude__multiple(c *check.C) { +func (s *Suite) Test_Package_checkDirent__errors(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - ".include \"001.mk\"", - ".include \"002.mk\"") - t.CreateFileLines("category/package/001.mk", - MkCvsID) - t.CreateFileLines("category/package/002.mk", - MkCvsID) + t.SetUpCommandLine("-Call", "-Wall,no-space") + t.SetUpPkgsrc() + t.CreateFileLines("category/package/files/subdir/file") + t.CreateFileLines("category/package/files/subdir/subsub/file") t.FinishSetUp() - t.EnableTracingToLog() - G.Check(t.File("category/package")) - t.EnableSilentTracing() + pkg := NewPackage(t.File("category/package")) + pkg.checkDirent(t.File("category/package/options.mk"), 0444) + pkg.checkDirent(t.File("category/package/files/subdir"), 0555|os.ModeDir) + pkg.checkDirent(t.File("category/package/files/subdir/subsub"), 0555|os.ModeDir) + pkg.checkDirent(t.File("category/package/files"), 0555|os.ModeDir) - // TODO: It's not necessary to trace this message three times. - t.CheckOutputLinesMatching(`^TRACE: .*seenInclude`, - "TRACE: 1 2 3 4 Including \"suppress-varorder.mk\" sets seenInclude.", - "TRACE: 1 2 3 4 Including \"001.mk\" sets seenInclude.", - "TRACE: 1 2 3 4 Including \"002.mk\" sets seenInclude.") + t.CheckOutputLines( + "ERROR: ~/category/package/options.mk: Cannot be read.", + "WARN: ~/category/package/files/subdir/subsub: Unknown directory name.") } -// Just for code coverage. -func (s *Suite) Test_Package_resolveIncludedFile__no_tracing(c *check.C) { +func (s *Suite) Test_Package_checkDirent__file_selection(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - ".include \"../../mk/${UNKNOWN_PKGPATH}.mk\"", - ".include \"../../${UNKNOWN_PKGPATH}/buildlink3.mk\"", - ".include \"../../lang/language/buildlink3.mk\"") - t.CreateFileLines("lang/language/buildlink3.mk", + t.SetUpCommandLine("-Call", "-Wall,no-space") + t.SetUpPkgsrc() + t.CreateFileLines("doc/CHANGES-2018", + CvsID) + t.CreateFileLines("category/package/buildlink3.mk", MkCvsID) + t.CreateFileLines("category/package/unexpected.txt", + CvsID) t.FinishSetUp() - pkg := NewPackage(t.File("category/package")) - t.DisableTracing() - pkg.included.Trace = true - pkg.loadPackageMakefile() + pkg := NewPackage(t.File("category/package")) + pkg.checkDirent(t.File("doc/CHANGES-2018"), 0444) + pkg.checkDirent(t.File("category/package/buildlink3.mk"), 0444) + pkg.checkDirent(t.File("category/package/unexpected.txt"), 0444) t.CheckOutputLines( - "FirstTime: suppress-varorder.mk", - "FirstTime: ../../lang/language/buildlink3.mk", - "FirstTime: ../../lang/language/builtin.mk") + "WARN: ~/category/package/buildlink3.mk:EOF: Expected a BUILDLINK_TREE line.", + "WARN: ~/category/package/unexpected.txt: Unexpected file found.") } -func (s *Suite) Test_Package_resolveIncludedFile__skipping(c *check.C) { +// Since all required information is passed to G.checkDirent via parameters, +// this test produces the expected results even though none of these files actually exists. +func (s *Suite) Test_Package_checkDirent__skipped(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - ".include \"../../mk/known.mk\"", - ".include \"../../${UNKNOWN_PKGPATH}/buildlink3.mk\"", - ".include \"../../lang/language/buildlink3.mk\"") - t.CreateFileLines("mk/known.mk", - MkCvsID, - ".include \"${UNKNOWN}.mk\"") - t.CreateFileLines("lang/language/buildlink3.mk", - MkCvsID) + t.SetUpPackage("category/package") t.FinishSetUp() - pkg := NewPackage(t.File("category/package")) + t.Chdir("category/package") + pkg := NewPackage(".") - t.EnableTracingToLog() - pkg.loadPackageMakefile() + pkg.checkDirent("work", os.ModeSymlink) + pkg.checkDirent("work.i386", os.ModeSymlink) + pkg.checkDirent("work.hostname", os.ModeSymlink) + pkg.checkDirent("other", os.ModeSymlink) - // The trace log does not contain the message that mk/known.mk includes - // a file that is skipped. This is because most package authors are not - // involved in the pkgsrc infrastructure, therefore there's no point in - // logging anything about these files. - t.CheckOutputLinesMatching(`.*Skipping.*`, - "TRACE: 1 2 ~/category/package/Makefile:21: "+ - "Skipping unresolvable include file \"../../${UNKNOWN_PKGPATH}/buildlink3.mk\".") + pkg.checkDirent("device", os.ModeDevice) + + t.CheckOutputLines( + "WARN: other: Invalid symlink name.", + "ERROR: device: Only files and directories are allowed in pkgsrc.") } // In packages without specific MAINTAINER, everyone may commit changes. @@ -2829,243 +2755,384 @@ func (s *Suite) Test_Package_checkFreeze__freeze_ended(c *check.C) { t.CheckOutputEmpty() } -// In practice the distinfo file can always be autofixed since it has -// just been read successfully and the corresponding patch file could -// also be autofixed right before. -func (s *Suite) Test_Package_AutofixDistinfo__missing_file(c *check.C) { +func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__file_but_not_package(c *check.C) { t := s.Init(c) - t.SetUpPkgsrc() + t.CreateFileLines("category/dependency/buildlink3.mk") + t.CreateFileLines("category/dependency/module.mk") G.Pkg = NewPackage(t.File("category/package")) - t.FinishSetUp() + mklines := t.NewMkLines("category/package/buildlink3.mk", + MkCvsID, + "", + ".include \"../../category/dependency/buildlink3.mk\"", + ".include \"../../category/dependency/module.mk\"") - G.Pkg.AutofixDistinfo("old", "new") + G.Pkg.checkLinesBuildlink3Inclusion(mklines) t.CheckOutputLines( - "ERROR: ~/category/package/distinfo: Cannot be read.") + "WARN: category/package/buildlink3.mk:3: " + + "../../category/dependency/buildlink3.mk is included by this file " + + "but not by the package.") } -func (s *Suite) Test_Package__using_common_Makefile_overriding_DISTINFO_FILE(c *check.C) { +// Several files from the pkgsrc infrastructure are named *.buildlink3.mk, +// even though they don't follow the typical file format for buildlink3.mk +// files. Therefore they are ignored by this check. +func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__infra_buildlink_file(c *check.C) { t := s.Init(c) - t.SetUpPackage("security/pinentry") - t.CreateFileLines("security/pinentry/Makefile.common", - MkCvsID, - "DISTINFO_FILE=\t${.CURDIR}/../../security/pinentry/distinfo") - t.SetUpPackage("security/pinentry-fltk", - ".include \"../../security/pinentry/Makefile.common\"", - "DISTINFO_FILE=\t${.CURDIR}/distinfo") - t.CreateFileDummyPatch("security/pinentry-fltk/patches/patch-aa") - t.CreateFileLines("security/pinentry-fltk/distinfo", - CvsID, - "", - "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") - t.FinishSetUp() + t.SetUpPackage("category/package", + ".include \"../../mk/motif.buildlink3.mk\"") + t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", + ".include \"../../mk/motif.buildlink3.mk\"") + t.CreateFileLines("mk/motif.buildlink3.mk", + MkCvsID) - G.Check(t.File("security/pinentry")) + t.Main("--quiet", "-Wall", "category/package") t.CheckOutputEmpty() +} - G.Check(t.File("security/pinentry-fltk")) +func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__package_but_not_file(c *check.C) { + t := s.Init(c) - // The DISTINFO_FILE definition from pinentry-fltk overrides - // the one from pinentry since it appears later. - // Therefore the patch is searched for at the right location. - t.CheckOutputEmpty() + t.CreateFileLines("category/dependency/buildlink3.mk") + G.Pkg = NewPackage(t.File("category/package")) + G.Pkg.bl3["../../category/dependency/buildlink3.mk"] = + t.NewMkLine("../../category/dependency/buildlink3.mk", 1, "") + mklines := t.NewMkLines("category/package/buildlink3.mk", + MkCvsID) + + t.EnableTracingToLog() + G.Pkg.checkLinesBuildlink3Inclusion(mklines) + + // This is only traced but not logged as a regular warning since + // several packages have build dependencies that are not needed + // for building other packages. These cannot be flagged as warnings. + t.CheckOutputLines( + "TRACE: + (*Package).checkLinesBuildlink3Inclusion()", + "TRACE: 1 ../../category/dependency/buildlink3.mk "+ + "is included by the package but not by the buildlink3.mk file.", + "TRACE: - (*Package).checkLinesBuildlink3Inclusion()") } -func (s *Suite) Test_Package__redundant_variable_in_unrelated_files(c *check.C) { +// Just for code coverage. +func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__no_tracing(c *check.C) { t := s.Init(c) - t.SetUpPackage("databases/py-trytond-ldap-authentication", - ".include \"../../devel/py-trytond/Makefile.common\"", - ".include \"../../lang/python/egg.mk\"") - t.CreateFileLines("devel/py-trytond/Makefile.common", - MkCvsID, - "PY_PATCHPLIST=\tyes") - t.CreateFileLines("lang/python/egg.mk", - MkCvsID, - "PY_PATCHPLIST=\tyes") + t.SetUpPackage("category/package") + t.CreateFileDummyBuildlink3("category/package/buildlink3.mk") t.FinishSetUp() - G.Check(t.File("databases/py-trytond-ldap-authentication")) + t.DisableTracing() + G.Check(t.File("category/package")) - // Since egg.mk and Makefile.common are unrelated, the definition of - // PY_PATCHPLIST is not redundant in these files. t.CheckOutputEmpty() } -// Pkglint loads some files from the pkgsrc infrastructure and skips others. -// -// When a buildlink3.mk file from the infrastructure is included, it should -// be allowed to include its corresponding builtin.mk file in turn. -// -// This is necessary to load the correct variable assignments for the -// redundancy check, in particular variable assignments that serve as -// arguments to "procedure calls", such as mk/find-files.mk. -func (s *Suite) Test_Package_parse__include_infrastructure(c *check.C) { +func (s *Suite) Test_Package_checkIncludeConditionally__conditional_and_unconditional_include(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("--dumpmakefile") + t.SetUpOption("zlib", "") t.SetUpPackage("category/package", - ".include \"../../mk/dlopen.buildlink3.mk\"", - ".include \"../../mk/pthread.buildlink3.mk\"") - t.CreateFileLines("mk/dlopen.buildlink3.mk", - ".include \"dlopen.builtin.mk\"") - t.CreateFileLines("mk/dlopen.builtin.mk", - ".include \"pthread.builtin.mk\"") - t.CreateFileLines("mk/pthread.buildlink3.mk", - ".include \"pthread.builtin.mk\"") - t.CreateFileLines("mk/pthread.builtin.mk", - "# This should be included by pthread.buildlink3.mk") + ".include \"../../devel/zlib/buildlink3.mk\"", + ".if ${OPSYS} == \"Linux\"", + ".include \"../../sysutils/coreutils/buildlink3.mk\"", + ".endif") + t.CreateFileLines("mk/bsd.options.mk", "") + t.CreateFileLines("devel/zlib/buildlink3.mk", "") + t.CreateFileLines("sysutils/coreutils/buildlink3.mk", "") + + t.CreateFileLines("category/package/options.mk", + MkCvsID, + "", + "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", + "PKG_SUPPORTED_OPTIONS=\tzlib", + "", + ".include \"../../mk/bsd.options.mk\"", + "", + ".if !empty(PKG_OPTIONS:Mzlib)", + ". include \"../../devel/zlib/buildlink3.mk\"", + ".endif", + ".include \"../../sysutils/coreutils/buildlink3.mk\"") + t.Chdir("category/package") t.FinishSetUp() - G.Check(t.File("category/package")) + G.checkdirPackage(".") t.CheckOutputLines( - "Whole Makefile (with all included files) follows:", - "~/category/package/Makefile:1: "+MkCvsID, - "~/category/package/Makefile:2: ", - "~/category/package/Makefile:3: DISTNAME=\tpackage-1.0", - "~/category/package/Makefile:4: #PKGNAME=\tpackage-1.0", - "~/category/package/Makefile:5: CATEGORIES=\tcategory", - "~/category/package/Makefile:6: MASTER_SITES=\t# none", - "~/category/package/Makefile:7: ", - "~/category/package/Makefile:8: MAINTAINER=\tpkgsrc-users@NetBSD.org", - "~/category/package/Makefile:9: HOMEPAGE=\t# none", - "~/category/package/Makefile:10: COMMENT=\tDummy package", - "~/category/package/Makefile:11: LICENSE=\t2-clause-bsd", - "~/category/package/Makefile:12: ", - "~/category/package/Makefile:13: .include \"suppress-varorder.mk\"", - "~/category/package/suppress-varorder.mk:1: "+MkCvsID, - "~/category/package/Makefile:14: ", - "~/category/package/Makefile:15: # filler", - "~/category/package/Makefile:16: # filler", - "~/category/package/Makefile:17: # filler", - "~/category/package/Makefile:18: # filler", - "~/category/package/Makefile:19: ", - "~/category/package/Makefile:20: .include \"../../mk/dlopen.buildlink3.mk\"", - "~/category/package/../../mk/dlopen.buildlink3.mk:1: .include \"dlopen.builtin.mk\"", - "~/mk/dlopen.builtin.mk:1: .include \"pthread.builtin.mk\"", - "~/category/package/Makefile:21: .include \"../../mk/pthread.buildlink3.mk\"", - "~/category/package/../../mk/pthread.buildlink3.mk:1: .include \"pthread.builtin.mk\"", - "~/mk/pthread.builtin.mk:1: # This should be included by pthread.buildlink3.mk", - "~/category/package/Makefile:22: ", - "~/category/package/Makefile:23: .include \"../../mk/bsd.pkg.mk\"") + "WARN: Makefile:20: \"../../devel/zlib/buildlink3.mk\" is included "+ + "unconditionally here "+ + "and conditionally in options.mk:9 (depending on PKG_OPTIONS).", + "WARN: Makefile:22: \"../../sysutils/coreutils/buildlink3.mk\" is included "+ + "conditionally here (depending on OPSYS) and "+ + "unconditionally in options.mk:11.") } -// As of April 2019, there are only a few files in the whole pkgsrc tree -// that are called Makefile.*, except Makefile.common, which occurs more -// often. -// -// Using the file extension for variants of that Makefile is confusing, -// therefore they should be renamed to *.mk. -func (s *Suite) Test_Package__Makefile_files(c *check.C) { +func (s *Suite) Test_Package_checkIncludeConditionally__explain_PKG_OPTIONS_in_Makefile(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") - t.CreateFileLines("category/package/Makefile.common", + t.SetUpCommandLine("-Wall", "--explain") + t.SetUpOption("zlib", "use zlib compression") + + t.CreateFileLines("mk/bsd.options.mk", MkCvsID) - t.CreateFileLines("category/package/Makefile.orig", + t.CreateFileLines("devel/zlib/buildlink3.mk", MkCvsID) - t.CreateFileLines("category/package/Makefile.php", + t.SetUpPackage("category/package", + "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", + "PKG_SUPPORTED_OPTIONS=\tzlib", + "", + ".include \"../../mk/bsd.options.mk\"", + "", + ".if ${PKG_OPTIONS:Mzlib}", + ".include \"../../devel/zlib/buildlink3.mk\"", + ".endif") + t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", + ".include \"../../devel/zlib/buildlink3.mk\"") + t.Chdir("category/package") + t.FinishSetUp() + + G.checkdirPackage(".") + + t.CheckOutputLines( + "WARN: Makefile:26: "+ + "\"../../devel/zlib/buildlink3.mk\" is included conditionally here "+ + "(depending on PKG_OPTIONS) and unconditionally in buildlink3.mk:12.", + "", + "\tWhen including a dependent file, the conditions in the buildlink3.mk", + "\tfile should be the same as in options.mk or the Makefile.", + "", + "\tTo find out the PKG_OPTIONS of this package at build time, have a", + "\tlook at mk/pkg-build-options.mk.", + "") +} + +func (s *Suite) Test_Package_checkIncludeConditionally__no_explanation(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "--explain") + t.CreateFileLines("devel/zlib/buildlink3.mk", MkCvsID) - t.CreateFileLines("category/package/ext.mk", + t.SetUpPackage("category/package", + ".if ${OPSYS} == Linux", + ".include \"../../devel/zlib/buildlink3.mk\"", + ".endif") + t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", + ".include \"../../devel/zlib/buildlink3.mk\"") + t.Chdir("category/package") + t.FinishSetUp() + + G.checkdirPackage(".") + + t.CheckOutputLines( + "WARN: Makefile:21: " + + "\"../../devel/zlib/buildlink3.mk\" is included conditionally here " + + "(depending on OPSYS) and unconditionally in buildlink3.mk:12.") +} + +func (s *Suite) Test_Package_checkIncludeConditionally__explain_PKG_OPTIONS_in_options_mk(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "--explain") + t.SetUpOption("zlib", "use zlib compression") + + t.CreateFileLines("mk/bsd.options.mk", MkCvsID) + t.CreateFileLines("devel/zlib/buildlink3.mk", + MkCvsID) + t.SetUpPackage("category/package", + ".include \"options.mk\"") + t.CreateFileLines("category/package/options.mk", + MkCvsID, + "", + "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", + "PKG_SUPPORTED_OPTIONS=\tzlib", + "", + ".include \"../../mk/bsd.options.mk\"", + "", + ".if ${PKG_OPTIONS:Mzlib}", + ".include \"../../devel/zlib/buildlink3.mk\"", + ".endif") + t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", + ".include \"../../devel/zlib/buildlink3.mk\"") + t.Chdir("category/package") t.FinishSetUp() - G.Check(t.File("category/package")) + G.checkdirPackage(".") - // No warning for the Makefile.orig since the package is not - // being imported at the moment; see Pkglint.checkReg. t.CheckOutputLines( - "NOTE: ~/category/package/Makefile.php: " + - "Consider renaming \"Makefile.php\" to \"php.mk\".") + "WARN: buildlink3.mk:12: "+ + "\"../../devel/zlib/buildlink3.mk\" is included unconditionally here "+ + "and conditionally in options.mk:9 (depending on PKG_OPTIONS).", + "", + "\tWhen including a dependent file, the conditions in the buildlink3.mk", + "\tfile should be the same as in options.mk or the Makefile.", + "", + "\tTo find out the PKG_OPTIONS of this package at build time, have a", + "\tlook at mk/pkg-build-options.mk.", + "") } -func (s *Suite) Test_Package__patch_in_FILESDIR(c *check.C) { +func (s *Suite) Test_Package_checkIncludeConditionally__unconditionally_first(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Wall", "-Call") t.SetUpPackage("category/package") - t.CreateFileLines("category/package/files/patch-aa", - "This file can contain anything, no matter what the filename says.") + t.Chdir("category/package") + t.CreateFileLines("including.mk", + MkCvsID, + "", + ".include \"included.mk\"", + ".if ${OPSYS} == \"Linux\"", + ".include \"included.mk\"", + ".endif") + t.CreateFileLines("included.mk", + MkCvsID) t.FinishSetUp() - G.Check(t.File("category/package")) + G.Check(".") - // No warnings. The files in FILESDIR are independent of pkgsrc - // and may contain anything. There are no naming conventions or - // anything else. - t.CheckOutputEmpty() + t.CheckOutputLines( + "WARN: including.mk:3: \"included.mk\" is included " + + "unconditionally here and conditionally in line 5 (depending on OPSYS).") } -// When a package defines PLIST_SRC, it may or may not use the -// PLIST file from the package directory. Therefore the check -// is skipped completely. -func (s *Suite) Test_Package_checkPlist__PLIST_SRC(c *check.C) { +func (s *Suite) Test_Package_checkIncludeConditionally__only_conditionally(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package", - "PLIST_SRC=\t${WRKDIR}/PLIST") + ".if ${OPSYS} == \"Linux\"", + ".include \"included.mk\"", + ".endif") + t.Chdir("category/package") + t.CreateFileLines("included.mk", + MkCvsID) t.FinishSetUp() - G.Check(t.File("category/package")) + G.Check(".") t.CheckOutputEmpty() } -func (s *Suite) Test_Package_checkPlist__META_PACKAGE(c *check.C) { +func (s *Suite) Test_Package_checkIncludeConditionally__conditionally_first(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "META_PACKAGE=\tyes") + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.CreateFileLines("including.mk", + MkCvsID, + "", + ".if ${OPSYS} == \"Linux\"", + ".include \"included.mk\"", + ".endif", + ".include \"included.mk\"") + t.CreateFileLines("included.mk", + MkCvsID) t.FinishSetUp() - G.Check(t.File("category/package")) + G.Check(".") t.CheckOutputLines( - "WARN: ~/category/package/Makefile:20: This package should not have a PLIST file.", - "WARN: ~/category/package/distinfo: This file should not exist "+ - "since NO_CHECKSUM or META_PACKAGE is set.") + "WARN: including.mk:4: \"included.mk\" is included " + + "conditionally here (depending on OPSYS) and unconditionally in line 6.") } -func (s *Suite) Test_Package_checkPlist__Perl5_packlist(c *check.C) { +func (s *Suite) Test_Package_checkIncludeConditionally__included_multiple_times(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/p5-Packlist", - "PERL5_PACKLIST=\tauto/Packlist/.packlist") + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.CreateFileLines("including.mk", + MkCvsID, + "", + ".include \"included.mk\"", + ".if ${OPSYS} == \"Linux\"", + ".include \"included.mk\"", + ".endif", + "", + ".include \"included.mk\"", + ".if ${OPSYS} == \"Linux\"", + ".include \"included.mk\"", + ".endif") + t.CreateFileLines("included.mk", + MkCvsID) t.FinishSetUp() - G.Check(t.File("category/p5-Packlist")) + G.Check(".") t.CheckOutputLines( - "WARN: ~/category/p5-Packlist/Makefile:20: This package should not have a PLIST file.") + "WARN: including.mk:3: \"included.mk\" is included "+ + "unconditionally here and conditionally in line 10 (depending on OPSYS).", + "WARN: including.mk:5: \"included.mk\" is included "+ + "conditionally here (depending on OPSYS) and unconditionally in line 8.", + "WARN: including.mk:8: \"included.mk\" is included "+ + "unconditionally here and conditionally in line 10 (depending on OPSYS).") } -func (s *Suite) Test_Package_checkPlist__PERL5_USE_PACKLIST_no(c *check.C) { +// For preferences files, it doesn't matter whether they are included +// conditionally or unconditionally since at the end they are included +// anyway by the infrastructure. +func (s *Suite) Test_Package_checkIncludeConditionally__prefs(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/p5-NoPacklist", - "PERL5_USE_PACKLIST=\tno") + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.CreateFileLines("including.mk", + MkCvsID, + "", + ".include \"../../mk/bsd.prefs.mk\"", + ".if ${OPSYS} == \"Linux\"", + ".include \"../../mk/bsd.prefs.mk\"", + ".endif") t.FinishSetUp() - G.Check(t.File("category/p5-NoPacklist")) + G.Check(".") t.CheckOutputEmpty() } -func (s *Suite) Test_Package_checkPlist__PERL5_USE_PACKLIST_yes(c *check.C) { +func (s *Suite) Test_Package_checkIncludeConditionally__other_directory(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/p5-Packlist", - "PERL5_USE_PACKLIST=\tyes") + t.SetUpPackage("category/package", + ".include \"../../category/package-base/including.mk\"") + t.CreateFileLines("category/package-base/including.mk", + MkCvsID, + "", + ".include \"included.mk\"", + ".if ${OPSYS} == \"Linux\"", + ".include \"included.mk\"", + ".endif", + "", + ".include \"included.mk\"", + ".if ${OPSYS} == \"Linux\"", + ".include \"included.mk\"", + ".endif") + t.CreateFileLines("category/package-base/included.mk", + MkCvsID) + + t.Main("-Wall", "-Call", "category/package") + + // TODO: Understand why ../../category/package-base/including.mk is + // not checked for (un)conditional includes. + t.CheckOutputLines( + "Looks fine.") +} + +// In practice the distinfo file can always be autofixed since it has +// just been read successfully and the corresponding patch file could +// also be autofixed right before. +func (s *Suite) Test_Package_AutofixDistinfo__missing_file(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + G.Pkg = NewPackage(t.File("category/package")) t.FinishSetUp() - G.Check(t.File("category/p5-Packlist")) + G.Pkg.AutofixDistinfo("old", "new") t.CheckOutputLines( - "WARN: ~/category/p5-Packlist/Makefile:20: This package should not have a PLIST file.") + "ERROR: ~/category/package/distinfo: Cannot be read.") } func (s *Suite) Test_Package_Includes(c *check.C) { @@ -3100,23 +3167,3 @@ func (s *Suite) Test_Package_Includes(c *check.C) { // Indentation.IsConditional for the current implementation. t.CheckEquals(pkg.conditionalIncludes["never.mk"], (*MkLine)(nil)) } - -func (s *Suite) Test_Package__case_insensitive(c *check.C) { - t := s.Init(c) - - t.SetUpPkgsrc() - t.SetUpPackage("net/p5-Net-DNS") - t.SetUpPackage("category/package", - "DEPENDS+=\tp5-Net-DNS>=0:../../net/p5-net-dns") - t.FinishSetUp() - - // this test is only interesting on a case-insensitive filesystem - if !fileExists(t.File("mk/BSD.PKG.MK")) { - return - } - - G.Check(t.File("category/package")) - - // FIXME: On a case-sensitive filesystem, p5-net-dns would not be found. - t.CheckOutputEmpty() -} diff --git a/pkgtools/pkglint/files/patches_test.go b/pkgtools/pkglint/files/patches_test.go index 1b9afcf5930..415812a7f91 100644 --- a/pkgtools/pkglint/files/patches_test.go +++ b/pkgtools/pkglint/files/patches_test.go @@ -123,31 +123,6 @@ func (s *Suite) Test_CheckLinesPatch__without_comment(c *check.C) { "ERROR: patch-WithoutComment:3: Each patch must be documented.") } -// Autogenerated "comments" from Git or other tools don't count as real -// comments since they don't convey any intention of a human developer. -func (s *Suite) Test_PatchChecker_isEmptyLine(c *check.C) { - t := s.Init(c) - - lines := t.NewLines("patch-aa", - CvsID, - "", - "diff --git a/aa b/aa", - "index 1234567..1234567 100644", - "Index: from Subversion", - "============= separator or conflict marker", - "", - "--- a/aa", - "+++ b/aa", - "@@ -1,1 +1,1 @@", - "-old", - "+new") - - CheckLinesPatch(lines) - - t.CheckOutputLines( - "ERROR: patch-aa:8: Each patch must be documented.") -} - // The output of BSD Make typically contains "*** Error code". // In some really good patches, this output is included in the patch comment, // to document why the patch is necessary. @@ -447,7 +422,7 @@ func (s *Suite) Test_CheckLinesPatch__autofix_long_empty_patch(c *check.C) { t := s.Init(c) t.SetUpCommandLine("-Wall", "--autofix") - lines := t.NewLines("patch-aa", + lines := t.SetUpFileLines("patch-aa", CvsID, "") @@ -678,3 +653,28 @@ func (s *Suite) Test_PatchChecker_checktextCvsID(c *check.C) { "WARN: ~/patch-aa:8: Found CVS tag \"$"+"Id$\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".", "WARN: ~/patch-aa:11: Found CVS tag \"$"+"Author$\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".") } + +// Autogenerated "comments" from Git or other tools don't count as real +// comments since they don't convey any intention of a human developer. +func (s *Suite) Test_PatchChecker_isEmptyLine(c *check.C) { + t := s.Init(c) + + lines := t.NewLines("patch-aa", + CvsID, + "", + "diff --git a/aa b/aa", + "index 1234567..1234567 100644", + "Index: from Subversion", + "============= separator or conflict marker", + "", + "--- a/aa", + "+++ b/aa", + "@@ -1,1 +1,1 @@", + "-old", + "+new") + + CheckLinesPatch(lines) + + t.CheckOutputLines( + "ERROR: patch-aa:8: Each patch must be documented.") +} diff --git a/pkgtools/pkglint/files/pkglint.go b/pkgtools/pkglint/files/pkglint.go index db3f063bbc6..1b9f0888329 100644 --- a/pkgtools/pkglint/files/pkglint.go +++ b/pkgtools/pkglint/files/pkglint.go @@ -51,73 +51,24 @@ type Pkglint struct { InterPackage InterPackage } -func NewPkglint() Pkglint { +func NewPkglint(stdout io.Writer, stderr io.Writer) Pkglint { cwd, err := os.Getwd() assertNil(err, "os.Getwd") - return Pkglint{ + p := Pkglint{ res: regex.NewRegistry(), fileCache: NewFileCache(200), cwd: filepath.ToSlash(cwd), interner: NewStringInterner()} + p.Logger.out = NewSeparatorWriter(stdout) + p.Logger.err = NewSeparatorWriter(stderr) + return p } // unusablePkglint returns a pkglint object that crashes as early as possible. // This is to ensure that tests are properly initialized and shut down. func unusablePkglint() Pkglint { return Pkglint{} } -type InterPackage struct { - hashes map[string]*Hash // Maps "alg:filename" => hash (inter-package check). - usedLicenses map[string]struct{} // Maps "license name" => true (inter-package check). - bl3Names map[string]Location // Maps buildlink3 identifiers to their first occurrence. -} - -func (ip *InterPackage) Enable() { - *ip = InterPackage{ - make(map[string]*Hash), - make(map[string]struct{}), - make(map[string]Location)} -} - -func (ip *InterPackage) Enabled() bool { return ip.hashes != nil } - -func (ip *InterPackage) Hash(alg, filename string, hashBytes []byte, loc *Location) *Hash { - key := alg + ":" + filename - if otherHash := ip.hashes[key]; otherHash != nil { - return otherHash - } - - ip.hashes[key] = &Hash{hashBytes, *loc} - return nil -} - -func (ip *InterPackage) UseLicense(name string) { - if ip.usedLicenses != nil { - ip.usedLicenses[intern(name)] = struct{}{} - } -} - -func (ip *InterPackage) LicenseUsed(name string) bool { - _, used := ip.usedLicenses[name] - return used -} - -// Bl3 remembers that the given buildlink3 name is used at the given location. -// Since these names must be unique, there should be no other location where -// the same name is used. -func (ip *InterPackage) Bl3(name string, loc *Location) *Location { - if ip.bl3Names == nil { - return nil - } - - if prev, found := ip.bl3Names[name]; found { - return &prev - } - - ip.bl3Names[name] = *loc - return nil -} - type CmdOpts struct { CheckGlobal bool @@ -156,7 +107,7 @@ type pkglintFatal struct{} // G is the abbreviation for "global state"; // this and the tracer are the only global variables in this Go package. var ( - G = NewPkglint() + G = NewPkglint(nil, nil) trace tracePkg.Tracer ) @@ -190,7 +141,7 @@ func (pkglint *Pkglint) Main(stdout io.Writer, stderr io.Writer, args []string) pkglint.prepareMainLoop() - for !pkglint.Todo.Empty() { + for !pkglint.Todo.IsEmpty() { pkglint.Check(pkglint.Todo.Pop()) } @@ -334,7 +285,7 @@ func (pkglint *Pkglint) ParseCommandLine(args []string) int { for _, arg := range pkglint.Opts.args { pkglint.Todo.Push(filepath.ToSlash(arg)) } - if pkglint.Todo.Empty() { + if pkglint.Todo.IsEmpty() { pkglint.Todo.Push(".") } @@ -530,7 +481,7 @@ func CheckLinesMessage(lines *Lines) { // // If the need arises, some of the checks may be activated again, but // that requires more sophisticated code. - if G.Pkg != nil && G.Pkg.vars.Defined("MESSAGE_SRC") { + if G.Pkg != nil && G.Pkg.vars.IsDefined("MESSAGE_SRC") { return } @@ -831,3 +782,55 @@ func (pkglint *Pkglint) loadCvsEntries(filename string) map[string]CvsEntry { pkglint.cvsEntries = entries return entries } + +type InterPackage struct { + hashes map[string]*Hash // Maps "alg:filename" => hash (inter-package check). + usedLicenses map[string]struct{} // Maps "license name" => true (inter-package check). + bl3Names map[string]Location // Maps buildlink3 identifiers to their first occurrence. +} + +func (ip *InterPackage) Enable() { + *ip = InterPackage{ + make(map[string]*Hash), + make(map[string]struct{}), + make(map[string]Location)} +} + +func (ip *InterPackage) Enabled() bool { return ip.hashes != nil } + +func (ip *InterPackage) Hash(alg, filename string, hashBytes []byte, loc *Location) *Hash { + key := alg + ":" + filename + if otherHash := ip.hashes[key]; otherHash != nil { + return otherHash + } + + ip.hashes[key] = &Hash{hashBytes, *loc} + return nil +} + +func (ip *InterPackage) UseLicense(name string) { + if ip.usedLicenses != nil { + ip.usedLicenses[intern(name)] = struct{}{} + } +} + +func (ip *InterPackage) IsLicenseUsed(name string) bool { + _, used := ip.usedLicenses[name] + return used +} + +// Bl3 remembers that the given buildlink3 name is used at the given location. +// Since these names must be unique, there should be no other location where +// the same name is used. +func (ip *InterPackage) Bl3(name string, loc *Location) *Location { + if ip.bl3Names == nil { + return nil + } + + if prev, found := ip.bl3Names[name]; found { + return &prev + } + + ip.bl3Names[name] = *loc + return nil +} diff --git a/pkgtools/pkglint/files/pkglint_test.go b/pkgtools/pkglint/files/pkglint_test.go index 342e10391f3..f07dccadc88 100644 --- a/pkgtools/pkglint/files/pkglint_test.go +++ b/pkgtools/pkglint/files/pkglint_test.go @@ -9,7 +9,36 @@ import ( "strings" ) -func (pkglint *Pkglint) usable() bool { return pkglint.fileCache != nil } +func (pkglint *Pkglint) isUsable() bool { return pkglint.fileCache != nil } + +func (s *Suite) Test_Pkglint_Main(c *check.C) { + t := s.Init(c) + + out, err := os.Create(t.CreateFileLines("out")) + c.Check(err, check.IsNil) + outProfiling, err := os.Create(t.CreateFileLines("out.profiling")) + c.Check(err, check.IsNil) + + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.FinishSetUp() + + runMain := func(out *os.File, commandLine ...string) { + exitCode := G.Main(out, out, commandLine) + t.CheckEquals(exitCode, 0) + } + + runMain(out, "pkglint", ".") + runMain(outProfiling, "pkglint", "--profiling", ".") + + c.Check(out.Close(), check.IsNil) + c.Check(outProfiling.Close(), check.IsNil) + + t.CheckOutputEmpty() // Because all output is redirected. + t.CheckFileLines("../../out", // See the t.Chdir above. + "Looks fine.") + // outProfiling is not checked because it contains timing information. +} func (s *Suite) Test_Pkglint_Main__help(c *check.C) { t := s.Init(c) @@ -76,19 +105,6 @@ func (s *Suite) Test_Pkglint_Main__no_args(c *check.C) { "FATAL: \".\" must be inside a pkgsrc tree.") } -func (s *Suite) Test_Pkglint_ParseCommandLine__only(c *check.C) { - t := s.Init(c) - - exitcode := G.ParseCommandLine([]string{"pkglint", "-Wall", "--only", ":Q", "--version"}) - - if exitcode != -1 { - t.CheckEquals(exitcode, 0) - } - t.CheckDeepEquals(G.Opts.LogOnly, []string{":Q"}) - t.CheckOutputLines( - confVersion) -} - func (s *Suite) Test_Pkglint_Main__unknown_option(c *check.C) { t := s.Init(c) @@ -271,7 +287,7 @@ func (s *Suite) Test_Pkglint_Main__autofix_exitcode(c *check.C) { // > pkglint-pkgsrc.out // // See https://github.com/rillig/gobco for the tool to measure the branch coverage. -func (s *Suite) Test_Pkglint__realistic(c *check.C) { +func (s *Suite) Test_Pkglint_Main__realistic(c *check.C) { if cwd := os.Getenv("PKGLINT_TESTDIR"); cwd != "" { err := os.Chdir(cwd) c.Assert(err, check.IsNil) @@ -283,6 +299,54 @@ func (s *Suite) Test_Pkglint__realistic(c *check.C) { } } +func (s *Suite) Test_Pkglint_Main__profiling(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.Chdir(".") + + t.Main("--profiling") + + // Pkglint always writes the profiling data into the current directory. + // TODO: Make the location of the profiling log a mandatory parameter. + t.CheckEquals(fileExists("pkglint.pprof"), true) + + err := os.Remove("pkglint.pprof") + c.Check(err, check.IsNil) + + // Everything but the first few lines of output is not easily testable + // or not interesting enough, since that info includes the exact timing + // that the top time-consuming regular expressions took. + firstOutput := strings.Split(t.Output(), "\n")[0] + t.CheckEquals(firstOutput, "ERROR: Makefile: Cannot be read.") +} + +func (s *Suite) Test_Pkglint_Main__profiling_error(c *check.C) { + t := s.Init(c) + + t.Chdir(".") + t.CreateFileLines("pkglint.pprof/file") + + exitcode := t.Main("--profiling") + + t.CheckEquals(exitcode, 1) + t.CheckOutputMatches( + `FATAL: Cannot create profiling file: open pkglint\.pprof: .*`) +} + +func (s *Suite) Test_Pkglint_ParseCommandLine__only(c *check.C) { + t := s.Init(c) + + exitcode := G.ParseCommandLine([]string{"pkglint", "-Wall", "--only", ":Q", "--version"}) + + if exitcode != -1 { + t.CheckEquals(exitcode, 0) + } + t.CheckDeepEquals(G.Opts.LogOnly, []string{":Q"}) + t.CheckOutputLines( + confVersion) +} + func (s *Suite) Test_Pkglint_Check__outside(c *check.C) { t := s.Init(c) @@ -417,6 +481,26 @@ func (s *Suite) Test_Pkglint_Check(c *check.C) { "ERROR: ~/category/package/nonexistent: No such file or directory.") } +func (s *Suite) Test_Pkglint_Check__invalid_files_before_import(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Call", "-Wall,no-space", "--import") + pkg := t.SetUpPackage("category/package") + t.CreateFileLines("category/package/work/log") + t.CreateFileLines("category/package/Makefile~") + t.CreateFileLines("category/package/Makefile.orig") + t.CreateFileLines("category/package/Makefile.rej") + t.FinishSetUp() + + G.Check(pkg) + + t.CheckOutputLines( + "ERROR: ~/category/package/Makefile.orig: Must be cleaned up before committing the package.", + "ERROR: ~/category/package/Makefile.rej: Must be cleaned up before committing the package.", + "ERROR: ~/category/package/Makefile~: Must be cleaned up before committing the package.", + "ERROR: ~/category/package/work: Must be cleaned up before committing the package.") +} + func (s *Suite) Test_Pkglint_checkMode__neither_file_nor_directory(c *check.C) { t := s.Init(c) @@ -426,6 +510,155 @@ func (s *Suite) Test_Pkglint_checkMode__neither_file_nor_directory(c *check.C) { "ERROR: /dev/null: No such file or directory.") } +// A package that is very incomplete may produce lots of warnings. +// This case is unrealistic since most packages are either generated by url2pkg +// or copied from an existing working package. +func (s *Suite) Test_Pkglint_checkdirPackage(c *check.C) { + t := s.Init(c) + + t.Chdir("category/package") + t.CreateFileLines("Makefile", + MkCvsID) + + G.checkdirPackage(".") + + t.CheckOutputLines( + "WARN: Makefile: This package should have a PLIST file.", + "WARN: distinfo: A package that downloads files should have a distinfo file.", + "ERROR: Makefile: Each package must define its LICENSE.", + "WARN: Makefile: Each package should define a COMMENT.") +} + +func (s *Suite) Test_Pkglint_checkdirPackage__PKGDIR(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("category/Makefile") + t.CreateFileLines("other/package/Makefile", + MkCvsID) + t.CreateFileLines("other/package/PLIST", + PlistCvsID, + "bin/program") + t.CreateFileLines("other/package/distinfo", + CvsID, + "", + "SHA1 (patch-aa) = da39a3ee5e6b4b0d3255bfef95601890afd80709") + t.CreateFileLines("category/package/patches/patch-aa", + CvsID) + t.Chdir("category/package") + t.CreateFileLines("Makefile", + MkCvsID, + "", + "CATEGORIES=\tcategory", + "", + "COMMENT=\tComment", + "LICENSE=\t2-clause-bsd", + "PKGDIR=\t\t../../other/package") + t.FinishSetUp() + + // DISTINFO_FILE is resolved relative to PKGDIR, + // the other locations are resolved relative to the package base directory. + G.checkdirPackage(".") + + t.CheckOutputLines( + "ERROR: patches/patch-aa:1: Patch files must not be empty.") +} + +func (s *Suite) Test_Pkglint_checkdirPackage__patch_without_distinfo(c *check.C) { + t := s.Init(c) + + pkg := t.SetUpPackage("category/package") + t.CreateFileDummyPatch("category/package/patches/patch-aa") + t.Remove("category/package/distinfo") + t.FinishSetUp() + + G.Check(pkg) + + t.CheckOutputLines( + "WARN: ~/category/package/distinfo: A package that downloads files should have a distinfo file.", + "WARN: ~/category/package/distinfo: A package with patches should have a distinfo file.") +} + +func (s *Suite) Test_Pkglint_checkdirPackage__meta_package_without_license(c *check.C) { + t := s.Init(c) + + t.Chdir("category/package") + t.CreateFileLines("Makefile", + MkCvsID, + "", + "META_PACKAGE=\tyes") + t.SetUpVartypes() + + G.checkdirPackage(".") + + // No error about missing LICENSE since meta-packages don't need a license. + // They are so simple that there is no reason to have any license. + t.CheckOutputLines( + "WARN: Makefile: Each package should define a COMMENT.") +} + +func (s *Suite) Test_Pkglint_checkdirPackage__filename_with_variable(c *check.C) { + t := s.Init(c) + + pkg := t.SetUpPackage("category/package", + ".include \"../../mk/bsd.prefs.mk\"", + "", + "RUBY_VERSIONS_ACCEPTED=\t22 23 24 25", // As of 2018. + ".for rv in ${RUBY_VERSIONS_ACCEPTED}", + "RUBY_VER?=\t\t${rv}", + ".endfor", + "", + "RUBY_PKGDIR=\t../../lang/ruby-${RUBY_VER}-base", + "DISTINFO_FILE=\t${RUBY_PKGDIR}/distinfo") + t.FinishSetUp() + + // As of January 2019, pkglint cannot resolve the location of DISTINFO_FILE completely + // because the variable \"rv\" comes from a .for loop. + // + // TODO: iterate over variables in simple .for loops like the above. + // TODO: when implementing the above, take care of deeply nested loops (42.zip). + G.Check(pkg) + + t.CheckOutputEmpty() + + // Just for code coverage. + t.DisableTracing() + G.Check(pkg) + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Pkglint_checkdirPackage__ALTERNATIVES(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall,no-space") + pkg := t.SetUpPackage("category/package") + t.CreateFileLines("category/package/ALTERNATIVES", + "bin/wrapper bin/wrapper-impl") + t.FinishSetUp() + + G.Check(pkg) + + t.CheckOutputLines( + "ERROR: ~/category/package/ALTERNATIVES:1: "+ + "Alternative implementation \"bin/wrapper-impl\" must appear in the PLIST.", + "ERROR: ~/category/package/ALTERNATIVES:1: "+ + "Alternative implementation \"bin/wrapper-impl\" must be an absolute path.") +} + +func (s *Suite) Test_Pkglint_checkdirPackage__nonexistent_DISTINFO_FILE(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "DISTINFO_FILE=\tnonexistent") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/package/nonexistent: A package that downloads files should have a distinfo file.", + "ERROR: ~/category/package/Makefile:20: Relative path \"nonexistent\" does not exist.") +} + // Pkglint must never be trapped in an endless loop, even when // resolving the value of a variable that refers back to itself. func (s *Suite) Test_resolveVariableRefs__circular_reference(c *check.C) { @@ -478,6 +711,18 @@ func (s *Suite) Test_resolveVariableRefs__special_chars(c *check.C) { t.CheckEquals(resolved, "gst-plugins0.10-x11/distinfo") } +// Just for code coverage. +func (s *Suite) Test_CheckFileOther__no_tracing(c *check.C) { + t := s.Init(c) + + t.DisableTracing() + + CheckFileOther(t.File("filename.mk")) + + t.CheckOutputLines( + "ERROR: ~/filename.mk: Cannot be read.") +} + func (s *Suite) Test_CheckLinesDescr(c *check.C) { t := s.Init(c) @@ -597,6 +842,15 @@ func (s *Suite) Test_CheckLinesMessage__common(c *check.C) { "Looks fine.") } +func (s *Suite) Test_CheckFileMk__enoent(c *check.C) { + t := s.Init(c) + + CheckFileMk(t.File("filename.mk")) + + t.CheckOutputLines( + "ERROR: ~/filename.mk: Cannot be read.") +} + // Demonstrates that an ALTERNATIVES file can be tested individually, // without any dependencies on a whole package or a PLIST file. func (s *Suite) Test_Pkglint_checkReg__alternatives(c *check.C) { @@ -648,41 +902,6 @@ func (s *Suite) Test_Pkglint_checkReg__no_tracing(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_Pkglint__profiling(c *check.C) { - t := s.Init(c) - - t.SetUpPkgsrc() - t.Chdir(".") - - t.Main("--profiling") - - // Pkglint always writes the profiling data into the current directory. - // TODO: Make the location of the profiling log a mandatory parameter. - t.CheckEquals(fileExists("pkglint.pprof"), true) - - err := os.Remove("pkglint.pprof") - c.Check(err, check.IsNil) - - // Everything but the first few lines of output is not easily testable - // or not interesting enough, since that info includes the exact timing - // that the top time-consuming regular expressions took. - firstOutput := strings.Split(t.Output(), "\n")[0] - t.CheckEquals(firstOutput, "ERROR: Makefile: Cannot be read.") -} - -func (s *Suite) Test_Pkglint__profiling_error(c *check.C) { - t := s.Init(c) - - t.Chdir(".") - t.CreateFileLines("pkglint.pprof/file") - - exitcode := t.Main("--profiling") - - t.CheckEquals(exitcode, 1) - t.CheckOutputMatches( - `FATAL: Cannot create profiling file: open pkglint\.pprof: .*`) -} - func (s *Suite) Test_Pkglint_checkReg__in_current_working_directory(c *check.C) { t := s.Init(c) @@ -697,120 +916,6 @@ func (s *Suite) Test_Pkglint_checkReg__in_current_working_directory(c *check.C) "1 warning found.") } -func (s *Suite) Test_Pkglint_Tool__prefer_mk_over_pkgsrc(c *check.C) { - t := s.Init(c) - - mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") - mklines := t.NewMkLines("Makefile", MkCvsID) - global := G.Pkgsrc.Tools.Define("tool", "TOOL", mkline) - local := mklines.Tools.Define("tool", "TOOL", mkline) - - global.Validity = Nowhere - local.Validity = AtRunTime - - loadTimeTool, loadTimeUsable := G.Tool(mklines, "tool", LoadTime) - runTimeTool, runTimeUsable := G.Tool(mklines, "tool", RunTime) - - t.CheckEquals(loadTimeTool, local) - t.CheckEquals(loadTimeUsable, false) - t.CheckEquals(runTimeTool, local) - t.CheckEquals(runTimeUsable, true) -} - -func (s *Suite) Test_Pkglint_Tool__lookup_by_name_fallback(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("Makefile", MkCvsID) - t.SetUpTool("tool", "", Nowhere) - - loadTimeTool, loadTimeUsable := G.Tool(mklines, "tool", LoadTime) - runTimeTool, runTimeUsable := G.Tool(mklines, "tool", RunTime) - - // The tool is returned even though it cannot be used at the moment. - // The calling code must explicitly check for usability. - - t.CheckEquals(loadTimeTool.String(), "tool:::Nowhere") - t.CheckEquals(loadTimeUsable, false) - t.CheckEquals(runTimeTool.String(), "tool:::Nowhere") - t.CheckEquals(runTimeUsable, false) -} - -// TODO: Document the purpose of this test. -func (s *Suite) Test_Pkglint_Tool__lookup_by_varname(c *check.C) { - t := s.Init(c) - - mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") - mklines := t.NewMkLines("Makefile", MkCvsID) - global := G.Pkgsrc.Tools.Define("tool", "TOOL", mkline) - local := mklines.Tools.Define("tool", "TOOL", mkline) - - global.Validity = Nowhere - local.Validity = AtRunTime - - loadTimeTool, loadTimeUsable := G.Tool(mklines, "${TOOL}", LoadTime) - runTimeTool, runTimeUsable := G.Tool(mklines, "${TOOL}", RunTime) - - t.CheckEquals(loadTimeTool, local) - t.CheckEquals(loadTimeUsable, false) - t.CheckEquals(runTimeTool, local) - t.CheckEquals(runTimeUsable, true) -} - -// TODO: Document the purpose of this test. -func (s *Suite) Test_Pkglint_Tool__lookup_by_varname_fallback(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("Makefile", MkCvsID) - G.Pkgsrc.Tools.def("tool", "TOOL", false, Nowhere, nil) - - loadTimeTool, loadTimeUsable := G.Tool(mklines, "${TOOL}", LoadTime) - runTimeTool, runTimeUsable := G.Tool(mklines, "${TOOL}", RunTime) - - t.CheckEquals(loadTimeTool.String(), "tool:TOOL::Nowhere") - t.CheckEquals(loadTimeUsable, false) - t.CheckEquals(runTimeTool.String(), "tool:TOOL::Nowhere") - t.CheckEquals(runTimeUsable, false) -} - -// TODO: Document the purpose of this test. -func (s *Suite) Test_Pkglint_Tool__lookup_by_varname_fallback_runtime(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("Makefile", MkCvsID) - G.Pkgsrc.Tools.def("tool", "TOOL", false, AtRunTime, nil) - - loadTimeTool, loadTimeUsable := G.Tool(mklines, "${TOOL}", LoadTime) - runTimeTool, runTimeUsable := G.Tool(mklines, "${TOOL}", RunTime) - - t.CheckEquals(loadTimeTool.String(), "tool:TOOL::AtRunTime") - t.CheckEquals(loadTimeUsable, false) - t.CheckEquals(runTimeTool.String(), "tool:TOOL::AtRunTime") - t.CheckEquals(runTimeUsable, true) -} - -func (s *Suite) Test_Pkglint_ToolByVarname__prefer_mk_over_pkgsrc(c *check.C) { - t := s.Init(c) - - mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") - mklines := t.NewMkLines("Makefile", MkCvsID) - global := G.Pkgsrc.Tools.Define("tool", "TOOL", mkline) - local := mklines.Tools.Define("tool", "TOOL", mkline) - - global.Validity = Nowhere - local.Validity = AtRunTime - - t.CheckEquals(G.ToolByVarname(mklines, "TOOL"), local) -} - -func (s *Suite) Test_Pkglint_ToolByVarname(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("Makefile", MkCvsID) - G.Pkgsrc.Tools.def("tool", "TOOL", false, AtRunTime, nil) - - t.CheckEquals(G.ToolByVarname(mklines, "TOOL").String(), "tool:TOOL::AtRunTime") -} - func (s *Suite) Test_Pkglint_checkReg__other(c *check.C) { t := s.Init(c) @@ -827,26 +932,6 @@ func (s *Suite) Test_Pkglint_checkReg__other(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_Pkglint_Check__invalid_files_before_import(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("-Call", "-Wall,no-space", "--import") - pkg := t.SetUpPackage("category/package") - t.CreateFileLines("category/package/work/log") - t.CreateFileLines("category/package/Makefile~") - t.CreateFileLines("category/package/Makefile.orig") - t.CreateFileLines("category/package/Makefile.rej") - t.FinishSetUp() - - G.Check(pkg) - - t.CheckOutputLines( - "ERROR: ~/category/package/Makefile.orig: Must be cleaned up before committing the package.", - "ERROR: ~/category/package/Makefile.rej: Must be cleaned up before committing the package.", - "ERROR: ~/category/package/Makefile~: Must be cleaned up before committing the package.", - "ERROR: ~/category/package/work: Must be cleaned up before committing the package.") -} - func (s *Suite) Test_Pkglint_checkReg__readme_and_todo(c *check.C) { t := s.Init(c) @@ -958,176 +1043,6 @@ func (s *Suite) Test_Pkglint_checkReg__spec(c *check.C) { "WARN: ~/category/package/spec: Only packages in regress/ may have spec files.") } -// A package that is very incomplete may produce lots of warnings. -// This case is unrealistic since most packages are either generated by url2pkg -// or copied from an existing working package. -func (s *Suite) Test_Pkglint_checkdirPackage(c *check.C) { - t := s.Init(c) - - t.Chdir("category/package") - t.CreateFileLines("Makefile", - MkCvsID) - - G.checkdirPackage(".") - - t.CheckOutputLines( - "WARN: Makefile: This package should have a PLIST file.", - "WARN: distinfo: A package that downloads files should have a distinfo file.", - "ERROR: Makefile: Each package must define its LICENSE.", - "WARN: Makefile: Each package should define a COMMENT.") -} - -func (s *Suite) Test_Pkglint_checkdirPackage__PKGDIR(c *check.C) { - t := s.Init(c) - - t.SetUpPkgsrc() - t.CreateFileLines("category/Makefile") - t.CreateFileLines("other/package/Makefile", - MkCvsID) - t.CreateFileLines("other/package/PLIST", - PlistCvsID, - "bin/program") - t.CreateFileLines("other/package/distinfo", - CvsID, - "", - "SHA1 (patch-aa) = da39a3ee5e6b4b0d3255bfef95601890afd80709") - t.CreateFileLines("category/package/patches/patch-aa", - CvsID) - t.Chdir("category/package") - t.CreateFileLines("Makefile", - MkCvsID, - "", - "CATEGORIES=\tcategory", - "", - "COMMENT=\tComment", - "LICENSE=\t2-clause-bsd", - "PKGDIR=\t\t../../other/package") - t.FinishSetUp() - - // DISTINFO_FILE is resolved relative to PKGDIR, - // the other locations are resolved relative to the package base directory. - G.checkdirPackage(".") - - t.CheckOutputLines( - "ERROR: patches/patch-aa:1: Patch files must not be empty.") -} - -func (s *Suite) Test_Pkglint_checkdirPackage__patch_without_distinfo(c *check.C) { - t := s.Init(c) - - pkg := t.SetUpPackage("category/package") - t.CreateFileDummyPatch("category/package/patches/patch-aa") - t.Remove("category/package/distinfo") - t.FinishSetUp() - - G.Check(pkg) - - t.CheckOutputLines( - "WARN: ~/category/package/distinfo: A package that downloads files should have a distinfo file.", - "WARN: ~/category/package/distinfo: A package with patches should have a distinfo file.") -} - -func (s *Suite) Test_Pkglint_checkdirPackage__meta_package_without_license(c *check.C) { - t := s.Init(c) - - t.Chdir("category/package") - t.CreateFileLines("Makefile", - MkCvsID, - "", - "META_PACKAGE=\tyes") - t.SetUpVartypes() - - G.checkdirPackage(".") - - // No error about missing LICENSE since meta-packages don't need a license. - // They are so simple that there is no reason to have any license. - t.CheckOutputLines( - "WARN: Makefile: Each package should define a COMMENT.") -} - -func (s *Suite) Test_Pkglint_checkdirPackage__filename_with_variable(c *check.C) { - t := s.Init(c) - - pkg := t.SetUpPackage("category/package", - ".include \"../../mk/bsd.prefs.mk\"", - "", - "RUBY_VERSIONS_ACCEPTED=\t22 23 24 25", // As of 2018. - ".for rv in ${RUBY_VERSIONS_ACCEPTED}", - "RUBY_VER?=\t\t${rv}", - ".endfor", - "", - "RUBY_PKGDIR=\t../../lang/ruby-${RUBY_VER}-base", - "DISTINFO_FILE=\t${RUBY_PKGDIR}/distinfo") - t.FinishSetUp() - - // As of January 2019, pkglint cannot resolve the location of DISTINFO_FILE completely - // because the variable \"rv\" comes from a .for loop. - // - // TODO: iterate over variables in simple .for loops like the above. - // TODO: when implementing the above, take care of deeply nested loops (42.zip). - G.Check(pkg) - - t.CheckOutputEmpty() - - // Just for code coverage. - t.DisableTracing() - G.Check(pkg) - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Pkglint_checkdirPackage__ALTERNATIVES(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("-Wall,no-space") - pkg := t.SetUpPackage("category/package") - t.CreateFileLines("category/package/ALTERNATIVES", - "bin/wrapper bin/wrapper-impl") - t.FinishSetUp() - - G.Check(pkg) - - t.CheckOutputLines( - "ERROR: ~/category/package/ALTERNATIVES:1: "+ - "Alternative implementation \"bin/wrapper-impl\" must appear in the PLIST.", - "ERROR: ~/category/package/ALTERNATIVES:1: "+ - "Alternative implementation \"bin/wrapper-impl\" must be an absolute path.") -} - -func (s *Suite) Test_Pkglint_checkdirPackage__nonexistent_DISTINFO_FILE(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package", - "DISTINFO_FILE=\tnonexistent") - t.FinishSetUp() - - G.Check(t.File("category/package")) - - t.CheckOutputLines( - "WARN: ~/category/package/nonexistent: A package that downloads files should have a distinfo file.", - "ERROR: ~/category/package/Makefile:20: Relative path \"nonexistent\" does not exist.") -} - -func (s *Suite) Test_CheckFileMk__enoent(c *check.C) { - t := s.Init(c) - - CheckFileMk(t.File("filename.mk")) - - t.CheckOutputLines( - "ERROR: ~/filename.mk: Cannot be read.") -} - -// Just for code coverage. -func (s *Suite) Test_CheckFileOther__no_tracing(c *check.C) { - t := s.Init(c) - - t.DisableTracing() - - CheckFileOther(t.File("filename.mk")) - - t.CheckOutputLines( - "ERROR: ~/filename.mk: Cannot be read.") -} - func (s *Suite) Test_Pkglint_checkExecutable(c *check.C) { t := s.Init(c) @@ -1188,54 +1103,118 @@ func (s *Suite) Test_Pkglint_checkExecutable__already_committed(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_Pkglint_Main(c *check.C) { +func (s *Suite) Test_Pkglint_Tool__prefer_mk_over_pkgsrc(c *check.C) { t := s.Init(c) - out, err := os.Create(t.CreateFileLines("out")) - c.Check(err, check.IsNil) - outProfiling, err := os.Create(t.CreateFileLines("out.profiling")) - c.Check(err, check.IsNil) + mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") + mklines := t.NewMkLines("Makefile", MkCvsID) + global := G.Pkgsrc.Tools.Define("tool", "TOOL", mkline) + local := mklines.Tools.Define("tool", "TOOL", mkline) - t.SetUpPackage("category/package") - t.Chdir("category/package") - t.FinishSetUp() + global.Validity = Nowhere + local.Validity = AtRunTime - runMain := func(out *os.File, commandLine ...string) { - exitCode := G.Main(out, out, commandLine) - t.CheckEquals(exitCode, 0) - } + loadTimeTool, loadTimeUsable := G.Tool(mklines, "tool", LoadTime) + runTimeTool, runTimeUsable := G.Tool(mklines, "tool", RunTime) - runMain(out, "pkglint", ".") - runMain(outProfiling, "pkglint", "--profiling", ".") + t.CheckEquals(loadTimeTool, local) + t.CheckEquals(loadTimeUsable, false) + t.CheckEquals(runTimeTool, local) + t.CheckEquals(runTimeUsable, true) +} - c.Check(out.Close(), check.IsNil) - c.Check(outProfiling.Close(), check.IsNil) +func (s *Suite) Test_Pkglint_Tool__lookup_by_name_fallback(c *check.C) { + t := s.Init(c) - t.CheckOutputEmpty() // Because all output is redirected. - t.CheckFileLines("../../out", // See the t.Chdir above. - "Looks fine.") - // outProfiling is not checked because it contains timing information. + mklines := t.NewMkLines("Makefile", MkCvsID) + t.SetUpTool("tool", "", Nowhere) + + loadTimeTool, loadTimeUsable := G.Tool(mklines, "tool", LoadTime) + runTimeTool, runTimeUsable := G.Tool(mklines, "tool", RunTime) + + // The tool is returned even though it cannot be used at the moment. + // The calling code must explicitly check for usability. + + t.CheckEquals(loadTimeTool.String(), "tool:::Nowhere") + t.CheckEquals(loadTimeUsable, false) + t.CheckEquals(runTimeTool.String(), "tool:::Nowhere") + t.CheckEquals(runTimeUsable, false) } -func (s *Suite) Test_InterPackage_Bl3__same_identifier(c *check.C) { +// TODO: Document the purpose of this test. +func (s *Suite) Test_Pkglint_Tool__lookup_by_varname(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package1", - "PKGNAME=\t${DISTNAME:@v@${v}@}") // Make the package name non-obvious. - t.SetUpPackage("category/package2", - "PKGNAME=\t${DISTNAME:@v@${v}@}") // Make the package name non-obvious. - t.CreateFileDummyBuildlink3("category/package1/buildlink3.mk") - t.Copy("category/package1/buildlink3.mk", "category/package2/buildlink3.mk") - t.Chdir(".") - t.FinishSetUp() + mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") + mklines := t.NewMkLines("Makefile", MkCvsID) + global := G.Pkgsrc.Tools.Define("tool", "TOOL", mkline) + local := mklines.Tools.Define("tool", "TOOL", mkline) - G.InterPackage.Enable() - G.Check("category/package1") - G.Check("category/package2") + global.Validity = Nowhere + local.Validity = AtRunTime - t.CheckOutputLines( - "ERROR: category/package2/buildlink3.mk:3: Duplicate package identifier " + - "\"package1\" already appeared in ../../category/package1/buildlink3.mk:3.") + loadTimeTool, loadTimeUsable := G.Tool(mklines, "${TOOL}", LoadTime) + runTimeTool, runTimeUsable := G.Tool(mklines, "${TOOL}", RunTime) + + t.CheckEquals(loadTimeTool, local) + t.CheckEquals(loadTimeUsable, false) + t.CheckEquals(runTimeTool, local) + t.CheckEquals(runTimeUsable, true) +} + +// TODO: Document the purpose of this test. +func (s *Suite) Test_Pkglint_Tool__lookup_by_varname_fallback(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", MkCvsID) + G.Pkgsrc.Tools.def("tool", "TOOL", false, Nowhere, nil) + + loadTimeTool, loadTimeUsable := G.Tool(mklines, "${TOOL}", LoadTime) + runTimeTool, runTimeUsable := G.Tool(mklines, "${TOOL}", RunTime) + + t.CheckEquals(loadTimeTool.String(), "tool:TOOL::Nowhere") + t.CheckEquals(loadTimeUsable, false) + t.CheckEquals(runTimeTool.String(), "tool:TOOL::Nowhere") + t.CheckEquals(runTimeUsable, false) +} + +// TODO: Document the purpose of this test. +func (s *Suite) Test_Pkglint_Tool__lookup_by_varname_fallback_runtime(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", MkCvsID) + G.Pkgsrc.Tools.def("tool", "TOOL", false, AtRunTime, nil) + + loadTimeTool, loadTimeUsable := G.Tool(mklines, "${TOOL}", LoadTime) + runTimeTool, runTimeUsable := G.Tool(mklines, "${TOOL}", RunTime) + + t.CheckEquals(loadTimeTool.String(), "tool:TOOL::AtRunTime") + t.CheckEquals(loadTimeUsable, false) + t.CheckEquals(runTimeTool.String(), "tool:TOOL::AtRunTime") + t.CheckEquals(runTimeUsable, true) +} + +func (s *Suite) Test_Pkglint_ToolByVarname__prefer_mk_over_pkgsrc(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") + mklines := t.NewMkLines("Makefile", MkCvsID) + global := G.Pkgsrc.Tools.Define("tool", "TOOL", mkline) + local := mklines.Tools.Define("tool", "TOOL", mkline) + + global.Validity = Nowhere + local.Validity = AtRunTime + + t.CheckEquals(G.ToolByVarname(mklines, "TOOL"), local) +} + +func (s *Suite) Test_Pkglint_ToolByVarname(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("Makefile", MkCvsID) + G.Pkgsrc.Tools.def("tool", "TOOL", false, AtRunTime, nil) + + t.CheckEquals(G.ToolByVarname(mklines, "TOOL").String(), "tool:TOOL::AtRunTime") } func (s *Suite) Test_Pkglint_loadCvsEntries(c *check.C) { @@ -1277,3 +1256,24 @@ func (s *Suite) Test_Pkglint_loadCvsEntries__with_Entries_Log(c *check.C) { "ERROR: ~/CVS/Entries.Log:1: Invalid line: A /invalid/", "ERROR: ~/CVS/Entries.Log:4: Invalid line: R /invalid/") } + +func (s *Suite) Test_InterPackage_Bl3__same_identifier(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package1", + "PKGNAME=\t${DISTNAME:@v@${v}@}") // Make the package name non-obvious. + t.SetUpPackage("category/package2", + "PKGNAME=\t${DISTNAME:@v@${v}@}") // Make the package name non-obvious. + t.CreateFileDummyBuildlink3("category/package1/buildlink3.mk") + t.Copy("category/package1/buildlink3.mk", "category/package2/buildlink3.mk") + t.Chdir(".") + t.FinishSetUp() + + G.InterPackage.Enable() + G.Check("category/package1") + G.Check("category/package2") + + t.CheckOutputLines( + "ERROR: category/package2/buildlink3.mk:3: Duplicate package identifier " + + "\"package1\" already appeared in ../../category/package1/buildlink3.mk:3.") +} diff --git a/pkgtools/pkglint/files/pkgsrc.go b/pkgtools/pkglint/files/pkgsrc.go index afad3c26b7a..adfa762527c 100644 --- a/pkgtools/pkglint/files/pkgsrc.go +++ b/pkgtools/pkglint/files/pkgsrc.go @@ -72,78 +72,6 @@ func NewPkgsrc(dir string) Pkgsrc { NewVarTypeRegistry()} } -func (src *Pkgsrc) loadDefaultBuildDefs() { - - // Some user-defined variables do not influence the binary - // package at all and therefore do not have to be added to - // BUILD_DEFS; therefore they are marked as "already added". - src.addBuildDefs( - "DISTDIR", - "FETCH_CMD", - "FETCH_OUTPUT_ARGS", - "FETCH_USING", - "PKGSRC_RUN_TEST") - - // The following variables are used so often that not every - // package should need to add it to BUILD_DEFS manually. - src.addBuildDefs( - "PKGSRC_COMPILER", - "PKGSRC_USE_SSP", - "UNPRIVILEGED", - "USE_CROSS_COMPILE") - - // The following variables are so obscure that they are - // probably not used in practice. - src.addBuildDefs( - "MANINSTALL") - - // The following variables are added to _BUILD_DEFS by the pkgsrc - // infrastructure and thus don't need to be added by the package again. - // To regenerate the below list: - // grep -hr '^_BUILD_DEFS+=' mk/ | tr ' \t' '\n\n' | sed -e 's,.*=,,' -e '/^_/d' -e '/^$/d' -e 's,.*,"&"\,,' | sort -u - // TODO: Run the equivalent of the above command at startup. - src.addBuildDefs( - "ABI", - "BUILTIN_PKGS", - "CFLAGS", - "CMAKE_ARGS", - "CONFIGURE_ARGS", - "CONFIGURE_ENV", - "CPPFLAGS", - "FFLAGS", - "GAMEDATAMODE", - "GAMEDIRMODE", - "GAMEMODE", - "GAMES_GROUP", - "GAMES_USER", - "GLIBC_VERSION", - "INIT_SYSTEM", - "LDFLAGS", - "LICENSE", - "LOCALBASE", - "MACHINE_ARCH", - "MACHINE_GNU_ARCH", - "MULTI", - "NO_BIN_ON_CDROM", - "NO_BIN_ON_FTP", - "NO_SRC_ON_CDROM", - "NO_SRC_ON_FTP", - "OBJECT_FMT", - "OPSYS", - "OS_VERSION", - "OSVERSION_SPECIFIC", - "PKG_HACKS", - "PKG_OPTIONS", - "PKG_SYSCONFBASEDIR", - "PKG_SYSCONFDIR", - "PKGGNUDIR", - "PKGINFODIR", - "PKGMANDIR", - "PKGPATH", - "RESTRICTED", - "USE_ABI_DEPENDS") -} - // LoadInfrastructure reads the pkgsrc infrastructure files to // extract information like the tools, packages to update, // user-defined variables. @@ -164,336 +92,83 @@ func (src *Pkgsrc) LoadInfrastructure() { src.loadDefaultBuildDefs() } -// Latest returns the latest package matching the given pattern. -// It searches the category for subdirectories matching the given -// regular expression, takes the latest of them and replaces its -// name with repl. -// -// Example: -// Latest("lang", `^php[0-9]+$`, "../../lang/$0") -// => "../../lang/php72" -func (src *Pkgsrc) Latest(category string, re regex.Pattern, repl string) string { - versions := src.ListVersions(category, re, repl, true) - - if len(versions) > 0 { - return versions[len(versions)-1] - } - return "" -} - -// ListVersions searches the category for subdirectories matching the given -// regular expression, replaces their names with repl and returns a slice -// of them, properly sorted from early to late. -// -// Example: -// ListVersions("lang", `^php[0-9]+$`, "php-$0") -// => {"php-53", "php-56", "php-73"} -func (src *Pkgsrc) ListVersions(category string, re regex.Pattern, repl string, errorIfEmpty bool) []string { - if G.Testing { - // Regular expression must be anchored at both ends, to avoid typos. - assert(hasPrefix(string(re), "^")) - assert(hasSuffix(string(re), "$")) - } - - // TODO: Maybe convert cache key to a struct, to save allocations. - cacheKey := category + "/" + string(re) + " => " + repl - if latest, found := src.listVersions[cacheKey]; found { - return latest - } - - categoryDir := src.File(category) - - var names []string - for _, fileInfo := range src.ReadDir(category) { - name := fileInfo.Name() - if matches(name, re) { - names = append(names, name) - } - } - if len(names) == 0 { - if errorIfEmpty { - dummyLine.Errorf("Cannot find package versions of %q in %q.", re, categoryDir) - } - src.listVersions[cacheKey] = nil - return nil - } - - // In the pkgsrc directories, the major versions of packages are - // written without dots, which leads to ambiguities: - // - // databases/postgresql: 94 < 95 < 96 < 10 < 11 - // lang/go: 19 < 110 < 111 < 2 - keys := make(map[string]int) - for _, name := range names { - if m, pkgbase, versionStr := match2(name, `^(\D+)(\d+)$`); m { - version := toInt(versionStr, 0) - if pkgbase == "postgresql" && version < 60 { - version = 10 * version - } - if pkgbase == "go" { - major := toInt(versionStr[:1], 0) - minor := toInt(versionStr[1:], 0) - version = 100*major + minor - } - keys[name] = version - } - } - - sort.SliceStable(names, func(i, j int) bool { - if keyI, keyJ := keys[names[i]], keys[names[j]]; keyI != keyJ { - return keyI < keyJ - } - return naturalLess(names[i], names[j]) - }) - - var repls = make([]string, len(names)) - for i, name := range names { - repls[i] = replaceAll(name, re, repl) - } - - src.listVersions[cacheKey] = repls - return repls -} - -func (src *Pkgsrc) checkToplevelUnusedLicenses() { - if !G.InterPackage.Enabled() { - return - } - - licensesDir := src.File("licenses") - for _, licenseFile := range src.ReadDir("licenses") { - licenseName := licenseFile.Name() - if !G.InterPackage.LicenseUsed(licenseName) { - licensePath := joinPath(licensesDir, licenseName) - NewLineWhole(licensePath).Warnf("This license seems to be unused.") - } - } -} - -// loadTools loads the tool definitions from `mk/tools/*`. -func (src *Pkgsrc) loadTools() { - tools := src.Tools +func (src *Pkgsrc) loadMasterSites() { + mklines := src.LoadMk("mk/fetch/sites.mk", MustSucceed|NotEmpty) - toolFiles := []string{"defaults.mk"} - { - toc := src.File("mk/tools/bsd.tools.mk") - mklines := LoadMk(toc, MustSucceed|NotEmpty) - for _, mkline := range mklines.mklines { - if mkline.IsInclude() { - includedFile := mkline.IncludedFile() - if !contains(includedFile, "/") { - toolFiles = append(toolFiles, includedFile) + for _, mkline := range mklines.mklines { + if mkline.IsVarassign() { + varname := mkline.Varname() + // TODO: Give a plausible reason for the MASTER_SITE_BACKUP exception. + if hasPrefix(varname, "MASTER_SITE_") && varname != "MASTER_SITE_BACKUP" { + for _, url := range mkline.ValueFields(mkline.Value()) { + if matches(url, `^(?:http://|https://|ftp://)`) { + src.registerMasterSite(varname, url) + } } + + // TODO: register variable type, to avoid redundant definitions in vardefs.go. } } - if len(toolFiles) <= 1 { - NewLineWhole(toc).Fatalf("Too few tool files.") - } } - // TODO: parse bsd.prefs.mk and bsd.pkg.mk instead of hardcoding this. - tools.def("echo", "ECHO", true, AfterPrefsMk, nil) - tools.def("echo -n", "ECHO_N", true, AfterPrefsMk, nil) - tools.def("false", "FALSE", true, AtRunTime, nil) // from bsd.pkg.mk - tools.def("test", "TEST", true, AfterPrefsMk, nil) - tools.def("true", "TRUE", true, AfterPrefsMk, nil) - - for _, basename := range toolFiles { - mklines := src.LoadMk("mk/tools/"+basename, MustSucceed|NotEmpty) - mklines.ForEach(func(mkline *MkLine) { - tools.ParseToolLine(mklines, mkline, true, !mklines.indentation.IsConditional()) - }) - } - - for _, relativeName := range [...]string{"mk/bsd.prefs.mk", "mk/bsd.pkg.mk"} { - - mklines := src.LoadMk(relativeName, MustSucceed|NotEmpty) - mklines.ForEach(func(mkline *MkLine) { - if mkline.IsVarassign() { - varname := mkline.Varname() - switch varname { - case "USE_TOOLS": - tools.ParseToolLine(mklines, mkline, true, !mklines.indentation.IsConditional()) - - case "_BUILD_DEFS": - // TODO: Compare with src.loadDefaultBuildDefs; is it redundant? - for _, buildDefsVar := range mkline.Fields() { - src.addBuildDefs(buildDefsVar) - } - } - } - }) - } + // Explicitly allowed, although not defined in mk/fetch/sites.mk. + // TODO: Document where this definition comes from and why it is good. + src.registerMasterSite("MASTER_SITE_LOCAL", "ftp://ftp.NetBSD.org/pub/pkgsrc/distfiles/LOCAL_PORTS/") if trace.Tracing { - tools.Trace() + trace.Stepf("Loaded %d MASTER_SITE_* URLs.", len(src.MasterSiteURLToVar)) } } -// loadUntypedVars scans all pkgsrc infrastructure files in mk/ -// to find variable definitions that are not yet covered in -// Pkgsrc.InitVartypes. -// -// Even if pkglint cannot guess the type of each variable, -// at least prevent the "used but not defined" warnings. -func (src *Pkgsrc) loadUntypedVars() { - - // Setting guessed to false prevents the vartype.guessed case in MkLineChecker.CheckVaruse. - unknownType := NewVartype(BtUnknown, NoVartypeOptions, NewACLEntry("*", aclpAll)) - - define := func(varcanon string, mkline *MkLine) { - switch { - case src.vartypes.DefinedCanon(varcanon): - // Already defined, can also be a tool. - - case hasPrefix(varcanon, "_"): - // Variables starting with an underscore are reserved for the - // infrastructure and are not available for use by packages. - - case contains(varcanon, "$"): - // Indirect, but not the usual parameterized form. Variables of - // this form should not be unintentionally visible from outside - // the infrastructure since they don't follow the pkgsrc naming - // conventions. - - case hasSuffix(varcanon, "_MK"): - // Multiple-inclusion guards are internal to the infrastructure. - - default: - if trace.Tracing { - trace.Stepf("Untyped variable %q in %s", varcanon, mkline) - } - src.vartypes.DefineType(varcanon, unknownType) - } - } - - handleMkFile := func(path string) { - mklines := LoadMk(path, MustSucceed) - mklines.collectVariables() - mklines.collectUsedVariables() - for varname, mkline := range mklines.vars.firstDef { - define(varnameCanon(varname), mkline) - } - for varname, mkline := range mklines.vars.used { - define(varnameCanon(varname), mkline) - } - } +func (src *Pkgsrc) registerMasterSite(varname, url string) { + nameToURL := src.MasterSiteVarToURL + urlToName := src.MasterSiteURLToVar - handleFile := func(pathName string, info os.FileInfo, err error) error { - assertNil(err, "handleFile %q", pathName) - baseName := info.Name() - if info.Mode().IsRegular() && (hasSuffix(baseName, ".mk") || baseName == "mk.conf") { - handleMkFile(filepath.ToSlash(pathName)) - } - return nil + if nameToURL[varname] == "" { + nameToURL[varname] = url } - - err := filepath.Walk(src.File("mk"), handleFile) - assertNil(err, "Walk error in pkgsrc infrastructure") + urlToName[replaceAll(url, `^\w+://`, "")] = varname } -func (src *Pkgsrc) parseSuggestedUpdates(lines *Lines) []SuggestedUpdate { - if lines == nil { - return nil - } +func (src *Pkgsrc) loadPkgOptions() { + lines := src.Load("mk/defaults/options.description", MustSucceed) - var updates []SuggestedUpdate - state := 0 for _, line := range lines.Lines { - text := line.Text - - // TODO: Replace this state transition scheme with explicit code, - // hoping that the code will be easier to understand. - if state == 0 && text == "Suggested package updates" { - state = 1 - } else if state == 1 && text == "" { - state = 2 - } else if state == 2 { - state = 3 - } else if state == 3 && text == "" { - state = 4 - } - - if state == 3 { - if m, pkgname, comment := match2(text, `^\to[\t ]([^\t ]+)(?:[\t ]*(.+))?$`); m { - if m, pkgbase, pkgversion := match2(pkgname, rePkgname); m { - updates = append(updates, SuggestedUpdate{line.Location, intern(pkgbase), intern(pkgversion), intern(comment)}) - } else { - line.Warnf("Invalid package name %q.", pkgname) - } - } else { - line.Warnf("Invalid line format %q.", text) - } + if m, name, description := match2(line.Text, `^([-0-9a-z_+]+)(?:[\t ]+(.*))?$`); m { + src.PkgOptions[name] = description + } else { + line.Errorf("Invalid line format: %s", line.Text) } } - return updates } -func (src *Pkgsrc) loadSuggestedUpdates() { - src.suggestedUpdates = src.parseSuggestedUpdates(Load(src.File("doc/TODO"), MustSucceed)) - src.suggestedWipUpdates = src.parseSuggestedUpdates(Load(src.File("wip/TODO"), NotEmpty)) -} - -func (*Pkgsrc) parseDocChange(line *Line, warn bool) *Change { - lex := textproc.NewLexer(line.Text) - - space := lex.NextHspace() - if space == "" { - return nil +func (src *Pkgsrc) loadDocChanges() { + docDir := src.File("doc") + files := src.ReadDir("doc") + if len(files) == 0 { + NewLineWhole(docDir).Fatalf("Cannot be read for loading the package changes.") } - if space != "\t" { - if warn { - line.Warnf("Package changes should be indented using a single tab, not %q.", space) - line.Explain( - "To avoid this formatting mistake in the future, just run", - sprintf("%q", bmake("cce")), - "after committing the update to the package.") + var filenames []string + for _, file := range files { + filename := file.Name() + if matches(filename, `^CHANGES-20\d\d$`) && filename >= "CHANGES-2011" { // TODO: Why 2011? + filenames = append(filenames, filename) } - - return nil - } - - f := strings.Fields(lex.Rest()) - n := len(f) - if n != 4 && n != 6 { - return nil } - action := ParseChangeAction(f[0]) - pkgpath := f[1] - author := f[len(f)-2] - date := f[len(f)-1] - - if !hasPrefix(author, "[") || !hasSuffix(date, "]") { - return nil - } - author, date = author[1:], date[:len(date)-1] - - switch { - case - action == Added && f[2] == "version", - action == Updated && f[2] == "to", - action == Downgraded && f[2] == "to", - action == Removed && (f[2] == "successor" || n == 4), - (action == Renamed || action == Moved) && f[2] == "to": - return &Change{ - Location: line.Location, - Action: action, - Pkgpath: intern(pkgpath), - target: intern(condStr(n == 6, f[3], "")), - Author: intern(author), - Date: intern(date), + src.LastChange = make(map[string]*Change) + for _, filename := range filenames { + changes := src.loadDocChangesFromFile(joinPath(docDir, filename)) + for _, change := range changes { + src.LastChange[change.Pkgpath] = change + if change.Action == Renamed || change.Action == Moved { + src.LastChange[change.Target()] = change + } } } - if warn { - line.Warnf("Unknown doc/CHANGES line: %s", line.Text) - line.Explain( - "See mk/misc/developer.mk for the rules.") - } - - return nil + src.checkRemovedAfterLastFreeze() } func (src *Pkgsrc) loadDocChangesFromFile(filename string) []*Change { @@ -572,41 +247,66 @@ func (src *Pkgsrc) loadDocChangesFromFile(filename string) []*Change { return changes } -func (src *Pkgsrc) SuggestedUpdates() []SuggestedUpdate { - if G.Wip { - return src.suggestedWipUpdates - } else { - return src.suggestedUpdates - } -} +func (*Pkgsrc) parseDocChange(line *Line, warn bool) *Change { + lex := textproc.NewLexer(line.Text) -func (src *Pkgsrc) loadDocChanges() { - docDir := src.File("doc") - files := src.ReadDir("doc") - if len(files) == 0 { - NewLineWhole(docDir).Fatalf("Cannot be read for loading the package changes.") + space := lex.NextHspace() + if space == "" { + return nil } - var filenames []string - for _, file := range files { - filename := file.Name() - if matches(filename, `^CHANGES-20\d\d$`) && filename >= "CHANGES-2011" { // TODO: Why 2011? - filenames = append(filenames, filename) + if space != "\t" { + if warn { + line.Warnf("Package changes should be indented using a single tab, not %q.", space) + line.Explain( + "To avoid this formatting mistake in the future, just run", + sprintf("%q", bmake("cce")), + "after committing the update to the package.") } + + return nil } - src.LastChange = make(map[string]*Change) - for _, filename := range filenames { - changes := src.loadDocChangesFromFile(joinPath(docDir, filename)) - for _, change := range changes { - src.LastChange[change.Pkgpath] = change - if change.Action == Renamed || change.Action == Moved { - src.LastChange[change.Target()] = change - } + f := strings.Fields(lex.Rest()) + n := len(f) + if n != 4 && n != 6 { + return nil + } + + action := ParseChangeAction(f[0]) + pkgpath := f[1] + author := f[len(f)-2] + date := f[len(f)-1] + + if !hasPrefix(author, "[") || !hasSuffix(date, "]") { + return nil + } + author, date = author[1:], date[:len(date)-1] + + switch { + case + action == Added && f[2] == "version", + action == Updated && f[2] == "to", + action == Downgraded && f[2] == "to", + action == Removed && (f[2] == "successor" || n == 4), + (action == Renamed || action == Moved) && f[2] == "to": + return &Change{ + Location: line.Location, + Action: action, + Pkgpath: intern(pkgpath), + target: intern(condStr(n == 6, f[3], "")), + Author: intern(author), + Date: intern(date), } } - src.checkRemovedAfterLastFreeze() + if warn { + line.Warnf("Unknown doc/CHANGES line: %s", line.Text) + line.Explain( + "See mk/misc/developer.mk for the rules.") + } + + return nil } func (src *Pkgsrc) checkRemovedAfterLastFreeze() { @@ -624,7 +324,7 @@ func (src *Pkgsrc) checkRemovedAfterLastFreeze() { } } - sort.Slice(wrong, func(i, j int) bool { return wrong[i].Above(wrong[j]) }) + sort.Slice(wrong, func(i, j int) bool { return wrong[i].IsAbove(wrong[j]) }) for _, change := range wrong { // It's a bit cheated to construct a Line from only a Location, @@ -635,6 +335,48 @@ func (src *Pkgsrc) checkRemovedAfterLastFreeze() { } } +func (src *Pkgsrc) loadSuggestedUpdates() { + src.suggestedUpdates = src.parseSuggestedUpdates(Load(src.File("doc/TODO"), MustSucceed)) + src.suggestedWipUpdates = src.parseSuggestedUpdates(Load(src.File("wip/TODO"), NotEmpty)) +} + +func (src *Pkgsrc) parseSuggestedUpdates(lines *Lines) []SuggestedUpdate { + if lines == nil { + return nil + } + + var updates []SuggestedUpdate + state := 0 + for _, line := range lines.Lines { + text := line.Text + + // TODO: Replace this state transition scheme with explicit code, + // hoping that the code will be easier to understand. + if state == 0 && text == "Suggested package updates" { + state = 1 + } else if state == 1 && text == "" { + state = 2 + } else if state == 2 { + state = 3 + } else if state == 3 && text == "" { + state = 4 + } + + if state == 3 { + if m, pkgname, comment := match2(text, `^\to[\t ]([^\t ]+)(?:[\t ]*(.+))?$`); m { + if m, pkgbase, pkgversion := match2(pkgname, rePkgname); m { + updates = append(updates, SuggestedUpdate{line.Location, intern(pkgbase), intern(pkgversion), intern(comment)}) + } else { + line.Warnf("Invalid package name %q.", pkgname) + } + } else { + line.Warnf("Invalid line format %q.", text) + } + } + } + return updates +} + func (src *Pkgsrc) loadUserDefinedVars() { mklines := src.LoadMk("mk/defaults/mk.conf", MustSucceed|NotEmpty) @@ -645,6 +387,72 @@ func (src *Pkgsrc) loadUserDefinedVars() { } } +// loadTools loads the tool definitions from `mk/tools/*`. +func (src *Pkgsrc) loadTools() { + tools := src.Tools + + toolFiles := []string{"defaults.mk"} + { + toc := src.File("mk/tools/bsd.tools.mk") + mklines := LoadMk(toc, MustSucceed|NotEmpty) + for _, mkline := range mklines.mklines { + if mkline.IsInclude() { + includedFile := mkline.IncludedFile() + if !contains(includedFile, "/") { + toolFiles = append(toolFiles, includedFile) + } + } + } + if len(toolFiles) <= 1 { + NewLineWhole(toc).Fatalf("Too few tool files.") + } + } + + // TODO: parse bsd.prefs.mk and bsd.pkg.mk instead of hardcoding this. + tools.def("echo", "ECHO", true, AfterPrefsMk, nil) + tools.def("echo -n", "ECHO_N", true, AfterPrefsMk, nil) + tools.def("false", "FALSE", true, AtRunTime, nil) // from bsd.pkg.mk + tools.def("test", "TEST", true, AfterPrefsMk, nil) + tools.def("true", "TRUE", true, AfterPrefsMk, nil) + + for _, basename := range toolFiles { + mklines := src.LoadMk("mk/tools/"+basename, MustSucceed|NotEmpty) + mklines.ForEach(func(mkline *MkLine) { + tools.ParseToolLine(mklines, mkline, true, !mklines.indentation.IsConditional()) + }) + } + + for _, relativeName := range [...]string{"mk/bsd.prefs.mk", "mk/bsd.pkg.mk"} { + + mklines := src.LoadMk(relativeName, MustSucceed|NotEmpty) + mklines.ForEach(func(mkline *MkLine) { + if mkline.IsVarassign() { + varname := mkline.Varname() + switch varname { + case "USE_TOOLS": + tools.ParseToolLine(mklines, mkline, true, !mklines.indentation.IsConditional()) + + case "_BUILD_DEFS": + // TODO: Compare with src.loadDefaultBuildDefs; is it redundant? + for _, buildDefsVar := range mkline.Fields() { + src.addBuildDefs(buildDefsVar) + } + } + } + }) + } + + if trace.Tracing { + tools.Trace() + } +} + +func (src *Pkgsrc) addBuildDefs(varnames ...string) { + for _, varname := range varnames { + src.buildDefs[varname] = true + } +} + func (src *Pkgsrc) initDeprecatedVars() { src.Deprecated = map[string]string{ // December 2003 @@ -764,7 +572,9 @@ func (src *Pkgsrc) initDeprecatedVars() { "SKIP_PORTABILITY_CHECK": "Use CHECK_PORTABILITY_SKIP (a list of patterns) instead.", // January 2007 - "BUILDLINK_TRANSFORM.*": "Use BUILDLINK_FNAME_TRANSFORM.* instead.", + // Only applies to BUILDLINK_TRANSFORM.${pkgbase}. + // There is still BUILDLINK_TRANSFORM.${OPSYS}. + // "BUILDLINK_TRANSFORM.*": "Use BUILDLINK_FNAME_TRANSFORM.* instead.", // March 2007 "SCRIPTDIR": "You can just remove it.", @@ -777,7 +587,7 @@ func (src *Pkgsrc) initDeprecatedVars() { "LICENCE": "Use LICENSE instead.", // November 2007 - //USE_NCURSES: Include "../../devel/ncurses/buildlink3.mk" instead. + // USE_NCURSES: Include "../../devel/ncurses/buildlink3.mk" instead. // December 2007 "INSTALLATION_DIRS_FROM_PLIST": "Use AUTO_MKDIRS instead.", @@ -809,131 +619,229 @@ func (src *Pkgsrc) initDeprecatedVars() { } } -// Load loads the file relative to the pkgsrc top directory. -func (src *Pkgsrc) Load(filename string, options LoadOptions) *Lines { - return Load(src.File(filename), options) -} +// loadUntypedVars scans all pkgsrc infrastructure files in mk/ +// to find variable definitions that are not yet covered in +// Pkgsrc.InitVartypes. +// +// Even if pkglint cannot guess the type of each variable, +// at least prevent the "used but not defined" warnings. +func (src *Pkgsrc) loadUntypedVars() { -// LoadMk loads the Makefile relative to the pkgsrc top directory. -func (src *Pkgsrc) LoadMk(filename string, options LoadOptions) *MkLines { - return LoadMk(src.File(filename), options) -} + // Setting guessed to false prevents the vartype.guessed case in MkLineChecker.CheckVaruse. + unknownType := NewVartype(BtUnknown, NoVartypeOptions, NewACLEntry("*", aclpAll)) -func (src *Pkgsrc) LoadMkInfra(filename string, options LoadOptions) *MkLines { - if G.Testing { - // During testing, the infrastructure files don't have to exist. - // They are often emulated by setting their data structures manually. - options &^= MustSucceed + define := func(varcanon string, mkline *MkLine) { + switch { + case src.vartypes.IsDefinedCanon(varcanon): + // Already defined, can also be a tool. + + case hasPrefix(varcanon, "_"): + // Variables starting with an underscore are reserved for the + // infrastructure and are not available for use by packages. + + case contains(varcanon, "$"): + // Indirect, but not the usual parameterized form. Variables of + // this form should not be unintentionally visible from outside + // the infrastructure since they don't follow the pkgsrc naming + // conventions. + + case hasSuffix(varcanon, "_MK"): + // Multiple-inclusion guards are internal to the infrastructure. + + default: + if trace.Tracing { + trace.Stepf("Untyped variable %q in %s", varcanon, mkline) + } + src.vartypes.DefineType(varcanon, unknownType) + } } - return src.LoadMk(filename, options) -} -// ReadDir lists the files and subdirectories from the given directory -// (relative to the pkgsrc root), filtering out any ignored files (CVS/*) -// and empty directories. -func (src *Pkgsrc) ReadDir(dirName string) []os.FileInfo { - dir := src.File(dirName) - files, err := ioutil.ReadDir(dir) - if err != nil { - return nil + handleMkFile := func(path string) { + mklines := LoadMk(path, MustSucceed) + mklines.collectVariables() + mklines.collectUsedVariables() + for varname, mkline := range mklines.vars.firstDef { + define(varnameCanon(varname), mkline) + } + for varname, mkline := range mklines.vars.used { + define(varnameCanon(varname), mkline) + } } - var relevantFiles []os.FileInfo - for _, dirent := range files { - name := dirent.Name() - if !dirent.IsDir() || !isIgnoredFilename(name) && !isEmptyDir(joinPath(dir, name)) { - relevantFiles = append(relevantFiles, dirent) + handleFile := func(pathName string, info os.FileInfo, err error) error { + assertNil(err, "handleFile %q", pathName) + baseName := info.Name() + if info.Mode().IsRegular() && (hasSuffix(baseName, ".mk") || baseName == "mk.conf") { + handleMkFile(filepath.ToSlash(pathName)) } + return nil } - return relevantFiles + err := filepath.Walk(src.File("mk"), handleFile) + assertNil(err, "Walk error in pkgsrc infrastructure") } -// File resolves a filename relative to the pkgsrc top directory. -// -// Example: -// NewPkgsrc("/usr/pkgsrc").File("distfiles") => "/usr/pkgsrc/distfiles" -func (src *Pkgsrc) File(relativeName string) string { - // TODO: Package.File resolves variables, Pkgsrc.File doesn't. They should behave the same. - return cleanpath(joinPath(src.topdir, relativeName)) +func (src *Pkgsrc) loadDefaultBuildDefs() { + + // Some user-defined variables do not influence the binary + // package at all and therefore do not have to be added to + // BUILD_DEFS; therefore they are marked as "already added". + src.addBuildDefs( + "DISTDIR", + "FETCH_CMD", + "FETCH_OUTPUT_ARGS", + "FETCH_USING", + "PKGSRC_RUN_TEST") + + // The following variables are used so often that not every + // package should need to add it to BUILD_DEFS manually. + src.addBuildDefs( + "PKGSRC_COMPILER", + "PKGSRC_USE_SSP", + "UNPRIVILEGED", + "USE_CROSS_COMPILE") + + // The following variables are so obscure that they are + // probably not used in practice. + src.addBuildDefs( + "MANINSTALL") + + // The following variables are added to _BUILD_DEFS by the pkgsrc + // infrastructure and thus don't need to be added by the package again. + // To regenerate the below list: + // grep -hr '^_BUILD_DEFS+=' mk/ | tr ' \t' '\n\n' | sed -e 's,.*=,,' -e '/^_/d' -e '/^$/d' -e 's,.*,"&"\,,' | sort -u + // TODO: Run the equivalent of the above command at startup. + src.addBuildDefs( + "ABI", + "BUILTIN_PKGS", + "CFLAGS", + "CMAKE_ARGS", + "CONFIGURE_ARGS", + "CONFIGURE_ENV", + "CPPFLAGS", + "FFLAGS", + "GAMEDATAMODE", + "GAMEDIRMODE", + "GAMEMODE", + "GAMES_GROUP", + "GAMES_USER", + "GLIBC_VERSION", + "INIT_SYSTEM", + "LDFLAGS", + "LICENSE", + "LOCALBASE", + "MACHINE_ARCH", + "MACHINE_GNU_ARCH", + "MULTI", + "NO_BIN_ON_CDROM", + "NO_BIN_ON_FTP", + "NO_SRC_ON_CDROM", + "NO_SRC_ON_FTP", + "OBJECT_FMT", + "OPSYS", + "OS_VERSION", + "OSVERSION_SPECIFIC", + "PKG_HACKS", + "PKG_OPTIONS", + "PKG_SYSCONFBASEDIR", + "PKG_SYSCONFDIR", + "PKGGNUDIR", + "PKGINFODIR", + "PKGMANDIR", + "PKGPATH", + "RESTRICTED", + "USE_ABI_DEPENDS") } -// ToRel returns the path of `filename`, relative to the pkgsrc top directory. +// Latest returns the latest package matching the given pattern. +// It searches the category for subdirectories matching the given +// regular expression, takes the latest of them and replaces its +// name with repl. // // Example: -// NewPkgsrc("/usr/pkgsrc").ToRel("/usr/pkgsrc/distfiles") => "distfiles" -func (src *Pkgsrc) ToRel(filename string) string { - return relpath(src.topdir, filename) -} - -// IsInfra returns whether the given filename (relative to the pkglint -// working directory) is part of the pkgsrc infrastructure. -func (src *Pkgsrc) IsInfra(filename string) bool { - rel := src.ToRel(filename) - return hasPrefix(rel, "mk/") || hasPrefix(rel, "wip/mk/") -} +// Latest("lang", `^php[0-9]+$`, "../../lang/$0") +// => "../../lang/php72" +func (src *Pkgsrc) Latest(category string, re regex.Pattern, repl string) string { + versions := src.ListVersions(category, re, repl, true) -func (src *Pkgsrc) addBuildDefs(varnames ...string) { - for _, varname := range varnames { - src.buildDefs[varname] = true + if len(versions) > 0 { + return versions[len(versions)-1] } + return "" } -// IsBuildDef returns whether the given variable is automatically added -// to BUILD_DEFS by the pkgsrc infrastructure. In such a case, the -// package doesn't need to add the variable to BUILD_DEFS itself. -func (src *Pkgsrc) IsBuildDef(varname string) bool { - return src.buildDefs[varname] -} +// ListVersions searches the category for subdirectories matching the given +// regular expression, replaces their names with repl and returns a slice +// of them, properly sorted from early to late. +// +// Example: +// ListVersions("lang", `^php[0-9]+$`, "php-$0") +// => {"php-53", "php-56", "php-73"} +func (src *Pkgsrc) ListVersions(category string, re regex.Pattern, repl string, errorIfEmpty bool) []string { + if G.Testing { + // Regular expression must be anchored at both ends, to avoid typos. + assert(hasPrefix(string(re), "^")) + assert(hasSuffix(string(re), "$")) + } -func (src *Pkgsrc) loadMasterSites() { - mklines := src.LoadMk("mk/fetch/sites.mk", MustSucceed|NotEmpty) + // TODO: Maybe convert cache key to a struct, to save allocations. + cacheKey := category + "/" + string(re) + " => " + repl + if latest, found := src.listVersions[cacheKey]; found { + return latest + } - for _, mkline := range mklines.mklines { - if mkline.IsVarassign() { - varname := mkline.Varname() - // TODO: Give a plausible reason for the MASTER_SITE_BACKUP exception. - if hasPrefix(varname, "MASTER_SITE_") && varname != "MASTER_SITE_BACKUP" { - for _, url := range mkline.ValueFields(mkline.Value()) { - if matches(url, `^(?:http://|https://|ftp://)`) { - src.registerMasterSite(varname, url) - } - } + categoryDir := src.File(category) - // TODO: register variable type, to avoid redundant definitions in vardefs.go. - } + var names []string + for _, fileInfo := range src.ReadDir(category) { + name := fileInfo.Name() + if matches(name, re) { + names = append(names, name) } } - - // Explicitly allowed, although not defined in mk/fetch/sites.mk. - // TODO: Document where this definition comes from and why it is good. - src.registerMasterSite("MASTER_SITE_LOCAL", "ftp://ftp.NetBSD.org/pub/pkgsrc/distfiles/LOCAL_PORTS/") - - if trace.Tracing { - trace.Stepf("Loaded %d MASTER_SITE_* URLs.", len(src.MasterSiteURLToVar)) + if len(names) == 0 { + if errorIfEmpty { + dummyLine.Errorf("Cannot find package versions of %q in %q.", re, categoryDir) + } + src.listVersions[cacheKey] = nil + return nil } -} - -func (src *Pkgsrc) registerMasterSite(varname, url string) { - nameToURL := src.MasterSiteVarToURL - urlToName := src.MasterSiteURLToVar - if nameToURL[varname] == "" { - nameToURL[varname] = url + // In the pkgsrc directories, the major versions of packages are + // written without dots, which leads to ambiguities: + // + // databases/postgresql: 94 < 95 < 96 < 10 < 11 + // lang/go: 19 < 110 < 111 < 2 + keys := make(map[string]int) + for _, name := range names { + if m, pkgbase, versionStr := match2(name, `^(\D+)(\d+)$`); m { + version := toInt(versionStr, 0) + if pkgbase == "postgresql" && version < 60 { + version = 10 * version + } + if pkgbase == "go" { + major := toInt(versionStr[:1], 0) + minor := toInt(versionStr[1:], 0) + version = 100*major + minor + } + keys[name] = version + } } - urlToName[replaceAll(url, `^\w+://`, "")] = varname -} - -func (src *Pkgsrc) loadPkgOptions() { - lines := src.Load("mk/defaults/options.description", MustSucceed) - for _, line := range lines.Lines { - if m, name, description := match2(line.Text, `^([-0-9a-z_+]+)(?:[\t ]+(.*))?$`); m { - src.PkgOptions[name] = description - } else { - line.Errorf("Invalid line format: %s", line.Text) + sort.SliceStable(names, func(i, j int) bool { + if keyI, keyJ := keys[names[i]], keys[names[j]]; keyI != keyJ { + return keyI < keyJ } + return naturalLess(names[i], names[j]) + }) + + var repls = make([]string, len(names)) + for i, name := range names { + repls[i] = replaceAll(name, re, repl) } + + src.listVersions[cacheKey] = repls + return repls } // VariableType returns the type of the variable @@ -1042,6 +950,100 @@ func (src *Pkgsrc) guessVariableType(varname string) (vartype *Vartype) { return nil } +func (src *Pkgsrc) checkToplevelUnusedLicenses() { + if !G.InterPackage.Enabled() { + return + } + + licensesDir := src.File("licenses") + for _, licenseFile := range src.ReadDir("licenses") { + licenseName := licenseFile.Name() + if !G.InterPackage.IsLicenseUsed(licenseName) { + licensePath := joinPath(licensesDir, licenseName) + NewLineWhole(licensePath).Warnf("This license seems to be unused.") + } + } +} + +func (src *Pkgsrc) SuggestedUpdates() []SuggestedUpdate { + if G.Wip { + return src.suggestedWipUpdates + } else { + return src.suggestedUpdates + } +} + +// IsBuildDef returns whether the given variable is automatically added +// to BUILD_DEFS by the pkgsrc infrastructure. In such a case, the +// package doesn't need to add the variable to BUILD_DEFS itself. +func (src *Pkgsrc) IsBuildDef(varname string) bool { + return src.buildDefs[varname] +} + +// ReadDir lists the files and subdirectories from the given directory +// (relative to the pkgsrc root), filtering out any ignored files (CVS/*) +// and empty directories. +func (src *Pkgsrc) ReadDir(dirName string) []os.FileInfo { + dir := src.File(dirName) + files, err := ioutil.ReadDir(dir) + if err != nil { + return nil + } + + var relevantFiles []os.FileInfo + for _, dirent := range files { + name := dirent.Name() + if !dirent.IsDir() || !isIgnoredFilename(name) && !isEmptyDir(joinPath(dir, name)) { + relevantFiles = append(relevantFiles, dirent) + } + } + + return relevantFiles +} + +func (src *Pkgsrc) LoadMkInfra(filename string, options LoadOptions) *MkLines { + if G.Testing { + // During testing, the infrastructure files don't have to exist. + // They are often emulated by setting their data structures manually. + options &^= MustSucceed + } + return src.LoadMk(filename, options) +} + +// LoadMk loads the Makefile relative to the pkgsrc top directory. +func (src *Pkgsrc) LoadMk(filename string, options LoadOptions) *MkLines { + return LoadMk(src.File(filename), options) +} + +// Load loads the file relative to the pkgsrc top directory. +func (src *Pkgsrc) Load(filename string, options LoadOptions) *Lines { + return Load(src.File(filename), options) +} + +// File resolves a filename relative to the pkgsrc top directory. +// +// Example: +// NewPkgsrc("/usr/pkgsrc").File("distfiles") => "/usr/pkgsrc/distfiles" +func (src *Pkgsrc) File(relativeName string) string { + // TODO: Package.File resolves variables, Pkgsrc.File doesn't. They should behave the same. + return cleanpath(joinPath(src.topdir, relativeName)) +} + +// ToRel returns the path of `filename`, relative to the pkgsrc top directory. +// +// Example: +// NewPkgsrc("/usr/pkgsrc").ToRel("/usr/pkgsrc/distfiles") => "distfiles" +func (src *Pkgsrc) ToRel(filename string) string { + return relpath(src.topdir, filename) +} + +// IsInfra returns whether the given filename (relative to the pkglint +// working directory) is part of the pkgsrc infrastructure. +func (src *Pkgsrc) IsInfra(filename string) bool { + rel := src.ToRel(filename) + return hasPrefix(rel, "mk/") || hasPrefix(rel, "wip/mk/") +} + // Change describes a modification to a single package, from the doc/CHANGES-* files. type Change struct { Location Location @@ -1070,7 +1072,7 @@ func (ch *Change) Successor() string { return ch.target } -func (ch *Change) Above(other *Change) bool { +func (ch *Change) IsAbove(other *Change) bool { if ch.Date != other.Date { return ch.Date < other.Date } diff --git a/pkgtools/pkglint/files/pkgsrc_test.go b/pkgtools/pkglint/files/pkgsrc_test.go index 3de9622a31e..25ee853df29 100644 --- a/pkgtools/pkglint/files/pkgsrc_test.go +++ b/pkgtools/pkglint/files/pkgsrc_test.go @@ -2,6 +2,57 @@ package pkglint import "gopkg.in/check.v1" +func (s *Suite) Test_Pkgsrc__frozen(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("doc/CHANGES-2018", + "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2018Q2 branch [freezer 2018-03-25]") + t.FinishSetUp() + + t.CheckEquals(G.Pkgsrc.LastFreezeStart, "2018-03-25") +} + +func (s *Suite) Test_Pkgsrc__not_frozen(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("doc/CHANGES-2018", + "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2018Q2 branch [freezer 2018-03-25]", + "\tmk/bsd.pkg.mk: freeze ended for pkgsrc-2018Q2 branch [freezer 2018-03-27]") + t.FinishSetUp() + + t.CheckEquals(G.Pkgsrc.LastFreezeStart, "2018-03-25") + t.CheckEquals(G.Pkgsrc.LastFreezeEnd, "2018-03-27") +} + +func (s *Suite) Test_Pkgsrc__frozen_with_typo(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("doc/CHANGES-2018", + // The closing bracket is missing. + "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2018Q2 branch [freezer 2018-03-25") + t.FinishSetUp() + + t.CheckEquals(G.Pkgsrc.LastFreezeStart, "") +} + +func (s *Suite) Test_Pkgsrc__caching(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("lang/Makefile") + t.CreateFileLines("lang/python27/Makefile") + + latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") + + t.CheckEquals(latest, "../../lang/python27") + + cached := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") + + t.CheckEquals(cached, "../../lang/python27") +} + // Ensures that pkglint can handle MASTER_SITES definitions with and // without line continuations. // @@ -37,211 +88,21 @@ func (s *Suite) Test_Pkgsrc_loadMasterSites(c *check.C) { t.CheckEquals(G.Pkgsrc.MasterSiteVarToURL["MASTER_SITE_BACKUP"], "") } -func (s *Suite) Test_Pkgsrc_parseSuggestedUpdates(c *check.C) { - t := s.Init(c) - - lines := t.NewLines("doc/TODO", - "", - "Suggested package updates", - "==============", - "For Perl updates \u2026", - "", - "\t"+"o CSP-0.34", - "\t"+"o freeciv-client-2.5.0 (urgent)", - "", - "\t"+"o ignored-0.0") - - todo := G.Pkgsrc.parseSuggestedUpdates(lines) - - t.CheckDeepEquals(todo, []SuggestedUpdate{ - {lines.Lines[5].Location, "CSP", "0.34", ""}, - {lines.Lines[6].Location, "freeciv-client", "2.5.0", "(urgent)"}}) -} - -func (s *Suite) Test_Pkgsrc_checkToplevelUnusedLicenses(c *check.C) { - t := s.Init(c) - - t.SetUpPkgsrc() - t.CreateFileLines("mk/misc/category.mk") - t.CreateFileLines("licenses/2-clause-bsd") - t.CreateFileLines("licenses/gnu-gpl-v3") - - t.CreateFileLines("Makefile", - MkCvsID, - "SUBDIR+=\tcategory") - - t.CreateFileLines("category/Makefile", - MkCvsID, - "COMMENT=\tExample category", - "", - "SUBDIR+=\tpackage", - "SUBDIR+=\tpackage2", - "", - ".include \"../mk/misc/category.mk\"") - - t.SetUpPackage("category/package", - "LICENSE=\t2-clause-bsd") - t.SetUpPackage("category/package2", - "LICENSE=\tmissing") - - t.Main("-r", "-Cglobal", t.File(".")) - - t.CheckOutputLines( - "WARN: ~/category/package2/Makefile:11: License file ~/licenses/missing does not exist.", - "WARN: ~/licenses/gnu-gpl-v2: This license seems to be unused.", // Added by Tester.SetUpPkgsrc - "WARN: ~/licenses/gnu-gpl-v3: This license seems to be unused.", - "3 warnings found.") -} - -func (s *Suite) Test_Pkgsrc_loadUntypedVars(c *check.C) { - t := s.Init(c) - - t.SetUpPkgsrc() - t.SetUpTool("echo", "ECHO", AtRunTime) - t.CreateFileLines("mk/infra.mk", - MkCvsID, - "#", - "# System-provided variables:", - "#", - "# DOCUMENTED", - "#\tThis variable is not actually defined but still documented.", - "#\tThis may be because its definition is evaluated dynamically.", - "", - ".if !defined(INFRA_MK)", - "INFRA_MK:=", - "", - "UNTYPED.one=\tone", - "UNTYPED.two=\ttwo", - "ECHO=\t\techo", - "_UNTYPED=\tinfrastructure only", - ".for p in param", - "PARAMETERIZED.${p}=\tparameterized", - "INDIRECT_${p}=\tindirect", - ".endfor", - "#COMMENTED=\tcommented", - ".endif") - t.FinishSetUp() - - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "", - "do-build:", - "\t: ${INFRA_MK} ${UNTYPED.three} ${ECHO}", - "\t: ${_UNTYPED} ${PARAMETERIZED.param}", - "\t: ${INDIRECT_param}", - "\t: ${DOCUMENTED} ${COMMENTED}") - - mklines.Check() - - t.CheckOutputLines( - "WARN: filename.mk:4: INFRA_MK is used but not defined.", - "WARN: filename.mk:5: _UNTYPED is used but not defined.", - "WARN: filename.mk:6: INDIRECT_param is used but not defined.") -} - -func (s *Suite) Test_Pkgsrc_loadUntypedVars__badly_named_directory(c *check.C) { - t := s.Init(c) - - t.SetUpPkgsrc() - t.CreateFileLines("mk/subdir.mk/file.mk", - MkCvsID) - t.FinishSetUp() - - // Even when a directory is named *.mk, pkglint doesn't crash. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Pkgsrc_loadTools(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("mk/tools/bsd.tools.mk", - ".include \"flex.mk\"", - ".include \"gettext.mk\"", - ".include \"../nonexistent.mk\"", // Is skipped because of the slash. - ".include \"strip.mk\"", - ".include \"replace.mk\"") - t.CreateFileLines("mk/tools/defaults.mk", - "_TOOLS_VARNAME.chown=CHOWN", - "_TOOLS_VARNAME.gawk=AWK", - "_TOOLS_VARNAME.mv=MV", - "_TOOLS_VARNAME.pwd=PWD") - t.CreateFileLines("mk/tools/flex.mk", - "# empty") - t.CreateFileLines("mk/tools/gettext.mk", - ".if ${USE_TOOLS:Mgettext}", // This conditional prevents msgfmt from - "USE_TOOLS+=msgfmt", // being added to the default USE_TOOLS. - ".endif", - "TOOLS_CREATE+=msgfmt") - t.CreateFileLines("mk/tools/strip.mk", - ".if defined(_INSTALL_UNSTRIPPED) || !defined(TOOLS_PLATFORM.strip)", - "TOOLS_NOOP+= strip", - ".else", - "TOOLS_CREATE+= strip", - "TOOLS_PATH.strip= ${TOOLS_PLATFORM.strip}", - ".endif", - "STRIP?= strip") - t.CreateFileLines("mk/tools/replace.mk", - "_TOOLS.bzip2=\tbzip2 bzcat", - "#TOOLS_CREATE+=commented out", - "_UNRELATED_VAR=\t# empty") - t.CreateFileLines("mk/bsd.prefs.mk", - "USE_TOOLS+=\tpwd", - "USE_TOOLS+=\tm4:pkgsrc") - t.CreateFileLines("mk/bsd.pkg.mk", - "USE_TOOLS+=\tmv") - - G.Pkgsrc.loadTools() - - t.EnableTracingToLog() - G.Pkgsrc.Tools.Trace() - t.DisableTracing() - - t.CheckOutputLines( - "TRACE: + (*Tools).Trace()", - "TRACE: 1 tool bzcat:::Nowhere", - "TRACE: 1 tool bzip2:::Nowhere", - "TRACE: 1 tool chown:CHOWN::Nowhere", - "TRACE: 1 tool echo:ECHO:var:AfterPrefsMk", - "TRACE: 1 tool echo -n:ECHO_N:var:AfterPrefsMk", - "TRACE: 1 tool false:FALSE:var:AtRunTime", - "TRACE: 1 tool gawk:AWK::Nowhere", - "TRACE: 1 tool m4:::AfterPrefsMk", - "TRACE: 1 tool msgfmt:::AtRunTime", - "TRACE: 1 tool mv:MV::AtRunTime", - "TRACE: 1 tool pwd:PWD::AfterPrefsMk", - "TRACE: 1 tool strip:::AtRunTime", - "TRACE: 1 tool test:TEST:var:AfterPrefsMk", - "TRACE: 1 tool true:TRUE:var:AfterPrefsMk", - "TRACE: - (*Tools).Trace()") -} - -// As a side-benefit, loadTools also loads the _BUILD_DEFS. -func (s *Suite) Test_Pkgsrc_loadTools__BUILD_DEFS(c *check.C) { +func (s *Suite) Test_Pkgsrc_loadPkgOptions(c *check.C) { t := s.Init(c) - t.SetUpTool("echo", "ECHO", AtRunTime) - pkg := t.SetUpPackage("category/package", - "pre-configure:", - "\t@${ECHO} ${PKG_SYSCONFDIR} ${VARBASE}") - t.CreateFileLines("mk/bsd.pkg.mk", - MkCvsID, - "_BUILD_DEFS+=\tPKG_SYSCONFBASEDIR PKG_SYSCONFDIR") - t.CreateFileLines("mk/defaults/mk.conf", - MkCvsID, - "", - "VARBASE=\t\t/var/pkg", - "PKG_SYSCONFBASEDIR=\t/usr/pkg/etc", - "PKG_SYSCONFDIR=\t/usr/pkg/etc") - t.FinishSetUp() - - G.Check(pkg) + t.CreateFileLines("mk/defaults/options.description", + "option-name Description of the option", + "<<<<< Merge conflict", + "===== Merge conflict", + ">>>>> Merge conflict") - t.CheckEquals(G.Pkgsrc.IsBuildDef("PKG_SYSCONFDIR"), true) - t.CheckEquals(G.Pkgsrc.IsBuildDef("VARBASE"), false) + G.Pkgsrc.loadPkgOptions() t.CheckOutputLines( - "WARN: ~/category/package/Makefile:21: " + - "The user-defined variable VARBASE is used but not added to BUILD_DEFS.") + "ERROR: ~/mk/defaults/options.description:2: Invalid line format: <<<<< Merge conflict", + "ERROR: ~/mk/defaults/options.description:3: Invalid line format: ===== Merge conflict", + "ERROR: ~/mk/defaults/options.description:4: Invalid line format: >>>>> Merge conflict") } func (s *Suite) Test_Pkgsrc_loadDocChanges(c *check.C) { @@ -259,93 +120,6 @@ func (s *Suite) Test_Pkgsrc_loadDocChanges(c *check.C) { t.CheckEquals(G.Pkgsrc.LastChange["pkgpath"].Action, Moved) } -func (s *Suite) Test_Pkgsrc_checkRemovedAfterLastFreeze(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("-Wall", "--source") - t.CreateFileLines("doc/CHANGES-2019", - CvsID, - "", - "\tUpdated category/updated-before to 1.0 [updater 2019-04-01]", - "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2019Q1 branch [freezer 2019-06-21]", - "\tmk/bsd.pkg.mk: freeze ended for pkgsrc-2019Q1 branch [freezer 2019-06-25]", - "\tUpdated category/updated-after to 1.0 [updater 2019-07-01]", - "\tAdded category/added-after version 1.0 [updater 2019-07-01]", - "\tMoved category/moved-from to category/moved-to [author 2019-07-02]", - "\tDowngraded category/downgraded to 1.0 [author 2019-07-03]", - "\tUpdated category/still-there to 1.0 [updater 2019-07-04]") - t.SetUpPackage("category/still-there") - t.FinishSetUp() - - // No error message since -Cglobal is not given. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Pkgsrc_checkRemovedAfterLastFreeze__check_global(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("-Wall", "-Cglobal", "--source") - t.CreateFileLines("doc/CHANGES-2019", - CvsID, - "", - "\tUpdated category/updated-before to 1.0 [updater 2019-04-01]", - "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2019Q1 branch [freezer 2019-06-21]", - "\tmk/bsd.pkg.mk: freeze ended for pkgsrc-2019Q1 branch [freezer 2019-06-25]", - "\tUpdated category/updated-after to 1.0 [updater 2019-07-01]", - "\tAdded category/added-after version 1.0 [updater 2019-07-01]", - "\tMoved category/moved-from to category/moved-to [author 2019-07-02]", - "\tDowngraded category/downgraded to 1.0 [author 2019-07-03]", - "\tUpdated category/still-there to 1.0 [updater 2019-07-04]") - t.SetUpPackage("category/still-there") - t.FinishSetUp() - - // It doesn't matter whether the last visible package change was before - // or after the latest freeze. The crucial point is that the most - // interesting change is the invisible one, which is the removal. - // And for finding the removal reliably, it doesn't matter how long ago - // the last package change was. - - // The empty lines in the following output demonstrate the cheating - // by creating fake lines from Change.Location. - t.CheckOutputLines( - "ERROR: ~/doc/CHANGES-2019:3: Package category/updated-before "+ - "must either exist or be marked as removed.", - "", - "ERROR: ~/doc/CHANGES-2019:6: Package category/updated-after "+ - "must either exist or be marked as removed.", - "", - "ERROR: ~/doc/CHANGES-2019:7: Package category/added-after "+ - "must either exist or be marked as removed.", - "", - "ERROR: ~/doc/CHANGES-2019:9: Package category/downgraded "+ - "must either exist or be marked as removed.") -} - -func (s *Suite) Test_Pkgsrc_checkRemovedAfterLastFreeze__wip(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("wip/package") - t.CreateFileLines("doc/CHANGES-2019", - CvsID, - "", - "\tUpdated category/updated-before to 1.0 [updater 2019-04-01]", - "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2019Q1 branch [freezer 2019-06-21]", - "\tmk/bsd.pkg.mk: freeze ended for pkgsrc-2019Q1 branch [freezer 2019-06-25]", - "\tUpdated category/updated-after to 1.0 [updater 2019-07-01]", - "\tAdded category/added-after version 1.0 [updater 2019-07-01]", - "\tMoved category/moved-from to category/moved-to [author 2019-07-02]", - "\tDowngraded category/downgraded to 1.0 [author 2019-07-03]") - - t.Main("-Wall", "--source", "wip/package") - - // Since the first argument is in pkgsrc-wip, the check for doc/CHANGES - // is skipped. It may well be that a pkgsrc-wip developer doesn't have - // write access to main pkgsrc, and therefore cannot fix doc/CHANGES. - - t.CheckOutputLines( - "Looks fine.") -} - func (s *Suite) Test_Pkgsrc_loadDocChanges__not_found(c *check.C) { t := s.Init(c) @@ -523,6 +297,45 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__infrastructure(c *check.C) { "Looks fine.") } +func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__old(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Cglobal", "-Wall") + t.SetUpPkgsrc() + t.CreateFileLines("doc/CHANGES-2010", + CvsID, + "", + "Changes to the packages collection and infrastructure in 2015:", + "", + "\tInvalid line [3 4]") + t.CreateFileLines("doc/CHANGES-2015", + CvsID, + "", + "Changes to the packages collection and infrastructure in 2015:", + "", + "\tUpdated pkgpath to 1.0 [author 2015-07-01]", + "\tInvalid line [3 4]", + // The date of the below entry is earlier than that of the above entry; + // this error is ignored because the 2015 file is too old. + "\tUpdated pkgpath to 1.2 [author 2015-02-01]") + t.CreateFileLines("doc/CHANGES-2018", + CvsID, + "", + "Changes to the packages collection and infrastructure in 2018:", + "", + "\tUpdated pkgpath to 1.0 [author date]", + "\tUpdated pkgpath to 1.0 [author d]") + t.FinishSetUp() + + // The 2010 file is so old that it is skipped completely. + // The 2015 file is so old that the date is not checked. + // Since 2018, each date in the file must match the filename. + t.CheckOutputLines( + "WARN: ~/doc/CHANGES-2015:6: Unknown doc/CHANGES line: \tInvalid line [3 4]", + "WARN: ~/doc/CHANGES-2018:5: Year \"date\" for pkgpath does not match the filename ~/doc/CHANGES-2018.", + "WARN: ~/doc/CHANGES-2018:6: Date \"d\" for pkgpath is earlier than \"date\" in line 5.") +} + func (s *Suite) Test_Pkgsrc_parseDocChange(c *check.C) { t := s.Init(c) @@ -607,43 +420,112 @@ func (s *Suite) Test_Pkgsrc_parseDocChange(c *check.C) { nil...) } -func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__old(c *check.C) { +func (s *Suite) Test_Pkgsrc_checkRemovedAfterLastFreeze(c *check.C) { t := s.Init(c) - t.SetUpCommandLine("-Cglobal", "-Wall") - t.SetUpPkgsrc() - t.CreateFileLines("doc/CHANGES-2010", + t.SetUpCommandLine("-Wall", "--source") + t.CreateFileLines("doc/CHANGES-2019", CvsID, "", - "Changes to the packages collection and infrastructure in 2015:", - "", - "\tInvalid line [3 4]") - t.CreateFileLines("doc/CHANGES-2015", + "\tUpdated category/updated-before to 1.0 [updater 2019-04-01]", + "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2019Q1 branch [freezer 2019-06-21]", + "\tmk/bsd.pkg.mk: freeze ended for pkgsrc-2019Q1 branch [freezer 2019-06-25]", + "\tUpdated category/updated-after to 1.0 [updater 2019-07-01]", + "\tAdded category/added-after version 1.0 [updater 2019-07-01]", + "\tMoved category/moved-from to category/moved-to [author 2019-07-02]", + "\tDowngraded category/downgraded to 1.0 [author 2019-07-03]", + "\tUpdated category/still-there to 1.0 [updater 2019-07-04]") + t.SetUpPackage("category/still-there") + t.FinishSetUp() + + // No error message since -Cglobal is not given. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Pkgsrc_checkRemovedAfterLastFreeze__check_global(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "-Cglobal", "--source") + t.CreateFileLines("doc/CHANGES-2019", CvsID, "", - "Changes to the packages collection and infrastructure in 2015:", + "\tUpdated category/updated-before to 1.0 [updater 2019-04-01]", + "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2019Q1 branch [freezer 2019-06-21]", + "\tmk/bsd.pkg.mk: freeze ended for pkgsrc-2019Q1 branch [freezer 2019-06-25]", + "\tUpdated category/updated-after to 1.0 [updater 2019-07-01]", + "\tAdded category/added-after version 1.0 [updater 2019-07-01]", + "\tMoved category/moved-from to category/moved-to [author 2019-07-02]", + "\tDowngraded category/downgraded to 1.0 [author 2019-07-03]", + "\tUpdated category/still-there to 1.0 [updater 2019-07-04]") + t.SetUpPackage("category/still-there") + t.FinishSetUp() + + // It doesn't matter whether the last visible package change was before + // or after the latest freeze. The crucial point is that the most + // interesting change is the invisible one, which is the removal. + // And for finding the removal reliably, it doesn't matter how long ago + // the last package change was. + + // The empty lines in the following output demonstrate the cheating + // by creating fake lines from Change.Location. + t.CheckOutputLines( + "ERROR: ~/doc/CHANGES-2019:3: Package category/updated-before "+ + "must either exist or be marked as removed.", "", - "\tUpdated pkgpath to 1.0 [author 2015-07-01]", - "\tInvalid line [3 4]", - // The date of the below entry is earlier than that of the above entry; - // this error is ignored because the 2015 file is too old. - "\tUpdated pkgpath to 1.2 [author 2015-02-01]") - t.CreateFileLines("doc/CHANGES-2018", - CvsID, + "ERROR: ~/doc/CHANGES-2019:6: Package category/updated-after "+ + "must either exist or be marked as removed.", "", - "Changes to the packages collection and infrastructure in 2018:", + "ERROR: ~/doc/CHANGES-2019:7: Package category/added-after "+ + "must either exist or be marked as removed.", "", - "\tUpdated pkgpath to 1.0 [author date]", - "\tUpdated pkgpath to 1.0 [author d]") - t.FinishSetUp() + "ERROR: ~/doc/CHANGES-2019:9: Package category/downgraded "+ + "must either exist or be marked as removed.") +} + +func (s *Suite) Test_Pkgsrc_checkRemovedAfterLastFreeze__wip(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("wip/package") + t.CreateFileLines("doc/CHANGES-2019", + CvsID, + "", + "\tUpdated category/updated-before to 1.0 [updater 2019-04-01]", + "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2019Q1 branch [freezer 2019-06-21]", + "\tmk/bsd.pkg.mk: freeze ended for pkgsrc-2019Q1 branch [freezer 2019-06-25]", + "\tUpdated category/updated-after to 1.0 [updater 2019-07-01]", + "\tAdded category/added-after version 1.0 [updater 2019-07-01]", + "\tMoved category/moved-from to category/moved-to [author 2019-07-02]", + "\tDowngraded category/downgraded to 1.0 [author 2019-07-03]") + + t.Main("-Wall", "--source", "wip/package") + + // Since the first argument is in pkgsrc-wip, the check for doc/CHANGES + // is skipped. It may well be that a pkgsrc-wip developer doesn't have + // write access to main pkgsrc, and therefore cannot fix doc/CHANGES. - // The 2010 file is so old that it is skipped completely. - // The 2015 file is so old that the date is not checked. - // Since 2018, each date in the file must match the filename. t.CheckOutputLines( - "WARN: ~/doc/CHANGES-2015:6: Unknown doc/CHANGES line: \tInvalid line [3 4]", - "WARN: ~/doc/CHANGES-2018:5: Year \"date\" for pkgpath does not match the filename ~/doc/CHANGES-2018.", - "WARN: ~/doc/CHANGES-2018:6: Date \"d\" for pkgpath is earlier than \"date\" in line 5.") + "Looks fine.") +} + +func (s *Suite) Test_Pkgsrc_parseSuggestedUpdates(c *check.C) { + t := s.Init(c) + + lines := t.NewLines("doc/TODO", + "", + "Suggested package updates", + "==============", + "For Perl updates \u2026", + "", + "\t"+"o CSP-0.34", + "\t"+"o freeciv-client-2.5.0 (urgent)", + "", + "\t"+"o ignored-0.0") + + todo := G.Pkgsrc.parseSuggestedUpdates(lines) + + t.CheckDeepEquals(todo, []SuggestedUpdate{ + {lines.Lines[5].Location, "CSP", "0.34", ""}, + {lines.Lines[6].Location, "freeciv-client", "2.5.0", "(urgent)"}}) } func (s *Suite) Test_Pkgsrc_parseSuggestedUpdates__wip(c *check.C) { @@ -666,7 +548,121 @@ func (s *Suite) Test_Pkgsrc_parseSuggestedUpdates__wip(c *check.C) { "This package should be updated to 1.13 ([cool new features]).") } -func (s *Suite) Test_Pkgsrc__deprecated(c *check.C) { +func (s *Suite) Test_Pkgsrc_loadTools(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("mk/tools/bsd.tools.mk", + ".include \"flex.mk\"", + ".include \"gettext.mk\"", + ".include \"../nonexistent.mk\"", // Is skipped because of the slash. + ".include \"strip.mk\"", + ".include \"replace.mk\"") + t.CreateFileLines("mk/tools/defaults.mk", + "_TOOLS_VARNAME.chown=CHOWN", + "_TOOLS_VARNAME.gawk=AWK", + "_TOOLS_VARNAME.mv=MV", + "_TOOLS_VARNAME.pwd=PWD") + t.CreateFileLines("mk/tools/flex.mk", + "# empty") + t.CreateFileLines("mk/tools/gettext.mk", + ".if ${USE_TOOLS:Mgettext}", // This conditional prevents msgfmt from + "USE_TOOLS+=msgfmt", // being added to the default USE_TOOLS. + ".endif", + "TOOLS_CREATE+=msgfmt") + t.CreateFileLines("mk/tools/strip.mk", + ".if defined(_INSTALL_UNSTRIPPED) || !defined(TOOLS_PLATFORM.strip)", + "TOOLS_NOOP+= strip", + ".else", + "TOOLS_CREATE+= strip", + "TOOLS_PATH.strip= ${TOOLS_PLATFORM.strip}", + ".endif", + "STRIP?= strip") + t.CreateFileLines("mk/tools/replace.mk", + "_TOOLS.bzip2=\tbzip2 bzcat", + "#TOOLS_CREATE+=commented out", + "_UNRELATED_VAR=\t# empty") + t.CreateFileLines("mk/bsd.prefs.mk", + "USE_TOOLS+=\tpwd", + "USE_TOOLS+=\tm4:pkgsrc") + t.CreateFileLines("mk/bsd.pkg.mk", + "USE_TOOLS+=\tmv") + + G.Pkgsrc.loadTools() + + t.EnableTracingToLog() + G.Pkgsrc.Tools.Trace() + t.DisableTracing() + + t.CheckOutputLines( + "TRACE: + (*Tools).Trace()", + "TRACE: 1 tool bzcat:::Nowhere", + "TRACE: 1 tool bzip2:::Nowhere", + "TRACE: 1 tool chown:CHOWN::Nowhere", + "TRACE: 1 tool echo:ECHO:var:AfterPrefsMk", + "TRACE: 1 tool echo -n:ECHO_N:var:AfterPrefsMk", + "TRACE: 1 tool false:FALSE:var:AtRunTime", + "TRACE: 1 tool gawk:AWK::Nowhere", + "TRACE: 1 tool m4:::AfterPrefsMk", + "TRACE: 1 tool msgfmt:::AtRunTime", + "TRACE: 1 tool mv:MV::AtRunTime", + "TRACE: 1 tool pwd:PWD::AfterPrefsMk", + "TRACE: 1 tool strip:::AtRunTime", + "TRACE: 1 tool test:TEST:var:AfterPrefsMk", + "TRACE: 1 tool true:TRUE:var:AfterPrefsMk", + "TRACE: - (*Tools).Trace()") +} + +// As a side-benefit, loadTools also loads the _BUILD_DEFS. +func (s *Suite) Test_Pkgsrc_loadTools__BUILD_DEFS(c *check.C) { + t := s.Init(c) + + t.SetUpTool("echo", "ECHO", AtRunTime) + pkg := t.SetUpPackage("category/package", + "pre-configure:", + "\t@${ECHO} ${PKG_SYSCONFDIR} ${VARBASE}") + t.CreateFileLines("mk/bsd.pkg.mk", + MkCvsID, + "_BUILD_DEFS+=\tPKG_SYSCONFBASEDIR PKG_SYSCONFDIR") + t.CreateFileLines("mk/defaults/mk.conf", + MkCvsID, + "", + "VARBASE=\t\t/var/pkg", + "PKG_SYSCONFBASEDIR=\t/usr/pkg/etc", + "PKG_SYSCONFDIR=\t/usr/pkg/etc") + t.FinishSetUp() + + G.Check(pkg) + + t.CheckEquals(G.Pkgsrc.IsBuildDef("PKG_SYSCONFDIR"), true) + t.CheckEquals(G.Pkgsrc.IsBuildDef("VARBASE"), false) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:21: " + + "The user-defined variable VARBASE is used but not added to BUILD_DEFS.") +} + +func (s *Suite) Test_Pkgsrc_loadTools__no_tools_found(c *check.C) { + t := s.Init(c) + + t.ExpectFatal( + G.Pkgsrc.loadTools, + "FATAL: ~/mk/tools/bsd.tools.mk: Cannot be read.") + + t.CreateFileLines("mk/tools/bsd.tools.mk") + + t.ExpectFatal( + G.Pkgsrc.loadTools, + "FATAL: ~/mk/tools/bsd.tools.mk: Must not be empty.") + + t.CreateFileLines("mk/tools/bsd.tools.mk", + MkCvsID) + + t.ExpectFatal( + G.Pkgsrc.loadTools, + "FATAL: ~/mk/tools/bsd.tools.mk: Too few tool files.") +} + +func (s *Suite) Test_Pkgsrc_initDeprecatedVars(c *check.C) { t := s.Init(c) t.SetUpTool("echo", "ECHO", AtRunTime) @@ -689,58 +685,62 @@ func (s *Suite) Test_Pkgsrc__deprecated(c *check.C) { "Use PKG_DEFAULT_JVM instead.") } -func (s *Suite) Test_Pkgsrc_ListVersions__no_basedir(c *check.C) { - t := s.Init(c) - - versions := G.Pkgsrc.ListVersions("lang", `^python[0-9]+$`, "../../lang/$0", true) - - c.Check(versions, check.HasLen, 0) - t.CheckOutputLines( - "ERROR: Cannot find package versions of \"^python[0-9]+$\" in \"~/lang\".") -} - -func (s *Suite) Test_Pkgsrc_ListVersions__no_subdirs(c *check.C) { +func (s *Suite) Test_Pkgsrc_loadUntypedVars(c *check.C) { t := s.Init(c) - t.CreateFileLines("lang/Makefile") - - versions := G.Pkgsrc.ListVersions("lang", `^python[0-9]+$`, "../../lang/$0", true) - - c.Check(versions, check.HasLen, 0) - t.CheckOutputLines( - "ERROR: Cannot find package versions of \"^python[0-9]+$\" in \"~/lang\".") -} + t.SetUpPkgsrc() + t.SetUpTool("echo", "ECHO", AtRunTime) + t.CreateFileLines("mk/infra.mk", + MkCvsID, + "#", + "# System-provided variables:", + "#", + "# DOCUMENTED", + "#\tThis variable is not actually defined but still documented.", + "#\tThis may be because its definition is evaluated dynamically.", + "", + ".if !defined(INFRA_MK)", + "INFRA_MK:=", + "", + "UNTYPED.one=\tone", + "UNTYPED.two=\ttwo", + "ECHO=\t\techo", + "_UNTYPED=\tinfrastructure only", + ".for p in param", + "PARAMETERIZED.${p}=\tparameterized", + "INDIRECT_${p}=\tindirect", + ".endfor", + "#COMMENTED=\tcommented", + ".endif") + t.FinishSetUp() -// Ensures that failed lookups are also cached since they can be assumed -// not to change during a single pkglint run. -func (s *Suite) Test_Pkgsrc_ListVersions__error_is_cached(c *check.C) { - t := s.Init(c) + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + "do-build:", + "\t: ${INFRA_MK} ${UNTYPED.three} ${ECHO}", + "\t: ${_UNTYPED} ${PARAMETERIZED.param}", + "\t: ${INDIRECT_param}", + "\t: ${DOCUMENTED} ${COMMENTED}") - versions := G.Pkgsrc.ListVersions("lang", `^python[0-9]+$`, "../../lang/$0", true) + mklines.Check() - c.Check(versions, check.HasLen, 0) t.CheckOutputLines( - "ERROR: Cannot find package versions of \"^python[0-9]+$\" in \"~/lang\".") - - versions2 := G.Pkgsrc.ListVersions("lang", `^python[0-9]+$`, "../../lang/$0", true) - - c.Check(versions2, check.HasLen, 0) - t.CheckOutputEmpty() // No repeated error message + "WARN: filename.mk:4: INFRA_MK is used but not defined.", + "WARN: filename.mk:5: _UNTYPED is used but not defined.", + "WARN: filename.mk:6: INDIRECT_param is used but not defined.") } -func (s *Suite) Test_Pkgsrc__caching(c *check.C) { +func (s *Suite) Test_Pkgsrc_loadUntypedVars__badly_named_directory(c *check.C) { t := s.Init(c) - t.CreateFileLines("lang/Makefile") - t.CreateFileLines("lang/python27/Makefile") - - latest := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") - - t.CheckEquals(latest, "../../lang/python27") - - cached := G.Pkgsrc.Latest("lang", `^python[0-9]+$`, "../../lang/$0") + t.SetUpPkgsrc() + t.CreateFileLines("mk/subdir.mk/file.mk", + MkCvsID) + t.FinishSetUp() - t.CheckEquals(cached, "../../lang/python27") + // Even when a directory is named *.mk, pkglint doesn't crash. + t.CheckOutputEmpty() } func (s *Suite) Test_Pkgsrc_Latest__multiple_candidates(c *check.C) { @@ -887,42 +887,43 @@ func (s *Suite) Test_Pkgsrc_ListVersions__invalid_argument(c *check.C) { t.Check(versions, check.HasLen, 0) } -func (s *Suite) Test_Pkgsrc_loadPkgOptions(c *check.C) { +func (s *Suite) Test_Pkgsrc_ListVersions__no_basedir(c *check.C) { t := s.Init(c) - t.CreateFileLines("mk/defaults/options.description", - "option-name Description of the option", - "<<<<< Merge conflict", - "===== Merge conflict", - ">>>>> Merge conflict") - - G.Pkgsrc.loadPkgOptions() + versions := G.Pkgsrc.ListVersions("lang", `^python[0-9]+$`, "../../lang/$0", true) + c.Check(versions, check.HasLen, 0) t.CheckOutputLines( - "ERROR: ~/mk/defaults/options.description:2: Invalid line format: <<<<< Merge conflict", - "ERROR: ~/mk/defaults/options.description:3: Invalid line format: ===== Merge conflict", - "ERROR: ~/mk/defaults/options.description:4: Invalid line format: >>>>> Merge conflict") + "ERROR: Cannot find package versions of \"^python[0-9]+$\" in \"~/lang\".") } -func (s *Suite) Test_Pkgsrc_loadTools__no_tools_found(c *check.C) { +func (s *Suite) Test_Pkgsrc_ListVersions__no_subdirs(c *check.C) { t := s.Init(c) - t.ExpectFatal( - G.Pkgsrc.loadTools, - "FATAL: ~/mk/tools/bsd.tools.mk: Cannot be read.") + t.CreateFileLines("lang/Makefile") - t.CreateFileLines("mk/tools/bsd.tools.mk") + versions := G.Pkgsrc.ListVersions("lang", `^python[0-9]+$`, "../../lang/$0", true) - t.ExpectFatal( - G.Pkgsrc.loadTools, - "FATAL: ~/mk/tools/bsd.tools.mk: Must not be empty.") + c.Check(versions, check.HasLen, 0) + t.CheckOutputLines( + "ERROR: Cannot find package versions of \"^python[0-9]+$\" in \"~/lang\".") +} - t.CreateFileLines("mk/tools/bsd.tools.mk", - MkCvsID) +// Ensures that failed lookups are also cached since they can be assumed +// not to change during a single pkglint run. +func (s *Suite) Test_Pkgsrc_ListVersions__error_is_cached(c *check.C) { + t := s.Init(c) - t.ExpectFatal( - G.Pkgsrc.loadTools, - "FATAL: ~/mk/tools/bsd.tools.mk: Too few tool files.") + versions := G.Pkgsrc.ListVersions("lang", `^python[0-9]+$`, "../../lang/$0", true) + + c.Check(versions, check.HasLen, 0) + t.CheckOutputLines( + "ERROR: Cannot find package versions of \"^python[0-9]+$\" in \"~/lang\".") + + versions2 := G.Pkgsrc.ListVersions("lang", `^python[0-9]+$`, "../../lang/$0", true) + + c.Check(versions2, check.HasLen, 0) + t.CheckOutputEmpty() // No repeated error message } // See PR 46570, Ctrl+F "3. In lang/perl5". @@ -1035,7 +1036,7 @@ func (s *Suite) Test_Pkgsrc_guessVariableType__SKIP(c *check.C) { mklines.Check() vartype := G.Pkgsrc.VariableType(mklines, "MY_CHECK_SKIP") - t.CheckEquals(vartype.Guessed(), true) + t.CheckEquals(vartype.IsGuessed(), true) t.CheckEquals(vartype.EffectivePermissions("filename.mk"), aclpAllRuntime) // The permissions for MY_CHECK_SKIP say aclpAllRuntime, which excludes @@ -1051,40 +1052,59 @@ func (s *Suite) Test_Pkgsrc_guessVariableType__SKIP(c *check.C) { "contains the invalid characters \"\\\"\\\"\".") } -func (s *Suite) Test_Pkgsrc__frozen(c *check.C) { +func (s *Suite) Test_Pkgsrc_checkToplevelUnusedLicenses(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") - t.CreateFileLines("doc/CHANGES-2018", - "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2018Q2 branch [freezer 2018-03-25]") - t.FinishSetUp() + t.SetUpPkgsrc() + t.CreateFileLines("mk/misc/category.mk") + t.CreateFileLines("licenses/2-clause-bsd") + t.CreateFileLines("licenses/gnu-gpl-v3") - t.CheckEquals(G.Pkgsrc.LastFreezeStart, "2018-03-25") -} + t.CreateFileLines("Makefile", + MkCvsID, + "SUBDIR+=\tcategory") -func (s *Suite) Test_Pkgsrc__not_frozen(c *check.C) { - t := s.Init(c) + t.CreateFileLines("category/Makefile", + MkCvsID, + "COMMENT=\tExample category", + "", + "SUBDIR+=\tpackage", + "SUBDIR+=\tpackage2", + "", + ".include \"../mk/misc/category.mk\"") - t.SetUpPackage("category/package") - t.CreateFileLines("doc/CHANGES-2018", - "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2018Q2 branch [freezer 2018-03-25]", - "\tmk/bsd.pkg.mk: freeze ended for pkgsrc-2018Q2 branch [freezer 2018-03-27]") - t.FinishSetUp() + t.SetUpPackage("category/package", + "LICENSE=\t2-clause-bsd") + t.SetUpPackage("category/package2", + "LICENSE=\tmissing") - t.CheckEquals(G.Pkgsrc.LastFreezeStart, "2018-03-25") - t.CheckEquals(G.Pkgsrc.LastFreezeEnd, "2018-03-27") + t.Main("-r", "-Cglobal", t.File(".")) + + t.CheckOutputLines( + "WARN: ~/category/package2/Makefile:11: License file ~/licenses/missing does not exist.", + "WARN: ~/licenses/gnu-gpl-v2: This license seems to be unused.", // Added by Tester.SetUpPkgsrc + "WARN: ~/licenses/gnu-gpl-v3: This license seems to be unused.", + "3 warnings found.") } -func (s *Suite) Test_Pkgsrc__frozen_with_typo(c *check.C) { +func (s *Suite) Test_Pkgsrc_ReadDir(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package") - t.CreateFileLines("doc/CHANGES-2018", - // The closing bracket is missing. - "\tmk/bsd.pkg.mk: started freeze for pkgsrc-2018Q2 branch [freezer 2018-03-25") - t.FinishSetUp() + t.CreateFileLines("dir/aaa-subdir/file") + t.CreateFileLines("dir/subdir/file") + t.CreateFileLines("dir/file") + t.CreateFileLines("dir/.git/file") + t.CreateFileLines("dir/CVS/Entries") + t.CreateFileLines("dir/empty/empty/empty/empty/CVS/Entries") - t.CheckEquals(G.Pkgsrc.LastFreezeStart, "") + infos := G.Pkgsrc.ReadDir("dir") + + var names []string + for _, info := range infos { + names = append(names, info.Name()) + } + + t.CheckDeepEquals(names, []string{"aaa-subdir", "file", "subdir"}) } func (s *Suite) Test_Change_Version(c *check.C) { @@ -1128,7 +1148,7 @@ func (s *Suite) Test_Change_Successor(c *check.C) { t.ExpectAssert(func() { downgraded.Successor() }) } -func (s *Suite) Test_Change_Above(c *check.C) { +func (s *Suite) Test_Change_IsAbove(c *check.C) { t := s.Init(c) var changes = []*Change{ @@ -1137,7 +1157,7 @@ func (s *Suite) Test_Change_Above(c *check.C) { {Location{"", 1, 1}, 0, "", "", "", "2011-07-02"}} test := func(i int, chi *Change, j int, chj *Change) { - actual := chi.Above(chj) + actual := chi.IsAbove(chj) expected := i < j if actual != expected { t.CheckDeepEquals( @@ -1159,23 +1179,3 @@ func (s *Suite) Test_ChangeAction_String(c *check.C) { t.CheckEquals(Added.String(), "Added") t.CheckEquals(Removed.String(), "Removed") } - -func (s *Suite) Test_Pkgsrc_ReadDir(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("dir/aaa-subdir/file") - t.CreateFileLines("dir/subdir/file") - t.CreateFileLines("dir/file") - t.CreateFileLines("dir/.git/file") - t.CreateFileLines("dir/CVS/Entries") - t.CreateFileLines("dir/empty/empty/empty/empty/CVS/Entries") - - infos := G.Pkgsrc.ReadDir("dir") - - var names []string - for _, info := range infos { - names = append(names, info.Name()) - } - - t.CheckDeepEquals(names, []string{"aaa-subdir", "file", "subdir"}) -} diff --git a/pkgtools/pkglint/files/pkgver/vercmp_test.go b/pkgtools/pkglint/files/pkgver/vercmp_test.go index 05c2cb0fbda..e69eb546871 100644 --- a/pkgtools/pkglint/files/pkgver/vercmp_test.go +++ b/pkgtools/pkglint/files/pkgver/vercmp_test.go @@ -2,6 +2,7 @@ package pkgver import ( "gopkg.in/check.v1" + "netbsd.org/pkglint/intqa" "testing" ) @@ -12,35 +13,6 @@ func Test(t *testing.T) { check.TestingT(t) } -func (s *Suite) Test_newVersion(c *check.C) { - c.Check(newVersion("5.0"), check.DeepEquals, - &version{[]int{5, 0, 0}, 0}) - c.Check(newVersion("5.0nb5"), check.DeepEquals, - &version{[]int{5, 0, 0}, 5}) - c.Check(newVersion("0.0.1-SNAPSHOT"), check.DeepEquals, - &version{[]int{0, 0, 0, 0, 1, 19, 14, 1, 16, 19, 8, 15, 20}, 0}) - c.Check(newVersion("1.0alpha3"), check.DeepEquals, - &version{[]int{1, 0, 0, -3, 3}, 0}) - c.Check(newVersion("1_0alpha3"), check.DeepEquals, - &version{[]int{1, 0, 0, -3, 3}, 0}) - c.Check(newVersion("2.5beta"), check.DeepEquals, - &version{[]int{2, 0, 5, -2}, 0}) - c.Check(newVersion("20151110"), check.DeepEquals, - &version{[]int{20151110}, 0}) - c.Check(newVersion("0"), check.DeepEquals, - &version{[]int{0}, 0}) - c.Check(newVersion("nb1"), check.DeepEquals, - &version{nil, 1}) - c.Check(newVersion("1.0.1a"), check.DeepEquals, - &version{[]int{1, 0, 0, 0, 1, 1}, 0}) - c.Check(newVersion("1.0.1z"), check.DeepEquals, - &version{[]int{1, 0, 0, 0, 1, 26}, 0}) - c.Check(newVersion("0pre20160620"), check.DeepEquals, - &version{[]int{0, -1, 20160620}, 0}) - c.Check(newVersion("3.5.DEV1710"), check.DeepEquals, - &version{[]int{3, 0, 5, 0, 4, 5, 22, 1710}, 0}) -} - func (s *Suite) Test_Compare(c *check.C) { var versions = [][]string{ {"0pre20160620"}, @@ -91,3 +63,38 @@ func (s *Suite) Test_Compare(c *check.C) { } } } + +func (s *Suite) Test_newVersion(c *check.C) { + c.Check(newVersion("5.0"), check.DeepEquals, + &version{[]int{5, 0, 0}, 0}) + c.Check(newVersion("5.0nb5"), check.DeepEquals, + &version{[]int{5, 0, 0}, 5}) + c.Check(newVersion("0.0.1-SNAPSHOT"), check.DeepEquals, + &version{[]int{0, 0, 0, 0, 1, 19, 14, 1, 16, 19, 8, 15, 20}, 0}) + c.Check(newVersion("1.0alpha3"), check.DeepEquals, + &version{[]int{1, 0, 0, -3, 3}, 0}) + c.Check(newVersion("1_0alpha3"), check.DeepEquals, + &version{[]int{1, 0, 0, -3, 3}, 0}) + c.Check(newVersion("2.5beta"), check.DeepEquals, + &version{[]int{2, 0, 5, -2}, 0}) + c.Check(newVersion("20151110"), check.DeepEquals, + &version{[]int{20151110}, 0}) + c.Check(newVersion("0"), check.DeepEquals, + &version{[]int{0}, 0}) + c.Check(newVersion("nb1"), check.DeepEquals, + &version{nil, 1}) + c.Check(newVersion("1.0.1a"), check.DeepEquals, + &version{[]int{1, 0, 0, 0, 1, 1}, 0}) + c.Check(newVersion("1.0.1z"), check.DeepEquals, + &version{[]int{1, 0, 0, 0, 1, 26}, 0}) + c.Check(newVersion("0pre20160620"), check.DeepEquals, + &version{[]int{0, -1, 20160620}, 0}) + c.Check(newVersion("3.5.DEV1710"), check.DeepEquals, + &version{[]int{3, 0, 5, 0, 4, 5, 22, 1710}, 0}) +} + +func (s *Suite) Test__test_names(c *check.C) { + ck := intqa.NewTestNameChecker(c.Errorf) + ck.Enable(intqa.EAll, -intqa.EMissingTest) + ck.Check() +} diff --git a/pkgtools/pkglint/files/plist.go b/pkgtools/pkglint/files/plist.go index 6dae946c287..93b55d7890e 100644 --- a/pkgtools/pkglint/files/plist.go +++ b/pkgtools/pkglint/files/plist.go @@ -48,12 +48,6 @@ type PlistChecker struct { nonAsciiAllowed bool } -type PlistLine struct { - *Line - conditions []string // e.g. PLIST.docs - text string // Line.Text without any conditions of the form ${PLIST.cond} -} - func (ck *PlistChecker) Load(lines *Lines) []*PlistLine { plines := ck.NewLines(lines) ck.collectFilesAndDirs(plines) @@ -196,7 +190,7 @@ func (ck *PlistChecker) checkPath(pline *PlistLine) { ck.checkPathShare(pline) } - if contains(text, "${PKGLOCALEDIR}") && ck.pkg != nil && !ck.pkg.vars.Defined("USE_PKGLOCALEDIR") { + if contains(text, "${PKGLOCALEDIR}") && ck.pkg != nil && !ck.pkg.vars.IsDefined("USE_PKGLOCALEDIR") { pline.Warnf("PLIST contains ${PKGLOCALEDIR}, but USE_PKGLOCALEDIR is not set in the package Makefile.") } @@ -307,7 +301,7 @@ func (ck *PlistChecker) checkPathInfo(pline *PlistLine, dirname, basename string return } - if ck.pkg != nil && !ck.pkg.vars.Defined("INFO_FILES") { + if ck.pkg != nil && !ck.pkg.vars.IsDefined("INFO_FILES") { pline.Warnf("Packages that install info files should set INFO_FILES in the Makefile.") } } @@ -338,7 +332,7 @@ func (ck *PlistChecker) checkPathLib(pline *PlistLine, dirname, basename string) pline.Errorf("Only the libiconv package may install lib/charset.alias.") } - if hasSuffix(basename, ".la") && !pkg.vars.Defined("USE_LIBTOOL") { + if hasSuffix(basename, ".la") && !pkg.vars.IsDefined("USE_LIBTOOL") { if ck.once.FirstTime("USE_LIBTOOL") { pline.Warnf("Packages that install libtool libraries should define USE_LIBTOOL.") } @@ -433,11 +427,17 @@ func (ck *PlistChecker) checkPathShareIcons(pline *PlistLine) { } } - if contains(text[12:], "/") && !pkg.vars.Defined("ICON_THEMES") && ck.once.FirstTime("ICON_THEMES") { + if contains(text[12:], "/") && !pkg.vars.IsDefined("ICON_THEMES") && ck.once.FirstTime("ICON_THEMES") { pline.Warnf("Packages that install icon theme files should set ICON_THEMES.") } } +type PlistLine struct { + *Line + conditions []string // e.g. PLIST.docs + text string // Line.Text without any conditions of the form ${PLIST.cond} +} + func (pline *PlistLine) CheckTrailingWhitespace() { if hasSuffix(pline.text, " ") || hasSuffix(pline.text, "\t") { pline.Errorf("Pkgsrc does not support filenames ending in whitespace.") diff --git a/pkgtools/pkglint/files/plist_test.go b/pkgtools/pkglint/files/plist_test.go index 10ed1c86ed0..43cc041c3b9 100644 --- a/pkgtools/pkglint/files/plist_test.go +++ b/pkgtools/pkglint/files/plist_test.go @@ -156,136 +156,6 @@ func (s *Suite) Test_CheckLinesPlist__sort_common(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_plistLineSorter_Sort(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("--autofix") - lines := t.SetUpFileLines("PLIST", - PlistCvsID, - "@comment Do not remove", - "A", - "b", - "CCC", - "lib/${UNKNOWN}.la", - "C", - "ddd", - "@exec echo \"after ddd\"", // Makes the PLIST unsortable - "sbin/program", - "${PLIST.one}bin/program", - "man/man1/program.1", - "${PLIST.two}bin/program2", - "lib/before.la", - "${PLIST.linux}${PLIST.x86_64}lib/lib-linux-x86_64.so", // Double condition, see graphics/graphviz - "lib/after.la", - "@exec echo \"after lib/after.la\"") - ck := PlistChecker{nil, nil, nil, "", Once{}, false} - plines := ck.NewLines(lines) - - sorter1 := NewPlistLineSorter(plines) - t.CheckEquals(sorter1.unsortable, lines.Lines[5]) - - cleanedLines := append(append(lines.Lines[0:5], lines.Lines[6:8]...), lines.Lines[9:]...) // Remove ${UNKNOWN} and @exec - - sorter2 := NewPlistLineSorter((&PlistChecker{nil, nil, nil, "", Once{}, false}). - NewLines(NewLines(lines.Filename, cleanedLines))) - - c.Check(sorter2.unsortable, check.IsNil) - - sorter2.Sort() - - t.CheckOutputLines( - "AUTOFIX: ~/PLIST:3: Sorting the whole file.") - t.CheckFileLines("PLIST", - PlistCvsID, - "@comment Do not remove", // The header ends here - "A", - "C", - "CCC", - "b", - "${PLIST.one}bin/program", // Conditional lines are ignored during sorting - "${PLIST.two}bin/program2", - "ddd", - "lib/after.la", - "lib/before.la", - "${PLIST.linux}${PLIST.x86_64}lib/lib-linux-x86_64.so", - "man/man1/program.1", - "sbin/program", - "@exec echo \"after lib/after.la\"") // The footer starts here -} - -func (s *Suite) Test_PlistChecker_checkLine(c *check.C) { - t := s.Init(c) - - lines := t.NewLines("PLIST", - PlistCvsID, - "bin/program", - "${PLIST.var}bin/conditional-program", - "${PLIST.linux}${PLIST.arm}bin/arm-linux-only", - "${PLIST.linux}${PLIST.arm-64}@exec echo 'This is Linux/arm64'", - "${PLIST.ocaml-opt}share/ocaml", - "${PLIST.ocaml-opt}@exec echo 'This is OCaml'", - "${PLIST.ocaml-opt}@exec echo 'This is OCaml'", - "${PYSITELIB:S,lib,share}/modifiers don't work in PLISTs", - "${PLIST.empty}", - "", - "$prefix/bin", - - // This line does not count as a PLIST condition since it has - // a :Q modifier, which does not work in PLISTs. Therefore the - // ${PLIST.man:Q} is considered part of the filename. - "${PLIST.man:Q}man/cat3/strlcpy.3", - "<<<<<<<<< merge conflict") - - CheckLinesPlist(nil, lines) - - t.CheckOutputLines( - "WARN: PLIST:3: \"bin/conditional-program\" should be sorted before \"bin/program\".", - "WARN: PLIST:4: \"bin/arm-linux-only\" should be sorted before \"bin/conditional-program\".", - "WARN: PLIST:10: PLISTs should not contain empty lines.", - "WARN: PLIST:11: PLISTs should not contain empty lines.", - "WARN: PLIST:14: Invalid line type: <<<<<<<<< merge conflict") -} - -func (s *Suite) Test_PlistChecker_checkPathMan__gz(c *check.C) { - t := s.Init(c) - - G.Pkg = NewPackage(t.File("category/pkgbase")) - lines := t.NewLines("PLIST", - PlistCvsID, - "man/man3/strerror.3.gz") - - CheckLinesPlist(G.Pkg, lines) - - t.CheckOutputLines( - "NOTE: PLIST:2: The .gz extension is unnecessary for manual pages.") -} - -func (s *Suite) Test_PlistChecker_checkPath__PKGMANDIR(c *check.C) { - t := s.Init(c) - - lines := t.NewLines("PLIST", - PlistCvsID, - "${PKGMANDIR}/man1/sh.1") - - CheckLinesPlist(nil, lines) - - t.CheckOutputLines( - "NOTE: PLIST:2: PLIST files should use \"man/\" instead of \"${PKGMANDIR}\".") -} - -func (s *Suite) Test_PlistChecker_checkPath__python_egg(c *check.C) { - t := s.Init(c) - - lines := t.NewLines("PLIST", - PlistCvsID, - "${PYSITELIB}/gdspy-${PKGVERSION}-py${PYVERSSUFFIX}.egg-info/PKG-INFO") - - CheckLinesPlist(nil, lines) - - t.CheckOutputLines( - "WARN: PLIST:2: Include \"../../lang/python/egg.mk\" instead of listing .egg-info files directly.") -} - func (s *Suite) Test_PlistChecker__autofix(c *check.C) { t := s.Init(c) @@ -473,59 +343,6 @@ func (s *Suite) Test_PlistChecker__invalid_line_type(c *check.C) { "WARN: ~/PLIST:6: Invalid line type: >>>>>>>> merge conflict") } -func (s *Suite) Test_PlistChecker_checkPathNonAscii(c *check.C) { - t := s.Init(c) - - t.SetUpCommandLine("-Wall", "--explain") - lines := t.NewLines("PLIST", - PlistCvsID, - - "dir1/fr\xFCher", // German, "back then", encoded in ISO 8859-1 - - // Subsequent non-ASCII filenames do not generate further messages - // since these filenames typically appear in groups, and issuing - // too many warnings quickly gets boring. - "dir1/\u00C4thernetz", // German - - // This ASCII-only pathname enables the check again. - "dir2/aaa", - "dir2/\u0633\u0644\u0627\u0645", // Arabic: salaam - - "dir2/\uC548\uB148", // Korean: annyeong - - // This ASCII-only pathname enables the check again. - "dir3/ascii-only", - - // Any comment suppresses the check for the next contiguous - // sequence of non-ASCII filenames. - "@comment The next file is non-ASCII on purpose.", - "dir3/\U0001F603", // Smiling face with open mouth - - // This ASCII-only pathname enables the check again. - "sbin/iconv", - - "sbin/\U0001F603", // Smiling face with open mouth - ) - - CheckLinesPlist(nil, lines) - - t.CheckOutputLines( - "WARN: PLIST:2: Non-ASCII filename \"dir1/fr<0xFC>her\".", - "", - "\tThe great majority of filenames installed by pkgsrc packages are", - "\tASCII-only. Filenames containing non-ASCII characters can cause", - "\tvarious problems since their name may already be different when", - "\tanother character encoding is set in the locale.", - "", - "\tTo mark a filename as intentionally non-ASCII, insert a PLIST", - "\t@comment with a convincing reason directly above this line. That", - "\tcomment will allow this line and the lines directly below it to", - "\tcontain non-ASCII filenames.", - "", - "WARN: PLIST:5: Non-ASCII filename \"dir2/<U+0633><U+0644><U+0627><U+0645>\".", - "WARN: PLIST:11: Non-ASCII filename \"sbin/<U+1F603>\".") -} - func (s *Suite) Test_PlistChecker__doc(c *check.C) { t := s.Init(c) @@ -583,6 +400,65 @@ func (s *Suite) Test_PlistChecker__PKGLOCALEDIR_without_package(c *check.C) { t.CheckOutputEmpty() } +func (s *Suite) Test_PlistChecker_checkLine(c *check.C) { + t := s.Init(c) + + lines := t.NewLines("PLIST", + PlistCvsID, + "bin/program", + "${PLIST.var}bin/conditional-program", + "${PLIST.linux}${PLIST.arm}bin/arm-linux-only", + "${PLIST.linux}${PLIST.arm-64}@exec echo 'This is Linux/arm64'", + "${PLIST.ocaml-opt}share/ocaml", + "${PLIST.ocaml-opt}@exec echo 'This is OCaml'", + "${PLIST.ocaml-opt}@exec echo 'This is OCaml'", + "${PYSITELIB:S,lib,share}/modifiers don't work in PLISTs", + "${PLIST.empty}", + "", + "$prefix/bin", + + // This line does not count as a PLIST condition since it has + // a :Q modifier, which does not work in PLISTs. Therefore the + // ${PLIST.man:Q} is considered part of the filename. + "${PLIST.man:Q}man/cat3/strlcpy.3", + "<<<<<<<<< merge conflict") + + CheckLinesPlist(nil, lines) + + t.CheckOutputLines( + "WARN: PLIST:3: \"bin/conditional-program\" should be sorted before \"bin/program\".", + "WARN: PLIST:4: \"bin/arm-linux-only\" should be sorted before \"bin/conditional-program\".", + "WARN: PLIST:10: PLISTs should not contain empty lines.", + "WARN: PLIST:11: PLISTs should not contain empty lines.", + "WARN: PLIST:14: Invalid line type: <<<<<<<<< merge conflict") +} + +func (s *Suite) Test_PlistChecker_checkPath__PKGMANDIR(c *check.C) { + t := s.Init(c) + + lines := t.NewLines("PLIST", + PlistCvsID, + "${PKGMANDIR}/man1/sh.1") + + CheckLinesPlist(nil, lines) + + t.CheckOutputLines( + "NOTE: PLIST:2: PLIST files should use \"man/\" instead of \"${PKGMANDIR}\".") +} + +func (s *Suite) Test_PlistChecker_checkPath__python_egg(c *check.C) { + t := s.Init(c) + + lines := t.NewLines("PLIST", + PlistCvsID, + "${PYSITELIB}/gdspy-${PKGVERSION}-py${PYVERSSUFFIX}.egg-info/PKG-INFO") + + CheckLinesPlist(nil, lines) + + t.CheckOutputLines( + "WARN: PLIST:2: Include \"../../lang/python/egg.mk\" instead of listing .egg-info files directly.") +} + func (s *Suite) Test_PlistChecker_checkPath__unwanted_entries(c *check.C) { t := s.Init(c) @@ -600,6 +476,59 @@ func (s *Suite) Test_PlistChecker_checkPath__unwanted_entries(c *check.C) { "WARN: ~/PLIST:4: .orig files should not be in the PLIST.") } +func (s *Suite) Test_PlistChecker_checkPathNonAscii(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "--explain") + lines := t.NewLines("PLIST", + PlistCvsID, + + "dir1/fr\xFCher", // German, "back then", encoded in ISO 8859-1 + + // Subsequent non-ASCII filenames do not generate further messages + // since these filenames typically appear in groups, and issuing + // too many warnings quickly gets boring. + "dir1/\u00C4thernetz", // German + + // This ASCII-only pathname enables the check again. + "dir2/aaa", + "dir2/\u0633\u0644\u0627\u0645", // Arabic: salaam + + "dir2/\uC548\uB148", // Korean: annyeong + + // This ASCII-only pathname enables the check again. + "dir3/ascii-only", + + // Any comment suppresses the check for the next contiguous + // sequence of non-ASCII filenames. + "@comment The next file is non-ASCII on purpose.", + "dir3/\U0001F603", // Smiling face with open mouth + + // This ASCII-only pathname enables the check again. + "sbin/iconv", + + "sbin/\U0001F603", // Smiling face with open mouth + ) + + CheckLinesPlist(nil, lines) + + t.CheckOutputLines( + "WARN: PLIST:2: Non-ASCII filename \"dir1/fr<0xFC>her\".", + "", + "\tThe great majority of filenames installed by pkgsrc packages are", + "\tASCII-only. Filenames containing non-ASCII characters can cause", + "\tvarious problems since their name may already be different when", + "\tanother character encoding is set in the locale.", + "", + "\tTo mark a filename as intentionally non-ASCII, insert a PLIST", + "\t@comment with a convincing reason directly above this line. That", + "\tcomment will allow this line and the lines directly below it to", + "\tcontain non-ASCII filenames.", + "", + "WARN: PLIST:5: Non-ASCII filename \"dir2/<U+0633><U+0644><U+0627><U+0645>\".", + "WARN: PLIST:11: Non-ASCII filename \"sbin/<U+1F603>\".") +} + func (s *Suite) Test_PlistChecker_checkPathInfo(c *check.C) { t := s.Init(c) @@ -716,6 +645,20 @@ func (s *Suite) Test_PlistChecker_checkPathMan(c *check.C) { "WARN: ~/PLIST:5: Unknown section \"x\" for manual page.") } +func (s *Suite) Test_PlistChecker_checkPathMan__gz(c *check.C) { + t := s.Init(c) + + G.Pkg = NewPackage(t.File("category/pkgbase")) + lines := t.NewLines("PLIST", + PlistCvsID, + "man/man3/strerror.3.gz") + + CheckLinesPlist(G.Pkg, lines) + + t.CheckOutputLines( + "NOTE: PLIST:2: The .gz extension is unnecessary for manual pages.") +} + func (s *Suite) Test_PlistChecker_checkPathShare(c *check.C) { t := s.Init(c) @@ -879,18 +822,6 @@ func (s *Suite) Test_PlistLine_CheckDirective(c *check.C) { "WARN: ~/PLIST:13: Unknown PLIST directive \"@unknown\".") } -func (s *Suite) Test_NewPlistLineSorter__only_comments(c *check.C) { - t := s.Init(c) - - lines := t.NewLines("PLIST", - PlistCvsID, - "@comment intentionally left empty") - - CheckLinesPlist(nil, lines) - - t.CheckOutputEmpty() -} - func (s *Suite) Test_plistLineSorter__unsortable(c *check.C) { t := s.Init(c) @@ -913,3 +844,72 @@ func (s *Suite) Test_plistLineSorter__unsortable(c *check.C) { "TRACE: 1 - SaveAutofixChanges()", "TRACE: - CheckLinesPlist(\"~/PLIST\")") } + +func (s *Suite) Test_NewPlistLineSorter__only_comments(c *check.C) { + t := s.Init(c) + + lines := t.NewLines("PLIST", + PlistCvsID, + "@comment intentionally left empty") + + CheckLinesPlist(nil, lines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_plistLineSorter_Sort(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--autofix") + lines := t.SetUpFileLines("PLIST", + PlistCvsID, + "@comment Do not remove", + "A", + "b", + "CCC", + "lib/${UNKNOWN}.la", + "C", + "ddd", + "@exec echo \"after ddd\"", // Makes the PLIST unsortable + "sbin/program", + "${PLIST.one}bin/program", + "man/man1/program.1", + "${PLIST.two}bin/program2", + "lib/before.la", + "${PLIST.linux}${PLIST.x86_64}lib/lib-linux-x86_64.so", // Double condition, see graphics/graphviz + "lib/after.la", + "@exec echo \"after lib/after.la\"") + ck := PlistChecker{nil, nil, nil, "", Once{}, false} + plines := ck.NewLines(lines) + + sorter1 := NewPlistLineSorter(plines) + t.CheckEquals(sorter1.unsortable, lines.Lines[5]) + + cleanedLines := append(append(lines.Lines[0:5], lines.Lines[6:8]...), lines.Lines[9:]...) // Remove ${UNKNOWN} and @exec + + sorter2 := NewPlistLineSorter((&PlistChecker{nil, nil, nil, "", Once{}, false}). + NewLines(NewLines(lines.Filename, cleanedLines))) + + c.Check(sorter2.unsortable, check.IsNil) + + sorter2.Sort() + + t.CheckOutputLines( + "AUTOFIX: ~/PLIST:3: Sorting the whole file.") + t.CheckFileLines("PLIST", + PlistCvsID, + "@comment Do not remove", // The header ends here + "A", + "C", + "CCC", + "b", + "${PLIST.one}bin/program", // Conditional lines are ignored during sorting + "${PLIST.two}bin/program2", + "ddd", + "lib/after.la", + "lib/before.la", + "${PLIST.linux}${PLIST.x86_64}lib/lib-linux-x86_64.so", + "man/man1/program.1", + "sbin/program", + "@exec echo \"after lib/after.la\"") // The footer starts here +} diff --git a/pkgtools/pkglint/files/redundantscope.go b/pkgtools/pkglint/files/redundantscope.go index 104ad1a9560..ee272c42e94 100644 --- a/pkgtools/pkglint/files/redundantscope.go +++ b/pkgtools/pkglint/files/redundantscope.go @@ -73,7 +73,7 @@ func (s *RedundantScope) handleVarassign(mkline *MkLine, ind *Indentation) { // this variable assignment and the/any? previous one. // See Test_RedundantScope__overwrite_inside_conditional. // Anyway, too few warnings are better than wrong warnings. - if info.vari.Conditional() || ind.Depth("") > 0 { + if info.vari.IsConditional() || ind.Depth("") > 0 { return } @@ -147,7 +147,7 @@ func (s *RedundantScope) handleVarassign(mkline *MkLine, ind *Indentation) { // // Except when this line has the same value as the guaranteed // current value of the variable. Then it is redundant. - if info.vari.Constant() && info.vari.ConstantValue() == mkline.Value() { + if info.vari.IsConstant() && info.vari.ConstantValue() == mkline.Value() { s.onRedundant(prevWrites[len(prevWrites)-1], mkline) } } diff --git a/pkgtools/pkglint/files/redundantscope_test.go b/pkgtools/pkglint/files/redundantscope_test.go index 6dd7e57e58b..00877d58103 100644 --- a/pkgtools/pkglint/files/redundantscope_test.go +++ b/pkgtools/pkglint/files/redundantscope_test.go @@ -1427,7 +1427,7 @@ func (s *Suite) Test_RedundantScope__procedure_parameters(c *check.C) { t.CheckOutputEmpty() } -// Branch coverage for info.vari.Constant(). The other tests typically +// Branch coverage for info.vari.IsConstant(). The other tests typically // make a variable non-constant by adding conditional assignments between // .if/.endif. But there are other ways. The output of shell commands is // unpredictable for pkglint (as of March 2019), therefore it treats these @@ -1449,7 +1449,7 @@ func (s *Suite) Test_RedundantScope_handleVarassign__shell_followed_by_default(c t.CheckOutputEmpty() } -func (s *Suite) Test_RedundantScope__overwrite_definition_from_included_file(c *check.C) { +func (s *Suite) Test_RedundantScope_handleVarassign__overwrite_definition_from_included_file(c *check.C) { t := s.Init(c) include, get := t.SetUpHierarchy() @@ -1504,7 +1504,7 @@ func (s *Suite) Test_RedundantScope_handleVarassign__conditional(c *check.C) { } // Ensures that commented variables do not influence the redundancy check. -func (s *Suite) Test_RedundantScope__commented_variable_assignment(c *check.C) { +func (s *Suite) Test_RedundantScope_handleVarassign__commented_variable_assignment(c *check.C) { t := s.Init(c) include, get := t.SetUpHierarchy() diff --git a/pkgtools/pkglint/files/shell.go b/pkgtools/pkglint/files/shell.go index 97407de0749..739b96e5b6b 100644 --- a/pkgtools/pkglint/files/shell.go +++ b/pkgtools/pkglint/files/shell.go @@ -1,453 +1,12 @@ package pkglint -// Parsing and checking shell commands embedded in Makefiles - import ( "netbsd.org/pkglint/textproc" "path" "strings" ) -// TODO: Can ShellLine and ShellProgramChecker be merged into one type? - -// ShellLineChecker is either a line from a Makefile starting with a tab, -// thereby containing shell commands to be executed. -// -// Or it is a variable assignment line from a Makefile with a left-hand -// side variable that is of some shell-like type; see Vartype.IsShell. -type ShellLineChecker struct { - MkLines *MkLines - mkline *MkLine - - // checkVarUse is set to false when checking a single shell word - // in order to skip duplicate warnings in variable assignments. - checkVarUse bool -} - -func NewShellLineChecker(mklines *MkLines, mkline *MkLine) *ShellLineChecker { - assertNotNil(mklines) - return &ShellLineChecker{mklines, mkline, true} -} - -func (ck *ShellLineChecker) Warnf(format string, args ...interface{}) { - ck.mkline.Warnf(format, args...) -} -func (ck *ShellLineChecker) Explain(explanation ...string) { - ck.mkline.Explain(explanation...) -} - -var shellCommandsType = NewVartype(BtShellCommands, NoVartypeOptions, NewACLEntry("*", aclpAllRuntime)) -var shellWordVuc = &VarUseContext{shellCommandsType, VucUnknownTime, VucQuotPlain, false} - -func (ck *ShellLineChecker) CheckWord(token string, checkQuoting bool, time ToolTime) { - if trace.Tracing { - defer trace.Call(token, checkQuoting)() - } - - if token == "" || hasPrefix(token, "#") { - return - } - - var line = ck.mkline.Line - - // Delegate check for shell words consisting of a single variable use - // to the MkLineChecker. Examples for these are ${VAR:Mpattern} or $@. - if varuse := ToVarUse(token); varuse != nil { - if ck.checkVarUse { - MkLineChecker{ck.MkLines, ck.mkline}.CheckVaruse(varuse, shellWordVuc) - } - return - } - - if matches(token, `\$\{PREFIX\}/man(?:$|/)`) { - line.Warnf("Please use ${PKGMANDIR} instead of \"man\".") - } - - if contains(token, "etc/rc.d") { - line.Warnf("Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") - } - - ck.checkWordQuoting(token, checkQuoting, time) -} - -func (ck *ShellLineChecker) checkWordQuoting(token string, checkQuoting bool, time ToolTime) { - tok := NewShTokenizer(ck.mkline.Line, token, false) - - atoms := tok.ShAtoms() - quoting := shqPlain -outer: - for len(atoms) > 0 { - atom := atoms[0] - // Cutting off the first atom is done at the end of the loop since in - // some cases the called methods need to see the current atom. - - if trace.Tracing { - trace.Stepf("shell state %s: %q", quoting, atom) - } - - switch { - case atom.Quoting == shqBackt || atom.Quoting == shqDquotBackt: - backtCommand := ck.unescapeBackticks(&atoms, quoting) - if backtCommand != "" { - // TODO: Wrap the setE into a struct. - setE := true - ck.CheckShellCommand(backtCommand, &setE, time) - } - continue - - // Make(1) variables have the same syntax, no matter in which state the shell parser is currently. - case ck.checkVaruseToken(&atoms, quoting): - continue - - case quoting == shqPlain: - switch { - case atom.Type == shtShVarUse: - ck.checkShVarUsePlain(atom, checkQuoting) - - case atom.Type == shtSubshell: - ck.Warnf("Invoking subshells via $(...) is not portable enough.") - ck.Explain( - "The Solaris /bin/sh does not know this way to execute a command in a subshell.", - "Please use backticks (`...`) as a replacement.") - - // Early return to avoid further parse errors. - // As of December 2018, it might be worth continuing again since the - // shell parser has improved in 2018. - return - - case atom.Type == shtText: - break - - default: - break outer - } - } - - quoting = atom.Quoting - atoms = atoms[1:] - } - - if trimHspace(tok.Rest()) != "" { - ck.Warnf("Internal pkglint error in ShellLine.CheckWord at %q (quoting=%s), rest: %s", - token, quoting, tok.Rest()) - } -} - -func (ck *ShellLineChecker) checkShVarUsePlain(atom *ShAtom, checkQuoting bool) { - shVarname := atom.ShVarname() - - if shVarname == "@" { - ck.Warnf("The $@ shell variable should only be used in double quotes.") - - } else if G.Opts.WarnQuoting && checkQuoting && ck.variableNeedsQuoting(shVarname) { - ck.Warnf("Unquoted shell variable %q.", shVarname) - ck.Explain( - "When a shell variable contains whitespace, it is expanded (split into multiple words)", - "when it is written as $variable in a shell script.", - "If that is not intended, it should be surrounded by quotation marks, like \"$variable\".", - "This way it always expands to a single word, preserving all whitespace and other special characters.", - "", - "Example:", - "\tfname=\"Curriculum vitae.doc\"", - "\tcp $filename /tmp", - "\t# tries to copy the two files \"Curriculum\" and \"Vitae.doc\"", - "", - "\tcp \"$filename\" /tmp", - "\t# copies one file, as intended") - } - - if shVarname == "?" { - ck.Warnf("The $? shell variable is often not available in \"set -e\" mode.") - // TODO: Explain how to properly fix this warning. - // TODO: Make sure the warning is only shown when applicable. - } -} - -func (ck *ShellLineChecker) checkVaruseToken(atoms *[]*ShAtom, quoting ShQuoting) bool { - varuse := (*atoms)[0].VarUse() - if varuse == nil { - return false - } - - *atoms = (*atoms)[1:] - varname := varuse.varname - - if varname == "@" { - ck.Warnf("Please use \"${.TARGET}\" instead of \"$@\".") - ck.Explain( - "The variable $@ can easily be confused with the shell variable of", - "the same name, which has a completely different meaning.") - - varname = ".TARGET" - varuse = &MkVarUse{varname, varuse.modifiers} - } - - switch { - case quoting == shqPlain && varuse.IsQ(): - // Fine. - - case (quoting == shqSquot || quoting == shqDquot) && matches(varname, `^(?:.*DIR|.*FILE|.*PATH|.*_VAR|PREFIX|.*BASE|PKGNAME)$`): - // This is ok as long as these variables don't have embedded [$\\"'`]. - - case quoting != shqPlain && varuse.IsQ(): - ck.Warnf("The :Q modifier should not be used inside quotes.") - ck.Explain( - "The :Q modifier is intended for embedding a string into a shell program.", - "It escapes all characters that have a special meaning in shell programs.", - "It only works correctly when it appears outside of \"double\" or 'single'", - "quotes or `backticks`.", - "", - "When it is used inside double quotes or backticks, the resulting string may", - "contain more backslashes than intended.", - "", - "When it is used inside single quotes and the string contains a single quote", - "itself, it produces syntax errors in the shell.", - "", - "To fix this warning, either remove the :Q or the double quotes.", - "In most cases, it is more appropriate to remove the double quotes.", - "", - "A special case is for empty strings.", - "If the empty string should be preserved as an empty string,", - "the correct form is ${VAR:Q}'' with either leading or trailing single or double quotes.", - "If the empty string should just be skipped,", - "a simple ${VAR:Q} without any surrounding quotes is correct.") - - // TODO: What about single quotes? - // TODO: What about backticks? - } - - if ck.checkVarUse { - vuc := VarUseContext{shellCommandsType, VucUnknownTime, quoting.ToVarUseContext(), true} - MkLineChecker{ck.MkLines, ck.mkline}.CheckVaruse(varuse, &vuc) - } - - return true -} - -// unescapeBackticks takes a backticks expression like `echo \\"hello\\"` and -// returns the part inside the backticks, removing one level of backslashes. -// -// Backslashes are only removed before a dollar, a backslash or a backtick. -// Other backslashes generate a warning since it is easier to remember that -// all backslashes are unescaped. -// -// See http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_06_03 -func (ck *ShellLineChecker) unescapeBackticks(atoms *[]*ShAtom, quoting ShQuoting) string { - line := ck.mkline.Line - - // Skip the initial backtick. - *atoms = (*atoms)[1:] - - var unescaped strings.Builder - for len(*atoms) > 0 { - atom := (*atoms)[0] - *atoms = (*atoms)[1:] - - if atom.Quoting == quoting { - return unescaped.String() - } - - if atom.Type != shtText { - unescaped.WriteString(atom.MkText) - continue - } - - lex := textproc.NewLexer(atom.MkText) - for !lex.EOF() { - unescaped.WriteString(lex.NextBytesFunc(func(b byte) bool { return b != '\\' })) - if lex.SkipByte('\\') { - switch lex.PeekByte() { - case '"', '\\', '`', '$': - unescaped.WriteByte(byte(lex.PeekByte())) - lex.Skip(1) - default: - line.Warnf("Backslashes should be doubled inside backticks.") - unescaped.WriteByte('\\') - } - } - } - - // XXX: The regular expression is a bit cheated but is good enough until - // pkglint has a real parser for all shell constructs. - if atom.Quoting == shqDquotBackt && matches(atom.MkText, `(^|[^\\])"`) { - line.Warnf("Double quotes inside backticks inside double quotes are error prone.") - line.Explain( - "According to the SUSv3, they produce undefined results.", - "", - "See the paragraph starting \"Within the backquoted ...\" in", - "http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html.", - "", - "To avoid this uncertainty, escape the double quotes using \\\".") - } - } - - line.Errorf("Unfinished backticks after %q.", unescaped.String()) - return unescaped.String() -} - -func (ck *ShellLineChecker) variableNeedsQuoting(shVarname string) bool { - switch shVarname { - case "#", "?", "$": - return false // Definitely ok - case "d", "f", "i", "id", "file", "src", "dst", "prefix": - return false // Probably ok - } - return !hasSuffix(shVarname, "dir") // Probably ok -} - -func (ck *ShellLineChecker) CheckShellCommandLine(shelltext string) { - if trace.Tracing { - defer trace.Call1(shelltext)() - } - - line := ck.mkline.Line - - // TODO: Add sed and mv in addition to ${SED} and ${MV}. - // TODO: Now that a shell command parser is available, be more precise in the condition. - if contains(shelltext, "${SED}") && contains(shelltext, "${MV}") { - line.Notef("Please use the SUBST framework instead of ${SED} and ${MV}.") - line.Explain( - "Using the SUBST framework instead of explicit commands is easier", - "to understand, since all the complexity of using sed and mv is", - "hidden behind the scenes.", - "", - // TODO: Provide a copy-and-paste example. - sprintf("Run %q for more information.", bmakeHelp("subst"))) - if contains(shelltext, "#") { - line.Explain( - "When migrating to the SUBST framework, pay attention to \"#\" characters.", - "In shell commands, make(1) does not interpret them as", - "comment character, but in variable assignments it does.", - "Therefore, instead of the shell command", - "", - "\tsed -e 's,#define foo,,'", - "", - "you need to write", - "", - "\tSUBST_SED.foo+=\t's,\\#define foo,,'") - } - } - - lexer := textproc.NewLexer(shelltext) - lexer.NextHspace() - hiddenAndSuppress := lexer.NextBytesFunc(func(b byte) bool { return b == '-' || b == '@' }) - if hiddenAndSuppress != "" { - ck.checkHiddenAndSuppress(hiddenAndSuppress, lexer.Rest()) - } - setE := lexer.SkipString("${RUN}") - if !setE { - if lexer.NextString("${_PKG_SILENT}${_PKG_DEBUG}") != "" { - line.Warnf("Use of _PKG_SILENT and _PKG_DEBUG is deprecated. Use ${RUN} instead.") - } - } - - ck.CheckShellCommand(lexer.Rest(), &setE, RunTime) -} - -func (ck *ShellLineChecker) CheckShellCommand(shellcmd string, pSetE *bool, time ToolTime) { - if trace.Tracing { - defer trace.Call0()() - } - - line := ck.mkline.Line - program, err := parseShellProgram(line, shellcmd) - // FIXME: This code is duplicated in checkWordQuoting. - if err != nil && contains(shellcmd, "$$(") { // Hack until the shell parser can handle subshells. - line.Warnf("Invoking subshells via $(...) is not portable enough.") - return - } - if err != nil { - line.Warnf("Pkglint ShellLine.CheckShellCommand: %s", err) - return - } - - spc := ShellProgramChecker{ck} - spc.checkConditionalCd(program) - - walker := NewMkShWalker() - walker.Callback.SimpleCommand = func(command *MkShSimpleCommand) { - scc := NewSimpleCommandChecker(ck, command, time) - scc.Check() - // TODO: Implement getopt parsing for StrCommand. - if scc.strcmd.Name == "set" && scc.strcmd.AnyArgMatches(`^-.*e`) { - *pSetE = true - } - } - walker.Callback.AndOr = func(andor *MkShAndOr) { - if G.Opts.WarnExtra && !*pSetE && walker.Current().Index != 0 { - spc.checkSetE(walker.Parent(1).(*MkShList), walker.Current().Index) - } - } - walker.Callback.Pipeline = func(pipeline *MkShPipeline) { - spc.checkPipeExitcode(pipeline) - } - walker.Callback.Word = func(word *ShToken) { - // TODO: Try to replace false with true here; it had been set to false - // in 2016 for no apparent reason. - spc.CheckWord(word.MkText, false, time) - } - - walker.Walk(program) -} - -func (ck *ShellLineChecker) CheckShellCommands(shellcmds string, time ToolTime) { - setE := true - ck.CheckShellCommand(shellcmds, &setE, time) - if !hasSuffix(shellcmds, ";") { - ck.mkline.Warnf("This shell command list should end with a semicolon.") - } -} - -func (ck *ShellLineChecker) checkHiddenAndSuppress(hiddenAndSuppress, rest string) { - if trace.Tracing { - defer trace.Call(hiddenAndSuppress, rest)() - } - - switch { - case !contains(hiddenAndSuppress, "@"): - // Nothing is hidden at all. - - case hasPrefix(ck.MkLines.target, "show-") || hasSuffix(ck.MkLines.target, "-message"): - // In these targets, all commands may be hidden. - - case hasPrefix(rest, "#"): - // Shell comments may be hidden, since they cannot have side effects. - - default: - tokens, _ := splitIntoShellTokens(ck.mkline.Line, rest) - if len(tokens) > 0 { - cmd := tokens[0] - switch cmd { - case "${DELAYED_ERROR_MSG}", "${DELAYED_WARNING_MSG}", - "${DO_NADA}", - "${ECHO}", "${ECHO_MSG}", "${ECHO_N}", "${ERROR_CAT}", "${ERROR_MSG}", - "${FAIL_MSG}", - "${INFO_MSG}", - "${PHASE_MSG}", "${PRINTF}", - "${SHCOMMENT}", "${STEP_MSG}", - "${WARNING_CAT}", "${WARNING_MSG}": - break - default: - ck.mkline.Warnf("The shell command %q should not be hidden.", cmd) - ck.mkline.Explain( - "Hidden shell commands do not appear on the terminal", - "or in the log file when they are executed.", - "When they fail, the error message cannot be related to the command,", - "which makes debugging more difficult.", - "", - "It is better to insert ${RUN} at the beginning of the whole command line.", - "This will hide the command by default but shows it when PKG_DEBUG_LEVEL is set.") - } - } - } - - if contains(hiddenAndSuppress, "-") { - ck.mkline.Warnf("Using a leading \"-\" to suppress errors is deprecated.") - ck.mkline.Explain( - "If you really want to ignore any errors from this command, append \"|| ${TRUE}\" to the command.", - "This is more visible than a single hyphen, and it should be.") - } -} +// Parsing and checking shell commands embedded in Makefiles type SimpleCommandChecker struct { *ShellLineChecker @@ -512,6 +71,23 @@ func (scc *SimpleCommandChecker) checkCommandStart() { } } +func (scc *SimpleCommandChecker) handleForbiddenCommand() bool { + if trace.Tracing { + defer trace.Call0()() + } + + shellword := scc.strcmd.Name + switch path.Base(shellword) { + case "mktexlsr", "texconfig": + scc.Errorf("%q must not be used in Makefiles.", shellword) + scc.Explain( + "This command may only appear in INSTALL scripts, not in the package Makefile,", + "so that the package also works if it is installed as a binary package.") + return true + } + return false +} + // handleTool tests whether the shell command is one of the recognized pkgsrc tools // and whether the package has added it to USE_TOOLS. func (scc *SimpleCommandChecker) handleTool() bool { @@ -534,23 +110,6 @@ func (scc *SimpleCommandChecker) handleTool() bool { return tool != nil } -func (scc *SimpleCommandChecker) handleForbiddenCommand() bool { - if trace.Tracing { - defer trace.Call0()() - } - - shellword := scc.strcmd.Name - switch path.Base(shellword) { - case "mktexlsr", "texconfig": - scc.Errorf("%q must not be used in Makefiles.", shellword) - scc.Explain( - "This command may only appear in INSTALL scripts, not in the package Makefile,", - "so that the package also works if it is installed as a binary package.") - return true - } - return false -} - func (scc *SimpleCommandChecker) handleCommandVariable() bool { if trace.Tracing { defer trace.Call0()() @@ -571,11 +130,11 @@ func (scc *SimpleCommandChecker) handleCommandVariable() bool { // When the package author has explicitly defined a command // variable, assume it to be valid. - if scc.MkLines.vars.DefinedSimilar(varname) { + if scc.MkLines.vars.IsDefinedSimilar(varname) { return true } - return G.Pkg != nil && G.Pkg.vars.DefinedSimilar(varname) + return G.Pkg != nil && G.Pkg.vars.IsDefinedSimilar(varname) } func (scc *SimpleCommandChecker) handleShellBuiltin() bool { @@ -797,11 +356,21 @@ func (scc *SimpleCommandChecker) Explain(explanation ...string) { scc.mkline.Explain(explanation...) } -type ShellProgramChecker struct { - *ShellLineChecker +// ShellLineChecker is either a line from a Makefile starting with a tab, +// thereby containing shell commands to be executed. +// +// Or it is a variable assignment line from a Makefile with a left-hand +// side variable that is of some shell-like type; see Vartype.IsShell. +type ShellLineChecker struct { + MkLines *MkLines + mkline *MkLine + + // checkVarUse is set to false when checking a single shell word + // in order to skip duplicate warnings in variable assignments. + checkVarUse bool } -func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { +func (ck *ShellLineChecker) checkConditionalCd(list *MkShList) { if trace.Tracing { defer trace.Call0()() } @@ -819,8 +388,8 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { checkConditionalCd := func(cmd *MkShSimpleCommand) { if NewStrCommand(cmd).Name == "cd" { - spc.Errorf("The Solaris /bin/sh cannot handle \"cd\" inside conditionals.") - spc.Explain( + ck.Errorf("The Solaris /bin/sh cannot handle \"cd\" inside conditionals.") + ck.Explain( "When the Solaris shell is in \"set -e\" mode and \"cd\" fails, the", "shell will exit, no matter if it is protected by an \"if\" or the", "\"||\" operator.") @@ -844,8 +413,8 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { } walker.Callback.Pipeline = func(pipeline *MkShPipeline) { if pipeline.Negated { - spc.Warnf("The Solaris /bin/sh does not support negation of shell commands.") - spc.Explain( + ck.Warnf("The Solaris /bin/sh does not support negation of shell commands.") + ck.Explain( "The GNU Autoconf manual has many more details of what shell", "features to avoid for portable programs.", "It can be read at:", @@ -855,39 +424,39 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { walker.Walk(list) } -func (spc *ShellProgramChecker) checkPipeExitcode(pipeline *MkShPipeline) { +func (ck *ShellLineChecker) checkSetE(list *MkShList, index int) { if trace.Tracing { defer trace.Call0()() } - canFail := func() (bool, string) { - for _, cmd := range pipeline.Cmds[:len(pipeline.Cmds)-1] { - if spc.canFail(cmd) { - if cmd.Simple != nil && cmd.Simple.Name != nil { - return true, cmd.Simple.Name.MkText - } - return true, "" - } - } - return false, "" + command := list.AndOrs[index-1].Pipes[0].Cmds[0] + if command.Simple == nil || !ck.canFail(command) { + return } - if G.Opts.WarnExtra && len(pipeline.Cmds) > 1 { - if canFail, cmd := canFail(); canFail { - if cmd != "" { - spc.Warnf("The exitcode of %q at the left of the | operator is ignored.", cmd) - } else { - spc.Warnf("The exitcode of the command at the left of the | operator is ignored.") - } - spc.Explain( - "In a shell command like \"cat *.txt | grep keyword\", if the command", - "on the left side of the \"|\" fails, this failure is ignored.", - "", - "If you need to detect the failure of the left-hand-side command, use", - "temporary files to save the output of the command.", - "A good place to create those files is in ${WRKDIR}.") - } + line := ck.mkline.Line + if !line.once.FirstTime("switch to set -e") { + return } + + line.Warnf("Please switch to \"set -e\" mode before using a semicolon (after %q) to separate commands.", + NewStrCommand(command.Simple).String()) + line.Explain( + "Normally, when a shell command fails (returns non-zero),", + "the remaining commands are still executed.", + "For example, the following commands would remove", + "all files from the HOME directory:", + "", + "\tcd \"$HOME\"; cd /nonexistent; rm -rf *", + "", + "In \"set -e\" mode, the shell stops when a command fails.", + "", + "To fix this warning, you can:", + "", + "* insert ${RUN} at the beginning of the line", + " (which among other things does \"set -e\")", + "* insert \"set -e\" explicitly at the beginning of the line", + "* use \"&&\" instead of \";\" to separate the commands") } // canFail returns true if the given shell command can fail. @@ -904,7 +473,7 @@ func (spc *ShellProgramChecker) checkPipeExitcode(pipeline *MkShPipeline) { // echo "hello" // sed 's,$, world,,' // wc -l -func (spc *ShellProgramChecker) canFail(cmd *MkShCommand) bool { +func (ck *ShellLineChecker) canFail(cmd *MkShCommand) bool { simple := cmd.Simple if simple == nil { return true @@ -940,7 +509,7 @@ func (spc *ShellProgramChecker) canFail(cmd *MkShCommand) bool { case "set": } - tool, _ := G.Tool(spc.MkLines, cmdName, RunTime) + tool, _ := G.Tool(ck.MkLines, cmdName, RunTime) if tool == nil { return true } @@ -963,49 +532,456 @@ func (spc *ShellProgramChecker) canFail(cmd *MkShCommand) bool { return true } -func (spc *ShellProgramChecker) checkSetE(list *MkShList, index int) { +func (ck *ShellLineChecker) checkPipeExitcode(pipeline *MkShPipeline) { if trace.Tracing { defer trace.Call0()() } - command := list.AndOrs[index-1].Pipes[0].Cmds[0] - if command.Simple == nil || !spc.canFail(command) { + canFail := func() (bool, string) { + for _, cmd := range pipeline.Cmds[:len(pipeline.Cmds)-1] { + if ck.canFail(cmd) { + if cmd.Simple != nil && cmd.Simple.Name != nil { + return true, cmd.Simple.Name.MkText + } + return true, "" + } + } + return false, "" + } + + if G.Opts.WarnExtra && len(pipeline.Cmds) > 1 { + if canFail, cmd := canFail(); canFail { + if cmd != "" { + ck.Warnf("The exitcode of %q at the left of the | operator is ignored.", cmd) + } else { + ck.Warnf("The exitcode of the command at the left of the | operator is ignored.") + } + ck.Explain( + "In a shell command like \"cat *.txt | grep keyword\", if the command", + "on the left side of the \"|\" fails, this failure is ignored.", + "", + "If you need to detect the failure of the left-hand-side command, use", + "temporary files to save the output of the command.", + "A good place to create those files is in ${WRKDIR}.") + } + } +} + +var shellCommandsType = NewVartype(BtShellCommands, NoVartypeOptions, NewACLEntry("*", aclpAllRuntime)) +var shellWordVuc = &VarUseContext{shellCommandsType, VucUnknownTime, VucQuotPlain, false} + +func NewShellLineChecker(mklines *MkLines, mkline *MkLine) *ShellLineChecker { + assertNotNil(mklines) + return &ShellLineChecker{mklines, mkline, true} +} + +func (ck *ShellLineChecker) CheckShellCommands(shellcmds string, time ToolTime) { + setE := true + ck.CheckShellCommand(shellcmds, &setE, time) + if !hasSuffix(shellcmds, ";") { + ck.mkline.Warnf("This shell command list should end with a semicolon.") + } +} + +func (ck *ShellLineChecker) CheckShellCommandLine(shelltext string) { + if trace.Tracing { + defer trace.Call1(shelltext)() + } + + line := ck.mkline.Line + + // TODO: Add sed and mv in addition to ${SED} and ${MV}. + // TODO: Now that a shell command parser is available, be more precise in the condition. + if contains(shelltext, "${SED}") && contains(shelltext, "${MV}") { + line.Notef("Please use the SUBST framework instead of ${SED} and ${MV}.") + line.Explain( + "Using the SUBST framework instead of explicit commands is easier", + "to understand, since all the complexity of using sed and mv is", + "hidden behind the scenes.", + "", + // TODO: Provide a copy-and-paste example. + sprintf("Run %q for more information.", bmakeHelp("subst"))) + if contains(shelltext, "#") { + line.Explain( + "When migrating to the SUBST framework, pay attention to \"#\" characters.", + "In shell commands, make(1) does not interpret them as", + "comment character, but in variable assignments it does.", + "Therefore, instead of the shell command", + "", + "\tsed -e 's,#define foo,,'", + "", + "you need to write", + "", + "\tSUBST_SED.foo+=\t's,\\#define foo,,'") + } + } + + lexer := textproc.NewLexer(shelltext) + lexer.NextHspace() + hiddenAndSuppress := lexer.NextBytesFunc(func(b byte) bool { return b == '-' || b == '@' }) + if hiddenAndSuppress != "" { + ck.checkHiddenAndSuppress(hiddenAndSuppress, lexer.Rest()) + } + setE := lexer.SkipString("${RUN}") + if !setE { + if lexer.NextString("${_PKG_SILENT}${_PKG_DEBUG}") != "" { + line.Warnf("Use of _PKG_SILENT and _PKG_DEBUG is deprecated. Use ${RUN} instead.") + } + } + + ck.CheckShellCommand(lexer.Rest(), &setE, RunTime) +} + +func (ck *ShellLineChecker) checkHiddenAndSuppress(hiddenAndSuppress, rest string) { + if trace.Tracing { + defer trace.Call(hiddenAndSuppress, rest)() + } + + switch { + case !contains(hiddenAndSuppress, "@"): + // Nothing is hidden at all. + + case hasPrefix(ck.MkLines.target, "show-") || hasSuffix(ck.MkLines.target, "-message"): + // In these targets, all commands may be hidden. + + case hasPrefix(rest, "#"): + // Shell comments may be hidden, since they cannot have side effects. + + default: + tokens, _ := splitIntoShellTokens(ck.mkline.Line, rest) + if len(tokens) > 0 { + cmd := tokens[0] + switch cmd { + case "${DELAYED_ERROR_MSG}", "${DELAYED_WARNING_MSG}", + "${DO_NADA}", + "${ECHO}", "${ECHO_MSG}", "${ECHO_N}", "${ERROR_CAT}", "${ERROR_MSG}", + "${FAIL_MSG}", + "${INFO_MSG}", + "${PHASE_MSG}", "${PRINTF}", + "${SHCOMMENT}", "${STEP_MSG}", + "${WARNING_CAT}", "${WARNING_MSG}": + break + default: + ck.mkline.Warnf("The shell command %q should not be hidden.", cmd) + ck.mkline.Explain( + "Hidden shell commands do not appear on the terminal", + "or in the log file when they are executed.", + "When they fail, the error message cannot be related to the command,", + "which makes debugging more difficult.", + "", + "It is better to insert ${RUN} at the beginning of the whole command line.", + "This will hide the command by default but shows it when PKG_DEBUG_LEVEL is set.") + } + } + } + + if contains(hiddenAndSuppress, "-") { + ck.mkline.Warnf("Using a leading \"-\" to suppress errors is deprecated.") + ck.mkline.Explain( + "If you really want to ignore any errors from this command, append \"|| ${TRUE}\" to the command.", + "This is more visible than a single hyphen, and it should be.") + } +} + +func (ck *ShellLineChecker) CheckShellCommand(shellcmd string, pSetE *bool, time ToolTime) { + if trace.Tracing { + defer trace.Call0()() + } + + line := ck.mkline.Line + program, err := parseShellProgram(line, shellcmd) + // FIXME: This code is duplicated in checkWordQuoting. + if err != nil && contains(shellcmd, "$$(") { // Hack until the shell parser can handle subshells. + line.Warnf("Invoking subshells via $(...) is not portable enough.") + return + } + if err != nil { + line.Warnf("Pkglint ShellLine.CheckShellCommand: %s", err) return } - line := spc.mkline.Line - if !line.once.FirstTime("switch to set -e") { + ck.checkConditionalCd(program) + + walker := NewMkShWalker() + walker.Callback.SimpleCommand = func(command *MkShSimpleCommand) { + scc := NewSimpleCommandChecker(ck, command, time) + scc.Check() + // TODO: Implement getopt parsing for StrCommand. + if scc.strcmd.Name == "set" && scc.strcmd.AnyArgMatches(`^-.*e`) { + *pSetE = true + } + } + walker.Callback.AndOr = func(andor *MkShAndOr) { + if G.Opts.WarnExtra && !*pSetE && walker.Current().Index != 0 { + ck.checkSetE(walker.Parent(1).(*MkShList), walker.Current().Index) + } + } + walker.Callback.Pipeline = func(pipeline *MkShPipeline) { + ck.checkPipeExitcode(pipeline) + } + walker.Callback.Word = func(word *ShToken) { + // TODO: Try to replace false with true here; it had been set to false + // in 2016 for no apparent reason. + ck.CheckWord(word.MkText, false, time) + } + + walker.Walk(program) +} + +func (ck *ShellLineChecker) CheckWord(token string, checkQuoting bool, time ToolTime) { + if trace.Tracing { + defer trace.Call(token, checkQuoting)() + } + + if token == "" { return } - line.Warnf("Please switch to \"set -e\" mode before using a semicolon (after %q) to separate commands.", - NewStrCommand(command.Simple).String()) - line.Explain( - "Normally, when a shell command fails (returns non-zero),", - "the remaining commands are still executed.", - "For example, the following commands would remove", - "all files from the HOME directory:", - "", - "\tcd \"$HOME\"; cd /nonexistent; rm -rf *", - "", - "In \"set -e\" mode, the shell stops when a command fails.", - "", - "To fix this warning, you can:", - "", - "* insert ${RUN} at the beginning of the line", - " (which among other things does \"set -e\")", - "* insert \"set -e\" explicitly at the beginning of the line", - "* use \"&&\" instead of \";\" to separate the commands") + var line = ck.mkline.Line + + // Delegate check for shell words consisting of a single variable use + // to the MkLineChecker. Examples for these are ${VAR:Mpattern} or $@. + if varuse := ToVarUse(token); varuse != nil { + if ck.checkVarUse { + MkLineChecker{ck.MkLines, ck.mkline}.CheckVaruse(varuse, shellWordVuc) + } + return + } + + if matches(token, `\$\{PREFIX\}/man(?:$|/)`) { + line.Warnf("Please use ${PKGMANDIR} instead of \"man\".") + } + + if contains(token, "etc/rc.d") { + line.Warnf("Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") + } + + ck.checkWordQuoting(token, checkQuoting, time) +} + +func (ck *ShellLineChecker) checkWordQuoting(token string, checkQuoting bool, time ToolTime) { + tok := NewShTokenizer(ck.mkline.Line, token, false) + + atoms := tok.ShAtoms() + quoting := shqPlain +outer: + for len(atoms) > 0 { + atom := atoms[0] + // Cutting off the first atom is done at the end of the loop since in + // some cases the called methods need to see the current atom. + + if trace.Tracing { + trace.Stepf("shell state %s: %q", quoting, atom) + } + + switch { + case atom.Quoting == shqBackt || atom.Quoting == shqDquotBackt: + backtCommand := ck.unescapeBackticks(&atoms, quoting) + if backtCommand != "" { + // TODO: Wrap the setE into a struct. + setE := true + ck.CheckShellCommand(backtCommand, &setE, time) + } + continue + + // Make(1) variables have the same syntax, no matter in which state the shell parser is currently. + case ck.checkVaruseToken(&atoms, quoting): + continue + + case quoting == shqPlain: + switch { + case atom.Type == shtShVarUse: + ck.checkShVarUsePlain(atom, checkQuoting) + + case atom.Type == shtSubshell: + ck.Warnf("Invoking subshells via $(...) is not portable enough.") + ck.Explain( + "The Solaris /bin/sh does not know this way to execute a command in a subshell.", + "Please use backticks (`...`) as a replacement.") + + // Early return to avoid further parse errors. + // As of December 2018, it might be worth continuing again since the + // shell parser has improved in 2018. + return + + case atom.Type == shtText: + break + + default: + break outer + } + } + + quoting = atom.Quoting + atoms = atoms[1:] + } + + if trimHspace(tok.Rest()) != "" { + ck.Warnf("Internal pkglint error in ShellLine.CheckWord at %q (quoting=%s), rest: %s", + token, quoting, tok.Rest()) + } +} + +// unescapeBackticks takes a backticks expression like `echo \\"hello\\"` and +// returns the part inside the backticks, removing one level of backslashes. +// +// Backslashes are only removed before a dollar, a backslash or a backtick. +// Other backslashes generate a warning since it is easier to remember that +// all backslashes are unescaped. +// +// See http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_06_03 +func (ck *ShellLineChecker) unescapeBackticks(atoms *[]*ShAtom, quoting ShQuoting) string { + line := ck.mkline.Line + + // Skip the initial backtick. + *atoms = (*atoms)[1:] + + var unescaped strings.Builder + for len(*atoms) > 0 { + atom := (*atoms)[0] + *atoms = (*atoms)[1:] + + if atom.Quoting == quoting { + return unescaped.String() + } + + if atom.Type != shtText { + unescaped.WriteString(atom.MkText) + continue + } + + lex := textproc.NewLexer(atom.MkText) + for !lex.EOF() { + unescaped.WriteString(lex.NextBytesFunc(func(b byte) bool { return b != '\\' })) + if lex.SkipByte('\\') { + switch lex.PeekByte() { + case '"', '\\', '`', '$': + unescaped.WriteByte(byte(lex.PeekByte())) + lex.Skip(1) + default: + line.Warnf("Backslashes should be doubled inside backticks.") + unescaped.WriteByte('\\') + } + } + } + + // XXX: The regular expression is a bit cheated but is good enough until + // pkglint has a real parser for all shell constructs. + if atom.Quoting == shqDquotBackt && matches(atom.MkText, `(^|[^\\])"`) { + line.Warnf("Double quotes inside backticks inside double quotes are error prone.") + line.Explain( + "According to the SUSv3, they produce undefined results.", + "", + "See the paragraph starting \"Within the backquoted ...\" in", + "http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html.", + "", + "To avoid this uncertainty, escape the double quotes using \\\".") + } + } + + line.Errorf("Unfinished backticks after %q.", unescaped.String()) + return unescaped.String() } -func (spc *ShellProgramChecker) Errorf(format string, args ...interface{}) { - spc.mkline.Errorf(format, args...) +func (ck *ShellLineChecker) checkShVarUsePlain(atom *ShAtom, checkQuoting bool) { + shVarname := atom.ShVarname() + + if shVarname == "@" { + ck.Warnf("The $@ shell variable should only be used in double quotes.") + + } else if G.Opts.WarnQuoting && checkQuoting && ck.variableNeedsQuoting(shVarname) { + ck.Warnf("Unquoted shell variable %q.", shVarname) + ck.Explain( + "When a shell variable contains whitespace, it is expanded (split into multiple words)", + "when it is written as $variable in a shell script.", + "If that is not intended, it should be surrounded by quotation marks, like \"$variable\".", + "This way it always expands to a single word, preserving all whitespace and other special characters.", + "", + "Example:", + "\tfname=\"Curriculum vitae.doc\"", + "\tcp $filename /tmp", + "\t# tries to copy the two files \"Curriculum\" and \"Vitae.doc\"", + "", + "\tcp \"$filename\" /tmp", + "\t# copies one file, as intended") + } + + if shVarname == "?" { + ck.Warnf("The $? shell variable is often not available in \"set -e\" mode.") + // TODO: Explain how to properly fix this warning. + // TODO: Make sure the warning is only shown when applicable. + } } -func (spc *ShellProgramChecker) Warnf(format string, args ...interface{}) { - spc.mkline.Warnf(format, args...) + +func (ck *ShellLineChecker) variableNeedsQuoting(shVarname string) bool { + switch shVarname { + case "#", "?", "$": + return false // Definitely ok + case "d", "f", "i", "id", "file", "src", "dst", "prefix": + return false // Probably ok + } + return !hasSuffix(shVarname, "dir") // Probably ok } -func (spc *ShellProgramChecker) Explain(explanation ...string) { - spc.mkline.Explain(explanation...) + +func (ck *ShellLineChecker) checkVaruseToken(atoms *[]*ShAtom, quoting ShQuoting) bool { + varuse := (*atoms)[0].VarUse() + if varuse == nil { + return false + } + + *atoms = (*atoms)[1:] + varname := varuse.varname + + if varname == "@" { + ck.Warnf("Please use \"${.TARGET}\" instead of \"$@\".") + ck.Explain( + "The variable $@ can easily be confused with the shell variable of", + "the same name, which has a completely different meaning.") + + varname = ".TARGET" + varuse = &MkVarUse{varname, varuse.modifiers} + } + + switch { + case quoting == shqPlain && varuse.IsQ(): + // Fine. + + case (quoting == shqSquot || quoting == shqDquot) && matches(varname, `^(?:.*DIR|.*FILE|.*PATH|.*_VAR|PREFIX|.*BASE|PKGNAME)$`): + // This is ok as long as these variables don't have embedded [$\\"'`]. + + case quoting != shqPlain && varuse.IsQ(): + ck.Warnf("The :Q modifier should not be used inside quotes.") + ck.Explain( + "The :Q modifier is intended for embedding a string into a shell program.", + "It escapes all characters that have a special meaning in shell programs.", + "It only works correctly when it appears outside of \"double\" or 'single'", + "quotes or `backticks`.", + "", + "When it is used inside double quotes or backticks, the resulting string may", + "contain more backslashes than intended.", + "", + "When it is used inside single quotes and the string contains a single quote", + "itself, it produces syntax errors in the shell.", + "", + "To fix this warning, either remove the :Q or the double quotes.", + "In most cases, it is more appropriate to remove the double quotes.", + "", + "A special case is for empty strings.", + "If the empty string should be preserved as an empty string,", + "the correct form is ${VAR:Q}'' with either leading or trailing single or double quotes.", + "If the empty string should just be skipped,", + "a simple ${VAR:Q} without any surrounding quotes is correct.") + + // TODO: What about single quotes? + // TODO: What about backticks? + } + + if ck.checkVarUse { + vuc := VarUseContext{shellCommandsType, VucUnknownTime, quoting.ToVarUseContext(), true} + MkLineChecker{ck.MkLines, ck.mkline}.CheckVaruse(varuse, &vuc) + } + + return true } // Some shell commands should not be used in the install phase. @@ -1053,6 +1029,18 @@ func (ck *ShellLineChecker) checkInstallCommand(shellcmd string) { } } +func (ck *ShellLineChecker) Errorf(format string, args ...interface{}) { + ck.mkline.Errorf(format, args...) +} + +func (ck *ShellLineChecker) Warnf(format string, args ...interface{}) { + ck.mkline.Warnf(format, args...) +} + +func (ck *ShellLineChecker) Explain(explanation ...string) { + ck.mkline.Explain(explanation...) +} + // Example: "word1 word2;;;" => "word1", "word2", ";;", ";" // // TODO: Document what this function should be used for. diff --git a/pkgtools/pkglint/files/shell_test.go b/pkgtools/pkglint/files/shell_test.go index 7f6f93079bc..8a2bbb01ac8 100644 --- a/pkgtools/pkglint/files/shell_test.go +++ b/pkgtools/pkglint/files/shell_test.go @@ -5,148 +5,612 @@ import ( "strings" ) -func (s *Suite) Test_splitIntoShellTokens__line_continuation(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_handleForbiddenCommand(c *check.C) { t := s.Init(c) - words, rest := splitIntoShellTokens(dummyLine, "if true; then \\") + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "\t${RUN} mktexlsr; texconfig") - t.CheckDeepEquals(words, []string{"if", "true", ";", "then"}) - t.CheckEquals(rest, "\\") + mklines.Check() t.CheckOutputLines( - "WARN: Internal pkglint error in ShTokenizer.ShAtom at \"\\\\\" (quoting=plain).") + "ERROR: Makefile:3: \"mktexlsr\" must not be used in Makefiles.", + "ERROR: Makefile:3: \"texconfig\" must not be used in Makefiles.") } -func (s *Suite) Test_splitIntoShellTokens__dollar_slash(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable(c *check.C) { t := s.Init(c) - words, rest := splitIntoShellTokens(dummyLine, "pax -s /.*~$$//g") + t.SetUpTool("perl", "PERL5", AtRunTime) + t.SetUpTool("perl6", "PERL6", Nowhere) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "PERL5_VARS_CMD=\t${PERL5:Q}", + "PERL5_VARS_CMD=\t${PERL6:Q}", + "", + "pre-configure:", + "\t${PERL5_VARS_CMD} -e 'print 12345'") - t.CheckDeepEquals(words, []string{"pax", "-s", "/.*~$$//g"}) - t.CheckEquals(rest, "") + mklines.Check() + + // FIXME: In PERL5:Q and PERL6:Q, the :Q is wrong. + t.CheckOutputLines( + "WARN: Makefile:4: The \"${PERL6:Q}\" tool is used but not added to USE_TOOLS.") } -func (s *Suite) Test_splitIntoShellTokens__dollar_subshell(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__parameterized(c *check.C) { t := s.Init(c) - words, rest := splitIntoShellTokens(dummyLine, "id=$$(${AWK} '{print}' < ${WRKSRC}/idfile) && echo \"$$id\"") + t.SetUpPackage("category/package") + G.Pkg = NewPackage(t.File("category/package")) + t.FinishSetUp() - t.CheckDeepEquals(words, []string{"id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)", "&&", "echo", "\"$$id\""}) - t.CheckEquals(rest, "") + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "MY_TOOL.i386=\t${PREFIX}/bin/tool-i386", + "MY_TOOL.x86_64=\t${PREFIX}/bin/tool-x86_64", + "", + "pre-configure:", + "\t${MY_TOOL.amd64} -e 'print 12345'", + "\t${UNKNOWN_TOOL}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:8: Unknown shell command \"${UNKNOWN_TOOL}\".", + "WARN: Makefile:8: UNKNOWN_TOOL is used but not defined.") } -func (s *Suite) Test_splitIntoShellTokens__semicolons(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__followed_by_literal(c *check.C) { t := s.Init(c) - words, rest := splitIntoShellTokens(dummyLine, "word1 word2;;;") + t.SetUpPackage("category/package") + G.Pkg = NewPackage(t.File("category/package")) + t.FinishSetUp() - t.CheckDeepEquals(words, []string{"word1", "word2", ";;", ";"}) - t.CheckEquals(rest, "") + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "QTDIR=\t${PREFIX}", + "", + "pre-configure:", + "\t${QTDIR}/bin/release") + + mklines.Check() + + t.CheckOutputEmpty() } -func (s *Suite) Test_splitIntoShellTokens__whitespace(c *check.C) { +// The package Makefile and other .mk files in a package directory +// may use any shell commands defined by any included files. +// But only if the package is checked as a whole. +// +// On the contrary, when pkglint checks a single .mk file, these +// commands are not guaranteed to be defined, not even when the +// .mk file includes the file defining the command. +// FIXME: This paragraph sounds wrong. All commands from included files should be valid. +// +// The PYTHON_BIN variable below must not be called *_CMD, or another code path is taken. +func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__from_package(c *check.C) { t := s.Init(c) - text := "\t${RUN} cd ${WRKSRC}&&(${ECHO} ${PERL5:Q};${ECHO})|${BASH} ./install" - words, rest := splitIntoShellTokens(dummyLine, text) + pkg := t.SetUpPackage("category/package", + "post-install:", + "\t${PYTHON_BIN}", + "", + ".include \"extra.mk\"") + t.CreateFileLines("category/package/extra.mk", + MkCvsID, + "PYTHON_BIN=\tmy_cmd") + t.FinishSetUp() - t.CheckDeepEquals(words, []string{ - "${RUN}", - "cd", "${WRKSRC}", - "&&", "(", "${ECHO}", "${PERL5:Q}", ";", "${ECHO}", ")", - "|", "${BASH}", "./install"}) - t.CheckEquals(rest, "") + G.Check(pkg) + + t.CheckOutputEmpty() } -func (s *Suite) Test_splitIntoShellTokens__finished_dquot(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_checkRegexReplace(c *check.C) { t := s.Init(c) - text := "\"\"" - words, rest := splitIntoShellTokens(dummyLine, text) + test := func(cmd string, diagnostics ...string) { + t.SetUpTool("pax", "PAX", AtRunTime) + t.SetUpTool("sed", "SED", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "pre-configure:", + "\t"+cmd) - t.CheckDeepEquals(words, []string{"\"\""}) - t.CheckEquals(rest, "") + mklines.Check() + + t.CheckOutput(diagnostics) + } + + test("${PAX} -s s,.*,, src dst", + "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") + + test("pax -s s,.*,, src dst", + "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") + + test("${SED} -e s,.*,, src dst", + "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") + + test("sed -e s,.*,, src dst", + "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") + + // The * is properly enclosed in quotes. + test("sed -e 's,.*,,' -e \"s,-*,,\"", + nil...) + + // The * is properly escaped. + test("sed -e s,.\\*,,", + nil...) + + test("pax -s s,\\.orig,, src dst", + nil...) + + test("sed -e s,a,b,g src dst", + nil...) + + // TODO: Merge the code with BtSedCommands. + + // TODO: Finally, remove the G.Testing from the main code. + // Then, remove this test case. + G.Testing = false + test("sed -e s,.*,match,", + nil...) + G.Testing = true } -func (s *Suite) Test_splitIntoShellTokens__unfinished_dquot(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs(c *check.C) { t := s.Init(c) - text := "\t\"" - words, rest := splitIntoShellTokens(dummyLine, text) + t.SetUpVartypes() + // TODO: Check whether these tools are actually necessary for this test. + t.SetUpTool("awk", "AWK", AtRunTime) + t.SetUpTool("cp", "CP", AtRunTime) + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("mkdir", "MKDIR", AtRunTime) // This is actually "mkdir -p". + t.SetUpTool("unzip", "UNZIP_CMD", AtRunTime) - c.Check(words, check.IsNil) - t.CheckEquals(rest, "\"") + test := func(shellCommand string, diagnostics ...string) { + mklines := t.NewMkLines("filename.mk", + "\t"+shellCommand) + ck := NewShellLineChecker(mklines, mklines.mklines[0]) + + mklines.ForEach(func(mkline *MkLine) { + ck.CheckShellCommandLine(ck.mkline.ShellCommand()) + }) + + t.CheckOutput(diagnostics) + } + + // AUTO_MKDIRS applies only when installing directories. + test("${RUN} ${INSTALL} -c ${WRKSRC}/file ${PREFIX}/bin/", + nil...) + + // TODO: Warn that ${INSTALL} -d can only handle a single directory. + test("${RUN} ${INSTALL} -m 0755 -d ${PREFIX}/first ${PREFIX}/second", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= first\" instead of \"${INSTALL} -d\".", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= second\" instead of \"${INSTALL} -d\".") + + G.Pkg = NewPackage(t.File("category/pkgbase")) + G.Pkg.Plist.Dirs["share/pkgbase"] = &PlistLine{ + t.NewLine("PLIST", 123, "share/pkgbase/file"), + nil, + "share/pkgbase/file"} + + // A directory that is found in the PLIST. + // TODO: Add a test for using this command inside a conditional; + // the note should not appear then. + test("${RUN} ${INSTALL_DATA_DIR} share/pkgbase ${PREFIX}/share/pkgbase", + "NOTE: filename.mk:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" "+ + "instead of \"${INSTALL_DATA_DIR}\".", + "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") + + // Directories from .for loops are too dynamic to be replaced with AUTO_MKDIRS. + // TODO: Expand simple .for loops. + test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/${dir}", + "WARN: filename.mk:1: dir is used but not defined.") + + // A directory that is not found in the PLIST would not be created by AUTO_MKDIRS, + // therefore only INSTALLATION_DIRS is suggested. + test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/share/other", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= share/other\" instead of \"${INSTALL_DATA_DIR}\".") } -func (s *Suite) Test_splitIntoShellTokens__unescaped_dollar_in_dquot(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs__redundant(c *check.C) { t := s.Init(c) - text := "echo \"$$\"" - words, rest := splitIntoShellTokens(dummyLine, text) + t.SetUpPackage("category/package", + "AUTO_MKDIRS=\t\tyes", + "INSTALLATION_DIRS+=\tshare/redundant", + "INSTALLATION_DIRS+=\tnot/redundant ${EGDIR}") + t.CreateFileLines("category/package/PLIST", + PlistCvsID, + "share/redundant/file", + "${EGDIR}/file") - t.CheckDeepEquals(words, []string{"echo", "\"$$\""}) - t.CheckEquals(rest, "") + t.Main("-Wall", "-q", "category/package") - t.CheckOutputEmpty() + t.CheckOutputLines( + "NOTE: ~/category/package/Makefile:21: The directory \"share/redundant\" "+ + "is redundant in INSTALLATION_DIRS.", + // The below is not proven to be always correct. It assumes that a + // variable in the Makefile has the same value as the corresponding + // variable from PLIST_SUBST. Violating this assumption would be + // confusing to the pkgsrc developers, therefore it's a safe bet. + // A notable counterexample is PKGNAME in PLIST, which corresponds + // to PKGNAME_NOREV in the package Makefile. + "NOTE: ~/category/package/Makefile:22: The directory \"${EGDIR}\" "+ + "is redundant in INSTALLATION_DIRS.") } -func (s *Suite) Test_splitIntoShellTokens__varuse_with_embedded_space_and_other_vars(c *check.C) { +// The AUTO_MKDIRS code in mk/install/install.mk (install-dirs-from-PLIST) +// skips conditional directories, as well as directories with placeholders. +func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs__conditional_PLIST(c *check.C) { t := s.Init(c) - varuseWord := "${GCONF_SCHEMAS:@.s.@${INSTALL_DATA} ${WRKSRC}/src/common/dbus/${.s.} ${DESTDIR}${GCONF_SCHEMAS_DIR}/@}" - words, rest := splitIntoShellTokens(dummyLine, varuseWord) + t.SetUpPackage("category/package", + "LIB_SUBDIR=\tsubdir", + "", + "do-install:", + "\t${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/libexec/always", + "\t${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/libexec/conditional", + "\t${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/${LIB_SUBDIR}", + ) + t.Chdir("category/package") + t.CreateFileLines("PLIST", + PlistCvsID, + "libexec/always/always", + "${LIB_SUBDIR}/file", + "${PLIST.cond}libexec/conditional/conditional") + t.FinishSetUp() - t.CheckDeepEquals(words, []string{varuseWord}) - t.CheckEquals(rest, "") + G.checkdirPackage(".") + + // As libexec/conditional will not be created automatically, + // AUTO_MKDIRS must not be suggested in that line. + t.CheckOutputLines( + "NOTE: Makefile:23: You can use AUTO_MKDIRS=yes "+ + "or \"INSTALLATION_DIRS+= libexec/always\" "+ + "instead of \"${INSTALL_DATA_DIR}\".", + "NOTE: Makefile:24: You can use "+ + "\"INSTALLATION_DIRS+= libexec/conditional\" "+ + "instead of \"${INSTALL_DATA_DIR}\".", + "NOTE: Makefile:25: You can use "+ + "\"INSTALLATION_DIRS+= ${LIB_SUBDIR}\" "+ + "instead of \"${INSTALL_DATA_DIR}\".") } -// Two shell variables, next to each other, -// are two separate atoms but count as a single token. -func (s *Suite) Test_splitIntoShellTokens__two_shell_variables(c *check.C) { +// This test ensures that the command line options to INSTALL_*_DIR are properly +// parsed and do not lead to "can only handle one directory at a time" warnings. +func (s *Suite) Test_SimpleCommandChecker_checkInstallMulti(c *check.C) { t := s.Init(c) - code := "echo $$i$$j" - words, rest := splitIntoShellTokens(dummyLine, code) + t.SetUpVartypes() + mklines := t.NewMkLines("install.mk", + MkCvsID, + "", + "do-install:", + "\t${INSTALL_PROGRAM_DIR} -m 0555 -g ${APACHE_GROUP} -o ${APACHE_USER} \\", + "\t\t${DESTDIR}${PREFIX}/lib/apache-modules") - t.CheckDeepEquals(words, []string{"echo", "$$i$$j"}) - t.CheckEquals(rest, "") + mklines.Check() + + t.CheckOutputLines( + "NOTE: install.mk:4--5: You can use \"INSTALLATION_DIRS+= lib/apache-modules\" " + + "instead of \"${INSTALL_PROGRAM_DIR}\".") } -func (s *Suite) Test_splitIntoShellTokens__varuse_with_embedded_space(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_checkPaxPe(c *check.C) { t := s.Init(c) - words, rest := splitIntoShellTokens(dummyLine, "${VAR:S/ /_/g}") + t.SetUpVartypes() + t.SetUpTool("pax", "PAX", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "do-install:", + "\t${RUN} pax -pe ${WRKSRC} ${DESTDIR}${PREFIX}", + "\t${RUN} ${PAX} -pe ${WRKSRC} ${DESTDIR}${PREFIX}") - t.CheckDeepEquals(words, []string{"${VAR:S/ /_/g}"}) - t.CheckEquals(rest, "") + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:4: Please use the -pp option to pax(1) instead of -pe.", + "WARN: Makefile:5: Please use the -pp option to pax(1) instead of -pe.") } -func (s *Suite) Test_splitIntoShellTokens__redirect(c *check.C) { +func (s *Suite) Test_SimpleCommandChecker_checkEchoN(c *check.C) { t := s.Init(c) - words, rest := splitIntoShellTokens(dummyLine, "echo 1>output 2>>append 3>|clobber 4>&5 6<input >>append") + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpTool("echo -n", "ECHO_N", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "do-install:", + "\t${RUN} ${ECHO} -n 'Computing...'", + "\t${RUN} ${ECHO_N} 'Computing...'", + "\t${RUN} ${ECHO} 'Computing...'") - t.CheckDeepEquals(words, []string{ - "echo", - "1>", "output", - "2>>", "append", - "3>|", "clobber", - "4>&", "5", - "6<", "input", - ">>", "append"}) - t.CheckEquals(rest, "") + mklines.Check() - words, rest = splitIntoShellTokens(dummyLine, "echo 1> output 2>> append 3>| clobber 4>& 5 6< input >> append") + t.CheckOutputLines( + "WARN: Makefile:4: Please use ${ECHO_N} instead of \"echo -n\".") +} - t.CheckDeepEquals(words, []string{ - "echo", - "1>", "output", - "2>>", "append", - "3>|", "clobber", - "4>&", "5", - "6<", "input", - ">>", "append"}) - t.CheckEquals(rest, "") +func (s *Suite) Test_ShellLineChecker__shell_comment_with_line_continuation(c *check.C) { + t := s.Init(c) + + t.SetUpTool("echo", "", AtRunTime) + + test := func(lines ...string) { + i := 0 + for ; i < len(lines) && hasPrefix(lines[i], "\t"); i++ { + } + + mklines := t.SetUpFileMkLines("Makefile", + append([]string{MkCvsID, "pre-install:"}, + lines[:i]...)...) + + mklines.Check() + + t.CheckOutput(lines[i:]) + } + + // The comment can start at the beginning of a follow-up line. + test( + "\techo first; \\", + "\t# comment at the beginning of a command \\", + "\techo \"hello\"", + + // TODO: Warn that the "echo hello" is commented out. + ) + + // The comment can start at the beginning of a simple command. + test( + "\techo first; # comment at the beginning of a command \\", + "\techo \"hello\"", + + // TODO: Warn that the "echo hello" is commented out. + ) + + // The comment can start at a word in the middle of a command. + test( + // TODO: Warn that the "echo hello" is commented out. + "\techo # comment starts inside a command \\", + "\techo \"hello\"") + + // If the comment starts in the last line, there's no further + // line that might be commented out accidentally. + test( + "\techo 'first line'; \\", + "\t# comment in last line") +} + +func (s *Suite) Test_ShellLineChecker_checkConditionalCd(c *check.C) { + t := s.Init(c) + + t.SetUpTool("ls", "", AtRunTime) + t.SetUpTool("printf", "", AtRunTime) + t.SetUpTool("wc", "", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "pre-configure:", + "\t${RUN} while cd ..; do printf .; done", + "\t${RUN} while cd .. && cd ..; do printf .; done", // Unusual, therefore no warning. + "\t${RUN} if cd ..; then printf .; fi", + "\t${RUN} ! cd ..", + "\t${RUN} if cd ..; printf 'ok\\n'; then printf .; fi", + "\t${RUN} if cd .. | wc -l; then printf .; fi", // Unusual, therefore no warning. + "\t${RUN} if cd .. && cd ..; then printf .; fi") // Unusual, therefore no warning. + + mklines.Check() + + t.CheckOutputLines( + "ERROR: Makefile:3: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", + "ERROR: Makefile:5: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", + "WARN: Makefile:6: The Solaris /bin/sh does not support negation of shell commands.", + "WARN: Makefile:8: The exitcode of \"cd\" at the left of the | operator is ignored.") +} + +func (s *Suite) Test_ShellLineChecker_checkSetE__simple_commands(c *check.C) { + t := s.Init(c) + + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("rm", "", AtRunTime) + t.SetUpTool("touch", "", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "pre-configure:", + "\techo 1; echo 2; echo 3", + "\techo 1; touch file; rm file", + "\techo 1; var=value; echo 3") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:4: Please switch to \"set -e\" mode before using a semicolon " + + "(after \"touch file\") to separate commands.") +} + +func (s *Suite) Test_ShellLineChecker_checkSetE__compound_commands(c *check.C) { + t := s.Init(c) + + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("touch", "", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "pre-configure:", + "\ttouch file; for f in file; do echo \"$$f\"; done", + "\tfor f in file; do echo \"$$f\"; done; touch file", + "\ttouch 1; touch 2; touch 3; touch 4") + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"touch file\") to separate commands.", + "WARN: Makefile:5: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"touch 1\") to separate commands.") +} + +func (s *Suite) Test_ShellLineChecker_checkSetE__no_tracing(c *check.C) { + t := s.Init(c) + + t.SetUpTool("touch", "", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "pre-configure:", + "\ttouch 1; touch 2") + t.DisableTracing() + + mklines.Check() + + t.CheckOutputLines( + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon " + + "(after \"touch 1\") to separate commands.") +} + +func (s *Suite) Test_ShellLineChecker_canFail(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpTool("basename", "", AtRunTime) + t.SetUpTool("dirname", "", AtRunTime) + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("env", "", AtRunTime) + t.SetUpTool("grep", "GREP", AtRunTime) + t.SetUpTool("sed", "", AtRunTime) + t.SetUpTool("touch", "", AtRunTime) + t.SetUpTool("tr", "tr", AtRunTime) + t.SetUpTool("true", "TRUE", AtRunTime) + + test := func(cmd string, diagnostics ...string) { + mklines := t.NewMkLines("Makefile", + MkCvsID, + "pre-configure:", + "\t"+cmd+" ; echo 'done.'") + + mklines.Check() + + t.CheckOutput(diagnostics) + } + + test("socklen=`${GREP} 'expr' ${WRKSRC}/config.h`", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"socklen=`${GREP} 'expr' ${WRKSRC}/config.h`\") to separate commands.") + + test("socklen=`${GREP} 'expr' ${WRKSRC}/config.h || ${TRUE}`", + nil...) + + test("socklen=$$(expr 16)", + "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"socklen=$$(expr 16)\") to separate commands.") + + test("socklen=$$(expr 16 || true)", + "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.") + + test("socklen=$$(expr 16 || ${TRUE})", + "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.") + + test("${ECHO_MSG} \"Message\"", + nil...) + + test("${FAIL_MSG} \"Failure\"", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"${FAIL_MSG} \\\"Failure\\\"\") to separate commands.") + + test("set -x", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"set -x\") to separate commands.") + + test("echo 'input' | sed -e s,in,out,", + nil...) + + test("sed -e s,in,out,", + nil...) + + test("sed s,in,out,", + nil...) + + test("grep input", + nil...) + + test("grep pattern file...", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"grep pattern file...\") to separate commands.") + + test("touch file", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"touch file\") to separate commands.") + + test("echo 'starting'", + nil...) + + test("echo 'logging' > log", + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"echo 'logging'\") to separate commands.") + + test("echo 'to stderr' 1>&2", + nil...) + + test("echo 'hello' | tr -d 'aeiou'", + nil...) + + test("env | grep '^PATH='", + nil...) + + test("basename dir/file", + nil...) + + test("dirname dir/file", + nil...) +} + +func (s *Suite) Test_ShellLineChecker_checkPipeExitcode(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpTool("cat", "", AtRunTime) + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("printf", "", AtRunTime) + t.SetUpTool("sed", "", AtRunTime) + t.SetUpTool("right-side", "", AtRunTime) + mklines := t.NewMkLines("Makefile", + "\t echo | right-side", + "\t sed s,s,s, | right-side", + "\t printf | sed s,s,s, | right-side ", + "\t cat | right-side", + "\t cat | echo | right-side", + "\t echo | cat | right-side", + "\t sed s,s,s, filename | right-side", + "\t sed s,s,s < input | right-side", + "\t ./unknown | right-side", + "\t var=value | right-side", + "\t if :; then :; fi | right-side", + "\t var=`cat file` | right-side") + + for _, mkline := range mklines.mklines { + ck := NewShellLineChecker(mklines, mkline) + ck.CheckShellCommandLine(mkline.ShellCommand()) + } + + t.CheckOutputLines( + "WARN: Makefile:4: The exitcode of \"cat\" at the left of the | operator is ignored.", + "WARN: Makefile:5: The exitcode of \"cat\" at the left of the | operator is ignored.", + "WARN: Makefile:6: The exitcode of \"cat\" at the left of the | operator is ignored.", + "WARN: Makefile:7: The exitcode of \"sed\" at the left of the | operator is ignored.", + "WARN: Makefile:8: The exitcode of \"sed\" at the left of the | operator is ignored.", + "WARN: Makefile:9: The exitcode of \"./unknown\" at the left of the | operator is ignored.", + "WARN: Makefile:11: The exitcode of the command at the left of the | operator is ignored.", + "WARN: Makefile:12: The exitcode of the command at the left of the | operator is ignored.") } func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine(c *check.C) { @@ -362,45 +826,6 @@ func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__autofix(c *check.C) // default, --show-autofix, --autofix. } -func (s *Suite) Test_ShellProgramChecker_checkPipeExitcode(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - t.SetUpTool("cat", "", AtRunTime) - t.SetUpTool("echo", "", AtRunTime) - t.SetUpTool("printf", "", AtRunTime) - t.SetUpTool("sed", "", AtRunTime) - t.SetUpTool("right-side", "", AtRunTime) - mklines := t.NewMkLines("Makefile", - "\t echo | right-side", - "\t sed s,s,s, | right-side", - "\t printf | sed s,s,s, | right-side ", - "\t cat | right-side", - "\t cat | echo | right-side", - "\t echo | cat | right-side", - "\t sed s,s,s, filename | right-side", - "\t sed s,s,s < input | right-side", - "\t ./unknown | right-side", - "\t var=value | right-side", - "\t if :; then :; fi | right-side", - "\t var=`cat file` | right-side") - - for _, mkline := range mklines.mklines { - ck := NewShellLineChecker(mklines, mkline) - ck.CheckShellCommandLine(mkline.ShellCommand()) - } - - t.CheckOutputLines( - "WARN: Makefile:4: The exitcode of \"cat\" at the left of the | operator is ignored.", - "WARN: Makefile:5: The exitcode of \"cat\" at the left of the | operator is ignored.", - "WARN: Makefile:6: The exitcode of \"cat\" at the left of the | operator is ignored.", - "WARN: Makefile:7: The exitcode of \"sed\" at the left of the | operator is ignored.", - "WARN: Makefile:8: The exitcode of \"sed\" at the left of the | operator is ignored.", - "WARN: Makefile:9: The exitcode of \"./unknown\" at the left of the | operator is ignored.", - "WARN: Makefile:11: The exitcode of the command at the left of the | operator is ignored.", - "WARN: Makefile:12: The exitcode of the command at the left of the | operator is ignored.") -} - // TODO: Document the exact purpose of this test, or split it into useful tests. func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__implementation(c *check.C) { t := s.Init(c) @@ -445,395 +870,390 @@ func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__dollar_without_vari "WARN: filename.mk:1: Substitution commands like \"/.*~$$//g\" should always be quoted.") } -func (s *Suite) Test_ShellLineChecker_CheckWord(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__echo(c *check.C) { t := s.Init(c) - t.SetUpVartypes() + echo := t.SetUpTool("echo", "ECHO", AtRunTime) + echo.MustUseVarForm = true + mklines := t.NewMkLines("filename.mk", + "# dummy") + mkline := t.NewMkLine("filename.mk", 3, "# dummy") - test := func(shellWord string, checkQuoting bool, diagnostics ...string) { - // See checkVaruseUndefined and checkVarassignLeftNotUsed. - ck := t.NewShellLineChecker("\t echo " + shellWord) - ck.CheckWord(shellWord, checkQuoting, RunTime) - t.CheckOutput(diagnostics) - } + MkLineChecker{mklines, mkline}.checkText("echo \"hello, world\"") - // No warning for the outer variable since it is completely indirect. - // The inner variable ${list} must still be defined, though. - test("${${list}}", false, - "WARN: filename.mk:1: list is used but not defined.") + t.CheckOutputEmpty() - // No warning for variables that are partly indirect. - // TODO: Why not? - test("${SED_FILE.${id}}", false, - "WARN: filename.mk:1: id is used but not defined.") + NewShellLineChecker(mklines, mkline).CheckShellCommandLine("echo \"hello, world\"") - // TODO: Since $@ refers to ${.TARGET} and not sh.argv, there is no point in checking for quotes. - // TODO: Having the same tests for $$@ would be much more interesting. + t.CheckOutputLines( + "WARN: filename.mk:3: Please use \"${ECHO}\" instead of \"echo\".") +} - // The unquoted $@ takes a different code path in pkglint than the quoted $@. - test("$@", false, - "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__shell_variables(c *check.C) { + t := s.Init(c) - // When $@ appears as part of a shell token, it takes another code path in pkglint. - test("-$@-", false, - "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") + t.SetUpVartypes() + t.SetUpTool("install", "INSTALL", AtRunTime) + t.SetUpTool("cp", "CP", AtRunTime) + t.SetUpTool("mv", "MV", AtRunTime) + t.SetUpTool("sed", "SED", AtRunTime) + text := "for f in *.pl; do ${SED} s,@PREFIX@,${PREFIX}, < $f > $f.tmp && ${MV} $f.tmp $f; done" + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "\t"+text) - // The unquoted $@ takes a different code path in pkglint than the quoted $@. - test("\"$@\"", false, - "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") + ck := NewShellLineChecker(mklines, mklines.mklines[2]) + ck.CheckShellCommandLine(text) - test("${COMMENT:Q}", true, - nil...) + t.CheckOutputLines( + "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Make variable or $$f if you mean a shell variable.", + "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Make variable or $$f if you mean a shell variable.", + "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Make variable or $$f if you mean a shell variable.", + "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Make variable or $$f if you mean a shell variable.", + "NOTE: Makefile:3: Please use the SUBST framework instead of ${SED} and ${MV}.", + "WARN: Makefile:3: f is used but not defined.") - test("\"${DISTINFO_FILE:Q}\"", true, - "NOTE: filename.mk:1: The :Q modifier isn't necessary for ${DISTINFO_FILE} here.") + ck.CheckShellCommandLine("install -c manpage.1 ${PREFIX}/man/man1/manpage.1") - test("embed${DISTINFO_FILE:Q}ded", true, - "NOTE: filename.mk:1: The :Q modifier isn't necessary for ${DISTINFO_FILE} here.") + t.CheckOutputLines( + "WARN: Makefile:3: Please use ${PKGMANDIR} instead of \"man\".") - test("s,\\.,,", true, - nil...) + ck.CheckShellCommandLine("cp init-script ${PREFIX}/etc/rc.d/service") - test("\"s,\\.,,\"", true, - nil...) + t.CheckOutputLines( + "WARN: Makefile:3: Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") } -func (s *Suite) Test_ShellLineChecker_CheckWord__dollar_without_variable(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__sed_and_mv(c *check.C) { t := s.Init(c) - ck := t.NewShellLineChecker("# dummy") + t.SetUpVartypes() + t.SetUpTool("sed", "SED", AtRunTime) + t.SetUpTool("mv", "MV", AtRunTime) + ck := t.NewShellLineChecker("\t${RUN} ${SED} 's,#,// comment:,g' filename > filename.tmp; ${MV} filename.tmp filename") - ck.CheckWord("/.*~$$//g", false, RunTime) // Typical argument to pax(1). + ck.CheckShellCommandLine(ck.mkline.ShellCommand()) - t.CheckOutputEmpty() + t.CheckOutputLines( + "NOTE: filename.mk:1: Please use the SUBST framework instead of ${SED} and ${MV}.") } -func (s *Suite) Test_ShellLineChecker_CheckWord__backslash_plus(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__subshell(c *check.C) { t := s.Init(c) - t.SetUpTool("find", "FIND", AtRunTime) - ck := t.NewShellLineChecker("\tfind . -exec rm -rf {} \\+") + ck := t.NewShellLineChecker("\t${RUN} uname=$$(uname)") ck.CheckShellCommandLine(ck.mkline.ShellCommand()) - // A backslash before any other character than " \ ` is discarded by the parser. - t.CheckOutputEmpty() + t.CheckOutputLines( + "WARN: filename.mk:1: Invoking subshells via $(...) is not portable enough.") } -func (s *Suite) Test_ShellLineChecker_CheckWord__squot_dollar(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__install_dir(c *check.C) { t := s.Init(c) - ck := t.NewShellLineChecker("\t'$") + t.SetUpVartypes() + ck := t.NewShellLineChecker("\t${RUN} ${INSTALL_DATA_DIR} ${DESTDIR}${PREFIX}/dir1 ${DESTDIR}${PREFIX}/dir2") - ck.CheckWord(ck.mkline.ShellCommand(), false, RunTime) + ck.CheckShellCommandLine(ck.mkline.ShellCommand()) - // FIXME: Should be parsed correctly. Make passes the dollar through (probably), - // and the shell parser should complain about the unfinished string literal. t.CheckOutputLines( - "WARN: filename.mk:1: Internal pkglint error in ShTokenizer.ShAtom at \"$\" (quoting=s).", - "WARN: filename.mk:1: Internal pkglint error in ShellLine.CheckWord at \"'$\" (quoting=s), rest: $") -} + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir1\" instead of \"${INSTALL_DATA_DIR}\".", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir2\" instead of \"${INSTALL_DATA_DIR}\".", + "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") -func (s *Suite) Test_ShellLineChecker_CheckWord__dquot_dollar(c *check.C) { - t := s.Init(c) + ck.CheckShellCommandLine("${INSTALL_DATA_DIR} -d -m 0755 ${DESTDIR}${PREFIX}/share/examples/gdchart") - ck := t.NewShellLineChecker("\t\"$") + // No warning about multiple directories, since 0755 is an option, not an argument. + t.CheckOutputLines( + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= share/examples/gdchart\" instead of \"${INSTALL_DATA_DIR}\".") - ck.CheckWord(ck.mkline.ShellCommand(), false, RunTime) + ck.CheckShellCommandLine("${INSTALL_DATA_DIR} -d -m 0755 ${DESTDIR}${PREFIX}/dir1 ${PREFIX}/dir2") - // FIXME: Make consumes the dollar silently. - // This could be worth another pkglint warning. - t.CheckOutputEmpty() + t.CheckOutputLines( + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir1\" instead of \"${INSTALL_DATA_DIR}\".", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir2\" instead of \"${INSTALL_DATA_DIR}\".", + "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") } -func (s *Suite) Test_ShellLineChecker_CheckWord__dollar_subshell(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__install_option_d(c *check.C) { t := s.Init(c) - ck := t.NewShellLineChecker("\t$$(echo output)") + t.SetUpVartypes() + ck := t.NewShellLineChecker("\t${RUN} ${INSTALL} -d ${DESTDIR}${PREFIX}/dir1 ${DESTDIR}${PREFIX}/dir2") - ck.CheckWord(ck.mkline.ShellCommand(), false, RunTime) + ck.CheckShellCommandLine(ck.mkline.ShellCommand()) t.CheckOutputLines( - "WARN: filename.mk:1: Invoking subshells via $(...) is not portable enough.") + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir1\" instead of \"${INSTALL} -d\".", + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir2\" instead of \"${INSTALL} -d\".") } -func (s *Suite) Test_ShellLineChecker_CheckWord__PKGMANDIR(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkHiddenAndSuppress(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - mklines := t.NewMkLines("chat/ircII/Makefile", + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpTool("ls", "LS", AtRunTime) + mklines := t.NewMkLines("Makefile", MkCvsID, - "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/man", - "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/${PKGMANDIR}") + "", + "show-all-targets: .PHONY", + "\t@echo 'hello'", + "\t@ls -l", + "", + "anything-message: .PHONY", + "\t@echo 'may be hidden'", + "\t@ls 'may be hidden'", + "", + "pre-configure:", + "\t@") mklines.Check() - t.CheckOutputLines( - "WARN: chat/ircII/Makefile:2: Please use ${PKGMANDIR} instead of \"man\".", - "NOTE: chat/ircII/Makefile:2: This variable value should be aligned to column 25.", - "NOTE: chat/ircII/Makefile:3: This variable value should be aligned to column 25.") + // No warning about the hidden ls since the target names start + // with "show-" or end with "-message". + t.CheckOutputEmpty() } -func (s *Suite) Test_ShellLineChecker_CheckWord__empty(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkHiddenAndSuppress__no_tracing(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - + t.SetUpTool("ls", "LS", AtRunTime) mklines := t.NewMkLines("Makefile", MkCvsID, "", - "JAVA_CLASSPATH=\t# empty") + "pre-configure:", + "\t@ls -l") + t.DisableTracing() mklines.Check() - t.CheckOutputEmpty() + t.CheckOutputLines( + "WARN: Makefile:4: The shell command \"ls\" should not be hidden.") } -func (s *Suite) Test_ShellLineChecker_unescapeBackticks__unfinished(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommand__cd_inside_if(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("filename.mk", + t.SetUpVartypes() + t.SetUpTool("echo", "ECHO", AtRunTime) + mklines := t.NewMkLines("Makefile", MkCvsID, "", - "pre-configure:", - "\t`${VAR}", // Error in first shell word - "\techo `${VAR}") // Error after first shell word + "\t${RUN} if cd /bin; then echo \"/bin exists.\"; fi") - // Breakpoint in ShellLine.CheckShellCommand - // Breakpoint in ShellLine.CheckToken - // Breakpoint in ShellLine.unescapeBackticks mklines.Check() t.CheckOutputLines( - "WARN: filename.mk:4: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`${VAR}\"", - "WARN: filename.mk:5: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`${VAR}\"") + "ERROR: Makefile:3: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.") } -func (s *Suite) Test_ShellLineChecker_unescapeBackticks__unfinished_direct(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommand__negated_pipe(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("dummy.mk", + t.SetUpVartypes() + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpTool("test", "TEST", AtRunTime) + mklines := t.NewMkLines("Makefile", MkCvsID, - "\t# shell command") + "", + "\t${RUN} if ! test -f /etc/passwd; then echo \"passwd is missing.\"; fi") - // This call is unrealistic. It doesn't happen in practice, and this - // direct, forcing test is only to reach the code coverage. - atoms := []*ShAtom{ - NewShAtom(shtText, "`", shqBackt)} - NewShellLineChecker(mklines, mklines.mklines[1]). - unescapeBackticks(&atoms, shqBackt) + mklines.Check() t.CheckOutputLines( - "ERROR: dummy.mk:2: Unfinished backticks after \"\".") + "WARN: Makefile:3: The Solaris /bin/sh does not support negation of shell commands.") } -func (s *Suite) Test_ShellLineChecker_variableNeedsQuoting(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommand__subshell(c *check.C) { t := s.Init(c) - test := func(shVarname string, expected bool) { - t.CheckEquals((*ShellLineChecker).variableNeedsQuoting(nil, shVarname), expected) - } - - test("#", false) // A length is always an integer. - test("?", false) // The exit code is always an integer. - test("$", false) // The PID is always an integer. - - // In most cases, file and directory names don't contain special characters, - // and if they do, the package will probably not build. Therefore pkglint - // doesn't require them to be quoted, but doing so does not hurt. - test("d", false) // Typically used for directories. - test("f", false) // Typically used for files. - test("i", false) // Typically used for literal values without special characters. - test("id", false) // Identifiers usually don't use special characters. - test("dir", false) // See d above. - test("file", false) // See f above. - test("src", false) // Typically used when copying files or directories. - test("dst", false) // Typically used when copying files or directories. + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpTool("expr", "EXPR", AtRunTime) + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + "pre-configure:", + "\t@(echo ok)", + "\techo $$(uname -r); echo $$(expr 4 '*' $$(echo 1024))", + "\t@(echo nb$$(uname -r) $$(${EXPR} 4 \\* $$(echo 1024)))") - test("bindir", false) // A typical GNU-style directory. - test("mandir", false) // A typical GNU-style directory. - test("prefix", false) // + mklines.Check() - test("bindirs", true) // A list of directories is typically separated by spaces. - test("var", true) // Other variables are unknown, so they should be quoted. - test("0", true) // The program name may contain special characters when given as full path. - test("1", true) // Command line arguments can be arbitrary strings. + // FIXME: Fix the parse errors (nested subshells). + // FIXME: Fix the duplicate diagnostic in line 6. + // FIXME: "(" is not a shell command, it's an operator. + t.CheckOutputLines( + "WARN: Makefile:4: The shell command \"(\" should not be hidden.", + "WARN: Makefile:5: Internal pkglint error in ShTokenizer.ShAtom at \"$$(echo 1024))\" (quoting=S).", + "WARN: Makefile:5: Invoking subshells via $(...) is not portable enough.", + "WARN: Makefile:6: Internal pkglint error in ShTokenizer.ShAtom at \"$$(echo 1024)))\" (quoting=S).", + "WARN: Makefile:6: The shell command \"(\" should not be hidden.", + "WARN: Makefile:6: Internal pkglint error in ShTokenizer.ShAtom at \"$$(echo 1024)))\" (quoting=S).", + "WARN: Makefile:6: Invoking subshells via $(...) is not portable enough.") } -func (s *Suite) Test_ShellLineChecker_variableNeedsQuoting__integration(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckShellCommand__case_patterns_from_variable(c *check.C) { t := s.Init(c) t.SetUpVartypes() - t.SetUpTool("cp", "", AtRunTime) - mklines := t.NewMkLines("filename.mk", + mklines := t.NewMkLines("Makefile", MkCvsID, "", - // It's a bit silly to use shell variables in CONFIGURE_ARGS, - // but as of January 2019 that's the only way to run ShellLine.variableNeedsQuoting. - "CONFIGURE_ARGS+=\t; cp $$dir $$\\# $$target", "pre-configure:", - "\tcp $$dir $$\\# $$target") + "\tcase $$file in ${CHECK_PERMS_SKIP:@pattern@${pattern}) ;;@} *) continue; esac") mklines.Check() - // As of January 2019, the quoting check is disabled for real shell commands. - // See ShellLine.CheckShellCommand, spc.checkWord. - t.CheckOutputLines( - "WARN: filename.mk:3: Unquoted shell variable \"target\".") + // TODO: Ensure that the shell word is really only one variable use. + // TODO: Ensure that the last modifier is :@@@. + // TODO: Ensure that the replacement is a well-formed case-item. + // TODO: Ensure that the replacement contains ";;" as the last shell token. + t.CheckOutputEmpty() } -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__echo(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord(c *check.C) { t := s.Init(c) - echo := t.SetUpTool("echo", "ECHO", AtRunTime) - echo.MustUseVarForm = true - mklines := t.NewMkLines("filename.mk", - "# dummy") - mkline := t.NewMkLine("filename.mk", 3, "# dummy") + t.SetUpVartypes() - MkLineChecker{mklines, mkline}.checkText("echo \"hello, world\"") + test := func(shellWord string, checkQuoting bool, diagnostics ...string) { + // See checkVaruseUndefined and checkVarassignLeftNotUsed. + ck := t.NewShellLineChecker("\t echo " + shellWord) + ck.CheckWord(shellWord, checkQuoting, RunTime) + t.CheckOutput(diagnostics) + } - t.CheckOutputEmpty() + // No warning for the outer variable since it is completely indirect. + // The inner variable ${list} must still be defined, though. + test("${${list}}", false, + "WARN: filename.mk:1: list is used but not defined.") - NewShellLineChecker(mklines, mkline).CheckShellCommandLine("echo \"hello, world\"") + // No warning for variables that are partly indirect. + // TODO: Why not? + test("${SED_FILE.${id}}", false, + "WARN: filename.mk:1: id is used but not defined.") - t.CheckOutputLines( - "WARN: filename.mk:3: Please use \"${ECHO}\" instead of \"echo\".") -} + // TODO: Since $@ refers to ${.TARGET} and not sh.argv, there is no point in checking for quotes. + // TODO: Having the same tests for $$@ would be much more interesting. -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__shell_variables(c *check.C) { - t := s.Init(c) + // The unquoted $@ takes a different code path in pkglint than the quoted $@. + test("$@", false, + "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") - t.SetUpVartypes() - t.SetUpTool("install", "INSTALL", AtRunTime) - t.SetUpTool("cp", "CP", AtRunTime) - t.SetUpTool("mv", "MV", AtRunTime) - t.SetUpTool("sed", "SED", AtRunTime) - text := "for f in *.pl; do ${SED} s,@PREFIX@,${PREFIX}, < $f > $f.tmp && ${MV} $f.tmp $f; done" - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "\t"+text) + // When $@ appears as part of a shell token, it takes another code path in pkglint. + test("-$@-", false, + "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") - ck := NewShellLineChecker(mklines, mklines.mklines[2]) - ck.CheckShellCommandLine(text) + // The unquoted $@ takes a different code path in pkglint than the quoted $@. + test("\"$@\"", false, + "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") - t.CheckOutputLines( - "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Make variable or $$f if you mean a shell variable.", - "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Make variable or $$f if you mean a shell variable.", - "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Make variable or $$f if you mean a shell variable.", - "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Make variable or $$f if you mean a shell variable.", - "NOTE: Makefile:3: Please use the SUBST framework instead of ${SED} and ${MV}.", - "WARN: Makefile:3: f is used but not defined.") + test("${COMMENT:Q}", true, + nil...) - ck.CheckShellCommandLine("install -c manpage.1 ${PREFIX}/man/man1/manpage.1") + test("\"${DISTINFO_FILE:Q}\"", true, + "NOTE: filename.mk:1: The :Q modifier isn't necessary for ${DISTINFO_FILE} here.") - t.CheckOutputLines( - "WARN: Makefile:3: Please use ${PKGMANDIR} instead of \"man\".") + test("embed${DISTINFO_FILE:Q}ded", true, + "NOTE: filename.mk:1: The :Q modifier isn't necessary for ${DISTINFO_FILE} here.") - ck.CheckShellCommandLine("cp init-script ${PREFIX}/etc/rc.d/service") + test("s,\\.,,", true, + nil...) - t.CheckOutputLines( - "WARN: Makefile:3: Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") + test("\"s,\\.,,\"", true, + nil...) } -func (s *Suite) Test_ShellLineChecker_checkInstallCommand(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord__dollar_without_variable(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("filename.mk", - "\t# dummy") - mklines.target = "do-install" - - ck := NewShellLineChecker(mklines, mklines.mklines[0]) - - ck.checkInstallCommand("sed") - - t.CheckOutputLines( - "WARN: filename.mk:1: The shell command \"sed\" should not be used in the install phase.") + ck := t.NewShellLineChecker("# dummy") - ck.checkInstallCommand("cp") + ck.CheckWord("/.*~$$//g", false, RunTime) // Typical argument to pax(1). - t.CheckOutputLines( - "WARN: filename.mk:1: ${CP} should not be used to install files.") + t.CheckOutputEmpty() } -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__sed_and_mv(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord__backslash_plus(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - t.SetUpTool("sed", "SED", AtRunTime) - t.SetUpTool("mv", "MV", AtRunTime) - ck := t.NewShellLineChecker("\t${RUN} ${SED} 's,#,// comment:,g' filename > filename.tmp; ${MV} filename.tmp filename") + t.SetUpTool("find", "FIND", AtRunTime) + ck := t.NewShellLineChecker("\tfind . -exec rm -rf {} \\+") ck.CheckShellCommandLine(ck.mkline.ShellCommand()) - t.CheckOutputLines( - "NOTE: filename.mk:1: Please use the SUBST framework instead of ${SED} and ${MV}.") + // A backslash before any other character than " \ ` is discarded by the parser. + t.CheckOutputEmpty() } -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__subshell(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord__squot_dollar(c *check.C) { t := s.Init(c) - ck := t.NewShellLineChecker("\t${RUN} uname=$$(uname)") + ck := t.NewShellLineChecker("\t'$") - ck.CheckShellCommandLine(ck.mkline.ShellCommand()) + ck.CheckWord(ck.mkline.ShellCommand(), false, RunTime) + // FIXME: Should be parsed correctly. Make passes the dollar through (probably), + // and the shell parser should complain about the unfinished string literal. t.CheckOutputLines( - "WARN: filename.mk:1: Invoking subshells via $(...) is not portable enough.") + "WARN: filename.mk:1: Internal pkglint error in ShTokenizer.ShAtom at \"$\" (quoting=s).", + "WARN: filename.mk:1: Internal pkglint error in ShellLine.CheckWord at \"'$\" (quoting=s), rest: $") } -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__install_dir(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord__dquot_dollar(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - ck := t.NewShellLineChecker("\t${RUN} ${INSTALL_DATA_DIR} ${DESTDIR}${PREFIX}/dir1 ${DESTDIR}${PREFIX}/dir2") + ck := t.NewShellLineChecker("\t\"$") - ck.CheckShellCommandLine(ck.mkline.ShellCommand()) + ck.CheckWord(ck.mkline.ShellCommand(), false, RunTime) - t.CheckOutputLines( - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir1\" instead of \"${INSTALL_DATA_DIR}\".", - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir2\" instead of \"${INSTALL_DATA_DIR}\".", - "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") + // FIXME: Make consumes the dollar silently. + // This could be worth another pkglint warning. + t.CheckOutputEmpty() +} - ck.CheckShellCommandLine("${INSTALL_DATA_DIR} -d -m 0755 ${DESTDIR}${PREFIX}/share/examples/gdchart") +func (s *Suite) Test_ShellLineChecker_CheckWord__dollar_subshell(c *check.C) { + t := s.Init(c) - // No warning about multiple directories, since 0755 is an option, not an argument. - t.CheckOutputLines( - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= share/examples/gdchart\" instead of \"${INSTALL_DATA_DIR}\".") + ck := t.NewShellLineChecker("\t$$(echo output)") - ck.CheckShellCommandLine("${INSTALL_DATA_DIR} -d -m 0755 ${DESTDIR}${PREFIX}/dir1 ${PREFIX}/dir2") + ck.CheckWord(ck.mkline.ShellCommand(), false, RunTime) t.CheckOutputLines( - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir1\" instead of \"${INSTALL_DATA_DIR}\".", - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir2\" instead of \"${INSTALL_DATA_DIR}\".", - "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") + "WARN: filename.mk:1: Invoking subshells via $(...) is not portable enough.") } -func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__install_option_d(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord__PKGMANDIR(c *check.C) { t := s.Init(c) t.SetUpVartypes() - ck := t.NewShellLineChecker("\t${RUN} ${INSTALL} -d ${DESTDIR}${PREFIX}/dir1 ${DESTDIR}${PREFIX}/dir2") + mklines := t.NewMkLines("chat/ircII/Makefile", + MkCvsID, + "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/man", + "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/${PKGMANDIR}") - ck.CheckShellCommandLine(ck.mkline.ShellCommand()) + mklines.Check() t.CheckOutputLines( - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir1\" instead of \"${INSTALL} -d\".", - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= dir2\" instead of \"${INSTALL} -d\".") + "WARN: chat/ircII/Makefile:2: Please use ${PKGMANDIR} instead of \"man\".", + "NOTE: chat/ircII/Makefile:2: This variable value should be aligned to column 25.", + "NOTE: chat/ircII/Makefile:3: This variable value should be aligned to column 25.") } -func (s *Suite) Test_ShellLineChecker__shell_comment_with_line_continuation(c *check.C) { +func (s *Suite) Test_ShellLineChecker_CheckWord__empty(c *check.C) { t := s.Init(c) - mklines := t.SetUpFileMkLines("Makefile", + t.SetUpVartypes() + + mklines := t.NewMkLines("Makefile", MkCvsID, - "pre-install:", - "\t"+"# comment\\", - "\t"+"echo \"hello\"") + "", + "JAVA_CLASSPATH=\t# empty") mklines.Check() - // TODO: "WARN: ~/Makefile:3--4: A shell comment does not stop at the end of line." t.CheckOutputEmpty() } @@ -895,51 +1315,42 @@ func (s *Suite) Test_ShellLineChecker_checkWordQuoting(c *check.C) { nil...) } -func (s *Suite) Test_ShellLineChecker_checkShVarUsePlain__default_warning_level(c *check.C) { +func (s *Suite) Test_ShellLineChecker_unescapeBackticks__unfinished(c *check.C) { t := s.Init(c) - t.SetUpCommandLine( /* none */ ) - t.SetUpVartypes() - t.SetUpTool("echo", "", AtRunTime) - mklines := t.NewMkLines("filename.mk", MkCvsID, - "CONFIGURE_ARGS+=\techo $$@ $$var", "", "pre-configure:", - "\techo $$@ $$var") + "\t`${VAR}", // Error in first shell word + "\techo `${VAR}") // Error after first shell word + // Breakpoint in ShellLine.CheckShellCommand + // Breakpoint in ShellLine.CheckToken + // Breakpoint in ShellLine.unescapeBackticks mklines.Check() - // Using $@ outside of double quotes is so obviously wrong that - // the warning is issued by default. t.CheckOutputLines( - "WARN: filename.mk:2: The $@ shell variable should only be used in double quotes.", - "WARN: filename.mk:5: The $@ shell variable should only be used in double quotes.") + "WARN: filename.mk:4: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`${VAR}\"", + "WARN: filename.mk:5: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`${VAR}\"") } -func (s *Suite) Test_ShellLineChecker_checkShVarUsePlain__Wall(c *check.C) { +func (s *Suite) Test_ShellLineChecker_unescapeBackticks__unfinished_direct(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - t.SetUpTool("echo", "", AtRunTime) - - mklines := t.NewMkLines("filename.mk", + mklines := t.NewMkLines("dummy.mk", MkCvsID, - "CONFIGURE_ARGS+=\techo $$@ $$var", - "", - "pre-configure:", - "\techo $$@ $$var") + "\t# shell command") - mklines.Check() + // This call is unrealistic. It doesn't happen in practice, and this + // direct, forcing test is only to reach the code coverage. + atoms := []*ShAtom{ + NewShAtom(shtText, "`", shqBackt)} + NewShellLineChecker(mklines, mklines.mklines[1]). + unescapeBackticks(&atoms, shqBackt) - // FIXME: It is inconsistent that the check for unquoted shell - // variables is enabled for CONFIGURE_ARGS (where shell variables - // don't make sense at all) but not for real shell commands. t.CheckOutputLines( - "WARN: filename.mk:2: The $@ shell variable should only be used in double quotes.", - "WARN: filename.mk:2: Unquoted shell variable \"var\".", - "WARN: filename.mk:5: The $@ shell variable should only be used in double quotes.") + "ERROR: dummy.mk:2: Unfinished backticks after \"\".") } func (s *Suite) Test_ShellLineChecker_unescapeBackticks(c *check.C) { @@ -1017,661 +1428,269 @@ func (s *Suite) Test_ShellLineChecker_unescapeBackticks__dquotBacktDquot(c *chec "WARN: dummy.mk:2: Double quotes inside backticks inside double quotes are error prone.") } -func (s *Suite) Test_ShellLineChecker__variable_outside_quotes(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkShVarUsePlain__default_warning_level(c *check.C) { t := s.Init(c) + t.SetUpCommandLine( /* none */ ) t.SetUpVartypes() - mklines := t.NewMkLines("dummy.mk", - MkCvsID, - "GZIP=\t${ECHO} $$comment") - - mklines.Check() - - t.CheckOutputLines( - "WARN: dummy.mk:2: The variable GZIP should not be set by any package.", - "WARN: dummy.mk:2: Unquoted shell variable \"comment\".", - "WARN: dummy.mk:2: ECHO should not be used indirectly at load time (via GZIP).") -} - -func (s *Suite) Test_ShellLineChecker_CheckShellCommand__cd_inside_if(c *check.C) { - t := s.Init(c) + t.SetUpTool("echo", "", AtRunTime) - t.SetUpVartypes() - t.SetUpTool("echo", "ECHO", AtRunTime) - mklines := t.NewMkLines("Makefile", + mklines := t.NewMkLines("filename.mk", MkCvsID, + "CONFIGURE_ARGS+=\techo $$@ $$var", "", - "\t${RUN} if cd /bin; then echo \"/bin exists.\"; fi") + "pre-configure:", + "\techo $$@ $$var") mklines.Check() + // Using $@ outside of double quotes is so obviously wrong that + // the warning is issued by default. t.CheckOutputLines( - "ERROR: Makefile:3: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.") + "WARN: filename.mk:2: The $@ shell variable should only be used in double quotes.", + "WARN: filename.mk:5: The $@ shell variable should only be used in double quotes.") } -func (s *Suite) Test_ShellLineChecker_CheckShellCommand__negated_pipe(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkShVarUsePlain__Wall(c *check.C) { t := s.Init(c) t.SetUpVartypes() - t.SetUpTool("echo", "ECHO", AtRunTime) - t.SetUpTool("test", "TEST", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "\t${RUN} if ! test -f /etc/passwd; then echo \"passwd is missing.\"; fi") - - mklines.Check() - - t.CheckOutputLines( - "WARN: Makefile:3: The Solaris /bin/sh does not support negation of shell commands.") -} - -func (s *Suite) Test_ShellLineChecker_CheckShellCommand__subshell(c *check.C) { - t := s.Init(c) + t.SetUpTool("echo", "", AtRunTime) - t.SetUpTool("echo", "ECHO", AtRunTime) - t.SetUpTool("expr", "EXPR", AtRunTime) - mklines := t.NewMkLines("Makefile", + mklines := t.NewMkLines("filename.mk", MkCvsID, + "CONFIGURE_ARGS+=\techo $$@ $$var", "", "pre-configure:", - "\t@(echo ok)", - "\techo $$(uname -r); echo $$(expr 4 '*' $$(echo 1024))", - "\t@(echo nb$$(uname -r) $$(${EXPR} 4 \\* $$(echo 1024)))") + "\techo $$@ $$var") mklines.Check() - // FIXME: Fix the parse errors (nested subshells). - // FIXME: Fix the duplicate diagnostic in line 6. - // FIXME: "(" is not a shell command, it's an operator. + // FIXME: It is inconsistent that the check for unquoted shell + // variables is enabled for CONFIGURE_ARGS (where shell variables + // don't make sense at all) but not for real shell commands. t.CheckOutputLines( - "WARN: Makefile:4: The shell command \"(\" should not be hidden.", - "WARN: Makefile:5: Internal pkglint error in ShTokenizer.ShAtom at \"$$(echo 1024))\" (quoting=S).", - "WARN: Makefile:5: Invoking subshells via $(...) is not portable enough.", - "WARN: Makefile:6: Internal pkglint error in ShTokenizer.ShAtom at \"$$(echo 1024)))\" (quoting=S).", - "WARN: Makefile:6: The shell command \"(\" should not be hidden.", - "WARN: Makefile:6: Internal pkglint error in ShTokenizer.ShAtom at \"$$(echo 1024)))\" (quoting=S).", - "WARN: Makefile:6: Invoking subshells via $(...) is not portable enough.") + "WARN: filename.mk:2: The $@ shell variable should only be used in double quotes.", + "WARN: filename.mk:2: Unquoted shell variable \"var\".", + "WARN: filename.mk:5: The $@ shell variable should only be used in double quotes.") } -func (s *Suite) Test_ShellLineChecker_CheckShellCommand__case_patterns_from_variable(c *check.C) { +func (s *Suite) Test_ShellLineChecker_variableNeedsQuoting(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "pre-configure:", - "\tcase $$file in ${CHECK_PERMS_SKIP:@pattern@${pattern}) ;;@} *) continue; esac") - - mklines.Check() - - // TODO: Ensure that the shell word is really only one variable use. - // TODO: Ensure that the last modifier is :@@@. - // TODO: Ensure that the replacement is a well-formed case-item. - // TODO: Ensure that the replacement contains ";;" as the last shell token. - t.CheckOutputEmpty() -} + test := func(shVarname string, expected bool) { + t.CheckEquals((*ShellLineChecker).variableNeedsQuoting(nil, shVarname), expected) + } -func (s *Suite) Test_ShellLineChecker_checkHiddenAndSuppress(c *check.C) { - t := s.Init(c) + test("#", false) // A length is always an integer. + test("?", false) // The exit code is always an integer. + test("$", false) // The PID is always an integer. - t.SetUpTool("echo", "ECHO", AtRunTime) - t.SetUpTool("ls", "LS", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "show-all-targets: .PHONY", - "\t@echo 'hello'", - "\t@ls -l", - "", - "anything-message: .PHONY", - "\t@echo 'may be hidden'", - "\t@ls 'may be hidden'", - "", - "pre-configure:", - "\t@") + // In most cases, file and directory names don't contain special characters, + // and if they do, the package will probably not build. Therefore pkglint + // doesn't require them to be quoted, but doing so does not hurt. + test("d", false) // Typically used for directories. + test("f", false) // Typically used for files. + test("i", false) // Typically used for literal values without special characters. + test("id", false) // Identifiers usually don't use special characters. + test("dir", false) // See d above. + test("file", false) // See f above. + test("src", false) // Typically used when copying files or directories. + test("dst", false) // Typically used when copying files or directories. - mklines.Check() + test("bindir", false) // A typical GNU-style directory. + test("mandir", false) // A typical GNU-style directory. + test("prefix", false) // - // No warning about the hidden ls since the target names start - // with "show-" or end with "-message". - t.CheckOutputEmpty() + test("bindirs", true) // A list of directories is typically separated by spaces. + test("var", true) // Other variables are unknown, so they should be quoted. + test("0", true) // The program name may contain special characters when given as full path. + test("1", true) // Command line arguments can be arbitrary strings. + test("comment", true) // Comments can be arbitrary strings. } -func (s *Suite) Test_ShellLineChecker_checkHiddenAndSuppress__no_tracing(c *check.C) { +func (s *Suite) Test_ShellLineChecker_variableNeedsQuoting__integration(c *check.C) { t := s.Init(c) - t.SetUpTool("ls", "LS", AtRunTime) - mklines := t.NewMkLines("Makefile", + t.SetUpVartypes() + t.SetUpTool("cp", "", AtRunTime) + mklines := t.NewMkLines("filename.mk", MkCvsID, "", + // It's a bit silly to use shell variables in CONFIGURE_ARGS, + // but as of January 2019 that's the only way to run ShellLine.variableNeedsQuoting. + "CONFIGURE_ARGS+=\t; cp $$dir $$\\# $$target", "pre-configure:", - "\t@ls -l") - t.DisableTracing() + "\tcp $$dir $$\\# $$target") mklines.Check() + // As of January 2019, the quoting check is disabled for real shell commands. + // See ShellLine.CheckShellCommand, spc.checkWord. t.CheckOutputLines( - "WARN: Makefile:4: The shell command \"ls\" should not be hidden.") + "WARN: filename.mk:3: Unquoted shell variable \"target\".") } -func (s *Suite) Test_SimpleCommandChecker_handleForbiddenCommand(c *check.C) { +func (s *Suite) Test_ShellLineChecker_checkInstallCommand(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "\t${RUN} mktexlsr; texconfig") - - mklines.Check() - - t.CheckOutputLines( - "ERROR: Makefile:3: \"mktexlsr\" must not be used in Makefiles.", - "ERROR: Makefile:3: \"texconfig\" must not be used in Makefiles.") -} - -func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable(c *check.C) { - t := s.Init(c) + mklines := t.NewMkLines("filename.mk", + "\t# dummy") + mklines.target = "do-install" - t.SetUpTool("perl", "PERL5", AtRunTime) - t.SetUpTool("perl6", "PERL6", Nowhere) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "PERL5_VARS_CMD=\t${PERL5:Q}", - "PERL5_VARS_CMD=\t${PERL6:Q}", - "", - "pre-configure:", - "\t${PERL5_VARS_CMD} -e 'print 12345'") + ck := NewShellLineChecker(mklines, mklines.mklines[0]) - mklines.Check() + ck.checkInstallCommand("sed") - // FIXME: In PERL5:Q and PERL6:Q, the :Q is wrong. t.CheckOutputLines( - "WARN: Makefile:4: The \"${PERL6:Q}\" tool is used but not added to USE_TOOLS.") -} - -func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__parameterized(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package") - G.Pkg = NewPackage(t.File("category/package")) - t.FinishSetUp() - - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "MY_TOOL.i386=\t${PREFIX}/bin/tool-i386", - "MY_TOOL.x86_64=\t${PREFIX}/bin/tool-x86_64", - "", - "pre-configure:", - "\t${MY_TOOL.amd64} -e 'print 12345'", - "\t${UNKNOWN_TOOL}") + "WARN: filename.mk:1: The shell command \"sed\" should not be used in the install phase.") - mklines.Check() + ck.checkInstallCommand("cp") t.CheckOutputLines( - "WARN: Makefile:8: Unknown shell command \"${UNKNOWN_TOOL}\".", - "WARN: Makefile:8: UNKNOWN_TOOL is used but not defined.") -} - -func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__followed_by_literal(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package") - G.Pkg = NewPackage(t.File("category/package")) - t.FinishSetUp() - - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "QTDIR=\t${PREFIX}", - "", - "pre-configure:", - "\t${QTDIR}/bin/release") - - mklines.Check() - - t.CheckOutputEmpty() -} - -// The package Makefile and other .mk files in a package directory -// may use any shell commands defined by any included files. -// But only if the package is checked as a whole. -// -// On the contrary, when pkglint checks a single .mk file, these -// commands are not guaranteed to be defined, not even when the -// .mk file includes the file defining the command. -// FIXME: This paragraph sounds wrong. All commands from included files should be valid. -// -// The PYTHON_BIN variable below must not be called *_CMD, or another code path is taken. -func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__from_package(c *check.C) { - t := s.Init(c) - - pkg := t.SetUpPackage("category/package", - "post-install:", - "\t${PYTHON_BIN}", - "", - ".include \"extra.mk\"") - t.CreateFileLines("category/package/extra.mk", - MkCvsID, - "PYTHON_BIN=\tmy_cmd") - t.FinishSetUp() - - G.Check(pkg) - - t.CheckOutputEmpty() + "WARN: filename.mk:1: ${CP} should not be used to install files.") } -// This test ensures that the command line options to INSTALL_*_DIR are properly -// parsed and do not lead to "can only handle one directory at a time" warnings. -func (s *Suite) Test_SimpleCommandChecker_checkInstallMulti(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__line_continuation(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - mklines := t.NewMkLines("install.mk", - MkCvsID, - "", - "do-install:", - "\t${INSTALL_PROGRAM_DIR} -m 0555 -g ${APACHE_GROUP} -o ${APACHE_USER} \\", - "\t\t${DESTDIR}${PREFIX}/lib/apache-modules") + words, rest := splitIntoShellTokens(dummyLine, "if true; then \\") - mklines.Check() + t.CheckDeepEquals(words, []string{"if", "true", ";", "then"}) + t.CheckEquals(rest, "\\") t.CheckOutputLines( - "NOTE: install.mk:4--5: You can use \"INSTALLATION_DIRS+= lib/apache-modules\" " + - "instead of \"${INSTALL_PROGRAM_DIR}\".") + "WARN: Internal pkglint error in ShTokenizer.ShAtom at \"\\\\\" (quoting=plain).") } -func (s *Suite) Test_SimpleCommandChecker_checkPaxPe(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__dollar_slash(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - t.SetUpTool("pax", "PAX", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "do-install:", - "\t${RUN} pax -pe ${WRKSRC} ${DESTDIR}${PREFIX}", - "\t${RUN} ${PAX} -pe ${WRKSRC} ${DESTDIR}${PREFIX}") - - mklines.Check() + words, rest := splitIntoShellTokens(dummyLine, "pax -s /.*~$$//g") - t.CheckOutputLines( - "WARN: Makefile:4: Please use the -pp option to pax(1) instead of -pe.", - "WARN: Makefile:5: Please use the -pp option to pax(1) instead of -pe.") + t.CheckDeepEquals(words, []string{"pax", "-s", "/.*~$$//g"}) + t.CheckEquals(rest, "") } -func (s *Suite) Test_SimpleCommandChecker_checkEchoN(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__dollar_subshell(c *check.C) { t := s.Init(c) - t.SetUpTool("echo", "ECHO", AtRunTime) - t.SetUpTool("echo -n", "ECHO_N", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - "do-install:", - "\t${RUN} ${ECHO} -n 'Computing...'", - "\t${RUN} ${ECHO_N} 'Computing...'", - "\t${RUN} ${ECHO} 'Computing...'") - - mklines.Check() + words, rest := splitIntoShellTokens(dummyLine, "id=$$(${AWK} '{print}' < ${WRKSRC}/idfile) && echo \"$$id\"") - t.CheckOutputLines( - "WARN: Makefile:4: Please use ${ECHO_N} instead of \"echo -n\".") + t.CheckDeepEquals(words, []string{"id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)", "&&", "echo", "\"$$id\""}) + t.CheckEquals(rest, "") } -func (s *Suite) Test_ShellProgramChecker_checkConditionalCd(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__semicolons(c *check.C) { t := s.Init(c) - t.SetUpTool("ls", "", AtRunTime) - t.SetUpTool("printf", "", AtRunTime) - t.SetUpTool("wc", "", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "pre-configure:", - "\t${RUN} while cd ..; do printf .; done", - "\t${RUN} while cd .. && cd ..; do printf .; done", // Unusual, therefore no warning. - "\t${RUN} if cd ..; then printf .; fi", - "\t${RUN} ! cd ..", - "\t${RUN} if cd ..; printf 'ok\\n'; then printf .; fi", - "\t${RUN} if cd .. | wc -l; then printf .; fi", // Unusual, therefore no warning. - "\t${RUN} if cd .. && cd ..; then printf .; fi") // Unusual, therefore no warning. - - mklines.Check() + words, rest := splitIntoShellTokens(dummyLine, "word1 word2;;;") - t.CheckOutputLines( - "ERROR: Makefile:3: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", - "ERROR: Makefile:5: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", - "WARN: Makefile:6: The Solaris /bin/sh does not support negation of shell commands.", - "WARN: Makefile:8: The exitcode of \"cd\" at the left of the | operator is ignored.") + t.CheckDeepEquals(words, []string{"word1", "word2", ";;", ";"}) + t.CheckEquals(rest, "") } -func (s *Suite) Test_SimpleCommandChecker_checkRegexReplace(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__whitespace(c *check.C) { t := s.Init(c) - test := func(cmd string, diagnostics ...string) { - t.SetUpTool("pax", "PAX", AtRunTime) - t.SetUpTool("sed", "SED", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "pre-configure:", - "\t"+cmd) - - mklines.Check() - - t.CheckOutput(diagnostics) - } - - test("${PAX} -s s,.*,, src dst", - "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") - - test("pax -s s,.*,, src dst", - "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") - - test("${SED} -e s,.*,, src dst", - "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") - - test("sed -e s,.*,, src dst", - "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.") - - // The * is properly enclosed in quotes. - test("sed -e 's,.*,,' -e \"s,-*,,\"", - nil...) - - // The * is properly escaped. - test("sed -e s,.\\*,,", - nil...) - - test("pax -s s,\\.orig,, src dst", - nil...) - - test("sed -e s,a,b,g src dst", - nil...) - - // TODO: Merge the code with BtSedCommands. + text := "\t${RUN} cd ${WRKSRC}&&(${ECHO} ${PERL5:Q};${ECHO})|${BASH} ./install" + words, rest := splitIntoShellTokens(dummyLine, text) - // TODO: Finally, remove the G.Testing from the main code. - // Then, remove this test case. - G.Testing = false - test("sed -e s,.*,match,", - nil...) - G.Testing = true + t.CheckDeepEquals(words, []string{ + "${RUN}", + "cd", "${WRKSRC}", + "&&", "(", "${ECHO}", "${PERL5:Q}", ";", "${ECHO}", ")", + "|", "${BASH}", "./install"}) + t.CheckEquals(rest, "") } -func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__finished_dquot(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - // TODO: Check whether these tools are actually necessary for this test. - t.SetUpTool("awk", "AWK", AtRunTime) - t.SetUpTool("cp", "CP", AtRunTime) - t.SetUpTool("echo", "", AtRunTime) - t.SetUpTool("mkdir", "MKDIR", AtRunTime) // This is actually "mkdir -p". - t.SetUpTool("unzip", "UNZIP_CMD", AtRunTime) - - test := func(shellCommand string, diagnostics ...string) { - mklines := t.NewMkLines("filename.mk", - "\t"+shellCommand) - ck := NewShellLineChecker(mklines, mklines.mklines[0]) - - mklines.ForEach(func(mkline *MkLine) { - ck.CheckShellCommandLine(ck.mkline.ShellCommand()) - }) - - t.CheckOutput(diagnostics) - } - - // AUTO_MKDIRS applies only when installing directories. - test("${RUN} ${INSTALL} -c ${WRKSRC}/file ${PREFIX}/bin/", - nil...) - - // TODO: Warn that ${INSTALL} -d can only handle a single directory. - test("${RUN} ${INSTALL} -m 0755 -d ${PREFIX}/first ${PREFIX}/second", - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= first\" instead of \"${INSTALL} -d\".", - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= second\" instead of \"${INSTALL} -d\".") - - G.Pkg = NewPackage(t.File("category/pkgbase")) - G.Pkg.Plist.Dirs["share/pkgbase"] = &PlistLine{ - t.NewLine("PLIST", 123, "share/pkgbase/file"), - nil, - "share/pkgbase/file"} - - // A directory that is found in the PLIST. - // TODO: Add a test for using this command inside a conditional; - // the note should not appear then. - test("${RUN} ${INSTALL_DATA_DIR} share/pkgbase ${PREFIX}/share/pkgbase", - "NOTE: filename.mk:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" "+ - "instead of \"${INSTALL_DATA_DIR}\".", - "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") - - // Directories from .for loops are too dynamic to be replaced with AUTO_MKDIRS. - // TODO: Expand simple .for loops. - test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/${dir}", - "WARN: filename.mk:1: dir is used but not defined.") + text := "\"\"" + words, rest := splitIntoShellTokens(dummyLine, text) - // A directory that is not found in the PLIST would not be created by AUTO_MKDIRS, - // therefore only INSTALLATION_DIRS is suggested. - test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/share/other", - "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= share/other\" instead of \"${INSTALL_DATA_DIR}\".") + t.CheckDeepEquals(words, []string{"\"\""}) + t.CheckEquals(rest, "") } -func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs__redundant(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__unfinished_dquot(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "AUTO_MKDIRS=\t\tyes", - "INSTALLATION_DIRS+=\tshare/redundant", - "INSTALLATION_DIRS+=\tnot/redundant ${EGDIR}") - t.CreateFileLines("category/package/PLIST", - PlistCvsID, - "share/redundant/file", - "${EGDIR}/file") - - t.Main("-Wall", "-q", "category/package") + text := "\t\"" + words, rest := splitIntoShellTokens(dummyLine, text) - t.CheckOutputLines( - "NOTE: ~/category/package/Makefile:21: The directory \"share/redundant\" "+ - "is redundant in INSTALLATION_DIRS.", - // The below is not proven to be always correct. It assumes that a - // variable in the Makefile has the same value as the corresponding - // variable from PLIST_SUBST. Violating this assumption would be - // confusing to the pkgsrc developers, therefore it's a safe bet. - // A notable counterexample is PKGNAME in PLIST, which corresponds - // to PKGNAME_NOREV in the package Makefile. - "NOTE: ~/category/package/Makefile:22: The directory \"${EGDIR}\" "+ - "is redundant in INSTALLATION_DIRS.") + c.Check(words, check.IsNil) + t.CheckEquals(rest, "\"") } -// The AUTO_MKDIRS code in mk/install/install.mk (install-dirs-from-PLIST) -// skips conditional directories, as well as directories with placeholders. -func (s *Suite) Test_SimpleCommandChecker_checkAutoMkdirs__conditional_PLIST(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__unescaped_dollar_in_dquot(c *check.C) { t := s.Init(c) - t.SetUpPackage("category/package", - "LIB_SUBDIR=\tsubdir", - "", - "do-install:", - "\t${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/libexec/always", - "\t${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/libexec/conditional", - "\t${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/${LIB_SUBDIR}", - ) - t.Chdir("category/package") - t.CreateFileLines("PLIST", - PlistCvsID, - "libexec/always/always", - "${LIB_SUBDIR}/file", - "${PLIST.cond}libexec/conditional/conditional") - t.FinishSetUp() + text := "echo \"$$\"" + words, rest := splitIntoShellTokens(dummyLine, text) - G.checkdirPackage(".") + t.CheckDeepEquals(words, []string{"echo", "\"$$\""}) + t.CheckEquals(rest, "") - // As libexec/conditional will not be created automatically, - // AUTO_MKDIRS must not be suggested in that line. - t.CheckOutputLines( - "NOTE: Makefile:23: You can use AUTO_MKDIRS=yes "+ - "or \"INSTALLATION_DIRS+= libexec/always\" "+ - "instead of \"${INSTALL_DATA_DIR}\".", - "NOTE: Makefile:24: You can use "+ - "\"INSTALLATION_DIRS+= libexec/conditional\" "+ - "instead of \"${INSTALL_DATA_DIR}\".", - "NOTE: Makefile:25: You can use "+ - "\"INSTALLATION_DIRS+= ${LIB_SUBDIR}\" "+ - "instead of \"${INSTALL_DATA_DIR}\".") + t.CheckOutputEmpty() } -func (s *Suite) Test_ShellProgramChecker_checkSetE__simple_commands(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__varuse_with_embedded_space_and_other_vars(c *check.C) { t := s.Init(c) - t.SetUpTool("echo", "", AtRunTime) - t.SetUpTool("rm", "", AtRunTime) - t.SetUpTool("touch", "", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "pre-configure:", - "\techo 1; echo 2; echo 3", - "\techo 1; touch file; rm file", - "\techo 1; var=value; echo 3") - - mklines.Check() + varuseWord := "${GCONF_SCHEMAS:@.s.@${INSTALL_DATA} ${WRKSRC}/src/common/dbus/${.s.} ${DESTDIR}${GCONF_SCHEMAS_DIR}/@}" + words, rest := splitIntoShellTokens(dummyLine, varuseWord) - t.CheckOutputLines( - "WARN: Makefile:4: Please switch to \"set -e\" mode before using a semicolon " + - "(after \"touch file\") to separate commands.") + t.CheckDeepEquals(words, []string{varuseWord}) + t.CheckEquals(rest, "") } -func (s *Suite) Test_ShellProgramChecker_checkSetE__compound_commands(c *check.C) { +// Two shell variables, next to each other, +// are two separate atoms but count as a single token. +func (s *Suite) Test_splitIntoShellTokens__two_shell_variables(c *check.C) { t := s.Init(c) - t.SetUpTool("echo", "", AtRunTime) - t.SetUpTool("touch", "", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "pre-configure:", - "\ttouch file; for f in file; do echo \"$$f\"; done", - "\tfor f in file; do echo \"$$f\"; done; touch file", - "\ttouch 1; touch 2; touch 3; touch 4") - - mklines.Check() + code := "echo $$i$$j" + words, rest := splitIntoShellTokens(dummyLine, code) - t.CheckOutputLines( - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"touch file\") to separate commands.", - "WARN: Makefile:5: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"touch 1\") to separate commands.") + t.CheckDeepEquals(words, []string{"echo", "$$i$$j"}) + t.CheckEquals(rest, "") } -func (s *Suite) Test_ShellProgramChecker_checkSetE__no_tracing(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__varuse_with_embedded_space(c *check.C) { t := s.Init(c) - t.SetUpTool("touch", "", AtRunTime) - mklines := t.NewMkLines("Makefile", - MkCvsID, - "pre-configure:", - "\ttouch 1; touch 2") - t.DisableTracing() - - mklines.Check() + words, rest := splitIntoShellTokens(dummyLine, "${VAR:S/ /_/g}") - t.CheckOutputLines( - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon " + - "(after \"touch 1\") to separate commands.") + t.CheckDeepEquals(words, []string{"${VAR:S/ /_/g}"}) + t.CheckEquals(rest, "") } -func (s *Suite) Test_ShellProgramChecker_canFail(c *check.C) { +func (s *Suite) Test_splitIntoShellTokens__redirect(c *check.C) { t := s.Init(c) - t.SetUpVartypes() - t.SetUpTool("basename", "", AtRunTime) - t.SetUpTool("dirname", "", AtRunTime) - t.SetUpTool("echo", "", AtRunTime) - t.SetUpTool("env", "", AtRunTime) - t.SetUpTool("grep", "GREP", AtRunTime) - t.SetUpTool("sed", "", AtRunTime) - t.SetUpTool("touch", "", AtRunTime) - t.SetUpTool("tr", "tr", AtRunTime) - t.SetUpTool("true", "TRUE", AtRunTime) - - test := func(cmd string, diagnostics ...string) { - mklines := t.NewMkLines("Makefile", - MkCvsID, - "pre-configure:", - "\t"+cmd+" ; echo 'done.'") - - mklines.Check() - - t.CheckOutput(diagnostics) - } - - test("socklen=`${GREP} 'expr' ${WRKSRC}/config.h`", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"socklen=`${GREP} 'expr' ${WRKSRC}/config.h`\") to separate commands.") - - test("socklen=`${GREP} 'expr' ${WRKSRC}/config.h || ${TRUE}`", - nil...) - - test("socklen=$$(expr 16)", - "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"socklen=$$(expr 16)\") to separate commands.") - - test("socklen=$$(expr 16 || true)", - "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.") - - test("socklen=$$(expr 16 || ${TRUE})", - "WARN: Makefile:3: Invoking subshells via $(...) is not portable enough.") - - test("${ECHO_MSG} \"Message\"", - nil...) - - test("${FAIL_MSG} \"Failure\"", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"${FAIL_MSG} \\\"Failure\\\"\") to separate commands.") - - test("set -x", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"set -x\") to separate commands.") - - test("echo 'input' | sed -e s,in,out,", - nil...) - - test("sed -e s,in,out,", - nil...) - - test("sed s,in,out,", - nil...) - - test("grep input", - nil...) - - test("grep pattern file...", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"grep pattern file...\") to separate commands.") - - test("touch file", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"touch file\") to separate commands.") - - test("echo 'starting'", - nil...) - - test("echo 'logging' > log", - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ - "(after \"echo 'logging'\") to separate commands.") - - test("echo 'to stderr' 1>&2", - nil...) - - test("echo 'hello' | tr -d 'aeiou'", - nil...) + words, rest := splitIntoShellTokens(dummyLine, "echo 1>output 2>>append 3>|clobber 4>&5 6<input >>append") - test("env | grep '^PATH='", - nil...) + t.CheckDeepEquals(words, []string{ + "echo", + "1>", "output", + "2>>", "append", + "3>|", "clobber", + "4>&", "5", + "6<", "input", + ">>", "append"}) + t.CheckEquals(rest, "") - test("basename dir/file", - nil...) + words, rest = splitIntoShellTokens(dummyLine, "echo 1> output 2>> append 3>| clobber 4>& 5 6< input >> append") - test("dirname dir/file", - nil...) + t.CheckDeepEquals(words, []string{ + "echo", + "1>", "output", + "2>>", "append", + "3>|", "clobber", + "4>&", "5", + "6<", "input", + ">>", "append"}) + t.CheckEquals(rest, "") } diff --git a/pkgtools/pkglint/files/shtokenizer_test.go b/pkgtools/pkglint/files/shtokenizer_test.go index 024b42a4e37..14f08659591 100644 --- a/pkgtools/pkglint/files/shtokenizer_test.go +++ b/pkgtools/pkglint/files/shtokenizer_test.go @@ -5,6 +5,100 @@ import ( "strings" ) +// This test demonstrates that the shell tokenizer is not perfect yet. +// There are still some corner cases that trigger a parse error. +// To get 100% code coverage, they have been found using the fuzzer +// and trimmed down to minimal examples. +func (s *Suite) Test_ShTokenizer__examples_from_fuzzing(c *check.C) { + t := s.Init(c) + + test := func(input string, diagnostics ...string) { + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "\t"+input) + mklines.Check() + t.CheckOutput(diagnostics) + } + + // Covers shAtomBacktDquot: return nil. + // These are nested backticks with double quotes, + // which should be avoided since POSIX marks them as unspecified. + test( + "`\"`", + "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=bd).", + "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`\\\"`\"") + + // Covers shAtomBacktSquot: return nil + test( + "`'$`", + "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"$`\" (quoting=bs).", + "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`'$`\"", + "WARN: filename.mk:2: Internal pkglint error in MkLine.Tokenize at \"$`\".") + + // Covers shAtomDquotBacktSquot: return nil + test( + "\"`'`y", + "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`'`y\"") + + // Covers shAtomDquotBackt: return nil + // FIXME: Pkglint must parse unescaped dollar in the same way, everywhere. + test( + "\"`$|", + "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"$|\" (quoting=db).", + "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`$|\"", + "WARN: filename.mk:2: Internal pkglint error in MkLine.Tokenize at \"$|\".") + + // Covers shAtomDquotBacktDquot: return nil + // FIXME: Pkglint must support unlimited nesting. + test( + "\"`\"`", + "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=dbd).", + "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`\\\"`\"") + + // Covers shAtomSubshDquot: return nil + test( + "$$(\"'", + "WARN: filename.mk:2: Invoking subshells via $(...) is not portable enough.") + + // Covers shAtomSubsh: case lexer.AdvanceStr("`") + test( + "$$(`", + "WARN: filename.mk:2: Invoking subshells via $(...) is not portable enough.") + + // Covers shAtomSubshSquot: return nil + test( + "$$('$)", + "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"$)\" (quoting=Ss).", + "WARN: filename.mk:2: Invoking subshells via $(...) is not portable enough.", + "WARN: filename.mk:2: Internal pkglint error in MkLine.Tokenize at \"$)\".") + + // Covers shAtomDquotBackt: case lexer.AdvanceRegexp("^#[^`]*") + test( + "\"`# comment", + "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`# comment\"") +} + +// In order to get 100% code coverage for the shell tokenizer, a panic() statement has been +// added to each uncovered basic block. After that, this fuzzer quickly found relatively +// small example programs that led to the uncovered code. +// +// This test is not useful as-is. +func (s *Suite) Test_ShTokenizer__fuzzing(c *check.C) { + t := s.Init(c) + + fuzzer := NewFuzzer() + fuzzer.Char("\"'`$();|_#", 10) + fuzzer.Range('a', 'z', 5) + + defer fuzzer.CheckOk() + for i := 0; i < 1000; i++ { + tokenizer := NewShTokenizer(dummyLine, fuzzer.Generate(50), false) + tokenizer.ShAtoms() + t.Output() // Discard the output, only react on panics. + } + fuzzer.Ok() +} + func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { t := s.Init(c) @@ -450,6 +544,57 @@ func (s *Suite) Test_ShTokenizer_ShAtom__quoting(c *check.C) { test("x`x\\\"x\\'x\\`x\\\\", "x`[b]x\\\"x\\'x\\`x\\\\") } +func (s *Suite) Test_ShTokenizer_shVarUse(c *check.C) { + t := s.Init(c) + + test := func(input string, output *ShAtom, rest string) { + tok := NewShTokenizer(nil, input, false) + actual := tok.shVarUse(shqPlain) + + t.CheckDeepEquals(actual, output) + t.CheckEquals(tok.Rest(), rest) + } + + shvar := func(text, varname string) *ShAtom { + return &ShAtom{shtShVarUse, text, shqPlain, varname} + } + + test("$", nil, "$") + test("$$", nil, "$$") + test("${MKVAR}", nil, "${MKVAR}") + + test("$$a", shvar("$$a", "a"), "") + test("$$a.", shvar("$$a", "a"), ".") + test("$$a_b_123:", shvar("$$a_b_123", "a_b_123"), ":") + test("$$123", shvar("$$1", "1"), "23") + + test("$${varname}", shvar("$${varname}", "varname"), "") + test("$${varname}.", shvar("$${varname}", "varname"), ".") + test("$${0123}.", shvar("$${0123}", "0123"), ".") + test("$${varname", nil, "$${varname") + + test("$${var:=value}", shvar("$${var:=value}", "var"), "") + test("$${var#value}", shvar("$${var#value}", "var"), "") + test("$${var##value}", shvar("$${var##value}", "var"), "") + test("$${var##*}", shvar("$${var##*}", "var"), "") + test("$${var%\".gz\"}", shvar("$${var%\".gz\"}", "var"), "") + + // TODO: allow variables in patterns. + test("$${var%.${ext}}", nil, "$${var%.${ext}}") + + test("$${var##*", nil, "$${var##*") + test("$${var\"", nil, "$${var\"") + + // TODO: test("$${var%${EXT}}", shvar("$${var%${EXT}}", "var"), "") + test("$${var%${EXT}}", nil, "$${var%${EXT}}") + + // TODO: length of var + test("$${#var}", nil, "$${#var}") + + test("$${/}", nil, "$${/}") + test("$${\\}", nil, "$${\\}") +} + func (s *Suite) Test_ShTokenizer_ShToken(c *check.C) { t := s.Init(c) @@ -522,148 +667,3 @@ func (s *Suite) Test_ShTokenizer_ShToken(c *check.C) { test("id=`${AWK} '{print}' < ${WRKSRC}/idfile`", "id=`${AWK} '{print}' < ${WRKSRC}/idfile`") } - -func (s *Suite) Test_ShTokenizer_shVarUse(c *check.C) { - t := s.Init(c) - - test := func(input string, output *ShAtom, rest string) { - tok := NewShTokenizer(nil, input, false) - actual := tok.shVarUse(shqPlain) - - t.CheckDeepEquals(actual, output) - t.CheckEquals(tok.Rest(), rest) - } - - shvar := func(text, varname string) *ShAtom { - return &ShAtom{shtShVarUse, text, shqPlain, varname} - } - - test("$", nil, "$") - test("$$", nil, "$$") - test("${MKVAR}", nil, "${MKVAR}") - - test("$$a", shvar("$$a", "a"), "") - test("$$a.", shvar("$$a", "a"), ".") - test("$$a_b_123:", shvar("$$a_b_123", "a_b_123"), ":") - test("$$123", shvar("$$1", "1"), "23") - - test("$${varname}", shvar("$${varname}", "varname"), "") - test("$${varname}.", shvar("$${varname}", "varname"), ".") - test("$${0123}.", shvar("$${0123}", "0123"), ".") - test("$${varname", nil, "$${varname") - - test("$${var:=value}", shvar("$${var:=value}", "var"), "") - test("$${var#value}", shvar("$${var#value}", "var"), "") - test("$${var##value}", shvar("$${var##value}", "var"), "") - test("$${var##*}", shvar("$${var##*}", "var"), "") - test("$${var%\".gz\"}", shvar("$${var%\".gz\"}", "var"), "") - - // TODO: allow variables in patterns. - test("$${var%.${ext}}", nil, "$${var%.${ext}}") - - test("$${var##*", nil, "$${var##*") - test("$${var\"", nil, "$${var\"") - - // TODO: test("$${var%${EXT}}", shvar("$${var%${EXT}}", "var"), "") - test("$${var%${EXT}}", nil, "$${var%${EXT}}") - - // TODO: length of var - test("$${#var}", nil, "$${#var}") - - test("$${/}", nil, "$${/}") - test("$${\\}", nil, "$${\\}") -} - -// This test demonstrates that the shell tokenizer is not perfect yet. -// There are still some corner cases that trigger a parse error. -// To get 100% code coverage, they have been found using the fuzzer -// and trimmed down to minimal examples. -func (s *Suite) Test_ShTokenizer__examples_from_fuzzing(c *check.C) { - t := s.Init(c) - - test := func(input string, diagnostics ...string) { - mklines := t.NewMkLines("filename.mk", - MkCvsID, - "\t"+input) - mklines.Check() - t.CheckOutput(diagnostics) - } - - // Covers shAtomBacktDquot: return nil. - // These are nested backticks with double quotes, - // which should be avoided since POSIX marks them as unspecified. - test( - "`\"`", - "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=bd).", - "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`\\\"`\"") - - // Covers shAtomBacktSquot: return nil - test( - "`'$`", - "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"$`\" (quoting=bs).", - "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`'$`\"", - "WARN: filename.mk:2: Internal pkglint error in MkLine.Tokenize at \"$`\".") - - // Covers shAtomDquotBacktSquot: return nil - test( - "\"`'`y", - "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`'`y\"") - - // Covers shAtomDquotBackt: return nil - // FIXME: Pkglint must parse unescaped dollar in the same way, everywhere. - test( - "\"`$|", - "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"$|\" (quoting=db).", - "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`$|\"", - "WARN: filename.mk:2: Internal pkglint error in MkLine.Tokenize at \"$|\".") - - // Covers shAtomDquotBacktDquot: return nil - // FIXME: Pkglint must support unlimited nesting. - test( - "\"`\"`", - "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=dbd).", - "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`\\\"`\"") - - // Covers shAtomSubshDquot: return nil - test( - "$$(\"'", - "WARN: filename.mk:2: Invoking subshells via $(...) is not portable enough.") - - // Covers shAtomSubsh: case lexer.AdvanceStr("`") - test( - "$$(`", - "WARN: filename.mk:2: Invoking subshells via $(...) is not portable enough.") - - // Covers shAtomSubshSquot: return nil - test( - "$$('$)", - "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"$)\" (quoting=Ss).", - "WARN: filename.mk:2: Invoking subshells via $(...) is not portable enough.", - "WARN: filename.mk:2: Internal pkglint error in MkLine.Tokenize at \"$)\".") - - // Covers shAtomDquotBackt: case lexer.AdvanceRegexp("^#[^`]*") - test( - "\"`# comment", - "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`# comment\"") -} - -// In order to get 100% code coverage for the shell tokenizer, a panic() statement has been -// added to each uncovered basic block. After that, this fuzzer quickly found relatively -// small example programs that led to the uncovered code. -// -// This test is not useful as-is. -func (s *Suite) Test_ShTokenizer__fuzzing(c *check.C) { - t := s.Init(c) - - fuzzer := NewFuzzer() - fuzzer.Char("\"'`$();|_#", 10) - fuzzer.Range('a', 'z', 5) - - defer fuzzer.CheckOk() - for i := 0; i < 1000; i++ { - tokenizer := NewShTokenizer(dummyLine, fuzzer.Generate(50), false) - tokenizer.ShAtoms() - t.Output() // Discard the output, only react on panics. - } - fuzzer.Ok() -} diff --git a/pkgtools/pkglint/files/substcontext_test.go b/pkgtools/pkglint/files/substcontext_test.go index d5f18356f6a..041f2a233fb 100644 --- a/pkgtools/pkglint/files/substcontext_test.go +++ b/pkgtools/pkglint/files/substcontext_test.go @@ -503,6 +503,28 @@ func (s *Suite) Test_SubstContext__multiple_SUBST_VARS(c *check.C) { t.CheckOutputEmpty() } +// As of May 2019, pkglint does not check the order of the variables in +// a SUBST block. Enforcing this order, or at least suggesting it, would +// make pkgsrc packages more uniform, which is a good idea, but not urgent. +func (s *Suite) Test_SubstContext__unusual_variable_order(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + mklines := t.NewMkLines("subst.mk", + MkCvsID, + "", + "SUBST_CLASSES+=\t\tid", + "SUBST_SED.id=\t\t-e /deleteme/d", + "SUBST_FILES.id=\t\tfile", + "SUBST_MESSAGE.id=\tMessage", + "SUBST_STAGE.id=\t\tpre-configure") + + mklines.Check() + + t.CheckOutputEmpty() +} + // Since the SUBST_CLASSES definition starts the SUBST block, all // directives above it are ignored by the SUBST context. func (s *Suite) Test_SubstContext_Directive__before_SUBST_CLASSES(c *check.C) { @@ -857,28 +879,6 @@ func (s *Suite) Test_SubstContext_extractVarname(c *check.C) { test("s,@VAR@,${VAR}suffix,", "") } -// As of May 2019, pkglint does not check the order of the variables in -// a SUBST block. Enforcing this order, or at least suggesting it, would -// make pkgsrc packages more uniform, which is a good idea, but not urgent. -func (s *Suite) Test_SubstContext__unusual_variable_order(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - - mklines := t.NewMkLines("subst.mk", - MkCvsID, - "", - "SUBST_CLASSES+=\t\tid", - "SUBST_SED.id=\t\t-e /deleteme/d", - "SUBST_FILES.id=\t\tfile", - "SUBST_MESSAGE.id=\tMessage", - "SUBST_STAGE.id=\t\tpre-configure") - - mklines.Check() - - t.CheckOutputEmpty() -} - // simulateSubstLines only tests some of the inner workings of SubstContext. // It is not realistic for all cases. If in doubt, use MkLines.Check. func simulateSubstLines(t *Tester, texts ...string) { diff --git a/pkgtools/pkglint/files/testnames_test.go b/pkgtools/pkglint/files/testnames_test.go index 9d31cf6fa26..3a49e1661cd 100644 --- a/pkgtools/pkglint/files/testnames_test.go +++ b/pkgtools/pkglint/files/testnames_test.go @@ -9,18 +9,8 @@ import ( // // Test_${Type}_${Method}__${description_using_underscores} func (s *Suite) Test__test_names(c *check.C) { - ck := intqa.NewTestNameChecker(c) + ck := intqa.NewTestNameChecker(c.Errorf) ck.IgnoreFiles("*yacc.go") - ck.AllowPrefix("ShellParser", "mkshparser.go") - ck.AllowCamelCaseDescriptions( - "compared_to_splitIntoShellTokens", - "comparing_YesNo_variable_to_string", - "enumFrom", - "enumFromDirs", - "enumFromFiles", - "dquotBacktDquot", - "and_getSubdirs", - "SilentAutofixFormat") - ck.ShowWarnings(false) + ck.Enable(intqa.EAll, -intqa.EMissingTest) ck.Check() } diff --git a/pkgtools/pkglint/files/textproc/lexer_test.go b/pkgtools/pkglint/files/textproc/lexer_test.go index 867a7e2da0d..f37e6f514b0 100644 --- a/pkgtools/pkglint/files/textproc/lexer_test.go +++ b/pkgtools/pkglint/files/textproc/lexer_test.go @@ -99,6 +99,13 @@ func (s *Suite) Test_Lexer_NextString(c *check.C) { c.Check(lexer.NextString("xt"), equals, "xt") } +func (s *Suite) Test_Lexer_NextString__EOF(c *check.C) { + lexer := NewLexer("text") + lexer.NextString("text") + + c.Check(lexer.EOF(), equals, true) +} + func (s *Suite) Test_Lexer_SkipString(c *check.C) { lexer := NewLexer("text") @@ -291,13 +298,6 @@ func (s *Suite) Test_Lexer_Reset__multiple(c *check.C) { c.Check(lexer.Rest(), equals, "") } -func (s *Suite) Test_Lexer__NextString_then_EOF(c *check.C) { - lexer := NewLexer("text") - lexer.NextString("text") - - c.Check(lexer.EOF(), equals, true) -} - func (s *Suite) Test_Lexer_Since(c *check.C) { lexer := NewLexer("text") mark := lexer.Mark() @@ -414,9 +414,7 @@ func (s *Suite) Test__Alpha(c *check.C) { } func (s *Suite) Test__test_names(c *check.C) { - ck := intqa.NewTestNameChecker(c) - ck.AllowCamelCaseDescriptions( - "NextString_then_EOF") - ck.ShowWarnings(false) + ck := intqa.NewTestNameChecker(c.Errorf) + ck.Enable(intqa.EAll, -intqa.EMissingTest) ck.Check() } diff --git a/pkgtools/pkglint/files/tools_test.go b/pkgtools/pkglint/files/tools_test.go index d5dc0165a09..dea93d3afbc 100644 --- a/pkgtools/pkglint/files/tools_test.go +++ b/pkgtools/pkglint/files/tools_test.go @@ -31,96 +31,6 @@ func (s *Suite) Test_Tool_UsableAtRunTime(c *check.C) { t.CheckEquals(run.UsableAtRunTime(), true) } -// USE_TOOLS is an operating-system-dependent variable. -// Many other tool variables have the form VARNAME.${tool}, -// which confused an earlier version of pkglint into -// thinking that the below definition was about a tool -// called "NetBSD". -func (s *Suite) Test_Tools_ParseToolLine__opsys(c *check.C) { - t := s.Init(c) - - t.SetUpTool("tool1", "", Nowhere) - t.SetUpVartypes() - t.CreateFileLines("Makefile", - MkCvsID, - "", - "USE_TOOLS.NetBSD+=\ttool1") - - CheckdirToplevel(t.File(".")) - - // No error about "Unknown tool \"NetBSD\"." - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Tools_ParseToolLine__invalid_tool_name(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - mklines := t.NewMkLines("Makefile", - MkCvsID, - "", - ".for t in abc ${UNDEFINED}", - "TOOLS_CREATE+=\t\t${t}", - "_TOOLS_VARNAME.${t}=\tVARNAME", - "TOOLS_PATH.${t}=\t/bin/${t}", - "TOOLS_ALIASES.${t}=\t${t} ${u} ${t}-arm64", - "TOOLS_ALIASES.tool=\t${t} ${u} ${t}-arm64", - "_TOOLS.${t}=\t${t}", - ".endfor") - - mklines.collectVariables() - t.Check(mklines.Tools.byName, check.HasLen, 1) - t.CheckEquals(mklines.Tools.ByName("tool").String(), "tool:::Nowhere:abc") - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Tools_parseUseTools(c *check.C) { - t := s.Init(c) - - t.SetUpPkgsrc() - t.CreateFileLines("mk/triple-tool.mk", - MkCvsID, - "", - "USE_TOOLS+=\tunknown unknown unknown") - t.FinishSetUp() - - t.Check(G.Pkgsrc.Tools.ByName("unknown"), check.IsNil) - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_Tools_Define__invalid_tool_name(c *check.C) { - t := s.Init(c) - - mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") - reg := NewTools() - - t.Check(reg.Define("tool_name", "", mkline), check.IsNil) - t.Check(reg.Define("tool:dependency", "", mkline), check.IsNil) - t.Check(reg.Define("tool:build", "", mkline), check.IsNil) - - // As of October 2018, the underscore is not used in any tool name. - // If there should ever be such a case, just use a different character for testing. - t.CheckOutputLines( - "ERROR: dummy.mk:123: Invalid tool name \"tool_name\".", - "ERROR: dummy.mk:123: Invalid tool name \"tool:dependency\".", - "ERROR: dummy.mk:123: Invalid tool name \"tool:build\".") - - t.Check(reg.byName, check.HasLen, 0) -} - -func (s *Suite) Test_Tools_Trace__coverage(c *check.C) { - t := s.Init(c) - - t.DisableTracing() - - reg := NewTools() - reg.Trace() - - t.CheckOutputEmpty() -} - func (s *Suite) Test_Tools__USE_TOOLS_predefined_sed(c *check.C) { t := s.Init(c) @@ -443,13 +353,6 @@ func (s *Suite) Test_Tools__tools_having_the_same_variable_name(c *check.C) { "TRACE: - (*Tools).Trace()") } -func (s *Suite) Test_ToolTime_String(c *check.C) { - t := s.Init(c) - - t.CheckEquals(LoadTime.String(), "LoadTime") - t.CheckEquals(RunTime.String(), "RunTime") -} - func (s *Suite) Test_Tools__var(c *check.C) { t := s.Init(c) @@ -472,96 +375,6 @@ func (s *Suite) Test_Tools__var(c *check.C) { t.CheckOutputEmpty() } -// Demonstrates how the Tools type handles tools that share the same -// variable name. Of these tools, the GNU variant is preferred. -// -// In this realistic variant, the non-GNU tool is defined in bsd.prefs.mk -// and the GNU tool is only defined but not made available. -// -// See also Pkglint.Tool. -func (s *Suite) Test_Tools_Fallback__tools_having_the_same_variable_name_realistic(c *check.C) { - t := s.Init(c) - - nonGnu := NewTools() - nonGnu.def("sed", "SED", false, AfterPrefsMk, nil) - - gnu := NewTools() - gnu.def("gsed", "SED", false, Nowhere, nil) - - local1 := NewTools() - local1.def("sed", "SED", false, AfterPrefsMk, nil) - local1.Fallback(gnu) - - t.CheckEquals(local1.ByName("sed").Validity, AfterPrefsMk) - t.CheckEquals(local1.ByName("gsed").Validity, Nowhere) - - local2 := NewTools() - local2.def("gsed", "SED", false, Nowhere, nil) - local2.Fallback(nonGnu) - - t.CheckEquals(local2.ByName("sed").Validity, AfterPrefsMk) - t.CheckEquals(local2.ByName("gsed").Validity, Nowhere) - - // No matter in which order the tool definitions are encountered, - // the non-GNU version is always chosen since the GNU version is - // not available at all. - t.CheckEquals(local1.ByVarname("SED").String(), "sed:SED::AfterPrefsMk") - t.CheckEquals(local2.ByVarname("SED").String(), "sed:SED::AfterPrefsMk") -} - -// Demonstrates how the Tools type handles tools that share the same -// variable name. Of these tools, the GNU variant is preferred. -// -// In this unrealistic variant, the GNU tool is defined in bsd.prefs.mk -// and the non-GNU tool is only defined but not made available. -// -// See also Pkglint.Tool. -func (s *Suite) Test_Tools_Fallback__tools_having_the_same_variable_name_unrealistic(c *check.C) { - t := s.Init(c) - - // This simulates a tool defined in the tools framework but not added - // to USE_TOOLS, neither by bsd.prefs.mk nor by bsd.pkg.mk. - nonGnu := NewTools() - nonGnu.def("sed", "SED", false, Nowhere, nil) - - // This simulates a tool that is added to USE_TOOLS in bsd.prefs.mk. - gnu := NewTools() - gnu.def("gsed", "SED", false, AfterPrefsMk, nil) - gnu.ByName("gsed").Aliases = []string{"sed"} - - // This simulates a package that doesn't mention the sed tool at all. - // The call to .def() is therefore unrealistic. - // Nevertheless, since the GNU tools define the gsed tool as well, - // it is available even though not explicitly mentioned in the package. - local1 := NewTools() - local1.def("sed", "SED", false, Nowhere, nil) - local1.Fallback(gnu) - - t.CheckEquals(local1.ByName("sed").Validity, Nowhere) - t.CheckEquals(local1.ByName("gsed").Validity, AfterPrefsMk) - - local2 := NewTools() - local2.def("gsed", "SED", false, AfterPrefsMk, []string{"sed"}) - local2.Fallback(nonGnu) - - t.CheckEquals(local2.ByName("sed").Validity, AfterPrefsMk) - t.CheckEquals(local2.ByName("gsed").Validity, AfterPrefsMk) - - t.CheckEquals(local1.ByVarname("SED").String(), "sed:SED::AfterPrefsMk") - t.CheckEquals(local2.ByVarname("SED").String(), "sed:SED::AfterPrefsMk") -} - -func (s *Suite) Test_Tools_Fallback__called_twice(c *check.C) { - t := s.Init(c) - - tools := NewTools() - fallback := NewTools() - - tools.Fallback(fallback) - - t.ExpectAssert(func() { tools.Fallback(fallback) }) -} - func (s *Suite) Test_Tools__aliases(c *check.C) { t := s.Init(c) @@ -684,6 +497,218 @@ func (s *Suite) Test_Tools__autoconf213(c *check.C) { t.CheckOutputEmpty() } +func (s *Suite) Test_Tools_Define__invalid_tool_name(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") + reg := NewTools() + + t.Check(reg.Define("tool_name", "", mkline), check.IsNil) + t.Check(reg.Define("tool:dependency", "", mkline), check.IsNil) + t.Check(reg.Define("tool:build", "", mkline), check.IsNil) + + // As of October 2018, the underscore is not used in any tool name. + // If there should ever be such a case, just use a different character for testing. + t.CheckOutputLines( + "ERROR: dummy.mk:123: Invalid tool name \"tool_name\".", + "ERROR: dummy.mk:123: Invalid tool name \"tool:dependency\".", + "ERROR: dummy.mk:123: Invalid tool name \"tool:build\".") + + t.Check(reg.byName, check.HasLen, 0) +} + +func (s *Suite) Test_Tools_Trace__coverage(c *check.C) { + t := s.Init(c) + + t.DisableTracing() + + reg := NewTools() + reg.Trace() + + t.CheckOutputEmpty() +} + +// USE_TOOLS is an operating-system-dependent variable. +// Many other tool variables have the form VARNAME.${tool}, +// which confused an earlier version of pkglint into +// thinking that the below definition was about a tool +// called "NetBSD". +func (s *Suite) Test_Tools_ParseToolLine__opsys(c *check.C) { + t := s.Init(c) + + t.SetUpTool("tool1", "", Nowhere) + t.SetUpVartypes() + t.CreateFileLines("Makefile", + MkCvsID, + "", + "USE_TOOLS.NetBSD+=\ttool1") + + CheckdirToplevel(t.File(".")) + + // No error about "Unknown tool \"NetBSD\"." + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Tools_ParseToolLine__invalid_tool_name(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("Makefile", + MkCvsID, + "", + ".for t in abc ${UNDEFINED}", + "TOOLS_CREATE+=\t\t${t}", + "_TOOLS_VARNAME.${t}=\tVARNAME", + "TOOLS_PATH.${t}=\t/bin/${t}", + "TOOLS_ALIASES.${t}=\t${t} ${u} ${t}-arm64", + "TOOLS_ALIASES.tool=\t${t} ${u} ${t}-arm64", + "_TOOLS.${t}=\t${t}", + ".endfor") + + mklines.collectVariables() + t.Check(mklines.Tools.byName, check.HasLen, 1) + t.CheckEquals(mklines.Tools.ByName("tool").String(), "tool:::Nowhere:abc") + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Tools_ParseToolLine__private_tool_undefined(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + "\tmd5sum filename") + + mklines.Check() + + t.CheckOutputLines( + "WARN: filename.mk:3: Unknown shell command \"md5sum\".") +} + +// Tools that are defined by a package by adding to TOOLS_CREATE can +// be used without adding them to USE_TOOLS again. +func (s *Suite) Test_Tools_ParseToolLine__private_tool_defined(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "TOOLS_CREATE+=\tmd5sum", + "", + "\tmd5sum filename") + + mklines.Check() + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Tools_parseUseTools(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("mk/triple-tool.mk", + MkCvsID, + "", + "USE_TOOLS+=\tunknown unknown unknown") + t.FinishSetUp() + + t.Check(G.Pkgsrc.Tools.ByName("unknown"), check.IsNil) + + t.CheckOutputEmpty() +} + +// Demonstrates how the Tools type handles tools that share the same +// variable name. Of these tools, the GNU variant is preferred. +// +// In this realistic variant, the non-GNU tool is defined in bsd.prefs.mk +// and the GNU tool is only defined but not made available. +// +// See also Pkglint.Tool. +func (s *Suite) Test_Tools_Fallback__tools_having_the_same_variable_name_realistic(c *check.C) { + t := s.Init(c) + + nonGnu := NewTools() + nonGnu.def("sed", "SED", false, AfterPrefsMk, nil) + + gnu := NewTools() + gnu.def("gsed", "SED", false, Nowhere, nil) + + local1 := NewTools() + local1.def("sed", "SED", false, AfterPrefsMk, nil) + local1.Fallback(gnu) + + t.CheckEquals(local1.ByName("sed").Validity, AfterPrefsMk) + t.CheckEquals(local1.ByName("gsed").Validity, Nowhere) + + local2 := NewTools() + local2.def("gsed", "SED", false, Nowhere, nil) + local2.Fallback(nonGnu) + + t.CheckEquals(local2.ByName("sed").Validity, AfterPrefsMk) + t.CheckEquals(local2.ByName("gsed").Validity, Nowhere) + + // No matter in which order the tool definitions are encountered, + // the non-GNU version is always chosen since the GNU version is + // not available at all. + t.CheckEquals(local1.ByVarname("SED").String(), "sed:SED::AfterPrefsMk") + t.CheckEquals(local2.ByVarname("SED").String(), "sed:SED::AfterPrefsMk") +} + +// Demonstrates how the Tools type handles tools that share the same +// variable name. Of these tools, the GNU variant is preferred. +// +// In this unrealistic variant, the GNU tool is defined in bsd.prefs.mk +// and the non-GNU tool is only defined but not made available. +// +// See also Pkglint.Tool. +func (s *Suite) Test_Tools_Fallback__tools_having_the_same_variable_name_unrealistic(c *check.C) { + t := s.Init(c) + + // This simulates a tool defined in the tools framework but not added + // to USE_TOOLS, neither by bsd.prefs.mk nor by bsd.pkg.mk. + nonGnu := NewTools() + nonGnu.def("sed", "SED", false, Nowhere, nil) + + // This simulates a tool that is added to USE_TOOLS in bsd.prefs.mk. + gnu := NewTools() + gnu.def("gsed", "SED", false, AfterPrefsMk, nil) + gnu.ByName("gsed").Aliases = []string{"sed"} + + // This simulates a package that doesn't mention the sed tool at all. + // The call to .def() is therefore unrealistic. + // Nevertheless, since the GNU tools define the gsed tool as well, + // it is available even though not explicitly mentioned in the package. + local1 := NewTools() + local1.def("sed", "SED", false, Nowhere, nil) + local1.Fallback(gnu) + + t.CheckEquals(local1.ByName("sed").Validity, Nowhere) + t.CheckEquals(local1.ByName("gsed").Validity, AfterPrefsMk) + + local2 := NewTools() + local2.def("gsed", "SED", false, AfterPrefsMk, []string{"sed"}) + local2.Fallback(nonGnu) + + t.CheckEquals(local2.ByName("sed").Validity, AfterPrefsMk) + t.CheckEquals(local2.ByName("gsed").Validity, AfterPrefsMk) + + t.CheckEquals(local1.ByVarname("SED").String(), "sed:SED::AfterPrefsMk") + t.CheckEquals(local2.ByVarname("SED").String(), "sed:SED::AfterPrefsMk") +} + +func (s *Suite) Test_Tools_Fallback__called_twice(c *check.C) { + t := s.Init(c) + + tools := NewTools() + fallback := NewTools() + + tools.Fallback(fallback) + + t.ExpectAssert(func() { tools.Fallback(fallback) }) +} + func (s *Suite) Test_Tools_IsValidToolName(c *check.C) { t := s.Init(c) @@ -699,3 +724,10 @@ func (s *Suite) Test_Tools_IsValidToolName(c *check.C) { t.CheckOutputEmpty() } + +func (s *Suite) Test_ToolTime_String(c *check.C) { + t := s.Init(c) + + t.CheckEquals(LoadTime.String(), "LoadTime") + t.CheckEquals(RunTime.String(), "RunTime") +} diff --git a/pkgtools/pkglint/files/toplevel_test.go b/pkgtools/pkglint/files/toplevel_test.go index 518e6710b43..01b70cf4753 100644 --- a/pkgtools/pkglint/files/toplevel_test.go +++ b/pkgtools/pkglint/files/toplevel_test.go @@ -34,48 +34,6 @@ func (s *Suite) Test_CheckdirToplevel(c *check.C) { "NOTE: ~/Makefile:3: This variable value should be aligned with tabs, not spaces, to column 17.") } -func (s *Suite) Test_Toplevel_checkSubdir__sorting_x11(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("Makefile", - MkCvsID, - "", - "SUBDIR+=\tx11", - "SUBDIR+=\tsysutils", - "SUBDIR+=\tarchivers") - t.CreateFileLines("archivers/Makefile") - t.CreateFileLines("sysutils/Makefile") - t.CreateFileLines("x11/Makefile") - t.SetUpVartypes() - - CheckdirToplevel(t.File(".")) - - t.CheckOutputLines( - "WARN: ~/Makefile:4: sysutils should come before x11.", - "WARN: ~/Makefile:5: archivers should come before sysutils.") -} - -func (s *Suite) Test_Toplevel_checkSubdir__commented_without_reason(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("Makefile", - MkCvsID, - "", - "#SUBDIR+=\taaa", - "#SUBDIR+=\tbbb\t#", - "#SUBDIR+=\tccc\t# reason") - t.CreateFileLines("aaa/Makefile") - t.CreateFileLines("bbb/Makefile") - t.CreateFileLines("ccc/Makefile") - t.SetUpVartypes() - - CheckdirToplevel(t.File(".")) - - t.CheckOutputLines( - "WARN: ~/Makefile:3: \"aaa\" commented out without giving a reason.", - "WARN: ~/Makefile:4: \"bbb\" commented out without giving a reason.") -} - func (s *Suite) Test_CheckdirToplevel__recursive(c *check.C) { t := s.Init(c) @@ -154,3 +112,45 @@ func (s *Suite) Test_CheckdirToplevel__indentation(c *check.C) { t.Shquote("(Run \"pkglint -fs -Wall %s\" to show what can be fixed automatically.)", "."), t.Shquote("(Run \"pkglint -F -Wall %s\" to automatically fix some issues.)", ".")) } + +func (s *Suite) Test_Toplevel_checkSubdir__sorting_x11(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("Makefile", + MkCvsID, + "", + "SUBDIR+=\tx11", + "SUBDIR+=\tsysutils", + "SUBDIR+=\tarchivers") + t.CreateFileLines("archivers/Makefile") + t.CreateFileLines("sysutils/Makefile") + t.CreateFileLines("x11/Makefile") + t.SetUpVartypes() + + CheckdirToplevel(t.File(".")) + + t.CheckOutputLines( + "WARN: ~/Makefile:4: sysutils should come before x11.", + "WARN: ~/Makefile:5: archivers should come before sysutils.") +} + +func (s *Suite) Test_Toplevel_checkSubdir__commented_without_reason(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("Makefile", + MkCvsID, + "", + "#SUBDIR+=\taaa", + "#SUBDIR+=\tbbb\t#", + "#SUBDIR+=\tccc\t# reason") + t.CreateFileLines("aaa/Makefile") + t.CreateFileLines("bbb/Makefile") + t.CreateFileLines("ccc/Makefile") + t.SetUpVartypes() + + CheckdirToplevel(t.File(".")) + + t.CheckOutputLines( + "WARN: ~/Makefile:3: \"aaa\" commented out without giving a reason.", + "WARN: ~/Makefile:4: \"bbb\" commented out without giving a reason.") +} diff --git a/pkgtools/pkglint/files/trace/tracing_test.go b/pkgtools/pkglint/files/trace/tracing_test.go index 023a1b65fb6..e21b332ed84 100755 --- a/pkgtools/pkglint/files/trace/tracing_test.go +++ b/pkgtools/pkglint/files/trace/tracing_test.go @@ -144,8 +144,7 @@ func (str) String() string { } func (s *Suite) Test__test_names(c *check.C) { - ck := intqa.NewTestNameChecker(c) - ck.AllowCamelCaseDescriptions() - ck.ShowWarnings(false) + ck := intqa.NewTestNameChecker(c.Errorf) + ck.Enable(intqa.EAll, -intqa.EMissingTest) ck.Check() } diff --git a/pkgtools/pkglint/files/util.go b/pkgtools/pkglint/files/util.go index 329fa368fe3..0c832a21679 100644 --- a/pkgtools/pkglint/files/util.go +++ b/pkgtools/pkglint/files/util.go @@ -350,6 +350,14 @@ type CvsEntry struct { // a tabulator size of 8. func tabWidth(s string) int { return tabWidthAppend(0, s) } +func tabWidthSlice(strs ...string) int { + w := 0 + for _, str := range strs { + w = tabWidthAppend(w, str) + } + return w +} + func tabWidthAppend(width int, s string) int { for _, r := range s { assert(r != '\n') @@ -766,19 +774,19 @@ func (s *Scope) Mentioned(varname string) *MkLine { return s.firstDef[varname] } -// Defined tests whether the variable is defined. +// IsDefined tests whether the variable is defined. // It does NOT test the canonicalized variable name. // -// Even if Defined returns true, FirstDefinition doesn't necessarily return true +// Even if IsDefined returns true, FirstDefinition doesn't necessarily return true // since the latter ignores the default definitions from vardefs.go, keyword dummyVardefMkline. -func (s *Scope) Defined(varname string) bool { +func (s *Scope) IsDefined(varname string) bool { mkline := s.firstDef[varname] return mkline != nil && mkline.IsVarassign() } -// DefinedSimilar tests whether the variable or its canonicalized form is defined. -func (s *Scope) DefinedSimilar(varname string) bool { - if s.Defined(varname) { +// IsDefinedSimilar tests whether the variable or its canonicalized form is defined. +func (s *Scope) IsDefinedSimilar(varname string) bool { + if s.IsDefined(varname) { if trace.Tracing { trace.Step1("Variable %q is defined", varname) } @@ -786,7 +794,7 @@ func (s *Scope) DefinedSimilar(varname string) bool { } varcanon := varnameCanon(varname) - if s.Defined(varcanon) { + if s.IsDefined(varcanon) { if trace.Tracing { trace.Step2("Variable %q (similar to %q) is defined", varcanon, varname) } @@ -795,23 +803,23 @@ func (s *Scope) DefinedSimilar(varname string) bool { return false } -// Used tests whether the variable is used. +// IsUsed tests whether the variable is used. // It does NOT test the canonicalized variable name. -func (s *Scope) Used(varname string) bool { +func (s *Scope) IsUsed(varname string) bool { return s.used[varname] != nil } -// UsedSimilar tests whether the variable or its canonicalized form is used. -func (s *Scope) UsedSimilar(varname string) bool { +// IsUsedSimilar tests whether the variable or its canonicalized form is used. +func (s *Scope) IsUsedSimilar(varname string) bool { if s.used[varname] != nil { return true } return s.used[varnameCanon(varname)] != nil } -// UsedAtLoadTime returns true if the variable is used at load time +// IsUsedAtLoadTime returns true if the variable is used at load time // somewhere. -func (s *Scope) UsedAtLoadTime(varname string) bool { +func (s *Scope) IsUsedAtLoadTime(varname string) bool { return s.usedAtLoadTime[varname] } @@ -1402,7 +1410,7 @@ func (q *StringQueue) Push(entries ...string) { q.entries = append(q.entries, entries...) } -func (q *StringQueue) Empty() bool { +func (q *StringQueue) IsEmpty() bool { return len(q.entries) == 0 } diff --git a/pkgtools/pkglint/files/util_test.go b/pkgtools/pkglint/files/util_test.go index 08364e2c3e2..95f6c5f6795 100644 --- a/pkgtools/pkglint/files/util_test.go +++ b/pkgtools/pkglint/files/util_test.go @@ -8,6 +8,54 @@ import ( "time" ) +func (s *Suite) Test_YesNoUnknown_String(c *check.C) { + t := s.Init(c) + + t.CheckEquals(yes.String(), "yes") + t.CheckEquals(no.String(), "no") + t.CheckEquals(unknown.String(), "unknown") +} + +func (s *Suite) Test_trimHspace(c *check.C) { + t := s.Init(c) + + t.CheckEquals(trimHspace("a b"), "a b") + t.CheckEquals(trimHspace(" a b "), "a b") + t.CheckEquals(trimHspace("\ta b\t"), "a b") + t.CheckEquals(trimHspace(" \t a b\t \t"), "a b") +} + +func (s *Suite) Test_trimCommon(c *check.C) { + t := s.Init(c) + + test := func(a, b, trimmedA, trimmedB string) { + ta, tb := trimCommon(a, b) + t.CheckEquals(ta, trimmedA) + t.CheckEquals(tb, trimmedB) + } + + test("", "", + "", "") + + test("equal", "equal", + "", "") + + test("prefixA", "prefixB", + "A", "B") + + test("ASuffix", "BSuffix", + "A", "B") + + test("PreMiddlePost", "PreCenterPost", + "Middle", "Center") + + test("", "b", + "", "b") + + test("a", "", + "a", "") +} + func (s *Suite) Test_assertNil(c *check.C) { t := s.Init(c) @@ -31,12 +79,231 @@ func (s *Suite) Test_assertNotNil(c *check.C) { "Pkglint internal error: unexpected nil pointer") } -func (s *Suite) Test_YesNoUnknown_String(c *check.C) { +func (s *Suite) Test_isEmptyDir(c *check.C) { t := s.Init(c) - t.CheckEquals(yes.String(), "yes") - t.CheckEquals(no.String(), "no") - t.CheckEquals(unknown.String(), "unknown") + t.CreateFileLines("CVS/Entries", + "dummy") + t.CreateFileLines("subdir/CVS/Entries", + "dummy") + + t.CheckEquals(isEmptyDir(t.File(".")), true) + t.CheckEquals(isEmptyDir(t.File("CVS")), true) +} + +func (s *Suite) Test_isEmptyDir__and_getSubdirs(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("CVS/Entries", + "dummy") + + if dir := t.File("."); true { + t.CheckEquals(isEmptyDir(dir), true) + t.CheckDeepEquals(getSubdirs(dir), []string(nil)) + + t.CreateFileLines("somedir/file") + + t.CheckEquals(isEmptyDir(dir), false) + t.CheckDeepEquals(getSubdirs(dir), []string{"somedir"}) + } + + if absent := t.File("nonexistent"); true { + t.CheckEquals(isEmptyDir(absent), true) // Counts as empty. + + // The last group from the error message is localized, therefore the matching. + t.ExpectFatalMatches( + func() { getSubdirs(absent) }, + `FATAL: ~/nonexistent: Cannot be read: open ~/nonexistent: (.+)\n`) + } +} + +func (s *Suite) Test_getSubdirs(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("subdir/file") + t.CreateFileLines("empty/file") + c.Check(os.Remove(t.File("empty/file")), check.IsNil) + + t.CheckDeepEquals(getSubdirs(t.File(".")), []string{"subdir"}) +} + +func (s *Suite) Test_isLocallyModified(c *check.C) { + t := s.Init(c) + + unmodified := t.CreateFileLines("unmodified") + modTime := time.Unix(1136239445, 0).UTC() + + err := os.Chtimes(unmodified, modTime, modTime) + c.Check(err, check.IsNil) + + st, err := os.Lstat(unmodified) + c.Check(err, check.IsNil) + + // Make sure that the file system has second precision and accuracy. + t.CheckDeepEquals(st.ModTime().UTC(), modTime) + + modified := t.CreateFileLines("modified") + + t.CreateFileLines("CVS/Entries", + "/unmodified//"+modTime.Format(time.ANSIC)+"//", + "/modified//"+modTime.Format(time.ANSIC)+"//", + "/enoent//"+modTime.Format(time.ANSIC)+"//") + + t.CheckEquals(isLocallyModified(unmodified), false) + t.CheckEquals(isLocallyModified(modified), true) + t.CheckEquals(isLocallyModified(t.File("enoent")), true) + t.CheckEquals(isLocallyModified(t.File("not_mentioned")), false) + t.CheckEquals(isLocallyModified(t.File("subdir/file")), false) + + t.DisableTracing() + + t.CheckEquals(isLocallyModified(t.File("unmodified")), false) +} + +func (s *Suite) Test_tabWidth(c *check.C) { + t := s.Init(c) + + t.CheckEquals(tabWidth("12345"), 5) + t.CheckEquals(tabWidth("\t"), 8) + t.CheckEquals(tabWidth("123\t"), 8) + t.CheckEquals(tabWidth("1234567\t"), 8) + t.CheckEquals(tabWidth("12345678\t"), 16) +} + +// Since tabWidthAppend is used with logical lines (Line.Text) as well as with +// raw lines (RawLine.textnl or RawLine.orignl), and since the width only +// makes sense for a single line, better panic. +func (s *Suite) Test_tabWidthAppend__panic(c *check.C) { + t := s.Init(c) + + t.ExpectAssert(func() { tabWidthAppend(0, "\n") }) +} + +func (s *Suite) Test_detab(c *check.C) { + t := s.Init(c) + + t.CheckEquals(detab(""), "") + t.CheckEquals(detab("\t"), " ") + t.CheckEquals(detab("1234\t9"), "1234 9") + t.CheckEquals(detab("1234567\t"), "1234567 ") + t.CheckEquals(detab("12345678\t"), "12345678 ") +} + +func (s *Suite) Test_alignWith(c *check.C) { + t := s.Init(c) + + test := func(str, other, expected string) { + t.CheckEquals(alignWith(str, other), expected) + } + + // At least one tab is _always_ added. + test("", "", "\t") + + test("VAR=", "1234567", "VAR=\t") + test("VAR=", "12345678", "VAR=\t") + test("VAR=", "123456789", "VAR=\t\t") + + // At least one tab is added in any case, + // even if the other string is shorter. + test("1234567890=", "V=", "1234567890=\t") +} + +func (s *Suite) Test_indent(c *check.C) { + t := s.Init(c) + + test := func(width int, ind string) { + actual := indent(width) + + t.CheckEquals(actual, ind) + } + + test(0, "") + test(1, " ") + test(7, " ") + test(8, "\t") + test(15, "\t ") + test(16, "\t\t") + test(72, "\t\t\t\t\t\t\t\t\t") +} + +func (s *Suite) Test_alignmentAfter(c *check.C) { + t := s.Init(c) + + test := func(prefix string, width int, ind string) { + actual := alignmentAfter(prefix, width) + + t.CheckEquals(actual, ind) + } + + test("", 0, "") + test("", 15, "\t ") + + test(" ", 5, " ") + test(" ", 10, "\t ") + + test("\t", 15, " ") + test(" \t", 15, " ") + test(" \t", 15, " ") + test("\t ", 15, " ") + + test(" ", 16, "\t\t") + + // The desired width must be at least the width of the prefix. + t.ExpectAssert(func() { test("\t", 7, "") }) +} + +func (s *Suite) Test_shorten(c *check.C) { + t := s.Init(c) + + t.CheckEquals(shorten("aaaaa", 3), "aaa...") + t.CheckEquals(shorten("aaaaa", 5), "aaaaa") + t.CheckEquals(shorten("aaa", 5), "aaa") +} + +func (s *Suite) Test_varnameBase(c *check.C) { + t := s.Init(c) + + t.CheckEquals(varnameBase("VAR"), "VAR") + t.CheckEquals(varnameBase("VAR.param"), "VAR") + t.CheckEquals(varnameBase(".CURDIR"), ".CURDIR") +} + +func (s *Suite) Test_varnameCanon(c *check.C) { + t := s.Init(c) + + t.CheckEquals(varnameCanon("VAR"), "VAR") + t.CheckEquals(varnameCanon("VAR.param"), "VAR.*") + t.CheckEquals(varnameCanon(".CURDIR"), ".CURDIR") +} + +func (s *Suite) Test_varnameParam(c *check.C) { + t := s.Init(c) + + t.CheckEquals(varnameParam("VAR"), "") + t.CheckEquals(varnameParam("VAR.param"), "param") + t.CheckEquals(varnameParam(".CURDIR"), "") +} + +func (s *Suite) Test_fileExists(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("dir/file") + + t.CheckEquals(fileExists(t.File("nonexistent")), false) + t.CheckEquals(fileExists(t.File("dir")), false) + t.CheckEquals(fileExists(t.File("dir/nonexistent")), false) + t.CheckEquals(fileExists(t.File("dir/file")), true) +} + +func (s *Suite) Test_dirExists(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("dir/file") + + t.CheckEquals(dirExists(t.File("nonexistent")), false) + t.CheckEquals(dirExists(t.File("dir")), true) + t.CheckEquals(dirExists(t.File("dir/nonexistent")), false) + t.CheckEquals(dirExists(t.File("dir/file")), false) } func (s *Suite) Test_mkopSubst__middle(c *check.C) { @@ -87,31 +354,58 @@ func (s *Suite) Test__regex_ReplaceFirst(c *check.C) { t.CheckEquals(rest, "X+c+d") } -func (s *Suite) Test_shorten(c *check.C) { +func (s *Suite) Test_relpath(c *check.C) { t := s.Init(c) - t.CheckEquals(shorten("aaaaa", 3), "aaa...") - t.CheckEquals(shorten("aaaaa", 5), "aaaaa") - t.CheckEquals(shorten("aaa", 5), "aaa") -} + t.Chdir(".") + t.CheckEquals(G.Pkgsrc.topdir, t.tmpdir) -func (s *Suite) Test_tabWidth(c *check.C) { - t := s.Init(c) + test := func(from, to, result string) { + t.CheckEquals(relpath(from, to), result) + } - t.CheckEquals(tabWidth("12345"), 5) - t.CheckEquals(tabWidth("\t"), 8) - t.CheckEquals(tabWidth("123\t"), 8) - t.CheckEquals(tabWidth("1234567\t"), 8) - t.CheckEquals(tabWidth("12345678\t"), 16) + test("some/dir", "some/directory", "../../some/directory") + test("some/directory", "some/dir", "../../some/dir") + + test("category/package/.", ".", "../..") + + // This case is handled by one of the shortcuts that avoid file system access. + test( + "./.", + "x11/frameworkintegration/../../meta-pkgs/kde/kf5.mk", + "meta-pkgs/kde/kf5.mk") + + test(".hidden/dir", ".", "../..") + test("dir/.hidden", ".", "../..") + + // This happens when "pkglint -r x11" is run. + G.Pkgsrc.topdir = "x11/.." + + test( + "./.", + "x11/frameworkintegration/../../meta-pkgs/kde/kf5.mk", + "meta-pkgs/kde/kf5.mk") + test( + "x11/..", + "x11/frameworkintegration/../../meta-pkgs/kde/kf5.mk", + "meta-pkgs/kde/kf5.mk") } -// Since tabWidthAppend is used with logical lines (Line.Text) as well as with -// raw lines (RawLine.textnl or RawLine.orignl), and since the width only -// makes sense for a single line, better panic. -func (s *Suite) Test_tabWidthAppend__panic(c *check.C) { +// Relpath is called so often that handling the most common calls +// without file system IO makes sense. +func (s *Suite) Test_relpath__quick(c *check.C) { t := s.Init(c) - t.ExpectAssert(func() { tabWidthAppend(0, "\n") }) + test := func(from, to, result string) { + t.CheckEquals(relpath(from, to), result) + } + + test("some/dir", "some/dir/../..", "../..") + test("some/dir", "some/dir/./././../..", "../..") + test("some/dir", "some/dir/", ".") + + test("some/dir", ".", "../..") + test("some/dir/.", ".", "../..") } func (s *Suite) Test_cleanpath(c *check.C) { @@ -164,60 +458,6 @@ func (s *Suite) Test_cleanpath(c *check.C) { test(".././././././././", "..") } -func (s *Suite) Test_relpath(c *check.C) { - t := s.Init(c) - - t.Chdir(".") - t.CheckEquals(G.Pkgsrc.topdir, t.tmpdir) - - test := func(from, to, result string) { - t.CheckEquals(relpath(from, to), result) - } - - test("some/dir", "some/directory", "../../some/directory") - test("some/directory", "some/dir", "../../some/dir") - - test("category/package/.", ".", "../..") - - // This case is handled by one of the shortcuts that avoid file system access. - test( - "./.", - "x11/frameworkintegration/../../meta-pkgs/kde/kf5.mk", - "meta-pkgs/kde/kf5.mk") - - test(".hidden/dir", ".", "../..") - test("dir/.hidden", ".", "../..") - - // This happens when "pkglint -r x11" is run. - G.Pkgsrc.topdir = "x11/.." - - test( - "./.", - "x11/frameworkintegration/../../meta-pkgs/kde/kf5.mk", - "meta-pkgs/kde/kf5.mk") - test( - "x11/..", - "x11/frameworkintegration/../../meta-pkgs/kde/kf5.mk", - "meta-pkgs/kde/kf5.mk") -} - -// Relpath is called so often that handling the most common calls -// without file system IO makes sense. -func (s *Suite) Test_relpath__quick(c *check.C) { - t := s.Init(c) - - test := func(from, to, result string) { - t.CheckEquals(relpath(from, to), result) - } - - test("some/dir", "some/dir/../..", "../..") - test("some/dir", "some/dir/./././../..", "../..") - test("some/dir", "some/dir/", ".") - - test("some/dir", ".", "../..") - test("some/dir/.", ".", "../..") -} - func (s *Suite) Test_pathContains(c *check.C) { t := s.Init(c) @@ -292,105 +532,6 @@ func (s *Suite) Test_pathContainsDir(c *check.C) { test("aa/bb/cc", "c", false) } -func (s *Suite) Test_fileExists(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("dir/file") - - t.CheckEquals(fileExists(t.File("nonexistent")), false) - t.CheckEquals(fileExists(t.File("dir")), false) - t.CheckEquals(fileExists(t.File("dir/nonexistent")), false) - t.CheckEquals(fileExists(t.File("dir/file")), true) -} - -func (s *Suite) Test_dirExists(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("dir/file") - - t.CheckEquals(dirExists(t.File("nonexistent")), false) - t.CheckEquals(dirExists(t.File("dir")), true) - t.CheckEquals(dirExists(t.File("dir/nonexistent")), false) - t.CheckEquals(dirExists(t.File("dir/file")), false) -} - -func (s *Suite) Test_isEmptyDir__and_getSubdirs(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("CVS/Entries", - "dummy") - - if dir := t.File("."); true { - t.CheckEquals(isEmptyDir(dir), true) - t.CheckDeepEquals(getSubdirs(dir), []string(nil)) - - t.CreateFileLines("somedir/file") - - t.CheckEquals(isEmptyDir(dir), false) - t.CheckDeepEquals(getSubdirs(dir), []string{"somedir"}) - } - - if absent := t.File("nonexistent"); true { - t.CheckEquals(isEmptyDir(absent), true) // Counts as empty. - - // The last group from the error message is localized, therefore the matching. - t.ExpectFatalMatches( - func() { getSubdirs(absent) }, - `FATAL: ~/nonexistent: Cannot be read: open ~/nonexistent: (.+)\n`) - } -} - -func (s *Suite) Test_isEmptyDir(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("CVS/Entries", - "dummy") - t.CreateFileLines("subdir/CVS/Entries", - "dummy") - - t.CheckEquals(isEmptyDir(t.File(".")), true) - t.CheckEquals(isEmptyDir(t.File("CVS")), true) -} - -func (s *Suite) Test_getSubdirs(c *check.C) { - t := s.Init(c) - - t.CreateFileLines("subdir/file") - t.CreateFileLines("empty/file") - c.Check(os.Remove(t.File("empty/file")), check.IsNil) - - t.CheckDeepEquals(getSubdirs(t.File(".")), []string{"subdir"}) -} - -func (s *Suite) Test_detab(c *check.C) { - t := s.Init(c) - - t.CheckEquals(detab(""), "") - t.CheckEquals(detab("\t"), " ") - t.CheckEquals(detab("1234\t9"), "1234 9") - t.CheckEquals(detab("1234567\t"), "1234567 ") - t.CheckEquals(detab("12345678\t"), "12345678 ") -} - -func (s *Suite) Test_alignWith(c *check.C) { - t := s.Init(c) - - test := func(str, other, expected string) { - t.CheckEquals(alignWith(str, other), expected) - } - - // At least one tab is _always_ added. - test("", "", "\t") - - test("VAR=", "1234567", "VAR=\t") - test("VAR=", "12345678", "VAR=\t") - test("VAR=", "123456789", "VAR=\t\t") - - // At least one tab is added in any case, - // even if the other string is shorter. - test("1234567890=", "V=", "1234567890=\t") -} - const reMkIncludeBenchmark = `^\.([\t ]*)(s?include)[\t ]+\"([^\"]+)\"[\t ]*(?:#.*)?$` const reMkIncludeBenchmarkPositive = `^\.([\t ]*)(s?include)[\t ]+\"(.+)\"[\t ]*(?:#.*)?$` @@ -455,121 +596,72 @@ func emptyToNil(slice []string) []string { return slice } -func (s *Suite) Test_trimHspace(c *check.C) { +func (s *Suite) Test_hasAlnumPrefix(c *check.C) { t := s.Init(c) - t.CheckEquals(trimHspace("a b"), "a b") - t.CheckEquals(trimHspace(" a b "), "a b") - t.CheckEquals(trimHspace("\ta b\t"), "a b") - t.CheckEquals(trimHspace(" \t a b\t \t"), "a b") + t.CheckEquals(hasAlnumPrefix(""), false) + t.CheckEquals(hasAlnumPrefix("A"), true) + t.CheckEquals(hasAlnumPrefix(","), false) } -func (s *Suite) Test_trimCommon(c *check.C) { +func (s *Suite) Test_Once(c *check.C) { t := s.Init(c) - test := func(a, b, trimmedA, trimmedB string) { - ta, tb := trimCommon(a, b) - t.CheckEquals(ta, trimmedA) - t.CheckEquals(tb, trimmedB) - } - - test("", "", - "", "") - - test("equal", "equal", - "", "") - - test("prefixA", "prefixB", - "A", "B") - - test("ASuffix", "BSuffix", - "A", "B") - - test("PreMiddlePost", "PreCenterPost", - "Middle", "Center") - - test("", "b", - "", "b") + var once Once - test("a", "", - "a", "") + t.CheckEquals(once.FirstTime("str"), true) + t.CheckEquals(once.FirstTime("str"), false) + t.CheckEquals(once.FirstTimeSlice("str"), false) + t.CheckEquals(once.FirstTimeSlice("str", "str2"), true) + t.CheckEquals(once.FirstTimeSlice("str", "str2"), false) } -func (s *Suite) Test_indent(c *check.C) { +func (s *Suite) Test_Once__trace(c *check.C) { t := s.Init(c) - test := func(width int, ind string) { - actual := indent(width) + var once Once + once.Trace = true - t.CheckEquals(actual, ind) - } + t.CheckEquals(once.FirstTime("str"), true) + t.CheckEquals(once.FirstTime("str"), false) + t.CheckEquals(once.FirstTimeSlice("str"), false) + t.CheckEquals(once.FirstTimeSlice("str", "str2"), true) + t.CheckEquals(once.FirstTimeSlice("str", "str2"), false) - test(0, "") - test(1, " ") - test(7, " ") - test(8, "\t") - test(15, "\t ") - test(16, "\t\t") - test(72, "\t\t\t\t\t\t\t\t\t") + t.CheckOutputLines( + "FirstTime: str", + "FirstTime: str, str2") } -func (s *Suite) Test_alignmentAfter(c *check.C) { +func (s *Suite) Test_Scope__no_tracing(c *check.C) { t := s.Init(c) - test := func(prefix string, width int, ind string) { - actual := alignmentAfter(prefix, width) - - t.CheckEquals(actual, ind) - } - - test("", 0, "") - test("", 15, "\t ") - - test(" ", 5, " ") - test(" ", 10, "\t ") - - test("\t", 15, " ") - test(" \t", 15, " ") - test(" \t", 15, " ") - test("\t ", 15, " ") - - test(" ", 16, "\t\t") + scope := NewScope() + scope.Define("VAR.param", t.NewMkLine("fname.mk", 3, "VAR.param=\tvalue")) + t.DisableTracing() - // The desired width must be at least the width of the prefix. - t.ExpectAssert(func() { test("\t", 7, "") }) + t.CheckEquals(scope.IsDefinedSimilar("VAR.param"), true) + t.CheckEquals(scope.IsDefinedSimilar("VAR.other"), true) + t.CheckEquals(scope.IsDefinedSimilar("OTHER"), false) } -func (s *Suite) Test_isLocallyModified(c *check.C) { +func (s *Suite) Test_Scope__commented_varassign(c *check.C) { t := s.Init(c) - unmodified := t.CreateFileLines("unmodified") - modTime := time.Unix(1136239445, 0).UTC() - - err := os.Chtimes(unmodified, modTime, modTime) - c.Check(err, check.IsNil) - - st, err := os.Lstat(unmodified) - c.Check(err, check.IsNil) - - // Make sure that the file system has second precision and accuracy. - t.CheckDeepEquals(st.ModTime().UTC(), modTime) - - modified := t.CreateFileLines("modified") - - t.CreateFileLines("CVS/Entries", - "/unmodified//"+modTime.Format(time.ANSIC)+"//", - "/modified//"+modTime.Format(time.ANSIC)+"//", - "/enoent//"+modTime.Format(time.ANSIC)+"//") + mkline := t.NewMkLine("mk/defaults/mk.conf", 3, "#VAR=default") + scope := NewScope() + scope.Define("VAR", mkline) - t.CheckEquals(isLocallyModified(unmodified), false) - t.CheckEquals(isLocallyModified(modified), true) - t.CheckEquals(isLocallyModified(t.File("enoent")), true) - t.CheckEquals(isLocallyModified(t.File("not_mentioned")), false) - t.CheckEquals(isLocallyModified(t.File("subdir/file")), false) + t.CheckEquals(scope.IsDefined("VAR"), false) + t.Check(scope.FirstDefinition("VAR"), check.IsNil) + t.Check(scope.LastDefinition("VAR"), check.IsNil) - t.DisableTracing() + t.CheckEquals(scope.Mentioned("VAR"), mkline) + t.CheckEquals(scope.Commented("VAR"), mkline) - t.CheckEquals(isLocallyModified(t.File("unmodified")), false) + value, found := scope.LastValueFound("VAR") + t.CheckEquals(value, "") + t.CheckEquals(found, false) } func (s *Suite) Test_Scope_Define(c *check.C) { @@ -590,53 +682,53 @@ func (s *Suite) Test_Scope_Define(c *check.C) { t.CheckEquals(scope.LastValue("BUILD_DIRS"), "one two three four") } -func (s *Suite) Test_Scope_Defined(c *check.C) { +func (s *Suite) Test_Scope_Mentioned(c *check.C) { t := s.Init(c) - scope := NewScope() - scope.Define("VAR.param", t.NewMkLine("file.mk", 1, "VAR.param=value")) + assigned := t.NewMkLine("filename.mk", 3, "VAR=\tvalue") + commented := t.NewMkLine("filename.mk", 4, "#COMMENTED=\tvalue") + documented := t.NewMkLine("filename.mk", 5, "# DOCUMENTED is a variable.") - t.CheckEquals(scope.Defined("VAR.param"), true) - t.CheckEquals(scope.Defined("VAR.other"), false) - t.CheckEquals(scope.Defined("VARIABLE.*"), false) + scope := NewScope() + scope.Define("VAR", assigned) + scope.Define("COMMENTED", commented) + scope.Define("DOCUMENTED", documented) - t.CheckEquals(scope.DefinedSimilar("VAR.param"), true) - t.CheckEquals(scope.DefinedSimilar("VAR.other"), true) - t.CheckEquals(scope.DefinedSimilar("VARIABLE.*"), false) + t.CheckEquals(scope.Mentioned("VAR"), assigned) + t.CheckEquals(scope.Mentioned("COMMENTED"), commented) + t.CheckEquals(scope.Mentioned("DOCUMENTED"), documented) + t.Check(scope.Mentioned("UNKNOWN"), check.IsNil) } -func (s *Suite) Test_Scope_Used(c *check.C) { +func (s *Suite) Test_Scope_IsDefined(c *check.C) { t := s.Init(c) scope := NewScope() - mkline := t.NewMkLine("file.mk", 1, "\techo ${VAR.param}") - scope.Use("VAR.param", mkline, VucRunTime) + scope.Define("VAR.param", t.NewMkLine("file.mk", 1, "VAR.param=value")) - t.CheckEquals(scope.Used("VAR.param"), true) - t.CheckEquals(scope.Used("VAR.other"), false) - t.CheckEquals(scope.Used("VARIABLE.*"), false) + t.CheckEquals(scope.IsDefined("VAR.param"), true) + t.CheckEquals(scope.IsDefined("VAR.other"), false) + t.CheckEquals(scope.IsDefined("VARIABLE.*"), false) - t.CheckEquals(scope.UsedSimilar("VAR.param"), true) - t.CheckEquals(scope.UsedSimilar("VAR.other"), true) - t.CheckEquals(scope.UsedSimilar("VARIABLE.*"), false) + t.CheckEquals(scope.IsDefinedSimilar("VAR.param"), true) + t.CheckEquals(scope.IsDefinedSimilar("VAR.other"), true) + t.CheckEquals(scope.IsDefinedSimilar("VARIABLE.*"), false) } -func (s *Suite) Test_Scope_DefineAll(c *check.C) { +func (s *Suite) Test_Scope_IsUsed(c *check.C) { t := s.Init(c) - src := NewScope() - - dst := NewScope() - dst.DefineAll(src) - - c.Check(dst.firstDef, check.HasLen, 0) - c.Check(dst.lastDef, check.HasLen, 0) - c.Check(dst.used, check.HasLen, 0) + scope := NewScope() + mkline := t.NewMkLine("file.mk", 1, "\techo ${VAR.param}") + scope.Use("VAR.param", mkline, VucRunTime) - src.Define("VAR", t.NewMkLine("file.mk", 1, "VAR=value")) - dst.DefineAll(src) + t.CheckEquals(scope.IsUsed("VAR.param"), true) + t.CheckEquals(scope.IsUsed("VAR.other"), false) + t.CheckEquals(scope.IsUsed("VARIABLE.*"), false) - t.CheckEquals(dst.Defined("VAR"), true) + t.CheckEquals(scope.IsUsedSimilar("VAR.param"), true) + t.CheckEquals(scope.IsUsedSimilar("VAR.other"), true) + t.CheckEquals(scope.IsUsedSimilar("VARIABLE.*"), false) } func (s *Suite) Test_Scope_FirstDefinition(c *check.C) { @@ -658,6 +750,24 @@ func (s *Suite) Test_Scope_FirstDefinition(c *check.C) { t.Check(scope.FirstDefinition("SNEAKY"), check.IsNil) } +func (s *Suite) Test_Scope_Commented(c *check.C) { + t := s.Init(c) + + assigned := t.NewMkLine("filename.mk", 3, "VAR=\tvalue") + commented := t.NewMkLine("filename.mk", 4, "#COMMENTED=\tvalue") + documented := t.NewMkLine("filename.mk", 5, "# DOCUMENTED is a variable.") + + scope := NewScope() + scope.Define("VAR", assigned) + scope.Define("COMMENTED", commented) + scope.Define("DOCUMENTED", documented) + + t.Check(scope.Commented("VAR"), check.IsNil) + t.CheckEquals(scope.Commented("COMMENTED"), commented) + t.Check(scope.Commented("DOCUMENTED"), check.IsNil) + t.Check(scope.Commented("UNKNOWN"), check.IsNil) +} + func (s *Suite) Test_Scope_LastValue(c *check.C) { t := s.Init(c) @@ -677,71 +787,22 @@ func (s *Suite) Test_Scope_LastValue(c *check.C) { "WARN: file.mk:2: VAR is defined but not used.") } -func (s *Suite) Test_Scope__no_tracing(c *check.C) { - t := s.Init(c) - - scope := NewScope() - scope.Define("VAR.param", t.NewMkLine("fname.mk", 3, "VAR.param=\tvalue")) - t.DisableTracing() - - t.CheckEquals(scope.DefinedSimilar("VAR.param"), true) - t.CheckEquals(scope.DefinedSimilar("VAR.other"), true) - t.CheckEquals(scope.DefinedSimilar("OTHER"), false) -} - -func (s *Suite) Test_Scope__commented_varassign(c *check.C) { - t := s.Init(c) - - mkline := t.NewMkLine("mk/defaults/mk.conf", 3, "#VAR=default") - scope := NewScope() - scope.Define("VAR", mkline) - - t.CheckEquals(scope.Defined("VAR"), false) - t.Check(scope.FirstDefinition("VAR"), check.IsNil) - t.Check(scope.LastDefinition("VAR"), check.IsNil) - - t.CheckEquals(scope.Mentioned("VAR"), mkline) - t.CheckEquals(scope.Commented("VAR"), mkline) - - value, found := scope.LastValueFound("VAR") - t.CheckEquals(value, "") - t.CheckEquals(found, false) -} - -func (s *Suite) Test_Scope_Commented(c *check.C) { +func (s *Suite) Test_Scope_DefineAll(c *check.C) { t := s.Init(c) - assigned := t.NewMkLine("filename.mk", 3, "VAR=\tvalue") - commented := t.NewMkLine("filename.mk", 4, "#COMMENTED=\tvalue") - documented := t.NewMkLine("filename.mk", 5, "# DOCUMENTED is a variable.") - - scope := NewScope() - scope.Define("VAR", assigned) - scope.Define("COMMENTED", commented) - scope.Define("DOCUMENTED", documented) - - t.Check(scope.Commented("VAR"), check.IsNil) - t.CheckEquals(scope.Commented("COMMENTED"), commented) - t.Check(scope.Commented("DOCUMENTED"), check.IsNil) - t.Check(scope.Commented("UNKNOWN"), check.IsNil) -} + src := NewScope() -func (s *Suite) Test_Scope_Mentioned(c *check.C) { - t := s.Init(c) + dst := NewScope() + dst.DefineAll(src) - assigned := t.NewMkLine("filename.mk", 3, "VAR=\tvalue") - commented := t.NewMkLine("filename.mk", 4, "#COMMENTED=\tvalue") - documented := t.NewMkLine("filename.mk", 5, "# DOCUMENTED is a variable.") + c.Check(dst.firstDef, check.HasLen, 0) + c.Check(dst.lastDef, check.HasLen, 0) + c.Check(dst.used, check.HasLen, 0) - scope := NewScope() - scope.Define("VAR", assigned) - scope.Define("COMMENTED", commented) - scope.Define("DOCUMENTED", documented) + src.Define("VAR", t.NewMkLine("file.mk", 1, "VAR=value")) + dst.DefineAll(src) - t.CheckEquals(scope.Mentioned("VAR"), assigned) - t.CheckEquals(scope.Mentioned("COMMENTED"), commented) - t.CheckEquals(scope.Mentioned("DOCUMENTED"), documented) - t.Check(scope.Mentioned("UNKNOWN"), check.IsNil) + t.CheckEquals(dst.IsDefined("VAR"), true) } func (s *Suite) Test_naturalLess(c *check.C) { @@ -770,30 +831,6 @@ func (s *Suite) Test_naturalLess(c *check.C) { } } -func (s *Suite) Test_varnameBase(c *check.C) { - t := s.Init(c) - - t.CheckEquals(varnameBase("VAR"), "VAR") - t.CheckEquals(varnameBase("VAR.param"), "VAR") - t.CheckEquals(varnameBase(".CURDIR"), ".CURDIR") -} - -func (s *Suite) Test_varnameParam(c *check.C) { - t := s.Init(c) - - t.CheckEquals(varnameParam("VAR"), "") - t.CheckEquals(varnameParam("VAR.param"), "param") - t.CheckEquals(varnameParam(".CURDIR"), "") -} - -func (s *Suite) Test_varnameCanon(c *check.C) { - t := s.Init(c) - - t.CheckEquals(varnameCanon("VAR"), "VAR") - t.CheckEquals(varnameCanon("VAR.param"), "VAR.*") - t.CheckEquals(varnameCanon(".CURDIR"), ".CURDIR") -} - func (s *Suite) Test_FileCache(c *check.C) { t := s.Init(c) @@ -942,43 +979,6 @@ func (s *Suite) Test_bmakeHelp(c *check.C) { t.CheckEquals(bmakeHelp("subst"), confMake+" help topic=subst") } -func (s *Suite) Test_hasAlnumPrefix(c *check.C) { - t := s.Init(c) - - t.CheckEquals(hasAlnumPrefix(""), false) - t.CheckEquals(hasAlnumPrefix("A"), true) - t.CheckEquals(hasAlnumPrefix(","), false) -} - -func (s *Suite) Test_Once(c *check.C) { - t := s.Init(c) - - var once Once - - t.CheckEquals(once.FirstTime("str"), true) - t.CheckEquals(once.FirstTime("str"), false) - t.CheckEquals(once.FirstTimeSlice("str"), false) - t.CheckEquals(once.FirstTimeSlice("str", "str2"), true) - t.CheckEquals(once.FirstTimeSlice("str", "str2"), false) -} - -func (s *Suite) Test_Once__trace(c *check.C) { - t := s.Init(c) - - var once Once - once.Trace = true - - t.CheckEquals(once.FirstTime("str"), true) - t.CheckEquals(once.FirstTime("str"), false) - t.CheckEquals(once.FirstTimeSlice("str"), false) - t.CheckEquals(once.FirstTimeSlice("str", "str2"), true) - t.CheckEquals(once.FirstTimeSlice("str", "str2"), false) - - t.CheckOutputLines( - "FirstTime: str", - "FirstTime: str, str2") -} - func (s *Suite) Test_wrap(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/var.go b/pkgtools/pkglint/files/var.go index 13f2a278e94..6ac81b81edf 100644 --- a/pkgtools/pkglint/files/var.go +++ b/pkgtools/pkglint/files/var.go @@ -7,12 +7,12 @@ package pkglint // analysis, such as: // // * Whether the variable value is constant, and if so, what the constant value -// is (see Constant, ConstantValue). +// is (see IsConstant, ConstantValue). // // * What its (approximated) value is, either including values from the pkgsrc // infrastructure (see ValueInfra) or excluding them (Value). // -// * On which other variables this variable depends (see Conditional, +// * On which other variables this variable depends (see IsConditional, // ConditionalVars). // // TODO: Decide how to handle OPSYS-specific variables, such as LDFLAGS.SunOS. @@ -44,8 +44,8 @@ func NewVar(name string) *Var { return &Var{name, 0, "", "", "", nil, nil, false, NewStringSet(), NewStringSet()} } -// Conditional returns whether the variable value depends on other variables. -func (v *Var) Conditional() bool { +// IsConditional returns whether the variable value depends on other variables. +func (v *Var) IsConditional() bool { return v.conditional } @@ -79,7 +79,7 @@ func (v *Var) AddRef(varname string) { v.refs.Add(varname) } -// Constant returns whether the variable's value is a constant. +// IsConstant returns whether the variable's value is a constant. // It may reference other variables since these references are evaluated // lazily, when the variable value is actually needed. // @@ -96,17 +96,17 @@ func (v *Var) AddRef(varname string) { // // Variable assignments in the pkgsrc infrastructure are taken into account // for determining whether a variable is constant. -func (v *Var) Constant() bool { +func (v *Var) IsConstant() bool { return v.constantState == 1 || v.constantState == 2 } // ConstantValue returns the constant value of the variable. -// It is only allowed when Constant() returns true. +// It is only allowed when IsConstant() returns true. // // Variable assignments in the pkgsrc infrastructure are taken into account // for determining the constant value. func (v *Var) ConstantValue() string { - assert(v.Constant()) + assert(v.IsConstant()) return v.constantValue } @@ -117,7 +117,7 @@ func (v *Var) ConstantValue() string { // returned value is not reliable. It may be the value from either branch, or // even the combined value of both branches. // -// See Constant and ConstantValue for more reliable information. +// See IsConstant and ConstantValue for more reliable information. func (v *Var) Value() string { return v.value } @@ -130,7 +130,7 @@ func (v *Var) Value() string { // returned value is not reliable. It may be the value from either branch, or // even the combined value of both branches. // -// See Constant and ConstantValue for more reliable information, but these +// See IsConstant and ConstantValue for more reliable information, but these // ignore assignments from the infrastructure. func (v *Var) ValueInfra() string { return v.valueInfra @@ -194,7 +194,7 @@ func (v *Var) Write(mkline *MkLine, conditional bool, conditionVarnames ...strin func (v *Var) update(mkline *MkLine, update *string) { firstWrite := len(v.writeLocations) == 1 - if v.Conditional() && !firstWrite { + if v.IsConditional() && !firstWrite { return } @@ -232,7 +232,7 @@ func (v *Var) updateConstantValue(mkline *MkLine) { // (And even after that, because of the ::= modifier. But luckily // almost no one knows that modifier.) - if v.Conditional() { + if v.IsConditional() { v.constantState = 3 v.constantValue = "" return diff --git a/pkgtools/pkglint/files/var_test.go b/pkgtools/pkglint/files/var_test.go index 07127cb34cf..5d27f882575 100644 --- a/pkgtools/pkglint/files/var_test.go +++ b/pkgtools/pkglint/files/var_test.go @@ -2,6 +2,42 @@ package pkglint import "gopkg.in/check.v1" +func (s *Suite) Test_Var_ConditionalVars(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + t.CheckEquals(v.IsConditional(), false) + t.Check(v.ConditionalVars(), check.IsNil) + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tconditional"), true, "OPSYS") + + t.CheckEquals(v.IsConstant(), false) + t.CheckEquals(v.IsConditional(), true) + t.CheckDeepEquals(v.ConditionalVars(), []string{"OPSYS"}) + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME=\tconditional"), true, "OPSYS") + + t.CheckEquals(v.IsConditional(), true) + t.CheckDeepEquals(v.ConditionalVars(), []string{"OPSYS"}) +} + +func (s *Suite) Test_Var_Refs(c *check.C) { + t := s.Init(c) + + v := NewVar("VAR") + + t.Check(v.Refs(), check.IsNil) + + // The referenced variables are taken from the mkline. + // They don't need to be passed separately. + v.Write(t.NewMkLine("write.mk", 123, "VAR=${OTHER} ${${OPSYS} == NetBSD :? ${THEN} : ${ELSE}}"), true, "COND") + + v.AddRef("FOR") + + t.CheckDeepEquals(v.Refs(), []string{"OTHER", "OPSYS", "THEN", "ELSE", "COND", "FOR"}) +} + func (s *Suite) Test_Var_ConstantValue__assign(c *check.C) { t := s.Init(c) @@ -31,7 +67,7 @@ func (s *Suite) Test_Var_ConstantValue__assign_reference(c *check.C) { v.Write(t.NewMkLine("write.mk", 124, "VARNAME=\t${OTHER}"), false) - t.CheckEquals(v.Constant(), true) + t.CheckEquals(v.IsConstant(), true) } func (s *Suite) Test_Var_ConstantValue__assign_eval_reference(c *check.C) { @@ -51,7 +87,7 @@ func (s *Suite) Test_Var_ConstantValue__assign_eval_reference(c *check.C) { // // As of March 2019 this is not implemented, therefore pkglint // doesn't treat the variable as constant, to prevent wrong warnings. - t.CheckEquals(v.Constant(), false) + t.CheckEquals(v.IsConstant(), false) } func (s *Suite) Test_Var_ConstantValue__assign_conditional(c *check.C) { @@ -63,7 +99,7 @@ func (s *Suite) Test_Var_ConstantValue__assign_conditional(c *check.C) { v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tconditional"), true, "OPSYS") - t.CheckEquals(v.Constant(), false) + t.CheckEquals(v.IsConstant(), false) } func (s *Suite) Test_Var_ConstantValue__default(c *check.C) { @@ -134,7 +170,7 @@ func (s *Suite) Test_Var_ConstantValue__shell(c *check.C) { v.Write(t.NewMkLine("write.mk", 124, "VARNAME!=\techo hello"), false) - t.CheckEquals(v.Constant(), false) + t.CheckEquals(v.IsConstant(), false) } func (s *Suite) Test_Var_ConstantValue__referenced_before(c *check.C) { @@ -148,11 +184,11 @@ func (s *Suite) Test_Var_ConstantValue__referenced_before(c *check.C) { // condition. v.Read(t.NewMkLine("readwrite.mk", 123, "OTHER=\t${VARNAME}")) - t.CheckEquals(v.Constant(), false) + t.CheckEquals(v.IsConstant(), false) v.Write(t.NewMkLine("readwrite.mk", 124, "VARNAME=\tvalue"), false) - t.CheckEquals(v.Constant(), false) + t.CheckEquals(v.IsConstant(), false) } func (s *Suite) Test_Var_ConstantValue__referenced_in_between(c *check.C) { @@ -174,73 +210,7 @@ func (s *Suite) Test_Var_ConstantValue__referenced_in_between(c *check.C) { v.Write(t.NewMkLine("write.mk", 125, "VARNAME=\toverwritten"), false) - t.CheckEquals(v.Constant(), false) -} - -func (s *Suite) Test_Var_ConditionalVars(c *check.C) { - t := s.Init(c) - - v := NewVar("VARNAME") - - t.CheckEquals(v.Conditional(), false) - t.Check(v.ConditionalVars(), check.IsNil) - - v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tconditional"), true, "OPSYS") - - t.CheckEquals(v.Constant(), false) - t.CheckEquals(v.Conditional(), true) - t.CheckDeepEquals(v.ConditionalVars(), []string{"OPSYS"}) - - v.Write(t.NewMkLine("write.mk", 124, "VARNAME=\tconditional"), true, "OPSYS") - - t.CheckEquals(v.Conditional(), true) - t.CheckDeepEquals(v.ConditionalVars(), []string{"OPSYS"}) -} - -func (s *Suite) Test_Var_Value__initial_conditional_write(c *check.C) { - t := s.Init(c) - - v := NewVar("VARNAME") - - v.Write(t.NewMkLine("write.mk", 124, "VARNAME:=\toverwritten conditionally"), true, "OPSYS") - - // Since there is no previous value, the simplest choice is to just - // take the first seen value, no matter if that value is conditional - // or not. - t.CheckEquals(v.Conditional(), true) - t.CheckEquals(v.Constant(), false) - t.CheckEquals(v.Value(), "overwritten conditionally") -} - -func (s *Suite) Test_Var_Write__conditional_without_variables(c *check.C) { - t := s.Init(c) - - mklines := t.NewMkLines("filename.mk", - MkCvsID, - ".if exists(/usr/bin)", - "VAR=\tvalue", - ".endif") - - scope := NewRedundantScope() - mklines.ForEach(func(mkline *MkLine) { - if mkline.IsVarassign() { - t.CheckEquals(scope.get("VAR").vari.Conditional(), false) - } - - scope.checkLine(mklines, mkline) - - if mkline.IsVarassign() { - t.CheckEquals(scope.get("VAR").vari.Conditional(), true) - } - }) -} - -func (s *Suite) Test_Var_Write__assertion(c *check.C) { - t := s.Init(c) - - v := NewVar("VAR") - t.ExpectAssert( - func() { v.Write(t.NewMkLine("filename.mk", 1, "OTHER=value"), false, nil...) }) + t.CheckEquals(v.IsConstant(), false) } func (s *Suite) Test_Var_Value__conditional_write_after_unconditional(c *check.C) { @@ -267,8 +237,8 @@ func (s *Suite) Test_Var_Value__conditional_write_after_unconditional(c *check.C // .endif // The value stays the same, still it is marked as conditional and therefore // not constant anymore. - t.CheckEquals(v.Conditional(), true) - t.CheckEquals(v.Constant(), false) + t.CheckEquals(v.IsConditional(), true) + t.CheckEquals(v.IsConstant(), false) t.CheckEquals(v.Value(), "value appended") } @@ -290,6 +260,21 @@ func (s *Suite) Test_Var_Value__infrastructure(c *check.C) { t.CheckEquals(v.Value(), "value") } +func (s *Suite) Test_Var_Value__initial_conditional_write(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME:=\toverwritten conditionally"), true, "OPSYS") + + // Since there is no previous value, the simplest choice is to just + // take the first seen value, no matter if that value is conditional + // or not. + t.CheckEquals(v.IsConditional(), true) + t.CheckEquals(v.IsConstant(), false) + t.CheckEquals(v.Value(), "overwritten conditionally") +} + func (s *Suite) Test_Var_ValueInfra(c *check.C) { t := s.Init(c) @@ -351,18 +336,33 @@ func (s *Suite) Test_Var_WriteLocations(c *check.C) { t.CheckDeepEquals(v.WriteLocations(), []*MkLine{mkline123, mkline125, mkline125}) } -func (s *Suite) Test_Var_Refs(c *check.C) { +func (s *Suite) Test_Var_Write__conditional_without_variables(c *check.C) { t := s.Init(c) - v := NewVar("VAR") + mklines := t.NewMkLines("filename.mk", + MkCvsID, + ".if exists(/usr/bin)", + "VAR=\tvalue", + ".endif") - t.Check(v.Refs(), check.IsNil) + scope := NewRedundantScope() + mklines.ForEach(func(mkline *MkLine) { + if mkline.IsVarassign() { + t.CheckEquals(scope.get("VAR").vari.IsConditional(), false) + } - // The referenced variables are taken from the mkline. - // They don't need to be passed separately. - v.Write(t.NewMkLine("write.mk", 123, "VAR=${OTHER} ${${OPSYS} == NetBSD :? ${THEN} : ${ELSE}}"), true, "COND") + scope.checkLine(mklines, mkline) - v.AddRef("FOR") + if mkline.IsVarassign() { + t.CheckEquals(scope.get("VAR").vari.IsConditional(), true) + } + }) +} - t.CheckDeepEquals(v.Refs(), []string{"OTHER", "OPSYS", "THEN", "ELSE", "COND", "FOR"}) +func (s *Suite) Test_Var_Write__assertion(c *check.C) { + t := s.Init(c) + + v := NewVar("VAR") + t.ExpectAssert( + func() { v.Write(t.NewMkLine("filename.mk", 1, "OTHER=value"), false, nil...) }) } diff --git a/pkgtools/pkglint/files/varalignblock.go b/pkgtools/pkglint/files/varalignblock.go index 6eb502965fa..72ca6fd52bb 100644 --- a/pkgtools/pkglint/files/varalignblock.go +++ b/pkgtools/pkglint/files/varalignblock.go @@ -66,6 +66,80 @@ import ( // as opposed to the meaning of the variable assignment. // // FIXME: Implement each requirement from the above documentation. +// +// Next try for the spec, from November 2019. +// Completely built from the existing examples, striving to be short and clear. +// Needs some more time to mature. +// After implementing it, it will be translated into English. +// +// Ebenen: Datei > Absatz > MkZeile > Zeile +// +// ### Datei +// +// #. Ein einzelner Absatz, der einen Tab weniger eingerückt ist als die übrigen, +// darf auf die Einrückung der anderen Absätze angeglichen werden, +// sofern der Absatz dadurch nicht zu breit wird. +// +// ### Einzelner Absatz +// +// #. Jede Zeile besteht aus #, VarOp, Leerraum, Wert, Leerraum und Fortsetzung. +// +// #. Die Werte aller Zeilen sind mit Tabs an einer gemeinsamen vertikalen Linie +// (Ausrichtung) ausgerichtet. +// +// #. Das Ausrichten mit mehr als 1 Tab ist erlaubt, wenn die Ausrichtung einheitlich ist. +// +// #. Wenn VarOp über die Ausrichtung hinausragt (Ausreißer), +// darf zwischen VarOp und Wert statt der Ausrichtung 1 Leerzeichen sein. +// +// #. Die minimale Ausrichtung ergibt sich aus der maximalen Breite von # und VarOp +// aller Zeilen, gerundet zum nächsten Tabstopp. +// Dabei zählen auch Zeilen mit, die rechts von VarOp komplett leer sind. +// +// #. Die maximale Ausrichtung ergibt sich aus der maximalen Breite von Wert +// und Kommentar, abgezogen vom maximalen rechten Rand (in Spalte 73). +// +// #. Beim Umformatieren darf die Zeilenbreite die 73 Zeichen nicht überschreiten, +// damit am rechten Rand eindeutig ist, wo jede Zeile aufhört. +// Zeilen, die bereits vorher breiter waren, dürfen ruhig noch breiter werden. +// +// #. Das Verhältnis zwischen Tab-Zeilen und hinausragenden Zeilen muss ausgewogen sein. +// Nicht zu viele hinausragende Zeilen. (Noch zu definieren.) +// Möglicher Ansatz: Anteil der Leerfläche? +// +// ### Mehrzeilig +// +// #. Jede MkZeile hat für alle ihre Zeilen einen gemeinsamen rechten Rand. +// +// #. Die Fortsetzungen jeder MkZeile sind entweder alle durch je 1 Leerzeichen abgetrennt, +// oder alle Fortsetzungen sind am rechten Rand. +// +// #. Um den gemeinsamen rechten Rand zu bestimmen, werden alle Zeilen ignoriert, +// in denen die Fortsetzung durch 1 Leerzeichen abgetrennt ist. +// +// #. Einzelne Fortsetzungen dürfen über den rechten Rand hinausragen. +// Die Fortsetzung wird dann durch 1 Leerzeichen abgetrennt. +// +// ### Mehrzeilig, Erstzeile +// +// #. Die Fortsetzung der Erstzeile ist durch 1 Leerzeichen abgetrennt, +// wenn sie rechts von der Ausrichtung steht, +// andernfalls durch Tabs an der Ausrichtung. +// +// #. Eine leere Erstzeile mit 1 fortgesetzer Zeile ist nur zulässig, +// wenn die kombinierte Zeile breiter als 73 Zeichen wäre. +// Sonst werden die beiden Zeilen kombiniert. +// +// ### Mehrzeilig, fortgesetzte Zeilen +// +// #. Nach einer leeren Erstzeile ist die erste fortgesetzte Zeile an der +// Ausrichtung aller Zeilen eingerückt, wenn die Erstzeile über die +// Ausrichtung ragt und der Platz aller Zeilen es zulässt, andernfalls +// mit 1 Tab. +// +// #. Bei mehrzeiligen einrückbaren Werten (AWK, Shell, Listen aus Tupeln) +// dürfen die weiteren Fortsetzungszeilen weiter eingerückt sein als die erste. +// Ihre Einrückung besteht aus Tabs, gefolgt von 0 bis 7 Leerzeichen. type VaralignBlock struct { infos []*varalignLine skip bool @@ -219,7 +293,7 @@ func (*VaralignBlock) rightMargin(infos []*varalignLine) int { var min int for _, info := range infos { if info.isContinuation() { - mainWidth := tabWidth(info.beforeContinuation()) + mainWidth := info.uptoCommentWidth() if mainWidth > min { min = mainWidth } @@ -291,16 +365,26 @@ func (*VaralignBlock) optimalWidth(infos []*varalignLine) int { return (minVarnameOpWidth & -8) + 8 } +// adjustLong allows any follow-up line to start either in column 8 or at +// least in column newWidth. But only if there is at least one continuation +// line that starts in column 8 and needs the full width up to column 72. func (va *VaralignBlock) adjustLong(newWidth int, infos []*varalignLine) { - long := false - for _, info := range infos { + anyLong := false + for i, info := range infos { if info.rawIndex == 0 { - long = false - } - if !info.multiEmpty && info.spaceBeforeValue == "\t" && info.varnameOpSpaceWidth() != newWidth && info.widthAlignedAt(newWidth) > 72 { - long = true + anyLong = false + for _, follow := range infos[i+1:] { + if follow.rawIndex == 0 { + break + } + if !follow.multiEmpty && follow.spaceBeforeValue == "\t" && follow.varnameOpSpaceWidth() < newWidth && follow.widthAlignedAt(newWidth) > 72 { + anyLong = true + break + } + } } - info.long = long + + info.long = anyLong && info.varnameOpSpaceWidth() == 8 } } @@ -321,10 +405,10 @@ func (va *VaralignBlock) checkRightMargin(info *varalignLine, newWidth int, righ newSpace := " " fix := info.mkline.Autofix() - if oldSpace == "" || rightMargin == 0 || tabWidth(info.beforeContinuation()) >= rightMargin { + if oldSpace == "" || rightMargin == 0 || info.uptoCommentWidth() >= rightMargin { fix.Notef("The continuation backslash should be preceded by a single space or tab.") } else { - newSpace = alignmentAfter(info.beforeContinuation(), rightMargin) + newSpace = alignmentAfter(info.uptoComment(), rightMargin) fix.Notef( "The continuation backslash should be preceded by a single space or tab, "+ "or be in column %d, not %d.", @@ -374,7 +458,7 @@ func (*VaralignBlock) realignMultiEmptyInitial(info *varalignLine, newWidth int) hasSpace := strings.IndexByte(oldSpace, ' ') != -1 oldColumn := info.varnameOpSpaceWidth() - column := tabWidth(leadingComment + varnameOp + newSpace) + column := tabWidthSlice(leadingComment, varnameOp, newSpace) fix := info.mkline.Autofix() if hasSpace && column != oldColumn { @@ -395,7 +479,7 @@ func (va *VaralignBlock) realignMultiEmptyFollow(info *varalignLine, newWidth in if !*indentDiffSet { *indentDiffSet = true *indentDiff = condInt(newWidth != 0, newWidth-oldWidth, 0) - if *indentDiff > 0 && !info.commentedOut() { + if *indentDiff > 0 && !info.isCommentedOut() { *indentDiff = 0 } } @@ -432,7 +516,7 @@ func (va *VaralignBlock) realignMultiInitial(info *varalignLine, newWidth int, i } hasSpace := strings.IndexByte(oldSpace, ' ') != -1 - width := tabWidth(leadingComment + varnameOp + newSpace) + width := tabWidthSlice(leadingComment, varnameOp, newSpace) fix := info.mkline.Autofix() if hasSpace && width != oldWidth { @@ -458,7 +542,21 @@ func (va *VaralignBlock) realignMultiFollow(info *varalignLine, newWidth int, in fix := info.mkline.Autofix() fix.Notef("This continuation line should be indented with %q.", indent(newWidth)) - fix.ReplaceAt(info.rawIndex, info.spaceBeforeValueIndex(), oldSpace, newSpace) + modified, replaced := fix.ReplaceAt(info.rawIndex, info.spaceBeforeValueIndex(), oldSpace, newSpace) + assert(modified) + if info.continuation != "" && info.continuationColumn() == 72 { + orig := strings.TrimSuffix(replaced, "\n") + base := rtrimHspace(strings.TrimSuffix(orig, "\\")) + spaceIndex := len(base) + oldSuffix := orig[spaceIndex:] + newSuffix := " \\" + if tabWidth(base) < 72 { + newSuffix = alignmentAfter(base, 72) + "\\" + } + if newSuffix != oldSuffix { + fix.ReplaceAt(info.rawIndex, spaceIndex, oldSuffix, newSuffix) + } + } fix.Apply() } @@ -468,12 +566,12 @@ func (va *VaralignBlock) realignSingle(info *varalignLine, newWidth int) { oldSpace := info.spaceBeforeValue newSpace := "" - for tabWidth(leadingComment+varnameOp+newSpace) < newWidth { + for tabWidthSlice(leadingComment, varnameOp, newSpace) < newWidth { newSpace += "\t" } // Indent the outlier with a space instead of a tab to keep the line short. - if newSpace == "" && info.canonicalInitial(newWidth) { + if newSpace == "" && info.isCanonicalInitial(newWidth) { return } if newSpace == "" { @@ -485,8 +583,8 @@ func (va *VaralignBlock) realignSingle(info *varalignLine, newWidth int) { } hasSpace := strings.IndexByte(oldSpace, ' ') != -1 - oldColumn := tabWidth(leadingComment + varnameOp + oldSpace) - column := tabWidth(leadingComment + varnameOp + newSpace) + oldColumn := tabWidthSlice(leadingComment, varnameOp, oldSpace) + column := tabWidthSlice(leadingComment, varnameOp, newSpace) fix := info.mkline.Autofix() if newSpace == " " { @@ -542,8 +640,7 @@ func (s VaralignSplitter) split(rawText string, initial bool) varalignParts { leadingComment := s.parseLeadingComment(lexer, initial) varnameOp, spaceBeforeValue := s.parseVarnameOp(parser, initial) - value, spaceAfterValue := s.parseValue(lexer) - trailingComment, spaceAfterComment, continuation := s.parseComment(lexer.Rest()) + value, spaceAfterValue, continuation := s.parseValue(lexer) return varalignParts{ leadingComment, @@ -551,8 +648,6 @@ func (s VaralignSplitter) split(rawText string, initial bool) varalignParts { spaceBeforeValue, value, spaceAfterValue, - trailingComment, - spaceAfterComment, continuation, } } @@ -588,8 +683,8 @@ func (VaralignSplitter) parseVarnameOp(parser *MkParser, initial bool) (string, return lexer.Since(mark), lexer.NextHspace() } -func (VaralignSplitter) parseValue(lexer *textproc.Lexer) (string, string) { - mark := lexer.Mark() +func (VaralignSplitter) parseValue(lexer *textproc.Lexer) (string, string, string) { + rest := lexer.Rest() for !lexer.EOF() && lexer.PeekByte() != '#' && lexer.Rest() != "\\" { @@ -603,13 +698,6 @@ func (VaralignSplitter) parseValue(lexer *textproc.Lexer) (string, string) { lexer.Skip(1) } - valueSpace := lexer.Since(mark) - value := rtrimHspace(valueSpace) - space := valueSpace[len(value):] - return value, space -} - -func (VaralignSplitter) parseComment(rest string) (string, string, string) { end := len(rest) backslash := end @@ -617,26 +705,24 @@ func (VaralignSplitter) parseComment(rest string) (string, string, string) { backslash-- } - if (end-backslash)&1 == 0 { // see https://github.com/golang/go/issues/34166 + if (end-backslash)%2 == 0 { return rest[:end], "", "" } continuation := rest[backslash:] - commentSpace := rest[:backslash] - comment := rtrimHspace(commentSpace) - space := commentSpace[len(comment):] - return comment, space, continuation + valueAndSpace := rest[:backslash] + value := rtrimHspace(valueAndSpace) + space := valueAndSpace[len(value):] + return value, space, continuation } type varalignParts struct { - leadingComment string // either the # or some rarely used U+0020 spaces - varnameOp string // empty iff it is a follow-up line - spaceBeforeValue string // for follow-up lines, this is the indentation - value string - spaceAfterValue string // only set if there is a value - trailingComment string - spaceAfterComment string // only set if there is a trailing comment - continuation string // either a single backslash or empty + leadingComment string // either the # or some rarely used U+0020 spaces + varnameOp string // empty iff it is a follow-up line + spaceBeforeValue string // for follow-up lines, this is the indentation + value string // including any trailing comment + spaceAfterValue string + continuation string // either a single backslash or empty } // continuation returns whether this line ends with a backslash. @@ -645,19 +731,19 @@ func (p *varalignParts) isContinuation() bool { } func (p *varalignParts) isEmptyContinuation() bool { - return p.value == "" && p.trailingComment == "" && p.isContinuation() + return p.value == "" && p.isContinuation() } func (p *varalignParts) isEmpty() bool { - return p.value == "" && p.trailingComment == "" && !p.isContinuation() + return p.value == "" && !p.isContinuation() } func (p *varalignParts) varnameOpWidth() int { - return tabWidth(p.leadingComment + p.varnameOp) + return tabWidthSlice(p.leadingComment, p.varnameOp) } func (p *varalignParts) varnameOpSpaceWidth() int { - return tabWidth(p.leadingComment + p.varnameOp + p.spaceBeforeValue) + return tabWidthSlice(p.leadingComment, p.varnameOp, p.spaceBeforeValue) } // spaceBeforeValueIndex returns the string index at which the space before the value starts. @@ -669,44 +755,44 @@ func (p *varalignParts) spaceBeforeValueIndex() int { } func (p *varalignParts) spaceBeforeContinuation() string { - if p.trailingComment == "" { - if p.value == "" { - return p.spaceBeforeValue - } - return p.spaceAfterValue + if p.value == "" { + return p.spaceBeforeValue } - return p.spaceAfterComment + return p.spaceAfterValue } -func (p *varalignParts) beforeContinuation() string { +func (p *varalignParts) uptoCommentWidth() int { + return tabWidth(rtrimHspace(p.leadingComment + + p.varnameOp + p.spaceBeforeValue + + p.value)) +} + +func (p *varalignParts) uptoComment() string { return rtrimHspace(p.leadingComment + p.varnameOp + p.spaceBeforeValue + - p.value + p.spaceAfterValue + - p.trailingComment + p.spaceAfterComment) + p.value) } func (p *varalignParts) continuationColumn() int { - return tabWidth(p.leadingComment + - p.varnameOp + p.spaceBeforeValue + - p.value + p.spaceAfterValue + - p.trailingComment + p.spaceAfterComment) + return tabWidthSlice(p.leadingComment, + p.varnameOp, p.spaceBeforeValue, + p.value, p.spaceAfterValue) } func (p *varalignParts) continuationIndex() int { return len(p.leadingComment) + len(p.varnameOp) + len(p.spaceBeforeValue) + - len(p.value) + len(p.spaceAfterValue) + - len(p.trailingComment) + len(p.spaceAfterComment) + len(p.value) + len(p.spaceAfterValue) } -func (p *varalignParts) commentedOut() bool { +func (p *varalignParts) isCommentedOut() bool { return hasPrefix(p.leadingComment, "#") } -// canonicalInitial returns whether the space between the assignment +// isCanonicalInitial returns whether the space between the assignment // operator and the value has its canonical form, which is either // at least one tab, or a single space, but only for lines that stick out. -func (p *varalignParts) canonicalInitial(width int) bool { +func (p *varalignParts) isCanonicalInitial(width int) bool { space := p.spaceBeforeValue if space == "" { return false @@ -719,9 +805,9 @@ func (p *varalignParts) canonicalInitial(width int) bool { return strings.TrimLeft(space, "\t") == "" } -// canonicalFollow returns whether the space before the value has its +// isCanonicalFollow returns whether the space before the value has its // canonical form, which is at least one tab, followed by up to 7 spaces. -func (p *varalignParts) canonicalFollow() bool { +func (p *varalignParts) isCanonicalFollow() bool { lexer := textproc.NewLexer(p.spaceBeforeValue) tabs := 0 @@ -740,7 +826,7 @@ func (p *varalignParts) canonicalFollow() bool { func (p *varalignParts) widthAlignedAt(valueAlign int) int { return tabWidthAppend( valueAlign, - p.value+p.spaceAfterValue+p.trailingComment+p.spaceAfterComment+p.continuation) + p.value+p.spaceAfterValue+p.continuation) } type mklineInts struct { diff --git a/pkgtools/pkglint/files/varalignblock_test.go b/pkgtools/pkglint/files/varalignblock_test.go index 43f25a1a4de..d21ed279eb3 100644 --- a/pkgtools/pkglint/files/varalignblock_test.go +++ b/pkgtools/pkglint/files/varalignblock_test.go @@ -15,6 +15,7 @@ type VaralignTester struct { suite *Suite tester *Tester input []string // The actual input lines + inputDetab []string // The expected input lines with spaces internals []string // The expected internal state, the varalignBlockInfos diagnostics []string // The expected diagnostics in default mode autofixes []string // The expected diagnostics in --autofix mode @@ -31,10 +32,17 @@ func NewVaralignTester(s *Suite, c *check.C) *VaralignTester { // Input remembers the input lines that are checked and possibly realigned. func (vt *VaralignTester) Input(lines ...string) { vt.input = lines } +// InputDetab validates the input lines after replacing tabs with spaces. +// +// Calling it is optional. +func (vt *VaralignTester) InputDetab(lines ...string) { vt.inputDetab = lines } + // Internals remembers the expected internal state of the varalignBlockInfos, // to better trace down at which points the decisions are made. // -// Each line has the format "<min-width> <actual-width>". +// Each line has the format "<min-width> <actual-width> <right-margin>". +// +// Calling it is optional. func (vt *VaralignTester) Internals(lines ...string) { vt.internals = lines } // Diagnostics remembers the expected diagnostics. @@ -71,6 +79,9 @@ func (vt *VaralignTester) run(autofix bool) { t.SetUpCommandLine(cmdline...) mklines := t.SetUpFileMkLines("Makefile", vt.input...) + if len(vt.inputDetab) > 0 { + t.CheckFileLinesDetab("Makefile", vt.inputDetab...) + } var varalign VaralignBlock for _, mkline := range mklines.mklines { @@ -171,15 +182,9 @@ func (vt *VaralignTester) checkTestName() { describeHspace(parts.spaceBeforeValue) if parts.value != "" { describe(parts.value, "value") - if parts.trailingComment != "" || parts.continuation != "" { - describeHspace(parts.spaceAfterValue) - } } - if parts.trailingComment != "" { - describe(parts.trailingComment, "comment") - if parts.continuation != "" { - describeHspace(parts.spaceAfterComment) - } + if parts.value != "" && parts.continuation != "" { + describeHspace(parts.spaceAfterValue) } if parts.continuation != "" { describe(parts.continuation, "cont") @@ -1257,7 +1262,7 @@ func (s *Suite) Test_VaralignBlock__var_tab24_value_var20_tabs72_cont_tab_value_ // For escaped variable names, the number of actual characters in the // Makefile is relevant for indenting the source code. Therefore, the -// parsed an unescaped mkline.Varname cannot be used here. +// parsed and unescaped mkline.Varname cannot be used here. func (s *Suite) Test_VaralignBlock__escaped_varname(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -1271,7 +1276,7 @@ func (s *Suite) Test_VaralignBlock__escaped_varname(c *check.C) { vt.Autofixes( "AUTOFIX: Makefile:1: Replacing \"\\t\" with \"\\t\\t\".") vt.Fixed( - "V.${v:S,\\#,,g}= value", // looks misaligned because of the backslash + "V.${v:S,\\#,,g}= value", // looks misaligned because of the Go string literal "V2345678123456781234= value") vt.Run() } @@ -2245,6 +2250,7 @@ func (s *Suite) Test_VaralignBlock__mixed_indentation(c *check.C) { "05 08 15", " 17") vt.Diagnostics( + // FIXME: This diagnostic doesn't match the autofix. "NOTE: Makefile:3: This continuation line should be indented with \"\\t\".") vt.Autofixes( "AUTOFIX: Makefile:3: Replacing \" \\t \\t \" with \"\\t\\t \".") @@ -2255,6 +2261,9 @@ func (s *Suite) Test_VaralignBlock__mixed_indentation(c *check.C) { vt.Run() } +// The follow-up line is quite short in this case, therefore it is not +// necessary to indent it with a single tab. There's enough space to +// the right so that it can be aligned at the common alignment. func (s *Suite) Test_VaralignBlock__long_line_followed_by_short_line_with_small_indentation(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -2273,6 +2282,54 @@ func (s *Suite) Test_VaralignBlock__long_line_followed_by_short_line_with_small_ vt.Run() } +// Continuation lines that are indented 2 tabs are obviously not +// space-constrained, otherwise they would use only a single tab. +// Therefore they have to be aligned with the other values. +func (s *Suite) Test_VaralignBlock__commented_cont_tab16(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "#SITES.long-distfile.tar.gz= \\", + "#\t\t-https://example.org/", + "#PATCH_DIST_STRIP=\t-p1") + vt.Internals( + "28 29 29", + " 16", + "18 24") + vt.Diagnostics( + "NOTE: Makefile:2: This continuation line should be indented with \"\\t\\t\\t\".") + vt.Autofixes( + "AUTOFIX: Makefile:2: Replacing \"\\t\\t\" with \"\\t\\t\\t\".") + vt.Fixed( + "#SITES.long-distfile.tar.gz= \\", + "# -https://example.org/", + "#PATCH_DIST_STRIP= -p1") + vt.Run() +} + +func (s *Suite) Test_VaralignBlock__shift_already_long_line_to_the__right(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "INSTALLATION_DIRS+=\tvalue", + "CONF_FILES=\t--20 -------30 -------40 -------50 -------60 -------70 \\", + "\t\t--20") + vt.Internals( + "19 24", + "11 16 71", + " 16") + vt.Diagnostics( + // FIXME: No, it shouldn't, as that would make the continuation marker invisible on 80x25. + "NOTE: Makefile:2: This variable value should be aligned to column 25.", + "NOTE: Makefile:3: This continuation line should be indented with \"\\t\\t\\t\".") + vt.Autofixes( + "AUTOFIX: Makefile:2: Replacing \"\\t\" with \"\\t\\t\".", + "AUTOFIX: Makefile:3: Replacing \"\\t\\t\" with \"\\t\\t\\t\".") + vt.Fixed( + "INSTALLATION_DIRS+= value", + "CONF_FILES= --20 -------30 -------40 -------50 -------60 -------70 \\", + " --20") + vt.Run() +} + // Ensure that the end-of-line comment is properly aligned // to the variable values. // @@ -2299,6 +2356,8 @@ func (s *Suite) Test_VaralignBlock__eol_comment(c *check.C) { vt.Run() } +// Since CONF_FILES is a list of tuples, it makes sense to have different +// indentation for the two tuple elements. func (s *Suite) Test_VaralignBlock__follow_up_indentation(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -2323,6 +2382,8 @@ func (s *Suite) Test_VaralignBlock__follow_up_indentation(c *check.C) { vt.Run() } +// For shell commands, it makes sense to have the full flexibility of +// arbitrary indentation. func (s *Suite) Test_VaralignBlock__staircase(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -2353,6 +2414,10 @@ func (s *Suite) Test_VaralignBlock__staircase(c *check.C) { // The follow-up lines may only start in column 9 if they are longer than // 72 characters. Since this is not the case in this test, they are realigned // to match the initial line. +// +// Since the variable value is a shell command and the follow-up lines contain +// its arguments only, it would only be possible to indent them by one more +// tab. But that is probably getting too special-cased. func (s *Suite) Test_VaralignBlock__command_with_arguments(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -2402,6 +2467,370 @@ func (s *Suite) Test_VaralignBlock__empty_value(c *check.C) { vt.Run() } +func (s *Suite) Test_VaralignBlock__aligned(c *check.C) { + t := s.Init(c) + + test := func(data ...interface{}) { + var lineTexts []string + for _, text := range data[:len(data)-1] { + lineTexts = append(lineTexts, text.(string)) + } + expected := data[len(data)-1].(bool) + + mklines := t.NewMkLines("filename.mk", + lineTexts...) + assert(len(mklines.mklines) == 1) + + var varalign VaralignBlock + varalign.Process(mklines.mklines[0]) + varalign.Finish() + + output := t.Output() + if expected { + t.CheckEquals(output, "") + } else if output == "" { + t.Check(output, check.Not(check.Equals), "") + } + } + + // The first line uses a space for indentation, which is typical of + // the outlier line in VaralignBlock. + // + // The second line starts in column 0, which is too far to the left. + // For a human reader the second line looks like a variable assignment + // of its own. + test( + "CONFIGURE_ENV+= \\", + "AWK=${AWK:Q}", + false) + + // The second line is indented and therefore visually distinct from + // a Makefile assignment line. Everything's fine. + test( + "CONFIGURE_ENV+= \\", + "\tAWK=${AWK:Q}", + true) + + // The first line may also use a tab instead of a space for indentation. + // This is typical of variable assignments whose name is short enough + // to be aligned with the other lines. + test( + "CONFIGURE_ENV+=\t\\", + "AWK=${AWK:Q}", + false) + test( + "CONFIGURE_ENV+=\t\\", + "\tAWK=${AWK:Q}", + true) + + // The first line contains a value, and the second line has the same + // indentation as the first line. This looks nicely aligned. + test( + "CONFIGURE_ENV+=\tAWK=${AWK:Q} \\", + "\t\tSED=${SED:Q}", + true) + + // The second line is indented less than the first line. This looks + // confusing to the human reader because the actual values do not + // appear in a rectangular shape in the source code. + test( + "VAR.param=\tvalue \\", + "\t10........20........30........40........50........60...4", + false) + + // The second line is indented with a single tab because otherwise + // it would be longer than 72 characters. In this case it is ok to + // use the smaller indentation. + test( + "VAR.param=\tvalue \\", + "\t10........20........30........40........50........60....5", + true) + + // Having the continuation line in column 0 looks even more confusing. + test( + "CONFIGURE_ENV+=\tAWK=${AWK:Q} \\", + "SED=${SED:Q}", + false) + + // Longer continuation lines may use internal indentation to represent + // AWK or shell code. + test( + "GENERATE_PLIST+=\t/pattern/ { \\", + "\t\t\t action(); \\", + "\t\t\t}", + true) + + // If any of the continuation lines is indented less than the first + // line, it looks confusing. + test( + "GENERATE_PLIST+=\t/pattern/ { \\", + "\t action(); \\", + "\t}", + false) + + // If the first line is empty, the indentation may start in column 8, + // and the continuation lines have to be indented as least as far to + // the right as the second line. + test( + "GENERATE_PLIST+= \\", + "\t/pattern/ { \\", + "\t action(); \\", + "\t}", + true) + + // The very last line is indented at column 0, therefore the whole + // line is not indented properly. + test( + "GENERATE_PLIST+= \\", + "\t/pattern/ { \\", + "\t action(); \\", + "}", + false) + + // If there is no visible variable value at all, pkglint must not crash. + // This case doesn't occur in practice since the code is usually + // succinct enough to avoid these useless lines. + // + // The first line is empty, the second line is indented to column 8 and + // the remaining lines are all indented by at least 8, therefore the + // alignment is correct. + // + // A theoretical use case might be to have a long explaining comment + // in the continuation lines, but that is not possible syntactically. + // In the line "VAR= value \# comment", the \# is interpreted as + // an escaped number sign, and not as a continuation marker followed + // by a comment. In the line "VAR= value \ # comment", the backslash + // is not a continuation marker as well, since it is not the very + // last character of the line. + test( + "CONFIGURE_ENV+= \\", + "\t\\", + "\t\\", + "\t# nothing", + true) + + // Commented variable assignments can also be tested for alignment. + test( + "#CONFIGURE_ENV+= \\", + "\tvalue", + true) + + // In commented multilines, bmake doesn't care whether the + // continuation lines does or doesn't start with a comment character. + // For human readers though, it is confusing to omit the leading + // comment character. + // + // For determining whether a multiline is aligned, the initial comment + // character is ignored. + test( + "#CONFIGURE_ENV+= \\", + "#\tvalue", + true) + + // The indentation of the continuation line is neither 8 nor the + // indentation of the first line. Therefore the line is not aligned. + test( + "#CONFIGURE_ENV+= value1 \\", + "#\t\tvalue2", + false) +} + +// It's ok to have all backslashes in the same column, even if that column +// is not 73. +func (s *Suite) Test_VaralignBlock__continuation_backslashes_aligned(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VAR=\tvalue value value\t\\", + "\tvalue\t\t\t\\", + "\tvalue\t\t\t\\", + "\tvalue") + vt.Internals( + "04 08 32", + " 08 32", + " 08 32", + " 08") + vt.Diagnostics( + nil...) + vt.Autofixes( + nil...) + vt.Fixed( + "VAR= value value value \\", + " value \\", + " value \\", + " value") + vt.Run() +} + +// The backslash in the first line is separated by a single tab. +// This looks strange but pkglint considers it acceptable +// since there is a simple rule saying "a single tab is always ok". +// Any rule that would replace this simple rule +// would have to be similarly simple and intuitive. +func (s *Suite) Test_VaralignBlock__continuation_backslashes_aligned_except_initial(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VAR=\tvalue value value\t\\", + "\tvalue\t\t\t\t\\", + "\tvalue\t\t\t\t\\", + "\tvalue") + vt.Internals( + "04 08 32", + " 08 40", + " 08 40", + " 08") + vt.Diagnostics( + nil...) + vt.Autofixes( + nil...) + // TODO: The backslashes should be aligned by a _simple_ rule. + vt.Fixed( + "VAR= value value value \\", + " value \\", + " value \\", + " value") + vt.Run() +} + +// Lines whose continuation backslash is indented by a single space are +// usually those that stick out further than column 73. These are not +// touched by the realignment. +func (s *Suite) Test_VaralignBlock__continuation_backslashes_one_sticks_out(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VAR=\tvalue\t\\", + "\tvalue value value \\", + "\tvalue\t\\", + "\tvalue") + vt.Internals( + "04 08 16", + " 08 26", + " 08 16", + " 08") + vt.Diagnostics( + nil...) + vt.Autofixes( + nil...) + vt.Fixed( + "VAR= value \\", + " value value value \\", + " value \\", + " value") + vt.Run() +} + +func (s *Suite) Test_VaralignBlock__realign_continuation_backslashes(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VAR4567890.234567890=\t----30--------40--------50\t\t\t\\", + "\t\t--20--------30--------40--------50\t\t\t\\", + "\t\t--20--------30--------40--------50") + vt.InputDetab( + "VAR4567890.234567890= ----30--------40--------50 \\", + " --20--------30--------40--------50 \\", + " --20--------30--------40--------50") + vt.Internals( + "21 24 72", + " 16 72", + " 16") + vt.Diagnostics( + "NOTE: Makefile:2: This continuation line should be indented with \"\\t\\t\\t\".", + "NOTE: Makefile:3: This continuation line should be indented with \"\\t\\t\\t\".") + vt.Autofixes( + "AUTOFIX: Makefile:2: Replacing \"\\t\\t\" with \"\\t\\t\\t\".", + "AUTOFIX: Makefile:2: Replacing \"\\t\\t\\t\\\\\" with \"\\t\\t\\\\\".", + "AUTOFIX: Makefile:3: Replacing \"\\t\\t\" with \"\\t\\t\\t\".") + vt.Fixed( + "VAR4567890.234567890= ----30--------40--------50 \\", + " --20--------30--------40--------50 \\", + " --20--------30--------40--------50") + vt.Run() +} + +func (s *Suite) Test_VaralignBlock__initial_value_tab80(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VVVVVVVVVVVVVVVVVVV=\tvalue\t\t\t\t\t\t\t\\", + "\t\t\tvalue\t\t\t\t\t\t\\", + "\t\t\tvalue\t\t\t\t\t\t\\", + "\t\t\tvalue") + vt.Internals( + "20 24 80", + " 24 72", + " 24 72", + " 24") + vt.Diagnostics( + "NOTE: Makefile:1: The continuation backslash should be preceded " + + "by a single space or tab, or be in column 73, not 81.") + vt.Autofixes( + "AUTOFIX: Makefile:1: Replacing \"\\t\\t\\t\\t\\t\\t\\t\" with \"\\t\\t\\t\\t\\t\\t\".") + vt.Fixed( + "VVVVVVVVVVVVVVVVVVV= value \\", + " value \\", + " value \\", + " value") + vt.Run() +} + +func (s *Suite) Test_VaralignBlock__long_lines(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VAR=\t\t\t\t\t\tvalue\t\t \\", + "\tvalue \t \\", + "\tvalue") + vt.Internals( + "04 48 65", + " 08 17", + " 08") + vt.Diagnostics( + "NOTE: Makefile:1: The continuation backslash should be preceded "+ + "by a single space or tab, or be in column 57, not 66.", + "NOTE: Makefile:2: This continuation line should be indented with \"\\t\\t\\t\\t\\t\\t\".", + "NOTE: Makefile:3: This continuation line should be indented with \"\\t\\t\\t\\t\\t\\t\".") + vt.Autofixes( + "AUTOFIX: Makefile:1: Replacing \"\\t\\t \" with \"\\t\".", + "AUTOFIX: Makefile:2: Replacing \"\\t\" with \"\\t\\t\\t\\t\\t\\t\".", + "AUTOFIX: Makefile:3: Replacing \"\\t\" with \"\\t\\t\\t\\t\\t\\t\".") + vt.Fixed( + "VAR= value \\", + // FIXME: The backslash should be aligned properly. + " value \\", + " value") + vt.Run() +} + +// A practical chaotic test case, derived from wip/compat32_mit-krb5/Makefile. +// It made pkglint before 2019-09-03 panic. +func (s *Suite) Test_VaralignBlock__long_lines_2(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "INSTALLATION_DIRS=\t_____________________________________________________________________ \\"+ + "\t\t\t\t __________________________________________________________\t\t \\", + "\t\t\t__________________________________________________________\t\t \t\\", + "\t\t\t__________________________________________________________\t\t \t\\", + "\t\t\t_________________________") + vt.InputDetab( + "INSTALLATION_DIRS= _____________________________________________________________________ \\ __________________________________________________________ \\", + " __________________________________________________________ \\", + " __________________________________________________________ \\", + " _________________________") + vt.Internals( + "18 24 201", + " 24 104", + " 24 104", + " 24") + vt.Diagnostics( + "NOTE: Makefile:1: The continuation backslash should be preceded by a single space or tab.") + vt.Autofixes( + "AUTOFIX: Makefile:1: Replacing \"\\t\\t \" with \" \".") + vt.Fixed( + "INSTALLATION_DIRS= _____________________________________________________________________ \\"+ + " __________________________________________________________ \\", + " __________________________________________________________ \\", + " __________________________________________________________ \\", + " _________________________") + vt.Run() +} + func (s *Suite) Test_VaralignBlock_Process__var_spaces7_value(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -2614,26 +3043,6 @@ func (s *Suite) Test_VaralignBlock_realignMultiEmptyInitial__spaces(c *check.C) vt.Run() } -func (s *Suite) Test_VaralignBlock_realignMultiInitial__spaces(c *check.C) { - vt := NewVaralignTester(s, c) - vt.Input( - "VAR= value1 \\", - " value2") - vt.Internals( - "04 08 15", - " 08") - vt.Diagnostics( - "NOTE: Makefile:1: Variable values should be aligned with tabs, not spaces.", - "NOTE: Makefile:2: This continuation line should be indented with \"\\t\".") - vt.Autofixes( - "AUTOFIX: Makefile:1: Replacing \" \" with \"\\t\".", - "AUTOFIX: Makefile:2: Replacing \" \" with \"\\t\".") - vt.Fixed( - "VAR= value1 \\", - " value2") - vt.Run() -} - // This example is quite unrealistic since typically the first line is // the least indented. // @@ -2682,188 +3091,136 @@ func (s *Suite) Test_VaralignBlock_realignMultiEmptyFollow(c *check.C) { vt.Run() } -// It's ok to have all backslashes in the same column, even if that column -// is not 73. -func (s *Suite) Test_VaralignBlock__continuation_backslashes_aligned(c *check.C) { - vt := NewVaralignTester(s, c) - vt.Input( - "VAR=\tvalue value value\t\\", - "\tvalue\t\t\t\\", - "\tvalue\t\t\t\\", - "\tvalue") - vt.Internals( - "04 08 32", - " 08 32", - " 08 32", - " 08") - vt.Diagnostics( - nil...) - vt.Autofixes( - nil...) - vt.Fixed( - "VAR= value value value \\", - " value \\", - " value \\", - " value") - vt.Run() -} - -// The first line is indented with a single tab. This looks strange but -// pkglint considers it acceptable since there is a simple rule saying -// "a single tab is always ok". Any rule that would replace this simple -// rule would have to be similarly simple and intuitive. -func (s *Suite) Test_VaralignBlock__continuation_backslashes_aligned_except_initial(c *check.C) { - vt := NewVaralignTester(s, c) - vt.Input( - "VAR=\tvalue value value\t\\", - "\tvalue\t\t\t\t\\", - "\tvalue\t\t\t\t\\", - "\tvalue") - vt.Internals( - "04 08 32", - " 08 40", - " 08 40", - " 08") - vt.Diagnostics( - nil...) - vt.Autofixes( - nil...) - // TODO: The backslashes should be aligned by a _simple_ rule. - vt.Fixed( - "VAR= value value value \\", - " value \\", - " value \\", - " value") - vt.Run() -} - -// Lines whose continuation backslash is indented by a single space are -// usually those that stick out further than column 73. These are not -// touched by the realignment. -func (s *Suite) Test_VaralignBlock__continuation_backslashes_one_sticks_out(c *check.C) { - vt := NewVaralignTester(s, c) - vt.Input( - "VAR=\tvalue\t\\", - "\tvalue value value \\", - "\tvalue\t\\", - "\tvalue") - vt.Internals( - "04 08 16", - " 08 26", - " 08 16", - " 08") - vt.Diagnostics( - nil...) - vt.Autofixes( - nil...) - vt.Fixed( - "VAR= value \\", - " value value value \\", - " value \\", - " value") - vt.Run() -} - -func (s *Suite) Test_VaralignBlock__initial_value_tab80(c *check.C) { - vt := NewVaralignTester(s, c) - vt.Input( - "VVVVVVVVVVVVVVVVVVV=\tvalue\t\t\t\t\t\t\t\\", - "\t\t\tvalue\t\t\t\t\t\t\\", - "\t\t\tvalue\t\t\t\t\t\t\\", - "\t\t\tvalue") - vt.Internals( - "20 24 80", - " 24 72", - " 24 72", - " 24") - vt.Diagnostics( - "NOTE: Makefile:1: The continuation backslash should be preceded " + - "by a single space or tab, or be in column 73, not 81.") - vt.Autofixes( - "AUTOFIX: Makefile:1: Replacing \"\\t\\t\\t\\t\\t\\t\\t\" with \"\\t\\t\\t\\t\\t\\t\".") - vt.Fixed( - "VVVVVVVVVVVVVVVVVVV= value \\", - " value \\", - " value \\", - " value") - vt.Run() -} - -func (s *Suite) Test_VaralignBlock__long_lines(c *check.C) { +func (s *Suite) Test_VaralignBlock_realignMultiInitial__spaces(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( - "VAR=\t\t\t\t\t\tvalue\t\t \\", - "\tvalue \t \\", - "\tvalue") + "VAR= value1 \\", + " value2") vt.Internals( - "04 48 65", - " 08 17", + "04 08 15", " 08") vt.Diagnostics( - "NOTE: Makefile:1: The continuation backslash should be preceded "+ - "by a single space or tab, or be in column 57, not 66.", - "NOTE: Makefile:2: This continuation line should be indented with \"\\t\\t\\t\\t\\t\\t\".", - "NOTE: Makefile:3: This continuation line should be indented with \"\\t\\t\\t\\t\\t\\t\".") + "NOTE: Makefile:1: Variable values should be aligned with tabs, not spaces.", + "NOTE: Makefile:2: This continuation line should be indented with \"\\t\".") vt.Autofixes( - "AUTOFIX: Makefile:1: Replacing \"\\t\\t \" with \"\\t\".", - "AUTOFIX: Makefile:2: Replacing \"\\t\" with \"\\t\\t\\t\\t\\t\\t\".", - "AUTOFIX: Makefile:3: Replacing \"\\t\" with \"\\t\\t\\t\\t\\t\\t\".") + "AUTOFIX: Makefile:1: Replacing \" \" with \"\\t\".", + "AUTOFIX: Makefile:2: Replacing \" \" with \"\\t\".") vt.Fixed( - "VAR= value \\", - // FIXME: The backslash should be aligned properly. - " value \\", - " value") + "VAR= value1 \\", + " value2") vt.Run() } -// A practical chaotic test case, derived from wip/compat32_mit-krb5/Makefile. -// It made pkglint before 2019-09-03 panic. -func (s *Suite) Test_VaralignBlock__long_lines_2(c *check.C) { +func (s *Suite) Test_VaralignBlock_realignMultiFollow__unindent_long_lines(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( - "INSTALLATION_DIRS=\t_____________________________________________________________________ \\"+ - "\t\t\t\t __________________________________________________________\t\t \\", - "\t\t\t__________________________________________________________\t\t \t\\", - "\t\t\t__________________________________________________________\t\t \t\\", - "\t\t\t_________________________") + "SHORT=\tvalue", + "PROGRAM_AWK=\t\t\t\t--------50--------60--------70 \\", + "\t\t\t\t\t\t\t\t\t3 \\", + "\t\t\t\t\t\t\t\t\t74 \\", + "\t\t\t\t\t\t\t\t\t-75 \t\t\t \\", + "\t\t\t\t\t\t\t\t\t--76 \\", + "\t\t\t\t\t\t\t\t66 \\", + "\t\t\t\t\t\t\t\t1") + vt.InputDetab( + "SHORT= value", + "PROGRAM_AWK= --------50--------60--------70 \\", + " 3 \\", + " 74 \\", + " -75 \\", + " --76 \\", + " 66 \\", + " 1") vt.Internals( - "18 24 201", - " 24 104", - " 24 104", - " 24") + "06 08", + "12 40 71", + " 72 89", + " 72 89", + " 72 98", + " 72 77", + " 64 67", + " 64") vt.Diagnostics( - "NOTE: Makefile:1: The continuation backslash should be preceded by a single space or tab.") + "NOTE: Makefile:1: This variable value should be aligned to column 17.", + "NOTE: Makefile:2: This variable value should be aligned to column 17.", + "NOTE: Makefile:3: This continuation line should be indented with \"\\t\\t\".", + "NOTE: Makefile:4: This continuation line should be indented with \"\\t\\t\".", + "NOTE: Makefile:5: The continuation backslash should be preceded by a single space or tab, or be in column 90, not 99.", + "NOTE: Makefile:5: This continuation line should be indented with \"\\t\\t\".", + "NOTE: Makefile:6: This continuation line should be indented with \"\\t\\t\".", + "NOTE: Makefile:7: This continuation line should be indented with \"\\t\\t\".", + "NOTE: Makefile:8: This continuation line should be indented with \"\\t\\t\".") vt.Autofixes( - "AUTOFIX: Makefile:1: Replacing \"\\t\\t \" with \" \".") - vt.Fixed( - "INSTALLATION_DIRS= _____________________________________________________________________ \\"+ - " __________________________________________________________ \\", - " __________________________________________________________ \\", - " __________________________________________________________ \\", - " _________________________") - vt.Run() -} - -// I've never seen an intentionally continued comment in practice, -// but pkglint needs to be able to handle this situation anyway. -func (s *Suite) Test_varalignParts_spaceBeforeContinuation__continued_comment(c *check.C) { - vt := NewVaralignTester(s, c) - vt.Input( - "VAR=\tvalue # comment \\", - "\tstill comment \\", - "\tand still") - vt.Internals( - "04 08 24", - " 08 22", - " 08") + "AUTOFIX: Makefile:1: Replacing \"\\t\" with \"\\t\\t\".", + "AUTOFIX: Makefile:2: Replacing \"\\t\\t\\t\\t\" with \"\\t\".", + "AUTOFIX: Makefile:3: Replacing \"\\t\\t\\t\\t\\t\\t\\t\\t\\t\" with \"\\t\\t\\t\\t\\t\\t\".", + "AUTOFIX: Makefile:4: Replacing \"\\t\\t\\t\\t\\t\\t\\t\\t\\t\" with \"\\t\\t\\t\\t\\t\\t\".", + "AUTOFIX: Makefile:5: Replacing \" \\t\\t\\t \" with \"\\t\\t \".", + "AUTOFIX: Makefile:5: Replacing \"\\t\\t\\t\\t\\t\\t\\t\\t\\t\" with \"\\t\\t\\t\\t\\t\\t\".", + "AUTOFIX: Makefile:6: Replacing \"\\t\\t\\t\\t\\t\\t\\t\\t\\t\" with \"\\t\\t\\t\\t\\t\\t\".", + "AUTOFIX: Makefile:7: Replacing \"\\t\\t\\t\\t\\t\\t\\t\\t\" with \"\\t\\t\\t\\t\\t\".", + "AUTOFIX: Makefile:8: Replacing \"\\t\\t\\t\\t\\t\\t\\t\\t\" with \"\\t\\t\\t\\t\\t\".") + vt.Fixed( + // After shifting the lines to the left, none of the lines is + // considered "long" anymore, therefore the backslashes are not + // kept in column 72. Nevertheless they look unorganized right now. + "SHORT= value", + "PROGRAM_AWK= --------50--------60--------70 \\", + // FIXME: only use a single space before the backslash. + " 3 \\", + // FIXME: only use a single space before the backslash. + " 74 \\", + // FIXME: only use a single space before the backslash. + " -75 \\", + " --76 \\", + " 66 \\", + " 1") + vt.Run() +} + +func (s *Suite) Test_VaralignBlock_realignMultiFollow__unindent_long_initial_line(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VAR-----10!=\t\t----30--------40--------50-----6\t\t\t\\", + "\t\t --------30--------40-\t\t\t\t\\", + "\t\t --------30--------40--------50--------60-------8\t\\", + "\t\t ----5\t\t\t\t\t\t\\", + "\t\t-7") + vt.InputDetab( + "VAR-----10!= ----30--------40--------50-----6 \\", + " --------30--------40- \\", + " --------30--------40--------50--------60-------8 \\", + " ----5 \\", + " -7") + vt.Internals( + "12 24 80", + " 20 72", + " 20 72", + " 20 72", + " 16") vt.Diagnostics( - nil...) + "NOTE: Makefile:1: The continuation backslash should be preceded by a single space or tab, or be in column 73, not 81.", + "NOTE: Makefile:2: This continuation line should be indented with \"\\t\\t\\t\".", + "NOTE: Makefile:3: This continuation line should be indented with \"\\t\\t\\t\".", + "NOTE: Makefile:4: This continuation line should be indented with \"\\t\\t\\t\".", + "NOTE: Makefile:5: This continuation line should be indented with \"\\t\\t\\t\".") vt.Autofixes( - nil...) + // FIXME: Mention the continuation backslash in the replacement. + "AUTOFIX: Makefile:1: Replacing \"\\t\\t\\t\" with \"\\t\\t\".", + "AUTOFIX: Makefile:2: Replacing \"\\t\\t \" with \"\\t\\t\\t\".", + "AUTOFIX: Makefile:3: Replacing \"\\t\\t \" with \"\\t\\t\\t\".", + "AUTOFIX: Makefile:3: Replacing \"\\t\\\\\" with \" \\\\\".", + "AUTOFIX: Makefile:4: Replacing \"\\t\\t \" with \"\\t\\t\\t\".", + "AUTOFIX: Makefile:5: Replacing \"\\t\\t\" with \"\\t\\t\\t\".") vt.Fixed( - "VAR= value # comment \\", - " still comment \\", - " and still") + "VAR-----10!= ----30--------40--------50-----6 \\", + // FIXME: Preserve the original relative indentation. + " --------30--------40- \\", + // FIXME: Preserve the original relative indentation. + " --------30--------40--------50--------60-------8 \\", + // FIXME: Preserve the original relative indentation. + " ----5 \\", + " -7") vt.Run() } @@ -2876,8 +3233,8 @@ func (s *Suite) Test_VaralignSplitter_split(c *check.C) { t.CheckEquals(actual, expected) t.CheckEquals( actual.leadingComment+actual.varnameOp+ - actual.spaceBeforeValue+actual.value+actual.spaceAfterValue+ - actual.trailingComment+actual.spaceAfterComment+actual.continuation, + actual.spaceBeforeValue+actual.value+ + actual.spaceAfterValue+actual.continuation, rawText) } @@ -2885,177 +3242,147 @@ func (s *Suite) Test_VaralignSplitter_split(c *check.C) { // code in VaralignBlock.processVarassign, see INCLUSION_GUARD_MK. test("VAR=", true, varalignParts{ - leadingComment: "", - varnameOp: "VAR=", - spaceBeforeValue: "", - value: "", - spaceAfterValue: "", - trailingComment: "", - spaceAfterComment: "", - continuation: ""}) + leadingComment: "", + varnameOp: "VAR=", + spaceBeforeValue: "", + value: "", + spaceAfterValue: "", + continuation: ""}) test("VAR=value", true, varalignParts{ - leadingComment: "", - varnameOp: "VAR=", - spaceBeforeValue: "", - value: "value", - spaceAfterValue: "", - trailingComment: "", - spaceAfterComment: "", - continuation: ""}) + leadingComment: "", + varnameOp: "VAR=", + spaceBeforeValue: "", + value: "value", + spaceAfterValue: "", + continuation: ""}) test("#VAR=value", true, varalignParts{ - leadingComment: "#", - varnameOp: "VAR=", - spaceBeforeValue: "", - value: "value", - spaceAfterValue: "", - trailingComment: "", - spaceAfterComment: "", - continuation: ""}) + leadingComment: "#", + varnameOp: "VAR=", + spaceBeforeValue: "", + value: "value", + spaceAfterValue: "", + continuation: ""}) test("#VAR = value # comment \\", true, varalignParts{ - leadingComment: "#", - varnameOp: "VAR =", - spaceBeforeValue: " ", - value: "value", - spaceAfterValue: " ", - trailingComment: "# comment", - spaceAfterComment: " ", - continuation: "\\"}) + leadingComment: "#", + varnameOp: "VAR =", + spaceBeforeValue: " ", + value: "value # comment", + spaceAfterValue: " ", + continuation: "\\"}) test("VAR=value \\", true, varalignParts{ - leadingComment: "", - varnameOp: "VAR=", - spaceBeforeValue: "", - value: "value", - spaceAfterValue: " ", - trailingComment: "", - spaceAfterComment: "", - continuation: "\\"}) + leadingComment: "", + varnameOp: "VAR=", + spaceBeforeValue: "", + value: "value", + spaceAfterValue: " ", + continuation: "\\"}) test("VAR=value # comment \\", true, varalignParts{ - leadingComment: "", - varnameOp: "VAR=", - spaceBeforeValue: "", - value: "value", - spaceAfterValue: " ", - trailingComment: "# comment", - spaceAfterComment: " ", - continuation: "\\"}) + leadingComment: "", + varnameOp: "VAR=", + spaceBeforeValue: "", + value: "value # comment", + spaceAfterValue: " ", + continuation: "\\"}) test("VAR=value # comment \\\\", true, varalignParts{ - leadingComment: "", - varnameOp: "VAR=", - spaceBeforeValue: "", - value: "value", - spaceAfterValue: " ", - trailingComment: "# comment \\\\", - spaceAfterComment: "", - continuation: ""}) + leadingComment: "", + varnameOp: "VAR=", + spaceBeforeValue: "", + value: "value # comment \\\\", + spaceAfterValue: "", + continuation: ""}) test("VAR=\\# a [#] b # comment \\\\", true, varalignParts{ - leadingComment: "", - varnameOp: "VAR=", - spaceBeforeValue: "", - value: "\\# a [#] b", - spaceAfterValue: " ", - trailingComment: "# comment \\\\", - spaceAfterComment: "", - continuation: ""}) + leadingComment: "", + varnameOp: "VAR=", + spaceBeforeValue: "", + value: "\\# a [#] b # comment \\\\", + spaceAfterValue: "", + continuation: ""}) test("VAR.${param:[#]}=\tvalue", true, varalignParts{ - leadingComment: "", - varnameOp: "VAR.${param:[#]}=", - spaceBeforeValue: "\t", - value: "value", - spaceAfterValue: "", - trailingComment: "", - spaceAfterComment: "", - continuation: ""}) + leadingComment: "", + varnameOp: "VAR.${param:[#]}=", + spaceBeforeValue: "\t", + value: "value", + spaceAfterValue: "", + continuation: ""}) test("VAR=value", true, varalignParts{ - leadingComment: "", - varnameOp: "VAR=", - spaceBeforeValue: "", - value: "value", - spaceAfterValue: "", - trailingComment: "", - spaceAfterComment: "", - continuation: ""}) + leadingComment: "", + varnameOp: "VAR=", + spaceBeforeValue: "", + value: "value", + spaceAfterValue: "", + continuation: ""}) // Since this is a follow-up line, the text ends up in the variable // value, and varnameOp is necessarily empty. test("VAR=value", false, varalignParts{ - leadingComment: "", - varnameOp: "", - spaceBeforeValue: "", - value: "VAR=value", - spaceAfterValue: "", - trailingComment: "", - spaceAfterComment: "", - continuation: ""}) + leadingComment: "", + varnameOp: "", + spaceBeforeValue: "", + value: "VAR=value", + spaceAfterValue: "", + continuation: ""}) // In some edge cases the variable name is indented with ordinary spaces. // This must not lead to a panic. test(" VAR=value", true, varalignParts{ - leadingComment: " ", - varnameOp: "VAR=", - spaceBeforeValue: "", - value: "value", - spaceAfterValue: "", - trailingComment: "", - spaceAfterComment: "", - continuation: ""}) + leadingComment: " ", + varnameOp: "VAR=", + spaceBeforeValue: "", + value: "value", + spaceAfterValue: "", + continuation: ""}) // And in really edgy cases, the leading space may even be followed by tabs. // This should not happen in practice since it is really confusing. test(" \t VAR=value", true, varalignParts{ - leadingComment: " \t ", - varnameOp: "VAR=", - spaceBeforeValue: "", - value: "value", - spaceAfterValue: "", - trailingComment: "", - spaceAfterComment: "", - continuation: ""}) + leadingComment: " \t ", + varnameOp: "VAR=", + spaceBeforeValue: "", + value: "value", + spaceAfterValue: "", + continuation: ""}) test(" value", false, varalignParts{ - leadingComment: "", - varnameOp: "", - spaceBeforeValue: " ", - value: "value", - spaceAfterValue: "", - trailingComment: "", - spaceAfterComment: "", - continuation: ""}) + leadingComment: "", + varnameOp: "", + spaceBeforeValue: " ", + value: "value", + spaceAfterValue: "", + continuation: ""}) // In practice it doesn't really happen that the last line of a file // ends in a backslash and at the same time it doesn't have the usual // newline ending. test(" value \\", false, varalignParts{ - leadingComment: "", - varnameOp: "", - spaceBeforeValue: " ", - value: "value", - spaceAfterValue: " ", - trailingComment: "", - spaceAfterComment: "", - continuation: "\\"}) + leadingComment: "", + varnameOp: "", + spaceBeforeValue: " ", + value: "value", + spaceAfterValue: " ", + continuation: "\\"}) // A follow-up line may start with a comment character. There are // two possible interpretations: @@ -3073,47 +3400,39 @@ func (s *Suite) Test_VaralignSplitter_split(c *check.C) { test("#\tcomment", false, varalignParts{ - leadingComment: "#", - varnameOp: "", - spaceBeforeValue: "\t", - value: "comment", - spaceAfterValue: "", - trailingComment: "", - spaceAfterComment: "", - continuation: ""}) + leadingComment: "#", + varnameOp: "", + spaceBeforeValue: "\t", + value: "comment", + spaceAfterValue: "", + continuation: ""}) test("#\tcomment \\", false, varalignParts{ - leadingComment: "#", - varnameOp: "", - spaceBeforeValue: "\t", - value: "comment", - spaceAfterValue: " ", - trailingComment: "", - spaceAfterComment: "", - continuation: "\\"}) + leadingComment: "#", + varnameOp: "", + spaceBeforeValue: "\t", + value: "comment", + spaceAfterValue: " ", + continuation: "\\"}) test("# comment", false, varalignParts{ - leadingComment: "", - varnameOp: "", - spaceBeforeValue: "", - value: "", - spaceAfterValue: "", - trailingComment: "# comment", - spaceAfterComment: "", - continuation: ""}) + leadingComment: "", + varnameOp: "", + spaceBeforeValue: "", + value: "# comment", + spaceAfterValue: "", + continuation: ""}) test("# comment \\", false, varalignParts{ - leadingComment: "", - varnameOp: "", - spaceBeforeValue: "", - value: "", - spaceAfterValue: "", - trailingComment: "# comment", - spaceAfterComment: " ", - continuation: "\\"}) + leadingComment: "", + varnameOp: "", + spaceBeforeValue: "", + value: "# comment", + spaceAfterValue: " ", + continuation: "\\"}) // Commented variable assignments are only valid if they // directly follow the comment sign. @@ -3127,27 +3446,60 @@ func (s *Suite) Test_VaralignSplitter_split(c *check.C) { func() { test("VAR=\tvalue\n", true, varalignParts{}) }) } -// This test runs canonicalInitial directly since as of August 2019 +// This constellation doesn't occur in practice because the code in +// VaralignBlock.processVarassign skips it, see INCLUSION_GUARD_MK. +func (s *Suite) Test_varalignParts_isEmptyContinuation__edge_case(c *check.C) { + t := s.Init(c) + + parts := NewVaralignSplitter().split("VAR=", true) + + t.CheckEquals(parts.isEmptyContinuation(), false) +} + +// I've never seen an intentionally continued comment in practice, +// but pkglint needs to be able to handle this situation anyway. +func (s *Suite) Test_varalignParts_spaceBeforeContinuation__continued_comment(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VAR=\tvalue # comment \\", + "\tstill comment \\", + "\tand still") + vt.Internals( + "04 08 24", + " 08 22", + " 08") + vt.Diagnostics( + nil...) + vt.Autofixes( + nil...) + vt.Fixed( + "VAR= value # comment \\", + " still comment \\", + " and still") + vt.Run() +} + +// This test runs isCanonicalInitial directly since as of August 2019 // that function is only used in a single place, and from this place // varnameOpSpaceWidth is always bigger than width. -func (s *Suite) Test_varalignParts_canonicalInitial(c *check.C) { +func (s *Suite) Test_varalignParts_isCanonicalInitial(c *check.C) { t := s.Init(c) var v varalignLine v.varnameOp = "LONG.123456789=" v.spaceBeforeValue = " " - t.CheckEquals(v.canonicalInitial(16), false) + t.CheckEquals(v.isCanonicalInitial(16), false) v.varnameOp = "LONG.1234567890=" - t.CheckEquals(v.canonicalInitial(16), true) + t.CheckEquals(v.isCanonicalInitial(16), true) v.spaceBeforeValue = "" - t.CheckEquals(v.canonicalInitial(16), false) + t.CheckEquals(v.isCanonicalInitial(16), false) } -func (s *Suite) Test_varalignParts_canonicalFollow(c *check.C) { +func (s *Suite) Test_varalignParts_isCanonicalFollow(c *check.C) { t := s.Init(c) test := func(comment, space string, expected bool) { @@ -3156,7 +3508,7 @@ func (s *Suite) Test_varalignParts_canonicalFollow(c *check.C) { leadingComment: comment, spaceBeforeValue: space}} - actual := l.canonicalFollow() + actual := l.isCanonicalFollow() t.CheckEquals(actual, expected) } @@ -3186,13 +3538,3 @@ func (s *Suite) Test_varalignParts_canonicalFollow(c *check.C) { test("#", " ", false) test("#", "\t", true) } - -// This constellation doesn't occur in practice because the code in -// VaralignBlock.processVarassign skips it, see INCLUSION_GUARD_MK. -func (s *Suite) Test_varalignParts_isEmptyContinuation__edge_case(c *check.C) { - t := s.Init(c) - - parts := NewVaralignSplitter().split("VAR=", true) - - t.CheckEquals(parts.isEmptyContinuation(), false) -} diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go index 42ded03a3a0..cf312c606c2 100644 --- a/pkgtools/pkglint/files/vardefs.go +++ b/pkgtools/pkglint/files/vardefs.go @@ -44,11 +44,11 @@ func (reg *VarTypeRegistry) Canon(varname string) *Vartype { return vartype } -func (reg *VarTypeRegistry) DefinedExact(varname string) bool { +func (reg *VarTypeRegistry) IsDefinedExact(varname string) bool { return reg.types[varname] != nil } -func (reg *VarTypeRegistry) DefinedCanon(varname string) bool { +func (reg *VarTypeRegistry) IsDefinedCanon(varname string) bool { return reg.Canon(varname) != nil } @@ -95,7 +95,7 @@ func (reg *VarTypeRegistry) DefineParse(varname string, basicType *BasicType, op // - why the predefined permission set is not good enough // - which packages need this custom permission set. func (reg *VarTypeRegistry) acl(varname string, basicType *BasicType, options vartypeOptions, aclEntries ...string) { - assertf(!reg.DefinedExact(varname), "Variable %q must only be defined once.", varname) + assertf(!reg.IsDefinedExact(varname), "Variable %q must only be defined once.", varname) reg.DefineParse(varname, basicType, options, aclEntries...) } diff --git a/pkgtools/pkglint/files/vardefs_test.go b/pkgtools/pkglint/files/vardefs_test.go index e114a4dcd27..7d523959985 100644 --- a/pkgtools/pkglint/files/vardefs_test.go +++ b/pkgtools/pkglint/files/vardefs_test.go @@ -2,16 +2,6 @@ package pkglint import "gopkg.in/check.v1" -func (s *Suite) Test_VarTypeRegistry_Init(c *check.C) { - t := s.Init(c) - - src := NewPkgsrc(t.File(".")) - src.vartypes.Init(&src) - - t.CheckEquals(src.vartypes.Canon("BSD_MAKE_ENV").basicType.name, "ShellWord") - t.CheckEquals(src.vartypes.Canon("USE_BUILTIN.*").basicType.name, "YesNoIndirectly") -} - func (s *Suite) Test_VarTypeRegistry_compilerLanguages(c *check.C) { t := s.Init(c) @@ -140,40 +130,14 @@ func (s *Suite) Test_VarTypeRegistry_enumFromFiles(c *check.C) { test("OPSYS", "enum: NetBSD SunOS (system-provided)") } -func (s *Suite) Test_VarTypeRegistry_parseACLEntries__invalid_arguments(c *check.C) { +func (s *Suite) Test_VarTypeRegistry_Init(c *check.C) { t := s.Init(c) - reg := NewVarTypeRegistry() - parseACLEntries := reg.parseACLEntries - - t.ExpectPanic( - func() { parseACLEntries("VARNAME", "buildlink3.mk: *", "*: *") }, - "Pkglint internal error: "+ - "Invalid ACL permission \"*\" for \"VARNAME\" in \"buildlink3.mk\". "+ - "Remaining parts are \"*\". "+ - "Valid permissions are default, set, append, use, use-loadtime (in this order), or none.") - - t.ExpectPanic( - func() { parseACLEntries("VARNAME", "buildlink3.mk: use", "*: use") }, - "Pkglint internal error: Repeated permissions \"use\" for \"VARNAME\".") - - t.ExpectPanic( - func() { parseACLEntries("VARNAME", "*.txt: use") }, - "Pkglint internal error: Invalid ACL glob \"*.txt\" for \"VARNAME\".") - - t.ExpectPanic( - func() { parseACLEntries("VARNAME", "*.mk: use", "buildlink3.mk: append") }, - "Pkglint internal error: Unreachable ACL pattern \"buildlink3.mk\" for \"VARNAME\".") - - t.ExpectPanic( - func() { parseACLEntries("VARNAME", "no colon") }, - "Pkglint internal error: ACL entry \"no colon\" must have exactly 1 colon.") - - t.ExpectPanic( - func() { parseACLEntries("VARNAME", "too: many: colons") }, - "Pkglint internal error: ACL entry \"too: many: colons\" must have exactly 1 colon.") + src := NewPkgsrc(t.File(".")) + src.vartypes.Init(&src) - t.ExpectAssert(func() { parseACLEntries("VAR") }) + t.CheckEquals(src.vartypes.Canon("BSD_MAKE_ENV").basicType.name, "ShellWord") + t.CheckEquals(src.vartypes.Canon("USE_BUILTIN.*").basicType.name, "YesNoIndirectly") } func (s *Suite) Test_VarTypeRegistry_Init__LP64PLATFORMS(c *check.C) { @@ -232,3 +196,39 @@ func (s *Suite) Test_VarTypeRegistry_Init__MASTER_SITES(c *check.C) { vartype := G.Pkgsrc.VariableType(nil, "MASTER_SITE_GITHUB") t.CheckEquals(vartype.String(), "FetchURL (list, system-provided)") } + +func (s *Suite) Test_VarTypeRegistry_parseACLEntries__invalid_arguments(c *check.C) { + t := s.Init(c) + + reg := NewVarTypeRegistry() + parseACLEntries := reg.parseACLEntries + + t.ExpectPanic( + func() { parseACLEntries("VARNAME", "buildlink3.mk: *", "*: *") }, + "Pkglint internal error: "+ + "Invalid ACL permission \"*\" for \"VARNAME\" in \"buildlink3.mk\". "+ + "Remaining parts are \"*\". "+ + "Valid permissions are default, set, append, use, use-loadtime (in this order), or none.") + + t.ExpectPanic( + func() { parseACLEntries("VARNAME", "buildlink3.mk: use", "*: use") }, + "Pkglint internal error: Repeated permissions \"use\" for \"VARNAME\".") + + t.ExpectPanic( + func() { parseACLEntries("VARNAME", "*.txt: use") }, + "Pkglint internal error: Invalid ACL glob \"*.txt\" for \"VARNAME\".") + + t.ExpectPanic( + func() { parseACLEntries("VARNAME", "*.mk: use", "buildlink3.mk: append") }, + "Pkglint internal error: Unreachable ACL pattern \"buildlink3.mk\" for \"VARNAME\".") + + t.ExpectPanic( + func() { parseACLEntries("VARNAME", "no colon") }, + "Pkglint internal error: ACL entry \"no colon\" must have exactly 1 colon.") + + t.ExpectPanic( + func() { parseACLEntries("VARNAME", "too: many: colons") }, + "Pkglint internal error: ACL entry \"too: many: colons\" must have exactly 1 colon.") + + t.ExpectAssert(func() { parseACLEntries("VAR") }) +} diff --git a/pkgtools/pkglint/files/vargroups.go b/pkgtools/pkglint/files/vargroups.go index 26c31bc2bca..9e60b95b536 100644 --- a/pkgtools/pkglint/files/vargroups.go +++ b/pkgtools/pkglint/files/vargroups.go @@ -40,7 +40,7 @@ func NewVargroupsChecker(mklines *MkLines) *VargroupsChecker { func (ck *VargroupsChecker) init() { mklines := ck.mklines scope := mklines.vars - if !scope.Defined("_VARGROUPS") { + if !scope.IsDefined("_VARGROUPS") { ck.skip = true return } diff --git a/pkgtools/pkglint/files/vartype.go b/pkgtools/pkglint/files/vartype.go index 563cab95134..a48c4d38ab0 100644 --- a/pkgtools/pkglint/files/vartype.go +++ b/pkgtools/pkglint/files/vartype.go @@ -96,14 +96,14 @@ func (perms ACLPermissions) HumanString() string { condStr(perms.Contains(aclpUse), "used", "")) } -func (vt *Vartype) List() bool { return vt.options&List != 0 } -func (vt *Vartype) Guessed() bool { return vt.options&Guessed != 0 } -func (vt *Vartype) PackageSettable() bool { return vt.options&PackageSettable != 0 } -func (vt *Vartype) UserSettable() bool { return vt.options&UserSettable != 0 } -func (vt *Vartype) SystemProvided() bool { return vt.options&SystemProvided != 0 } -func (vt *Vartype) CommandLineProvided() bool { return vt.options&CommandLineProvided != 0 } -func (vt *Vartype) NeedsRationale() bool { return vt.options&NeedsRationale != 0 } -func (vt *Vartype) OnePerLine() bool { return vt.options&OnePerLine != 0 } +func (vt *Vartype) IsList() bool { return vt.options&List != 0 } +func (vt *Vartype) IsGuessed() bool { return vt.options&Guessed != 0 } +func (vt *Vartype) IsPackageSettable() bool { return vt.options&PackageSettable != 0 } +func (vt *Vartype) IsUserSettable() bool { return vt.options&UserSettable != 0 } +func (vt *Vartype) IsSystemProvided() bool { return vt.options&SystemProvided != 0 } +func (vt *Vartype) IsCommandLineProvided() bool { return vt.options&CommandLineProvided != 0 } +func (vt *Vartype) NeedsRationale() bool { return vt.options&NeedsRationale != 0 } +func (vt *Vartype) IsOnePerLine() bool { return vt.options&OnePerLine != 0 } func (vt *Vartype) EffectivePermissions(basename string) ACLPermissions { for _, aclEntry := range vt.aclEntries { @@ -186,7 +186,7 @@ func (vt *Vartype) AlternativeFiles(perms ACLPermissions) string { } func (vt *Vartype) MayBeAppendedTo() bool { - if vt.List() { + if vt.IsList() { return true } @@ -201,22 +201,22 @@ func (vt *Vartype) MayBeAppendedTo() bool { func (vt *Vartype) String() string { var opts []string - if vt.List() { + if vt.IsList() { opts = append(opts, "list") } - if vt.Guessed() { + if vt.IsGuessed() { opts = append(opts, "guessed") } - if vt.PackageSettable() { + if vt.IsPackageSettable() { opts = append(opts, "package-settable") } - if vt.UserSettable() { + if vt.IsUserSettable() { opts = append(opts, "user-settable") } - if vt.SystemProvided() { + if vt.IsSystemProvided() { opts = append(opts, "system-provided") } - if vt.CommandLineProvided() { + if vt.IsCommandLineProvided() { opts = append(opts, "command-line-provided") } diff --git a/pkgtools/pkglint/files/vartype_test.go b/pkgtools/pkglint/files/vartype_test.go index 0f20be16794..3398cd57e8e 100644 --- a/pkgtools/pkglint/files/vartype_test.go +++ b/pkgtools/pkglint/files/vartype_test.go @@ -4,6 +4,34 @@ import ( "gopkg.in/check.v1" ) +func (s *Suite) Test_ACLPermissions_Contains(c *check.C) { + t := s.Init(c) + + perms := aclpAllRuntime + + t.CheckEquals(perms.Contains(aclpAllRuntime), true) + t.CheckEquals(perms.Contains(aclpUse), true) + t.CheckEquals(perms.Contains(aclpUseLoadtime), false) +} + +func (s *Suite) Test_ACLPermissions_String(c *check.C) { + t := s.Init(c) + + t.CheckEquals(ACLPermissions(0).String(), "none") + t.CheckEquals(aclpAll.String(), "set, set-default, append, use-loadtime, use") +} + +func (s *Suite) Test_ACLPermissions_HumanString(c *check.C) { + t := s.Init(c) + + // Doesn't happen in practice + t.CheckEquals(ACLPermissions(0).HumanString(), "") + + t.CheckEquals( + aclpAll.HumanString(), + "set, given a default value, appended to, used at load time, or used") +} + func (s *Suite) Test_Vartype_EffectivePermissions(c *check.C) { t := s.Init(c) @@ -117,6 +145,16 @@ func (s *Suite) Test_Vartype_AlternativeFiles(c *check.C) { "builtin.mk, but not buildlink3.mk, Makefile or *.mk") } +func (s *Suite) Test_Vartype_MayBeAppendedTo(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + t.CheckEquals(G.Pkgsrc.VariableType(nil, "COMMENT").MayBeAppendedTo(), true) + t.CheckEquals(G.Pkgsrc.VariableType(nil, "DEPENDS").MayBeAppendedTo(), true) + t.CheckEquals(G.Pkgsrc.VariableType(nil, "PKG_FAIL_REASON").MayBeAppendedTo(), true) + t.CheckEquals(G.Pkgsrc.VariableType(nil, "CONF_FILES").MayBeAppendedTo(), true) +} func (s *Suite) Test_Vartype_String(c *check.C) { t := s.Init(c) @@ -140,42 +178,3 @@ func (s *Suite) Test_BasicType_HasEnum(c *check.C) { t.CheckEquals(vc.HasEnum("nd"), false) t.CheckEquals(vc.HasEnum("start middle"), false) } - -func (s *Suite) Test_ACLPermissions_Contains(c *check.C) { - t := s.Init(c) - - perms := aclpAllRuntime - - t.CheckEquals(perms.Contains(aclpAllRuntime), true) - t.CheckEquals(perms.Contains(aclpUse), true) - t.CheckEquals(perms.Contains(aclpUseLoadtime), false) -} - -func (s *Suite) Test_ACLPermissions_String(c *check.C) { - t := s.Init(c) - - t.CheckEquals(ACLPermissions(0).String(), "none") - t.CheckEquals(aclpAll.String(), "set, set-default, append, use-loadtime, use") -} - -func (s *Suite) Test_ACLPermissions_HumanString(c *check.C) { - t := s.Init(c) - - // Doesn't happen in practice - t.CheckEquals(ACLPermissions(0).HumanString(), "") - - t.CheckEquals( - aclpAll.HumanString(), - "set, given a default value, appended to, used at load time, or used") -} - -func (s *Suite) Test_Vartype_MayBeAppendedTo(c *check.C) { - t := s.Init(c) - - t.SetUpVartypes() - - t.CheckEquals(G.Pkgsrc.VariableType(nil, "COMMENT").MayBeAppendedTo(), true) - t.CheckEquals(G.Pkgsrc.VariableType(nil, "DEPENDS").MayBeAppendedTo(), true) - t.CheckEquals(G.Pkgsrc.VariableType(nil, "PKG_FAIL_REASON").MayBeAppendedTo(), true) - t.CheckEquals(G.Pkgsrc.VariableType(nil, "CONF_FILES").MayBeAppendedTo(), true) -} diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go index d76414007fd..47a637cf080 100644 --- a/pkgtools/pkglint/files/vartypecheck.go +++ b/pkgtools/pkglint/files/vartypecheck.go @@ -11,15 +11,7 @@ import ( // VartypeCheck groups together the various checks for variables of the different types. type VartypeCheck struct { MkLines *MkLines - - // Note: if "go vet" or "go test" complains about a "variable with invalid type", update to go1.11.4. - // See https://github.com/golang/go/issues/28972. - // That doesn't help though since pkglint contains these "more convoluted alias declarations" - // mentioned in https://github.com/golang/go/commit/6971090515ba. - // Therefore MkLine is declared as *MkLine here. - // Ideally the "more convoluted cyclic type declaration" should be broken up. - - MkLine *MkLine + MkLine *MkLine // The name of the variable being checked. // @@ -593,7 +585,7 @@ func (cv *VartypeCheck) FetchURL() { } if G.Pkgsrc.MasterSiteVarToURL[name] == "" { - if G.Pkg == nil || !G.Pkg.vars.Defined(name) { + if G.Pkg == nil || !G.Pkg.vars.IsDefined(name) { cv.Errorf("The site %s does not exist.", name) } } @@ -838,6 +830,47 @@ func (cv *VartypeCheck) MachineGnuPlatform() { } } +func (cv *VartypeCheck) MachinePlatform() { + cv.MachinePlatformPattern() +} + +func (cv *VartypeCheck) MachinePlatformPattern() { + if cv.Value != cv.ValueNoVar { + return + } + + const rePart = `(?:\[[^\]]+\]|[^-\[])+` + const rePair = `^(` + rePart + `)-(` + rePart + `)$` + const reTriple = `^(` + rePart + `)-(` + rePart + `)-(` + rePart + `)$` + + pattern := cv.Value + if matches(pattern, rePair) && hasSuffix(pattern, "*") { + pattern += "-*" + } + + if m, opsysPattern, versionPattern, archPattern := match3(pattern, reTriple); m { + opsysCv := cv.WithVarnameValueMatch("the operating system part of "+cv.Varname, opsysPattern) + enumMachineOpsys.checker(opsysCv) + + versionCv := cv.WithVarnameValueMatch("the version part of "+cv.Varname, versionPattern) + versionCv.Version() + + archCv := cv.WithVarnameValueMatch("the hardware architecture part of "+cv.Varname, archPattern) + enumMachineArch.checker(archCv) + + } else { + cv.Warnf("%q is not a valid platform pattern.", cv.Value) + cv.Explain( + "A platform pattern has the form <OPSYS>-<OS_VERSION>-<MACHINE_ARCH>.", + "Each of these components may be a shell globbing expression.", + "", + "Examples:", + "* NetBSD-[456].*-i386", + "* *-*-*", + "* Linux-*-*") + } +} + func (cv *VartypeCheck) MailAddress() { value := cv.Value @@ -1078,47 +1111,6 @@ func (cv *VartypeCheck) Pkgrevision() { } } -func (cv *VartypeCheck) MachinePlatform() { - cv.MachinePlatformPattern() -} - -func (cv *VartypeCheck) MachinePlatformPattern() { - if cv.Value != cv.ValueNoVar { - return - } - - const rePart = `(?:\[[^\]]+\]|[^-\[])+` - const rePair = `^(` + rePart + `)-(` + rePart + `)$` - const reTriple = `^(` + rePart + `)-(` + rePart + `)-(` + rePart + `)$` - - pattern := cv.Value - if matches(pattern, rePair) && hasSuffix(pattern, "*") { - pattern += "-*" - } - - if m, opsysPattern, versionPattern, archPattern := match3(pattern, reTriple); m { - opsysCv := cv.WithVarnameValueMatch("the operating system part of "+cv.Varname, opsysPattern) - enumMachineOpsys.checker(opsysCv) - - versionCv := cv.WithVarnameValueMatch("the version part of "+cv.Varname, versionPattern) - versionCv.Version() - - archCv := cv.WithVarnameValueMatch("the hardware architecture part of "+cv.Varname, archPattern) - enumMachineArch.checker(archCv) - - } else { - cv.Warnf("%q is not a valid platform pattern.", cv.Value) - cv.Explain( - "A platform pattern has the form <OPSYS>-<OS_VERSION>-<MACHINE_ARCH>.", - "Each of these components may be a shell globbing expression.", - "", - "Examples:", - "* NetBSD-[456].*-i386", - "* *-*-*", - "* Linux-*-*") - } -} - // PrefixPathname checks for a pathname relative to ${PREFIX}. func (cv *VartypeCheck) PrefixPathname() { if m, manSubdir := match1(cv.Value, `^man/(.+)`); m { diff --git a/pkgtools/pkglint/files/vartypecheck_test.go b/pkgtools/pkglint/files/vartypecheck_test.go index bf36d9d212c..92b6e304a2c 100644 --- a/pkgtools/pkglint/files/vartypecheck_test.go +++ b/pkgtools/pkglint/files/vartypecheck_test.go @@ -1872,7 +1872,7 @@ func (vt *VartypeCheckTester) Values(values ...string) { // See MkLineChecker.checkVartype. var lineValues []string - if vartype == nil || !vartype.List() { + if vartype == nil || !vartype.IsList() { lineValues = []string{effectiveValue} } else { lineValues = mkline.ValueFields(effectiveValue) |