diff options
| author | rillig <rillig@pkgsrc.org> | 2019-04-20 17:43:24 +0000 |
|---|---|---|
| committer | rillig <rillig@pkgsrc.org> | 2019-04-20 17:43:24 +0000 |
| commit | 32bb741a16c58b453bb7d7c1869e11ab14cd59ac (patch) | |
| tree | c02172a5676a532458ea8f58ee8c66dacdcf265e /pkgtools | |
| parent | 0bc679db66baaebc57e71000e060401914232007 (diff) | |
| download | pkgsrc-32bb741a16c58b453bb7d7c1869e11ab14cd59ac.tar.gz | |
pkgtools/pkglint: update to 5.7.5
Changes since 5.7.4:
* Warn about invalid variable uses in directives like
.if and .for
* Do not warn when a package-settable variable is assigned using the ?=
operator before including bsd.prefs.mk. This warning only makes sense
for user-settable and system-provided variables.
* The parser for variable uses like ${VAR:@v@${v:Q}} is more robust now,
which reduces the number of parse errors and leads to more appropriate
diagnostics, in cases like ${URL:Mftp://*}, which should really be
${URL:Mftp\://*}.
* The valid values for OPSYS are now determined by the files in
mk/platform instead of allowing arbitrary identifiers. This catches a
few instances where "Solaris" is used instead of the correct "SunOS".
* Setting USE_LANGUAGES only has an effect if mk/compiler.mk has not yet
been included. In all other cases, pkglint warns now.
* Missing entries in doc/CHANGES produce a note now. This will lead to
more accurate statistics for the release notes.
Diffstat (limited to 'pkgtools')
63 files changed, 2346 insertions, 1441 deletions
diff --git a/pkgtools/pkglint/Makefile b/pkgtools/pkglint/Makefile index edcd972a09a..1fb89f3b0fd 100644 --- a/pkgtools/pkglint/Makefile +++ b/pkgtools/pkglint/Makefile @@ -1,7 +1,6 @@ -# $NetBSD: Makefile,v 1.574 2019/04/16 18:41:18 bsiegert Exp $ +# $NetBSD: Makefile,v 1.575 2019/04/20 17:43:24 rillig Exp $ -PKGNAME= pkglint-5.7.4 -PKGREVISION= 1 +PKGNAME= pkglint-5.7.5 CATEGORIES= pkgtools DISTNAME= tools MASTER_SITES= ${MASTER_SITE_GITHUB:=golang/} diff --git a/pkgtools/pkglint/files/alternatives.go b/pkgtools/pkglint/files/alternatives.go index c87f6b0a89a..7818dc2a02d 100644 --- a/pkgtools/pkglint/files/alternatives.go +++ b/pkgtools/pkglint/files/alternatives.go @@ -48,7 +48,7 @@ func CheckFileAlternatives(filename string) { m, wrapper, space, alternative := match3(line.Text, `^([^\t ]+)([ \t]+)([^\t ]+)`) if !m { line.Errorf("Invalid line %q.", line.Text) - G.Explain( + line.Explain( sprintf("Run %q for more information.", makeHelp("alternatives"))) continue } diff --git a/pkgtools/pkglint/files/alternatives_test.go b/pkgtools/pkglint/files/alternatives_test.go index f17fecfe775..596f99fb6ab 100644 --- a/pkgtools/pkglint/files/alternatives_test.go +++ b/pkgtools/pkglint/files/alternatives_test.go @@ -21,6 +21,7 @@ func (s *Suite) Test_CheckFileAlternatives__PLIST(c *check.C) { "bin/echo", "bin/vim", "sbin/sendmail.exim${EXIMVER}") + t.FinishSetUp() G.Check(".") @@ -73,7 +74,7 @@ func (s *Suite) Test_CheckFileAlternatives__ALTERNATIVES_SRC(c *check.C) { t.CreateFileLines("category/package/ALTERNATIVES", "bin/pgm @PREFIX@/bin/gnu-program", "bin/pgm @PREFIX@/bin/nb-program") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) diff --git a/pkgtools/pkglint/files/autofix.go b/pkgtools/pkglint/files/autofix.go index 709f9ed96ba..25d0f49e4f4 100644 --- a/pkgtools/pkglint/files/autofix.go +++ b/pkgtools/pkglint/files/autofix.go @@ -299,7 +299,7 @@ func (fix *Autofix) Apply() { line.showSource(G.out) } if logDiagnostic && len(fix.explanation) > 0 { - G.Explain(fix.explanation...) + line.Explain(fix.explanation...) } if G.Logger.Opts.ShowSource { if !(G.Logger.Opts.Explain && logDiagnostic && len(fix.explanation) > 0) { @@ -330,7 +330,8 @@ func (fix *Autofix) Realign(mkline MkLine, newWidth int) { { // Parsing the continuation marker as variable value is cheating but works well. text := strings.TrimSuffix(mkline.raw[0].orignl, "\n") - _, a := MatchVarassign(text) + data := MkLineParser{}.split(nil, text) + _, a := MkLineParser{}.MatchVarassign(mkline.Line, text, data) if a.value != "\\" { oldWidth = tabWidth(a.valueAlign) } @@ -448,11 +449,11 @@ func SaveAutofixChanges(lines Lines) (autofixed bool) { G.fileCache.Evict(filename) changedLines := changes[filename] tmpName := filename + ".pkglint.tmp" - text := "" + var text strings.Builder for _, changedLine := range changedLines { - text += changedLine + text.WriteString(changedLine) } - err := ioutil.WriteFile(tmpName, []byte(text), 0666) + err := ioutil.WriteFile(tmpName, []byte(text.String()), 0666) if err != nil { G.Logf(Error, tmpName, "", "Cannot write: %s", "Cannot write: "+err.Error()) continue diff --git a/pkgtools/pkglint/files/autofix_test.go b/pkgtools/pkglint/files/autofix_test.go index 5d8bcc7de9f..590207f26fb 100644 --- a/pkgtools/pkglint/files/autofix_test.go +++ b/pkgtools/pkglint/files/autofix_test.go @@ -1167,8 +1167,8 @@ func (s *Suite) Test_Autofix__lonely_source(c *check.C) { "", ".for id in ${PRE_XORGPROTO_LIST_MISSING}", ".endfor") - G.Pkgsrc.LoadInfrastructure() t.Chdir(".") + t.FinishSetUp() G.Check("x11/xorg-cf-files") G.Check("x11/xorgproto") @@ -1188,8 +1188,8 @@ func (s *Suite) Test_Autofix__lonely_source_2(c *check.C) { t.SetUpPackage("print/tex-bibtex8", "MAKE_FLAGS+=\tCFLAGS=${CFLAGS.${PKGSRC_COMPILER}}") - G.Pkgsrc.LoadInfrastructure() t.Chdir(".") + t.FinishSetUp() G.Check("print/tex-bibtex8") diff --git a/pkgtools/pkglint/files/buildlink3.go b/pkgtools/pkglint/files/buildlink3.go index ac9559ad2f3..ced4ebd4d2f 100644 --- a/pkgtools/pkglint/files/buildlink3.go +++ b/pkgtools/pkglint/files/buildlink3.go @@ -230,7 +230,7 @@ func (ck *Buildlink3Checker) checkVaruseInPkgbase(pkgbase string, pkgbaseLine Mk token.Text) } - G.Explain( + pkgbaseLine.Explain( "The identifiers in the BUILDLINK_TREE variable should be plain", "strings that do not refer to any variable.", "", diff --git a/pkgtools/pkglint/files/buildlink3_test.go b/pkgtools/pkglint/files/buildlink3_test.go index 18a31a6fc8c..d496b797b17 100644 --- a/pkgtools/pkglint/files/buildlink3_test.go +++ b/pkgtools/pkglint/files/buildlink3_test.go @@ -19,6 +19,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__package(c *check.C) { t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", ".include \"../../category/dependency2/buildlink3.mk\"") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -85,6 +86,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_Haskell_incomplete(c ".endif\t# HS_X11_BUILDLINK3_MK", "", "BUILDLINK_TREE+=\t-hs-X11") + t.FinishSetUp() G.Check(".") @@ -124,6 +126,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_Haskell_complete(c *c ".endif\t# HS_X11_BUILDLINK3_MK", "", "BUILDLINK_TREE+=\t-hs-X11") + t.FinishSetUp() G.Check(".") @@ -150,6 +153,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch__Perl(c *check.C) { ".endif\t# P5_GTK2_BUILDLINK3_MK", "", "BUILDLINK_TREE+=\t-p5-gtk2") + t.FinishSetUp() G.Check(t.File("x11/p5-gtk2")) @@ -508,6 +512,8 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_unknown_variable(c *ch "it would be ok in Makefile, Makefile.* or *.mk, but not buildlink3.mk or builtin.mk.", "WARN: buildlink3.mk:3: The variable LICENSE should be quoted as part of a shell word.", "WARN: buildlink3.mk:8: The variable LICENSE should be quoted as part of a shell word.", + "WARN: buildlink3.mk:8: The variable LICENSE should be quoted as part of a shell word.", + "WARN: buildlink3.mk:9: The variable LICENSE should be quoted as part of a shell word.", "WARN: buildlink3.mk:9: The variable LICENSE should be quoted as part of a shell word.", "WARN: buildlink3.mk:13: The variable LICENSE should be quoted as part of a shell word.", "WARN: buildlink3.mk:3: Please replace \"${LICENSE}\" with a simple string "+ @@ -523,6 +529,7 @@ func (s *Suite) Test_Buildlink3Checker_checkMainPart__if_else_endif(c *check.C) ".if ${X11_TYPE} == modular", ".else", ".endif") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -537,6 +544,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__dependencies_with_path(c t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", "BUILDLINK_ABI_DEPENDS.package+=\tpackage>=1.0:../../category/package", "BUILDLINK_API_DEPENDS.package+=\tpackage>=1.5:../../category/package") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -570,6 +578,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_without_api(c *check. ".endif # PACKAGE_BUILDLINK3_MK", "", "BUILDLINK_TREE+=\t-package") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -589,6 +598,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_and_api_with_variable "", "ABI_VERSION=\t1.0", "API_VERSION=\t1.5") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -606,6 +616,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__api_with_variable(c *chec "BUILDLINK_API_DEPENDS.package+=\tpackage>=${API_VERSION}", "", "API_VERSION=\t1.5") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -621,6 +632,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_and_api_with_pattern( t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", "BUILDLINK_ABI_DEPENDS.package+=\tpackage-1.*", "BUILDLINK_API_DEPENDS.package+=\tpackage-2.*") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -637,6 +649,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__api_with_pattern(c *check t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", "BUILDLINK_ABI_DEPENDS.package+=\tpackage>=1", "BUILDLINK_API_DEPENDS.package+=\tpackage-1.*") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -658,6 +671,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__other_variables(c *check. "BUILDLINK_DEPMETHOD.other+=\tbuild", "", "BUILDLINK_API_DEPENDS.other+=\tother>=3") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -674,6 +688,7 @@ func (s *Suite) Test_Buildlink3Checker_Check__no_tracing(c *check.C) { t.SetUpPackage("category/package") t.CreateFileDummyBuildlink3("category/package/buildlink3.mk") t.DisableTracing() + t.FinishSetUp() G.Check(t.File("category/package/buildlink3.mk")) @@ -687,6 +702,7 @@ func (s *Suite) Test_Buildlink3Checker_checkSecondParagraph__missing_mkbase(c *c "DISTNAME=\t# empty", "PKGNAME=\t# empty, to force mkbase to be empty") t.CreateFileDummyBuildlink3("category/package/buildlink3.mk") + t.FinishSetUp() G.Check(t.File("category/package")) diff --git a/pkgtools/pkglint/files/category.go b/pkgtools/pkglint/files/category.go index f97afc2717c..46a0c2a0a61 100644 --- a/pkgtools/pkglint/files/category.go +++ b/pkgtools/pkglint/files/category.go @@ -1,6 +1,10 @@ package pkglint -import "netbsd.org/pkglint/textproc" +import ( + "fmt" + "netbsd.org/pkglint/textproc" + "strings" +) func CheckdirCategory(dir string) { if trace.Tracing { @@ -25,18 +29,18 @@ func CheckdirCategory(dir string) { lex := textproc.NewLexer(mkline.Value()) valid := textproc.NewByteSet("--- '(),/0-9A-Za-z") invalid := valid.Inverse() - uni := "" + var uni strings.Builder for !lex.EOF() { _ = lex.NextBytesSet(valid) ch := lex.NextByteSet(invalid) if ch != -1 { - uni += sprintf(" %U", ch) + _, _ = fmt.Fprintf(&uni, " %U", ch) } } - if uni != "" { - mkline.Warnf("%s contains invalid characters (%s).", mkline.Varname(), uni[1:]) + if uni.Len() > 0 { + mkline.Warnf("%s contains invalid characters (%s).", mkline.Varname(), uni.String()[1:]) } } else { diff --git a/pkgtools/pkglint/files/category_test.go b/pkgtools/pkglint/files/category_test.go index e56b7c2af4f..fb54aa0895e 100644 --- a/pkgtools/pkglint/files/category_test.go +++ b/pkgtools/pkglint/files/category_test.go @@ -83,6 +83,7 @@ func (s *Suite) Test_CheckdirCategory__wip(c *check.C) { "\t${RUN}wip-specific-command", "", ".include \"../mk/misc/category.mk\"") + t.FinishSetUp() G.Check(t.File("wip")) @@ -117,6 +118,7 @@ func (s *Suite) Test_CheckdirCategory__subdirs(c *check.C) { "#SUBDIR+=\tcommented-without-reason", "", ".include \"../mk/misc/category.mk\"") + t.FinishSetUp() CheckdirCategory(t.File("category")) @@ -147,6 +149,7 @@ func (s *Suite) Test_CheckdirCategory__only_in_Makefile(c *check.C) { "SUBDIR+=\tonly-in-makefile", "", ".include \"../mk/misc/category.mk\"") + t.FinishSetUp() CheckdirCategory(t.File("category")) @@ -174,6 +177,7 @@ func (s *Suite) Test_CheckdirCategory__only_in_file_system(c *check.C) { "SUBDIR+=\tboth", "", ".include \"../mk/misc/category.mk\"") + t.FinishSetUp() CheckdirCategory(t.File("category")) @@ -203,6 +207,7 @@ func (s *Suite) Test_CheckdirCategory__recursive(c *check.C) { "", ".include \"../mk/misc/category.mk\"") t.Chdir("category") + t.FinishSetUp() CheckdirCategory(".") @@ -238,6 +243,7 @@ func (s *Suite) Test_CheckdirCategory__subdirs_file_system_at_the_bottom(c *chec "SUBDIR+=\tmk-and-fs", "", ".include \"../mk/misc/category.mk\"") + t.FinishSetUp() CheckdirCategory(t.File("category")) @@ -263,6 +269,7 @@ func (s *Suite) Test_CheckdirCategory__indentation(c *check.C) { "SUBDIR+=\tpackage2", "", ".include \"../mk/misc/category.mk\"") + t.FinishSetUp() CheckdirCategory(t.File("category")) @@ -287,6 +294,7 @@ func (s *Suite) Test_CheckdirCategory__comment_at_the_top(c *check.C) { "SUBDIR+=\tpackage", "", ".include \"../mk/misc/category.mk\"") + t.FinishSetUp() CheckdirCategory(t.File("category")) @@ -318,6 +326,7 @@ func (s *Suite) Test_CheckdirCategory__unexpected_EOF_while_reading_SUBDIR(c *ch "COMMENT=\tCategory comment", "", "SUBDIR+=\tpackage") + t.FinishSetUp() CheckdirCategory(t.File("category")) @@ -333,6 +342,7 @@ func (s *Suite) Test_CheckdirCategory__no_Makefile(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("category/other-file") + t.FinishSetUp() G.Check(t.File("category")) diff --git a/pkgtools/pkglint/files/check_test.go b/pkgtools/pkglint/files/check_test.go index e4af984a840..496689ad8a8 100644 --- a/pkgtools/pkglint/files/check_test.go +++ b/pkgtools/pkglint/files/check_test.go @@ -56,7 +56,7 @@ func (s *Suite) Init(c *check.C) *Tester { } func (s *Suite) SetUpTest(c *check.C) { - t := Tester{c: c} + t := Tester{c: c, testName: c.TestName()} s.Tester = &t G = NewPkglint() @@ -89,7 +89,11 @@ func (s *Suite) TearDownTest(c *check.C) { t.c = nil // No longer usable; see https://github.com/go-check/check/issues/22 if err := os.Chdir(t.prevdir); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "Cannot chdir back to previous dir: %s", err) + t.Errorf("Cannot chdir back to previous dir: %s", err) + } + + if t.seenSetupPkgsrc > 0 && !t.seenFinish && !t.seenMain { + t.Errorf("After t.SetupPkgsrc(), t.FinishSetUp() or t.Main() must be called.") } if out := t.Output(); out != "" { @@ -121,12 +125,18 @@ func Test(t *testing.T) { check.TestingT(t) } // all the test methods, which makes it difficult to find // a method by auto-completion. type Tester struct { + c *check.C // Only usable during the test method itself + testName string + stdout bytes.Buffer stderr bytes.Buffer tmpdir string - c *check.C // Only usable during the test method itself - prevdir string // The current working directory before the test started - relCwd string // See Tester.Chdir + prevdir string // The current working directory before the test started + relCwd string // See Tester.Chdir + + seenSetupPkgsrc int + seenFinish bool + seenMain bool } // SetUpCommandLine simulates a command line for the remainder of the test. @@ -294,6 +304,8 @@ func (t *Tester) SetUpPkgsrc() { // Category Makefiles require this file for the common definitions. t.CreateFileLines("mk/misc/category.mk") + + t.seenSetupPkgsrc++ } // SetUpCategory makes the given category valid by creating a dummy Makefile. @@ -316,7 +328,8 @@ func (t *Tester) SetUpCategory(name string) { // Returns the path to the package, ready to be used with Pkglint.Check. // // After calling this method, individual files can be overwritten as necessary. -// Then, G.Pkgsrc.LoadInfrastructure should be called to load all the files. +// At the end of the setup phase, t.FinishSetUp() must be called to load all +// the files. func (t *Tester) SetUpPackage(pkgpath string, makefileLines ...string) string { category := path.Dir(pkgpath) @@ -375,7 +388,7 @@ func (t *Tester) SetUpPackage(pkgpath string, makefileLines ...string) string { line: for _, line := range makefileLines { if m, prefix := match1(line, `^#?(\w+=)`); m { - for i, existingLine := range mlines { + for i, existingLine := range mlines[:19] { if hasPrefix(strings.TrimPrefix(existingLine, "#"), prefix) { mlines[i] = line continue line @@ -619,6 +632,37 @@ func (s *Suite) Test_Tester_SetUpHierarchy(c *check.C) { "NOTE: subdir/env.mk:1: Text is: VAR= env") } +func (t *Tester) FinishSetUp() { + if t.seenSetupPkgsrc == 0 { + t.Errorf("Unnecessary t.FinishSetUp() since t.SetUpPkgsrc() has not been called.") + } + + if !t.seenFinish { + t.seenFinish = true + G.Pkgsrc.LoadInfrastructure() + } else { + t.Errorf("Redundant t.FinishSetup() since it was called multiple times.") + } +} + +// Main runs the pkglint main program with the given command line arguments. +func (t *Tester) Main(args ...string) int { + if t.seenFinish && !t.seenMain { + t.Errorf("Calling t.FinishSetup() before t.Main() is redundant " + + "since t.Main() loads the pkgsrc infrastructure.") + } + + t.seenMain = true + + // Reset the logger, for tests where t.Main is called multiple times. + G.errors = 0 + G.warnings = 0 + G.logged = Once{} + + argv := append([]string{"pkglint"}, args...) + return G.Main(argv...) +} + // Check delegates a check to the check.Check function. // Thereby, there is no need to distinguish between c.Check and t.Check // in the test code. @@ -626,6 +670,10 @@ func (t *Tester) Check(obj interface{}, checker check.Checker, args ...interface return t.c.Check(obj, checker, args...) } +func (t *Tester) Errorf(format string, args ...interface{}) { + _, _ = fmt.Fprintf(os.Stderr, "In %s: %s\n", t.testName, sprintf(format, args...)) +} + // ExpectFatal runs the given action and expects that this action calls // Line.Fatalf or uses some other way to panic with a pkglintFatal. // @@ -722,7 +770,7 @@ func (t *Tester) NewMkLine(filename string, lineno int, text string) MkLine { hasSuffix(basename, ".mk") || basename == "Makefile" || hasPrefix(basename, "Makefile."), "filename %q must be realistic, otherwise the variable permissions are wrong", filename) - return NewMkLine(t.NewLine(filename, lineno, text)) + return MkLineParser{}.Parse(t.NewLine(filename, lineno, text)) } func (t *Tester) NewShellLineChecker(mklines MkLines, filename string, lineno int, text string) *ShellLineChecker { @@ -890,3 +938,11 @@ func (t *Tester) CheckFileLinesDetab(relativeFileName string, lines ...string) { t.Check(detabbedLines, deepEquals, lines) } + +// Use marks all passed functions as used for the Go compiler. +// +// This means that the test cases that follow do not have to use each of them, +// and this in turn allows uninteresting test cases to be deleted during +// development. +func (t *Tester) Use(functions ...interface{}) { +} diff --git a/pkgtools/pkglint/files/distinfo.go b/pkgtools/pkglint/files/distinfo.go index 521c2aaf749..053d474c48b 100644 --- a/pkgtools/pkglint/files/distinfo.go +++ b/pkgtools/pkglint/files/distinfo.go @@ -153,7 +153,7 @@ func (ck *distinfoLinesChecker) checkAlgorithms(info distinfoFileInfo) { line.Warnf("Patch file %q does not exist in directory %q.", filename, line.PathToFile(ck.pkg.File(ck.patchdir))) - G.Explain( + line.Explain( "If the patches directory looks correct, the patch may have been", "removed without updating the distinfo file.", "In such a case please update the distinfo file.", diff --git a/pkgtools/pkglint/files/distinfo_test.go b/pkgtools/pkglint/files/distinfo_test.go index f2332601e32..9661f6bef5a 100644 --- a/pkgtools/pkglint/files/distinfo_test.go +++ b/pkgtools/pkglint/files/distinfo_test.go @@ -113,6 +113,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__wrong_patch_algorithm "", "MD5 (patch-aa) = 12345678901234567890123456789012", "SHA1 (patch-aa) = 1234567890123456789012345678901234567890") + t.FinishSetUp() G.Check(".") @@ -176,7 +177,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkGlobalDistfileMismatch(c *check.C "", ".include \"../mk/misc/category.mk\"") - G.Main("pkglint", "-r", "-Wall", "-Call", t.File(".")) + t.Main("-r", "-Wall", "-Call", t.File(".")) t.CheckOutputLines( "ERROR: ~/category/package1/distinfo:3: "+ @@ -240,6 +241,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__existing_patch_with_d "SHA512 (patch-aa) = ...", "Size (patch-aa) = ... bytes") t.CreateFileDummyPatch("category/package/patches/patch-aa") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -265,6 +267,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__missing_patch_with_wr RcsID, "", "RMD160 (patch-aa) = ...") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -288,6 +291,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch__bad(c *check.C) RcsID, "", "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") + t.FinishSetUp() G.checkdirPackage(".") @@ -309,6 +313,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch__good(c *check.C RcsID, "", "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") + t.FinishSetUp() G.checkdirPackage(".") @@ -330,6 +335,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkUnrecordedPatches(c *check.C) { "RMD160 (distfile.tar.gz) = ...", "SHA512 (distfile.tar.gz) = ...", "Size (distfile.tar.gz) = 1024 bytes") + t.FinishSetUp() G.checkdirPackage(".") @@ -357,6 +363,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1__relative_path_in_disti "SHA1 (patch-aa) = ...", "SHA1 (patch-only-in-distinfo) = ...") t.Chdir("category/package") + t.FinishSetUp() G.checkdirPackage(".") @@ -387,6 +394,7 @@ func (s *Suite) Test_CheckLinesDistinfo__distinfo_and_patches_in_separate_direct "SHA1 (patch-aa) = ...", "SHA1 (patch-only-in-distinfo) = ...") t.Chdir("category/package") + t.FinishSetUp() G.checkdirPackage(".") @@ -465,6 +473,7 @@ func (s *Suite) Test_CheckLinesDistinfo__missing_php_patches(c *check.C) { "", ".include \"../../lang/php/ext.mk\"", ".include \"../../mk/bsd.pkg.mk\"") + t.FinishSetUp() G.Check(t.File("archivers/php-bz2")) @@ -508,7 +517,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__add_missing_h "CRC32 (package-1.0.txt) = asdf") t.CreateFileLines("distfiles/package-1.0.txt", "hello, world") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() // This run is only used to verify that the RMD160 hash is correct, and if // it should ever differ, the correct hash will appear in an error message. @@ -584,7 +593,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__wrong_distfil "RMD160 (package-1.0.txt) = 1234wrongHash1234") t.CreateFileLines("distfiles/package-1.0.txt", "hello, world") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -607,7 +616,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__no_usual_algo "MD5 (package-1.0.txt) = 1234wrongHash1234") t.CreateFileLines("distfiles/package-1.0.txt", "hello, world") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -629,7 +638,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__top_algorithm "Size (package-1.0.txt) = 13 bytes") t.CreateFileLines("distfiles/package-1.0.txt", "hello, world") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -652,7 +661,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__bottom_algori "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a") t.CreateFileLines("distfiles/package-1.0.txt", "hello, world") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -691,7 +700,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__algorithms_in t.CreateFileLines("distfiles/package-1.0.txt", "hello, world") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -716,7 +725,7 @@ func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__some_algorith t.CreateFileLines("distfiles/package-1.0.txt", "hello, world") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) diff --git a/pkgtools/pkglint/files/files_test.go b/pkgtools/pkglint/files/files_test.go index 64638753ae5..813587ff884 100644 --- a/pkgtools/pkglint/files/files_test.go +++ b/pkgtools/pkglint/files/files_test.go @@ -98,6 +98,22 @@ func (s *Suite) Test_convertToLogicalLines__comments(c *check.C) { t.CheckOutputEmpty() } +func (s *Suite) Test_convertToLogicalLines__commented_multi(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("filename.mk", + "#COMMENTED= \\", + "#\tcontinuation 1 \\", + "#\tcontinuation 2") + mkline := mklines.mklines[0] + + // FIXME: It is more pragmatic to strip the leading comments from the + // continuation lines as well, so that the variable value is "continuation 1 continuation 2". + // See nextLogicalLine. + t.Check(mkline.Value(), equals, "") + t.Check(mkline.VarassignComment(), equals, "#\tcontinuation 1 #\tcontinuation 2") +} + func (s *Suite) Test_convertToLogicalLines__missing_newline_at_eof(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/licenses.go b/pkgtools/pkglint/files/licenses.go index c828725bce8..7988dc271e0 100644 --- a/pkgtools/pkglint/files/licenses.go +++ b/pkgtools/pkglint/files/licenses.go @@ -48,7 +48,7 @@ func (lc *LicenseChecker) checkName(license string) { "no-redistribution", "shareware": lc.MkLine.Errorf("License %q must not be used.", license) - G.Explain( + lc.MkLine.Explain( "Instead of using these deprecated licenses, extract the actual", "license from the package into the pkgsrc/licenses/ directory", "and define LICENSE to that filename.", @@ -65,7 +65,7 @@ func (lc *LicenseChecker) checkNode(cond *licenses.Condition) { if cond.And && cond.Or { lc.MkLine.Errorf("AND and OR operators in license conditions can only be combined using parentheses.") - G.Explain( + lc.MkLine.Explain( "Examples for valid license conditions are:", "", "\tlicense1 AND license2 AND (license3 OR license4)", diff --git a/pkgtools/pkglint/files/licenses_test.go b/pkgtools/pkglint/files/licenses_test.go index 59216f8490e..f3059d0846a 100644 --- a/pkgtools/pkglint/files/licenses_test.go +++ b/pkgtools/pkglint/files/licenses_test.go @@ -55,7 +55,7 @@ func (s *Suite) Test_LicenseChecker_checkName__LICENSE_FILE(c *check.C) { t.CreateFileLines("category/package/my-license", "An individual license file.") - G.Main("pkglint", t.File("category/package")) + t.Main(t.File("category/package")) // There is no warning about the unusual file name in the package directory. // If it were not mentioned in LICENSE_FILE, the file named my-license diff --git a/pkgtools/pkglint/files/linechecker.go b/pkgtools/pkglint/files/linechecker.go index 34ef14943e9..8fc3b18d6c5 100644 --- a/pkgtools/pkglint/files/linechecker.go +++ b/pkgtools/pkglint/files/linechecker.go @@ -18,7 +18,7 @@ func (ck LineChecker) CheckLength(maxLength int) { for i := 0; i < len(prefix); i++ { if isHspace(prefix[i]) { ck.line.Warnf("Line too long (should be no more than %d characters).", maxLength) - G.Explain( + ck.line.Explain( "Back in the old time, terminals with 80x25 characters were common.", "And this is still the default size of many terminal emulators.", "Moderately short lines also make reading easier.") diff --git a/pkgtools/pkglint/files/linelexer.go b/pkgtools/pkglint/files/linelexer.go index aa32309da70..b89b5da2131 100644 --- a/pkgtools/pkglint/files/linelexer.go +++ b/pkgtools/pkglint/files/linelexer.go @@ -12,6 +12,8 @@ func NewLinesLexer(lines Lines) *LinesLexer { return &LinesLexer{lines, 0} } +// CurrentLine returns the line that the lexer is currently looking at. +// If it is at the end of file, the line number of the line is EOF. func (llex *LinesLexer) CurrentLine() Line { if llex.index < llex.lines.Len() { return llex.lines.Lines[llex.index] diff --git a/pkgtools/pkglint/files/lines_test.go b/pkgtools/pkglint/files/lines_test.go index 7c2915efad0..2e758114cb1 100644 --- a/pkgtools/pkglint/files/lines_test.go +++ b/pkgtools/pkglint/files/lines_test.go @@ -42,6 +42,7 @@ func (s *Suite) Test_Lines_CheckRcsID__wip(c *check.C) { "# $"+"Id$") t.CreateFileLines("wip/package/file5.mk", "# $"+"FreeBSD$") + t.FinishSetUp() G.Check(t.File("wip/package")) diff --git a/pkgtools/pkglint/files/logging_test.go b/pkgtools/pkglint/files/logging_test.go index 6b1dcaad546..1672e1aea1b 100644 --- a/pkgtools/pkglint/files/logging_test.go +++ b/pkgtools/pkglint/files/logging_test.go @@ -288,10 +288,10 @@ func (s *Suite) Test_Logger_Explain__only(c *check.C) { // Neither the warning nor the corresponding explanation are logged. line.Warnf("Filtered warning.") - G.Explain("Explanation for the above warning.") + line.Explain("Explanation for the above warning.") line.Notef("What an interesting line.") - G.Explain("This explanation is logged.") + line.Explain("This explanation is logged.") t.CheckOutputLines( "NOTE: Makefile:27: What an interesting line.", @@ -417,7 +417,7 @@ func (s *Suite) Test_Logger_ShowSummary__explanations_with_only(c *check.C) { // Neither the warning nor the corresponding explanation are logged. line.Warnf("Filtered warning.") - G.Explain("Explanation for the above warning.") + line.Explain("Explanation for the above warning.") G.ShowSummary() // Since the above warning is filtered out by the --only option, @@ -429,7 +429,7 @@ func (s *Suite) Test_Logger_ShowSummary__explanations_with_only(c *check.C) { "Looks fine.") line.Warnf("This warning is interesting.") - G.Explain("This explanation is available.") + line.Explain("This explanation is available.") G.ShowSummary() c.Check(G.explanationsAvailable, equals, true) @@ -608,12 +608,12 @@ func (s *Suite) Test_Logger_Logf__duplicate_messages(c *check.C) { // Is logged because it is the first appearance of this warning. line.Warnf("The warning.") - G.Explain("Explanation 1") + line.Explain("Explanation 1") // Is suppressed because the warning is the same as above and LogVerbose // has been set to false for this test. line.Warnf("The warning.") - G.Explain("Explanation 2") + line.Explain("Explanation 2") t.CheckOutputLines( "WARN: README.txt:123: The warning.", @@ -630,9 +630,9 @@ func (s *Suite) Test_Logger_Logf__duplicate_explanations(c *check.C) { // In rare cases, different diagnostics may have the same explanation. line.Warnf("Warning 1.") - G.Explain("Explanation") + line.Explain("Explanation") line.Warnf("Warning 2.") - G.Explain("Explanation") // Is suppressed. + line.Explain("Explanation") // Is suppressed. t.CheckOutputLines( "WARN: README.txt:123: Warning 1.", @@ -745,7 +745,7 @@ func (s *Suite) Test_Logger_Diag__source_duplicates(c *check.C) { t.SetUpPackage("category/package2", "PATCHDIR=\t../../category/dependency/patches") - G.Main("pkglint", "--source", "-Wall", t.File("category/package1"), t.File("category/package2")) + t.Main("--source", "-Wall", t.File("category/package1"), t.File("category/package2")) t.CheckOutputLines( "ERROR: ~/category/package1/distinfo: "+ diff --git a/pkgtools/pkglint/files/mkline.go b/pkgtools/pkglint/files/mkline.go index 4af4173b3c6..138f855e856 100644 --- a/pkgtools/pkglint/files/mkline.go +++ b/pkgtools/pkglint/files/mkline.go @@ -62,57 +62,101 @@ type mkLineDependency struct { sources string } -// NewMkLine parses the text of a Makefile line to see what kind of line +type MkLineParser struct{} + +// Parse parses the text of a Makefile line to see what kind of line // it is: variable assignment, include, comment, etc. // // See devel/bmake/parse.c:/^Parse_File/ -func NewMkLine(line Line) *MkLineImpl { +func (p MkLineParser) Parse(line Line) *MkLineImpl { text := line.Text // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing. if hasPrefix(text, " ") && line.Basename != "bsd.buildlink3.mk" { line.Warnf("Makefile lines should not start with space characters.") - G.Explain( + line.Explain( "If this line should be a shell command connected to a target, use a tab character for indentation.", "Otherwise remove the leading whitespace.") } - if m, a := MatchVarassign(text); m { - if a.spaceAfterVarname != "" { - varname := a.varname - op := a.op - switch { - case hasSuffix(varname, "+") && (op == opAssign || op == opAssignAppend): - break - case matches(varname, `^[a-z]`) && op == opAssignEval: - break - default: - // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing. - fix := line.Autofix() - fix.Notef("Unnecessary space after variable name %q.", varname) - fix.Replace(varname+a.spaceAfterVarname+op.String(), varname+op.String()) - fix.Apply() - } - } + data := p.split(line, text) - // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing. - if a.comment != "" && a.value != "" && a.spaceAfterValue == "" { - line.Warnf("The # character starts a Makefile comment.") - G.Explain( - "In a variable assignment, an unescaped # starts a comment that", - "continues until the end of the line.", - "To escape the #, write \\#.") - } + // Check for shell commands first because these cannot have comments + // at the end of the line. + if hasPrefix(text, "\t") { + return p.parseShellcmd(line) + } - return &MkLineImpl{line, a} + if mkline := p.parseVarassign(line, data); mkline != nil { + return mkline + } + if mkline := p.parseCommentOrEmpty(line); mkline != nil { + return mkline + } + if mkline := p.parseDirective(line, data); mkline != nil { + return mkline + } + if mkline := p.parseInclude(line); mkline != nil { + return mkline + } + if mkline := p.parseSysinclude(line); mkline != nil { + return mkline + } + if mkline := p.parseDependency(line); mkline != nil { + return mkline + } + if mkline := p.parseMergeConflict(line); mkline != nil { + return mkline } - if hasPrefix(text, "\t") { - shellcmd := text[1:] - return &MkLineImpl{line, mkLineShell{shellcmd}} + // The %q is deliberate here since it shows possible strange characters. + line.Errorf("Unknown Makefile line format: %q.", text) + return &MkLineImpl{line, nil} +} + +func (p MkLineParser) parseVarassign(line Line, data mkLineSplitResult) MkLine { + m, a := p.MatchVarassign(line, line.Text, data) + if !m { + return nil + } + + if a.spaceAfterVarname != "" { + varname := a.varname + op := a.op + switch { + case hasSuffix(varname, "+") && (op == opAssign || op == opAssignAppend): + break + case matches(varname, `^[a-z]`) && op == opAssignEval: + break + default: + fix := line.Autofix() + fix.Notef("Unnecessary space after variable name %q.", varname) + fix.Replace(varname+a.spaceAfterVarname+op.String(), varname+op.String()) + fix.Apply() + } } - trimmedText := trimHspace(text) + if a.comment != "" && a.value != "" && a.spaceAfterValue == "" { + line.Warnf("The # character starts a Makefile comment.") + line.Explain( + "In a variable assignment, an unescaped # starts a comment that", + "continues until the end of the line.", + "To escape the #, write \\#.", + "", + "If this # character intentionally starts a comment,", + "it should be preceded by a space in order to make it more visible.") + } + + return &MkLineImpl{line, a} +} + +func (p MkLineParser) parseShellcmd(line Line) MkLine { + return &MkLineImpl{line, mkLineShell{line.Text[1:]}} +} + +func (p MkLineParser) parseCommentOrEmpty(line Line) MkLine { + trimmedText := trimHspace(line.Text) + if strings.HasPrefix(trimmedText, "#") { return &MkLineImpl{line, mkLineComment{}} } @@ -121,40 +165,46 @@ func NewMkLine(line Line) *MkLineImpl { return &MkLineImpl{line, mkLineEmpty{}} } - if m, indent, directive, args, comment := matchMkDirective(text); m { - - // In .if and .endif lines the space surrounding the comment is irrelevant. - // Especially for checking that the .endif comment matches the .if condition, - // it must be trimmed. - trimmedComment := trimHspace(comment) + return nil +} - return &MkLineImpl{line, &mkLineDirectiveImpl{indent, directive, args, trimmedComment, nil, nil, nil}} +func (p MkLineParser) parseInclude(line Line) MkLine { + m, indent, directive, includedFile := MatchMkInclude(line.Text) + if !m { + return nil } - if m, indent, directive, includedFile := MatchMkInclude(text); m { - return &MkLineImpl{line, &mkLineIncludeImpl{directive == "include", false, indent, includedFile, nil}} - } + return &MkLineImpl{line, &mkLineIncludeImpl{directive == "include", false, indent, includedFile, nil}} +} - if m, indent, directive, includedFile := match3(text, `^\.([\t ]*)(s?include)[\t ]+<([^>]+)>[\t ]*(?:#.*)?$`); m { - return &MkLineImpl{line, &mkLineIncludeImpl{directive == "include", true, indent, includedFile, nil}} +func (p MkLineParser) parseSysinclude(line Line) MkLine { + m, indent, directive, includedFile := match3(line.Text, `^\.([\t ]*)(s?include)[\t ]+<([^>]+)>[\t ]*(?:#.*)?$`) + if !m { + return nil } + return &MkLineImpl{line, &mkLineIncludeImpl{directive == "include", true, indent, includedFile, nil}} +} + +func (p MkLineParser) parseDependency(line Line) MkLine { // XXX: Replace this regular expression with proper parsing. // There might be a ${VAR:M*.c} in these variables, which the below regular expression cannot handle. - if m, targets, whitespace, sources := match3(text, `^([^\t :]+(?:[\t ]*[^\t :]+)*)([\t ]*):[\t ]*([^#]*?)(?:[\t ]*#.*)?$`); m { - // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing. - if whitespace != "" { - line.Notef("Space before colon in dependency line.") - } - return &MkLineImpl{line, mkLineDependency{targets, sources}} + m, targets, whitespace, sources := match3(line.Text, `^([^\t :]+(?:[\t ]*[^\t :]+)*)([\t ]*):[\t ]*([^#]*?)(?:[\t ]*#.*)?$`) + if !m { + return nil + } + + if whitespace != "" { + line.Notef("Space before colon in dependency line.") } + return &MkLineImpl{line, mkLineDependency{targets, sources}} +} - if matches(text, `^(<<<<<<<|=======|>>>>>>>)`) { - return &MkLineImpl{line, nil} +func (p MkLineParser) parseMergeConflict(line Line) MkLine { + if !matches(line.Text, `^(<<<<<<<|=======|>>>>>>>)`) { + return nil } - // The %q is deliberate here since it shows possible strange characters. - line.Errorf("Unknown Makefile line format: %q.", text) return &MkLineImpl{line, nil} } @@ -295,7 +345,7 @@ func (mkline *MkLineImpl) Args() string { return mkline.data.(mkLineDirective).a func (mkline *MkLineImpl) Cond() MkCond { cond := mkline.data.(mkLineDirective).cond if cond == nil { - cond = NewMkParser(nil, mkline.Args(), false).MkCond() + cond = NewMkParser(mkline.Line, mkline.Args(), true).MkCond() mkline.data.(mkLineDirective).cond = cond } return cond @@ -452,23 +502,24 @@ func (mkline *MkLineImpl) ValueFields(value string) []string { atoms = atoms[1:] } - word := "" + var word strings.Builder var words []string for _, atom := range atoms { if atom.Type == shtSpace && atom.Quoting == shqPlain { - words = append(words, word) - word = "" + words = append(words, word.String()) + word.Reset() } else { - word += atom.MkText + word.WriteString(atom.MkText) } } - if word != "" && atoms[len(atoms)-1].Quoting == shqPlain { - words = append(words, word) - word = "" + if word.Len() > 0 && atoms[len(atoms)-1].Quoting == shqPlain { + words = append(words, word.String()) + word.Reset() } // TODO: Handle parse errors - rest := word + p.parser.Rest() + word.WriteString(p.parser.Rest()) + rest := word.String() _ = rest return words @@ -485,7 +536,9 @@ func (mkline *MkLineImpl) ValueTokens() ([]*MkToken, string) { return assign.valueMk, assign.valueMkRest } - p := NewMkParser(mkline.Line, value, true) + // No error checking here since all this has already been done when the + // whole line was parsed in MkLineParser.Parse. + p := NewMkParser(nil, value, false) assign.valueMk = p.MkTokens() assign.valueMkRest = p.Rest() return assign.valueMk, assign.valueMkRest @@ -527,13 +580,13 @@ func (mkline *MkLineImpl) Fields() []string { } func (mkline *MkLineImpl) WithoutMakeVariables(value string) string { - valueNovar := "" + var valueNovar strings.Builder for _, token := range NewMkParser(nil, value, false).MkTokens() { if token.Varuse == nil { - valueNovar += token.Text + valueNovar.WriteString(token.Text) } } - return valueNovar + return valueNovar.String() } func (mkline *MkLineImpl) ResolveVarsInRelativePath(relativePath string) string { @@ -612,7 +665,7 @@ func (mkline *MkLineImpl) ResolveVarsInRelativePath(relativePath string) string } func (mkline *MkLineImpl) ExplainRelativeDirs() { - G.Explain( + mkline.Explain( "Directories in the form \"../../category/package\" make it easier to", "move a package around in pkgsrc, for example from pkgsrc-wip to the", "main pkgsrc repository.") @@ -633,7 +686,7 @@ var ( unescapeMkCommentSafeChars = textproc.NewByteSet("\\#[$").Inverse() ) -// unescapeMkComment takes a Makefile line, as written in a file, and splits +// unescapeComment takes a Makefile line, as written in a file, and splits // it into the main part and the comment. // // The comment starts at the first #. Except if it is preceded by an odd number @@ -644,7 +697,7 @@ var ( // // The comment is returned including the leading "#", if any. If the line has // no comment, it is an empty string. -func unescapeMkComment(text string) (main, comment string) { +func (p MkLineParser) unescapeComment(text string) (main, comment string) { var sb strings.Builder lexer := textproc.NewLexer(text) @@ -682,13 +735,21 @@ again: return main, lexer.Rest() } - G.Assertf(lexer.EOF(), "unescapeMkComment(%q): sb = %q, rest = %q", text, main, lexer.Rest()) + G.Assertf(lexer.EOF(), "unescapeComment(%q): sb = %q, rest = %q", text, main, lexer.Rest()) return main, "" } goto again } +type mkLineSplitResult struct { + main string + tokens []*MkToken + spaceBeforeComment string + hasComment bool + comment string +} + // splitMkLine parses a logical line from a Makefile (that is, after joining // the lines that end in a backslash) into two parts: the main part and the // comment. @@ -696,12 +757,12 @@ again: // This applies to all line types except those starting with a tab, which // contain the shell commands to be associated with make targets. These cannot // have comments. -func splitMkLine(text string) (main string, tokens []*MkToken, rest string, spaceBeforeComment string, hasComment bool, comment string) { +func (p MkLineParser) split(line Line, text string) mkLineSplitResult { - main, comment = unescapeMkComment(text) + main, comment := p.unescapeComment(text) - p := NewMkParser(nil, main, false) - lexer := p.lexer + parser := NewMkParser(line, main, line != nil) + lexer := parser.lexer rtrimHspace := func(s string) string { end := len(s) @@ -711,7 +772,7 @@ func splitMkLine(text string) (main string, tokens []*MkToken, rest string, spac return s[:end] } - parseToken := func() string { + parseOther := func() string { var sb strings.Builder for !lexer.EOF() { @@ -731,56 +792,53 @@ func splitMkLine(text string) (main string, tokens []*MkToken, rest string, spac return sb.String() } + var tokens []*MkToken for !lexer.EOF() { mark := lexer.Mark() - if varUse := p.VarUse(); varUse != nil { + if varUse := parser.VarUse(); varUse != nil { tokens = append(tokens, &MkToken{lexer.Since(mark), varUse}) - } else if token := parseToken(); token != "" { - tokens = append(tokens, &MkToken{token, nil}) + } else if other := parseOther(); other != "" { + tokens = append(tokens, &MkToken{other, nil}) } else { - break + G.Assertf(lexer.SkipByte('$'), "Parse error for %q.", text) + tokens = append(tokens, &MkToken{"$", nil}) } } - if comment != "" { - hasComment = true + hasComment := comment != "" + if hasComment { comment = comment[1:] } - rest = lexer.Rest() - main = main[:len(main)-len(rest)] - if rest == "" { - mainWithSpaces := main - main = rtrimHspace(main) - spaceBeforeComment = mainWithSpaces[len(main):] - } else { - restWithoutSpace := strings.TrimRightFunc(rest, func(r rune) bool { return isHspace(byte(r)) }) - if len(restWithoutSpace) < len(rest) { - spaceBeforeComment = rest[len(restWithoutSpace):] - rest = restWithoutSpace + G.Assertf(lexer.Rest() == "", "Parse error for %q.", text) + + mainWithSpaces := main + main = rtrimHspace(main) + spaceBeforeComment := ifelseStr(true, mainWithSpaces[len(main):], "") + if spaceBeforeComment != "" && len(tokens) > 0 { + tokenText := &tokens[len(tokens)-1].Text + *tokenText = rtrimHspace(*tokenText) + if *tokenText == "" { + tokens = tokens[:len(tokens)-1] } } - return + return mkLineSplitResult{main, tokens, spaceBeforeComment, hasComment, comment} } -func matchMkDirective(text string) (m bool, indent, directive, args, comment string) { +func (p MkLineParser) parseDirective(line Line, data mkLineSplitResult) MkLine { + text := line.Text if !hasPrefix(text, ".") { - return - } - - main, _, rest, _, hasComment, trailingComment := splitMkLine(text) - if rest != "" { - return + return nil } - lexer := textproc.NewLexer(main[1:]) + lexer := textproc.NewLexer(data.main[1:]) - indent = lexer.NextHspace() - directive = lexer.NextBytesSet(LowerDash) + indent := lexer.NextHspace() + directive := lexer.NextBytesSet(LowerDash) switch directive { case "if", "else", "elif", "endif", "ifdef", "ifndef", @@ -790,19 +848,19 @@ func matchMkDirective(text string) (m bool, indent, directive, args, comment str break default: // Intentionally not supported are: ifmake ifnmake elifdef elifndef elifmake elifnmake. - return + return nil } lexer.SkipHspace() - args = lexer.Rest() + args := lexer.Rest() - if hasComment { - comment = trailingComment - } + // In .if and .endif lines the space surrounding the comment is irrelevant. + // Especially for checking that the .endif comment matches the .if condition, + // it must be trimmed. + trimmedComment := trimHspace(data.comment) - m = true - return + return &MkLineImpl{line, &mkLineDirectiveImpl{indent, directive, args, trimmedComment, nil, nil, nil}} } // VariableNeedsQuoting determines whether the given variable needs the :Q operator @@ -825,8 +883,8 @@ func (mkline *MkLineImpl) VariableNeedsQuoting(mklines MkLines, varuse *MkVarUse } if !vartype.basicType.NeedsQ() { - if vartype.kindOfList == lkNone { - if vartype.guessed { + if !vartype.List() { + if vartype.Guessed() { return unknown } return no @@ -838,14 +896,14 @@ func (mkline *MkLineImpl) VariableNeedsQuoting(mklines MkLines, varuse *MkVarUse // A shell word may appear as part of a shell word, for example COMPILER_RPATH_FLAG. if vuc.IsWordPart && vuc.quoting == VucQuotPlain { - if vartype.kindOfList == lkNone && vartype.basicType == BtShellWord { + if !vartype.List() && vartype.basicType == BtShellWord { return no } } // Determine whether the context expects a list of shell words or not. - wantList := vucVartype.IsConsideredList() - haveList := vartype.IsConsideredList() + wantList := vucVartype.MayBeAppendedTo() + haveList := vartype.MayBeAppendedTo() if trace.Tracing { trace.Stepf("wantList=%v, haveList=%v", wantList, haveList) } @@ -1382,16 +1440,26 @@ var ( VarparamBytes = textproc.NewByteSet("A-Za-z_0-9#*+---.[") ) -func MatchVarassign(text string) (m bool, assignment mkLineAssign) { - commented := hasPrefix(text, "#") +func (p MkLineParser) MatchVarassign(line Line, text string, asdfData mkLineSplitResult) (m bool, assignment mkLineAssign) { + + // A commented variable assignment does not have leading whitespace. + // Otherwise line 1 of almost every Makefile fragment would need to + // be scanned for a variable assignment even though it only contains + // the $NetBSD CVS Id. + clex := textproc.NewLexer(text) + commented := clex.SkipByte('#') + if commented && clex.SkipHspace() || clex.EOF() { + return false, nil + } + withoutLeadingComment := text if commented { withoutLeadingComment = withoutLeadingComment[1:] } - main, tokens, rest, spaceBeforeComment, hasComment, comment := splitMkLine(withoutLeadingComment) + data := p.split(nil, withoutLeadingComment) - lexer := NewMkTokensLexer(tokens) + lexer := NewMkTokensLexer(data.tokens) mainStart := lexer.Mark() for !commented && lexer.SkipByte(' ') { @@ -1401,7 +1469,7 @@ func MatchVarassign(text string) (m bool, assignment mkLineAssign) { // TODO: duplicated code in MkParser.Varname for lexer.NextBytesSet(VarbaseBytes) != "" || lexer.NextVarUse() != nil { } - if lexer.SkipByte('.') || hasPrefix(main, "SITES_") { + if lexer.SkipByte('.') || hasPrefix(data.main, "SITES_") { for lexer.NextBytesSet(VarparamBytes) != "" || lexer.NextVarUse() != nil { } } @@ -1431,11 +1499,13 @@ func MatchVarassign(text string) (m bool, assignment mkLineAssign) { lexer.SkipHspace() - value := trimHspace(lexer.Rest() + rest) + value := trimHspace(lexer.Rest()) + valueAlign := ifelseStr(commented, "#", "") + lexer.Since(mainStart) + spaceBeforeComment := data.spaceBeforeComment if value == "" { + valueAlign += spaceBeforeComment spaceBeforeComment = "" } - valueAlign := ifelseStr(commented, "#", "") + lexer.Since(mainStart) return true, &mkLineAssignImpl{ commented: commented, @@ -1450,7 +1520,7 @@ func MatchVarassign(text string) (m bool, assignment mkLineAssign) { valueMkRest: "", // filled in lazily fields: nil, // filled in lazily spaceAfterValue: spaceBeforeComment, - comment: ifelseStr(hasComment, "#", "") + comment, + comment: ifelseStr(data.hasComment, "#", "") + data.comment, } } diff --git a/pkgtools/pkglint/files/mkline_test.go b/pkgtools/pkglint/files/mkline_test.go index f712de44be0..54d8e737d44 100644 --- a/pkgtools/pkglint/files/mkline_test.go +++ b/pkgtools/pkglint/files/mkline_test.go @@ -1,8 +1,11 @@ package pkglint -import "gopkg.in/check.v1" +import ( + "gopkg.in/check.v1" + "strings" +) -func (s *Suite) Test_NewMkLine__varassign(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__varassign(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("test.mk", 101, @@ -17,7 +20,7 @@ func (s *Suite) Test_NewMkLine__varassign(c *check.C) { c.Check(mkline.VarassignComment(), equals, "# varassign comment") } -func (s *Suite) Test_NewMkLine__varassign_space_around_operator(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__varassign_space_around_operator(c *check.C) { t := s.Init(c) t.SetUpCommandLine("--show-autofix", "--source") @@ -31,7 +34,7 @@ func (s *Suite) Test_NewMkLine__varassign_space_around_operator(c *check.C) { "+\tpkgbase= package") } -func (s *Suite) Test_NewMkLine__shellcmd(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__shellcmd(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("test.mk", 101, @@ -41,7 +44,7 @@ func (s *Suite) Test_NewMkLine__shellcmd(c *check.C) { c.Check(mkline.ShellCommand(), equals, "shell command # shell comment") } -func (s *Suite) Test_NewMkLine__comment(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__comment(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("test.mk", 101, @@ -50,7 +53,7 @@ func (s *Suite) Test_NewMkLine__comment(c *check.C) { c.Check(mkline.IsComment(), equals, true) } -func (s *Suite) Test_NewMkLine__empty(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__empty(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("test.mk", 101, "") @@ -58,7 +61,7 @@ func (s *Suite) Test_NewMkLine__empty(c *check.C) { c.Check(mkline.IsEmpty(), equals, true) } -func (s *Suite) Test_NewMkLine__directive(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__directive(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("test.mk", 101, @@ -71,7 +74,7 @@ func (s *Suite) Test_NewMkLine__directive(c *check.C) { c.Check(mkline.DirectiveComment(), equals, "directive comment") } -func (s *Suite) Test_NewMkLine__include(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__include(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("test.mk", 101, @@ -85,7 +88,7 @@ func (s *Suite) Test_NewMkLine__include(c *check.C) { c.Check(mkline.IsSysinclude(), equals, false) } -func (s *Suite) Test_NewMkLine__sysinclude(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__sysinclude(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("test.mk", 101, @@ -99,7 +102,7 @@ func (s *Suite) Test_NewMkLine__sysinclude(c *check.C) { c.Check(mkline.IsInclude(), equals, false) } -func (s *Suite) Test_NewMkLine__dependency(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__dependency(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("test.mk", 101, @@ -110,7 +113,7 @@ func (s *Suite) Test_NewMkLine__dependency(c *check.C) { c.Check(mkline.Sources(), equals, "source1 source2") } -func (s *Suite) Test_NewMkLine__dependency_space(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__dependency_space(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("test.mk", 101, @@ -122,7 +125,7 @@ func (s *Suite) Test_NewMkLine__dependency_space(c *check.C) { "NOTE: test.mk:101: Space before colon in dependency line.") } -func (s *Suite) Test_NewMkLine__varassign_append(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__varassign_append(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("test.mk", 101, @@ -134,7 +137,7 @@ func (s *Suite) Test_NewMkLine__varassign_append(c *check.C) { c.Check(mkline.Varparam(), equals, "") } -func (s *Suite) Test_NewMkLine__merge_conflict(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__merge_conflict(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("test.mk", 101, @@ -151,7 +154,7 @@ func (s *Suite) Test_NewMkLine__merge_conflict(c *check.C) { c.Check(mkline.IsSysinclude(), equals, false) } -func (s *Suite) Test_NewMkLine__autofix_space_after_varname(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__autofix_space_after_varname(c *check.C) { t := s.Init(c) t.SetUpCommandLine("-Wspace") @@ -187,7 +190,7 @@ func (s *Suite) Test_NewMkLine__autofix_space_after_varname(c *check.C) { "pkgbase := pkglint") } -func (s *Suite) Test_NewMkLine__varname_with_hash(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__varname_with_hash(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("Makefile", 123, "VARNAME.#=\tvalue") @@ -208,7 +211,7 @@ func (s *Suite) Test_NewMkLine__varname_with_hash(c *check.C) { // // To check that bmake parses them the same, set a breakpoint after the t.NewMkLines // and look in t.tmpdir for the location of the file. Then run bmake with that file. -func (s *Suite) Test_NewMkLine__escaped_hash_in_value(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__escaped_hash_in_value(c *check.C) { t := s.Init(c) mklines := t.SetUpFileMkLines("Makefile", @@ -278,13 +281,13 @@ func (s *Suite) Test_VarUseContext_String(c *check.C) { vartype := G.Pkgsrc.VariableType(nil, "PKGNAME") vuc := VarUseContext{vartype, vucTimeUnknown, VucQuotBackt, false} - c.Check(vuc.String(), equals, "(Pkgname time:unknown quoting:backt wordpart:false)") + c.Check(vuc.String(), equals, "(Pkgname (package-settable) time:unknown quoting:backt wordpart:false)") } // In variable assignments, a plain '#' introduces a line comment, unless // it is escaped by a backslash. In shell commands, on the other hand, it // is interpreted literally. -func (s *Suite) Test_NewMkLine__number_sign(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__number_sign(c *check.C) { t := s.Init(c) mklineVarassignEscaped := t.NewMkLine("filename.mk", 1, "SED_CMD=\t's,\\#,hash,g'") @@ -309,7 +312,7 @@ func (s *Suite) Test_NewMkLine__number_sign(c *check.C) { "WARN: filename.mk:1: The # character starts a Makefile comment.") } -func (s *Suite) Test_NewMkLine__varassign_leading_space(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__varassign_leading_space(c *check.C) { t := s.Init(c) _ = t.NewMkLine("rubyversion.mk", 427, " _RUBYVER=\t2.15") @@ -327,7 +330,7 @@ func (s *Suite) Test_NewMkLine__varassign_leading_space(c *check.C) { // be able to parse and check the infrastructure files as well. // // See Pkgsrc.loadUntypedVars. -func (s *Suite) Test_NewMkLine__infrastructure(c *check.C) { +func (s *Suite) Test_MkLineParser_Parse__infrastructure(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("infra.mk", @@ -413,7 +416,6 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__eval_shell(c *check.C) { MkLineChecker{nil, mkline}.checkVarassign() t.CheckOutputLines( - "WARN: builtin.mk:3: PKG_ADMIN should not be used at load time in any file.", "NOTE: builtin.mk:3: The :Q operator isn't necessary for ${BUILTIN_PKG.Xfixes} here.") } @@ -841,8 +843,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__uncovered_cases(c *check.C) { "\tname in which the variable is used or defined. The rules for PATH", "\tare:", "", - "\t* in buildlink3.mk, it should not be accessed at all", - "\t* in any file, it may be used", + "\t* in any file, it may be used at load time, or used", "", "\tIf these rules seem to be incorrect, please ask on the", "\ttech-pkg@NetBSD.org mailing list.", @@ -867,28 +868,6 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__uncovered_cases(c *check.C) { "", "\tIf these rules seem to be incorrect, please ask on the", "\ttech-pkg@NetBSD.org mailing list.", - "", - "WARN: ~/Makefile:6: PATH should not be used at load time in any file.", - "", - "\tMany variables, especially lists of something, get their values", - "\tincrementally. Therefore it is generally unsafe to rely on their", - "\tvalue until it is clear that it will never change again. This point", - "\tis reached when the whole package Makefile is loaded and execution", - "\tof the shell commands starts; in some cases earlier.", - "", - "\tAdditionally, when using the \":=\" operator, each $$ is replaced with", - "\ta single $, so variables that have references to shell variables or", - "\tregular expressions are modified in a subtle way.", - "", - "\tThe allowed actions for a variable are determined based on the file", - "\tname in which the variable is used or defined. The rules for PATH", - "\tare:", - "", - "\t* in buildlink3.mk, it should not be accessed at all", - "\t* in any file, it may be used", - "", - "\tIf these rules seem to be incorrect, please ask on the", - "\ttech-pkg@NetBSD.org mailing list.", "") // Just for branch coverage. @@ -1142,56 +1121,87 @@ func (s *Suite) Test_MkLine_ValueFields__compared_to_splitIntoShellTokens(c *che func (s *Suite) Test_MkLine_ValueTokens(c *check.C) { t := s.Init(c) - testTokens := func(value string, expected ...*MkToken) { + text := func(text string) *MkToken { return &MkToken{text, nil} } + varUseText := func(text string, varname string, modifiers ...string) *MkToken { + return &MkToken{text, NewMkVarUse(varname, modifiers...)} + } + tokens := func(tokens ...*MkToken) []*MkToken { return tokens } + test := func(value string, expected []*MkToken, diagnostics ...string) { mkline := t.NewMkLine("Makefile", 1, "PATH=\t"+value) - tokens, _ := mkline.ValueTokens() - c.Check(tokens, deepEquals, expected) + actualTokens, _ := mkline.ValueTokens() + c.Check(actualTokens, deepEquals, expected) + t.CheckOutput(diagnostics) } - testTokens("#empty", - []*MkToken(nil)...) + t.Use(text, varUseText, tokens, test) - testTokens("value", - &MkToken{"value", nil}) + test("#empty", + tokens()) - testTokens("value ${VAR} rest", - &MkToken{"value ", nil}, - &MkToken{"${VAR}", NewMkVarUse("VAR")}, - &MkToken{" rest", nil}) + test("value", + tokens(text("value"))) - testTokens("value ${UNFINISHED", - &MkToken{"value ", nil}) + test("value ${VAR} rest", + tokens( + text("value "), + varUseText("${VAR}", "VAR"), + text(" rest"))) + + test("value # comment", + tokens( + text("value"))) + + test("value ${UNFINISHED", + tokens( + text("value "), + varUseText("${UNFINISHED", "UNFINISHED")), + + "WARN: Makefile:1: Missing closing \"}\" for \"UNFINISHED\".") } func (s *Suite) Test_MkLine_ValueTokens__caching(c *check.C) { t := s.Init(c) + tokens := func(tokens ...*MkToken) []*MkToken { return tokens } + mkline := t.NewMkLine("Makefile", 1, "PATH=\tvalue ${UNFINISHED") - tokens, rest := mkline.ValueTokens() + valueTokens, rest := mkline.ValueTokens() - c.Check(tokens, deepEquals, []*MkToken{{"value ", nil}}) - c.Check(rest, equals, "${UNFINISHED") + c.Check(valueTokens, deepEquals, + tokens( + &MkToken{"value ", nil}, + &MkToken{"${UNFINISHED", NewMkVarUse("UNFINISHED")})) + c.Check(rest, equals, "") + t.CheckOutputLines( + "WARN: Makefile:1: Missing closing \"}\" for \"UNFINISHED\".") - tokens2, rest2 := mkline.ValueTokens() // This time the slice is taken from the cache. + // This time the slice is taken from the cache. + tokens2, rest2 := mkline.ValueTokens() - // In Go, it's not possible to compare slices for reference equality. - c.Check(tokens2, deepEquals, tokens) + c.Check(&tokens2[0], equals, &valueTokens[0]) c.Check(rest2, equals, rest) } func (s *Suite) Test_MkLine_ValueTokens__caching_parse_error(c *check.C) { t := s.Init(c) + tokens := func(tokens ...*MkToken) []*MkToken { return tokens } + varuseText := func(text, varname string, modifiers ...string) *MkToken { + return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)} + } + mkline := t.NewMkLine("Makefile", 1, "PATH=\t${UNFINISHED") - tokens, rest := mkline.ValueTokens() + valueTokens, rest := mkline.ValueTokens() - c.Check(tokens, check.IsNil) - c.Check(rest, equals, "${UNFINISHED") + c.Check(valueTokens, deepEquals, tokens(varuseText("${UNFINISHED", "UNFINISHED"))) + c.Check(rest, equals, "") + t.CheckOutputLines( + "WARN: Makefile:1: Missing closing \"}\" for \"UNFINISHED\".") - tokens2, rest2 := mkline.ValueTokens() // This time the slice is taken from the cache. + // This time the slice is taken from the cache. + tokens2, rest2 := mkline.ValueTokens() - // In Go, it's not possible to compare slices for reference equality. - c.Check(tokens2, deepEquals, tokens) + c.Check(&tokens2[0], equals, &valueTokens[0]) c.Check(rest2, equals, rest) } @@ -1266,14 +1276,16 @@ func (s *Suite) Test_MkLine_ResolveVarsInRelativePath__directory_depth(c *check. "WARN: ~/multimedia/totem/bla.mk:2: "+ "The variable BUILDLINK_PKGSRCDIR.totem should not be given a default value in this file; "+ "it would be ok in buildlink3.mk.", - "ERROR: ~/multimedia/totem/bla.mk:2: There is no package in \"multimedia/totem\".") + "ERROR: ~/multimedia/totem/bla.mk:2: Relative path \"../../multimedia/totem/Makefile\" does not exist.") } -func (s *Suite) Test_MatchVarassign(c *check.C) { - s.Init(c) +func (s *Suite) Test_MkLineParser_MatchVarassign(c *check.C) { + t := s.Init(c) - test := func(text string, commented bool, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment string) { - m, actual := MatchVarassign(text) + test := func(text string, commented bool, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment string, diagnostics ...string) { + line := t.NewLine("filename.mk", 123, text) + data := MkLineParser{}.split(line, text) + m, actual := MkLineParser{}.MatchVarassign(line, text, data) if !m { c.Errorf("Text %q doesn't match variable assignment", text) return @@ -1295,13 +1307,17 @@ func (s *Suite) Test_MatchVarassign(c *check.C) { comment: comment, } c.Check(*actual, deepEquals, expected) + t.CheckOutput(diagnostics) } - testInvalid := func(text string) { - m, _ := MatchVarassign(text) + testInvalid := func(text string, diagnostics ...string) { + line := t.NewLine("filename.mk", 123, text) + data := MkLineParser{}.split(nil, text) + m, _ := MkLineParser{}.MatchVarassign(line, text, data) if m { c.Errorf("Text %q matches variable assignment but shouldn't.", text) } + t.CheckOutput(diagnostics) } test("C++=c11", false, "C+", "", "+=", "C++=", "c11", "", "") @@ -1396,6 +1412,7 @@ func (s *Suite) Test_MatchVarassign(c *check.C) { "# none") test("EGDIRS=\t${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d", + false, "EGDIRS", "", @@ -1403,7 +1420,14 @@ func (s *Suite) Test_MatchVarassign(c *check.C) { "EGDIRS=\t", "${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d", "", - "") + "", + + "WARN: filename.mk:123: Missing closing \"}\" for \"EGDIR/pam.d\".", + "WARN: filename.mk:123: Invalid part \"/pam.d\" after variable name \"EGDIR\".", + "WARN: filename.mk:123: Missing closing \"}\" for \"EGDIR/dbus-1/system.d ${EGDIR/pam.d\".", + "WARN: filename.mk:123: Invalid part \"/dbus-1/system.d ${EGDIR/pam.d\" after variable name \"EGDIR\".", + "WARN: filename.mk:123: Missing closing \"}\" for \"EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".", + "WARN: filename.mk:123: Invalid part \"/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\" after variable name \"EGDIR\".") test("VAR:=\t${VAR:M-*:[\\#]}", false, @@ -1414,6 +1438,13 @@ func (s *Suite) Test_MatchVarassign(c *check.C) { "${VAR:M-*:[#]}", "", "") + + test("#VAR=value", + true, "VAR", "", "=", "#VAR=", "value", "", "") + + testInvalid("# VAR=value") + testInvalid("#\tVAR=value") + testInvalid(MkRcsID) } func (s *Suite) Test_NewMkOperator(c *check.C) { @@ -1532,6 +1563,7 @@ func (s *Suite) Test_Indentation_Varnames__repetition(c *check.C) { ". include \"../../category/other/buildlink3.mk\"", ". endif", ".endif") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -1590,6 +1622,9 @@ func (s *Suite) Test_MkLine_ForEachUsed(c *check.C) { "run <", "run @", "run x"}) + t.CheckOutputLines( + "WARN: Makefile:12: Please use curly braces {} instead of round parentheses () for ROUND_PARENTHESES.", + "WARN: Makefile:14: $x is ambiguous. Use ${x} if you mean a Make variable or $$x if you mean a shell variable.") } func (s *Suite) Test_MkLine_UnquoteShell(c *check.C) { @@ -1621,11 +1656,11 @@ func (s *Suite) Test_MkLine_UnquoteShell(c *check.C) { test("`", "`") } -func (s *Suite) Test_unescapeMkComment(c *check.C) { +func (s *Suite) Test_MkLineParser_unescapeComment(c *check.C) { t := s.Init(c) test := func(text string, main, comment string) { - aMain, aComment := unescapeMkComment(text) + aMain, aComment := MkLineParser{}.unescapeComment(text) t.Check( []interface{}{text, aMain, aComment}, deepEquals, @@ -1741,16 +1776,19 @@ func (s *Suite) Test_unescapeMkComment(c *check.C) { "#comment") } -func (s *Suite) Test_splitMkLine(c *check.C) { +func (s *Suite) Test_MkLineParser_split(c *check.C) { t := s.Init(c) varuse := func(varname string, modifiers ...string) *MkToken { - text := "${" + varname + var text strings.Builder + text.WriteString("${") + text.WriteString(varname) for _, modifier := range modifiers { - text += ":" + modifier + text.WriteString(":") + text.WriteString(modifier) } - text += "}" - return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)} + text.WriteString("}") + return &MkToken{Text: text.String(), Varuse: NewMkVarUse(varname, modifiers...)} } varuseText := func(text, varname string, modifiers ...string) *MkToken { return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)} @@ -1761,30 +1799,67 @@ func (s *Suite) Test_splitMkLine(c *check.C) { tokens := func(tokens ...*MkToken) []*MkToken { return tokens } - _, _, _, _ = text, varuse, varuseText, tokens - test := func(text string, main string, tokens []*MkToken, rest string, spaceBeforeComment string, hasComment bool, comment string) { - aMain, aTokens, aRest, aSpaceBeforeComment, aHasComment, aComment := splitMkLine(text) - t.Check( - []interface{}{text, aTokens, aMain, aRest, aSpaceBeforeComment, aHasComment, aComment}, - deepEquals, - []interface{}{text, tokens, main, rest, spaceBeforeComment, hasComment, comment}) + test := func(text string, data mkLineSplitResult, diagnostics ...string) { + line := t.NewLine("filename.mk", 123, text) + actualData := MkLineParser{}.split(line, text) + + t.CheckOutput(diagnostics) + t.Check([]interface{}{text, actualData}, deepEquals, []interface{}{text, data}) } - test("", - "", - tokens(), - "", + t.Use(text, varuse, varuseText, tokens) + + test( "", - false, - "") - test("text", + mkLineSplitResult{}) + + test( "text", - tokens(text("text")), - "", - "", - false, - "") + mkLineSplitResult{ + main: "text", + tokens: tokens(text("text")), + }) + + // Leading space is always kept. + test( + " text", + mkLineSplitResult{ + main: " text", + tokens: tokens(text(" text")), + }) + + // Trailing space does not end up in the tokens since it is usually + // ignored. + test( + "text\t", + mkLineSplitResult{ + main: "text", + tokens: tokens(text("text")), + spaceBeforeComment: "\t", + }) + + test( + "text\t# intended comment", + mkLineSplitResult{ + main: "text", + tokens: tokens(text("text")), + spaceBeforeComment: "\t", + hasComment: true, + comment: " intended comment", + }) + + // Trailing space is saved in a separate field to detect accidental + // unescaped # in the middle of a word, like the URL fragment in this + // example. + test( + "url#fragment", + mkLineSplitResult{ + main: "url", + tokens: tokens(text("url")), + hasComment: true, + comment: "fragment", + }) // The leading space from the comment is preserved to make parsing as exact // as possible. @@ -1792,161 +1867,165 @@ func (s *Suite) Test_splitMkLine(c *check.C) { // The difference between "#defined" and "# defined" is relevant in a few // cases, such as the API documentation of the infrastructure files. test("# comment", - "", - tokens(), - "", - "", - true, - " comment") + mkLineSplitResult{ + hasComment: true, + comment: " comment", + }) + test("#\tcomment", - "", - tokens(), - "", - "", - true, - "\tcomment") + mkLineSplitResult{ + hasComment: true, + comment: "\tcomment", + }) + test("# comment", - "", - tokens(), - "", - "", - true, - " comment") + mkLineSplitResult{ + hasComment: true, + comment: " comment", + }) // Other than in the shell, # also starts a comment in the middle of a word. test("COMMENT=\tThe C# compiler", - "COMMENT=\tThe C", - tokens(text("COMMENT=\tThe C")), - "", - "", - true, - " compiler") + mkLineSplitResult{ + main: "COMMENT=\tThe C", + tokens: tokens(text("COMMENT=\tThe C")), + hasComment: true, + comment: " compiler", + }) + test("COMMENT=\tThe C\\# compiler", - "COMMENT=\tThe C# compiler", - tokens(text("COMMENT=\tThe C# compiler")), - "", - "", - false, - "") + mkLineSplitResult{ + main: "COMMENT=\tThe C# compiler", + tokens: tokens(text("COMMENT=\tThe C# compiler")), + hasComment: false, + comment: "", + }) test("${TARGET}: ${SOURCES} # comment", - "${TARGET}: ${SOURCES}", - tokens(varuse("TARGET"), text(": "), varuse("SOURCES"), text(" ")), - "", - " ", - true, - " comment") + mkLineSplitResult{ + main: "${TARGET}: ${SOURCES}", + tokens: tokens(varuse("TARGET"), text(": "), varuse("SOURCES")), + spaceBeforeComment: " ", + hasComment: true, + comment: " comment", + }) // A # starts a comment, except if it immediately follows a [. // This is done so that the length modifier :[#] can be written without // escaping the #. test("VAR=\t${OTHER:[#]} # comment", - "VAR=\t${OTHER:[#]}", - tokens(text("VAR=\t"), varuse("OTHER", "[#]"), text(" ")), - "", - " ", - true, - " comment") + mkLineSplitResult{ + main: "VAR=\t${OTHER:[#]}", + tokens: tokens(text("VAR=\t"), varuse("OTHER", "[#]")), + spaceBeforeComment: " ", + hasComment: true, + comment: " comment", + }) // The # in the :[#] modifier may be escaped or not. Both forms are equivalent. test("VAR:=\t${VAR:M-*:[\\#]}", - "VAR:=\t${VAR:M-*:[#]}", - tokens(text("VAR:=\t"), varuse("VAR", "M-*", "[#]")), - "", - "", - false, - "") + mkLineSplitResult{ + main: "VAR:=\t${VAR:M-*:[#]}", + tokens: tokens(text("VAR:=\t"), varuse("VAR", "M-*", "[#]")), + }) // A backslash always escapes the next character, be it a # for a comment // or something else. This makes it difficult to write a literal \# in a // Makefile, but that's an edge case anyway. test("VAR0=\t#comment", - "VAR0=", - tokens(text("VAR0=\t")), - "", - // Later, when converting this result into a proper variable assignment, - // this "space before comment" is reclassified as "space before the value", - // in order to align the "#comment" with the other variable values. - "\t", - true, - "comment") + mkLineSplitResult{ + main: "VAR0=", + tokens: tokens(text("VAR0=")), + // Later, when converting this result into a proper variable assignment, + // this "space before comment" is reclassified as "space before the value", + // in order to align the "#comment" with the other variable values. + spaceBeforeComment: "\t", + hasComment: true, + comment: "comment", + }) + test("VAR1=\t\\#no-comment", - "VAR1=\t#no-comment", - tokens(text("VAR1=\t#no-comment")), - "", - "", - false, - "") + mkLineSplitResult{ + main: "VAR1=\t#no-comment", + tokens: tokens(text("VAR1=\t#no-comment")), + }) + test("VAR2=\t\\\\#comment", - "VAR2=\t\\\\", - tokens(text("VAR2=\t\\\\")), - "", - "", - true, - "comment") + mkLineSplitResult{ + main: "VAR2=\t\\\\", + tokens: tokens(text("VAR2=\t\\\\")), + hasComment: true, + comment: "comment", + }) // The backslash is only removed when it escapes a comment. // In particular, it cannot be used to escape a dollar that starts a // variable use. test("VAR0=\t$T", - "VAR0=\t$T", - tokens(text("VAR0=\t"), varuseText("$T", "T")), - "", - "", - false, - "") + mkLineSplitResult{ + main: "VAR0=\t$T", + tokens: tokens(text("VAR0=\t"), varuseText("$T", "T")), + }, + "WARN: filename.mk:123: $T is ambiguous. Use ${T} if you mean a Make variable or $$T if you mean a shell variable.") + test("VAR1=\t\\$T", - "VAR1=\t\\$T", - tokens(text("VAR1=\t\\"), varuseText("$T", "T")), - "", - "", - false, - "") + mkLineSplitResult{ + main: "VAR1=\t\\$T", + tokens: tokens(text("VAR1=\t\\"), varuseText("$T", "T")), + }, + "WARN: filename.mk:123: $T is ambiguous. Use ${T} if you mean a Make variable or $$T if you mean a shell variable.") + test("VAR2=\t\\\\$T", - "VAR2=\t\\\\$T", - tokens(text("VAR2=\t\\\\"), varuseText("$T", "T")), - "", - "", - false, - "") + mkLineSplitResult{ + main: "VAR2=\t\\\\$T", + tokens: tokens(text("VAR2=\t\\\\"), varuseText("$T", "T")), + }, + "WARN: filename.mk:123: $T is ambiguous. Use ${T} if you mean a Make variable or $$T if you mean a shell variable.") // To escape a dollar, write it twice. test("$$shellvar $${shellvar} \\${MKVAR} [] \\x", - "$$shellvar $${shellvar} \\${MKVAR} [] \\x", - tokens(text("$$shellvar $${shellvar} \\"), varuse("MKVAR"), text(" [] \\x")), - "", - "", - false, - "") + mkLineSplitResult{ + main: "$$shellvar $${shellvar} \\${MKVAR} [] \\x", + tokens: tokens(text("$$shellvar $${shellvar} \\"), varuse("MKVAR"), text(" [] \\x")), + }) // Parse errors are recorded in the rest return value. test("${UNCLOSED", - "", - tokens(), - "${UNCLOSED", - "", - false, - "") + mkLineSplitResult{ + main: "${UNCLOSED", + tokens: tokens(varuseText("${UNCLOSED", "UNCLOSED")), + }, + "WARN: filename.mk:123: Missing closing \"}\" for \"UNCLOSED\".") // Even if there is a parse error in the main part, // the comment is extracted. test("text before ${UNCLOSED# comment", - "text before ", - tokens(text("text before ")), - "${UNCLOSED", - "", - true, - " comment") + mkLineSplitResult{ + main: "text before ${UNCLOSED", + tokens: tokens( + text("text before "), + varuseText("${UNCLOSED", "UNCLOSED")), + hasComment: true, + comment: " comment", + }, + "WARN: filename.mk:123: Missing closing \"}\" for \"UNCLOSED\".") // Even in case of parse errors, the space before the comment is parsed // correctly. test("text before ${UNCLOSED # comment", - "text before ", - tokens(text("text before ")), - "${UNCLOSED", - " ", - true, - " comment") + mkLineSplitResult{ + main: "text before ${UNCLOSED", + tokens: tokens( + text("text before "), + // It's a bit inconsistent that the varname includes the space + // but the text doesn't; anyway, it's an edge case. + varuseText("${UNCLOSED", "UNCLOSED ")), + spaceBeforeComment: " ", + hasComment: true, + comment: " comment", + }, + "WARN: filename.mk:123: Missing closing \"}\" for \"UNCLOSED \".", + "WARN: filename.mk:123: Invalid part \" \" after variable name \"UNCLOSED\".") // The dollar-space refers to a normal Make variable named " ". // The lonely dollar at the very end refers to the variable named "", @@ -1957,58 +2036,73 @@ func (s *Suite) Test_splitMkLine(c *check.C) { // variable name, mainly because the empty variable name is not visible // outside of the bmake debugging mode. test("Lonely $ character $", - "Lonely $ character ", - tokens( - text("Lonely "), - varuseText("$ " /* instead of "${ }" */, " "), - text("character ")), - "$", - "", - false, - "") + mkLineSplitResult{ + main: "Lonely $ character $", + tokens: tokens( + text("Lonely "), + varuseText("$ " /* instead of "${ }" */, " "), + text("character "), + text("$")), + }) // The character [ prevents the following # from starting a comment, even // outside of variable modifiers. test("COMMENT=\t[#] $$\\# $$# comment", - "COMMENT=\t[#] $$# $$", - tokens(text("COMMENT=\t[#] $$# $$")), - "", - "", - true, - " comment") + mkLineSplitResult{ + main: "COMMENT=\t[#] $$# $$", + tokens: tokens(text("COMMENT=\t[#] $$# $$")), + hasComment: true, + comment: " comment", + }) test("VAR2=\t\\\\#comment", - "VAR2=\t\\\\", - tokens(text("VAR2=\t\\\\")), - "", - "", - true, - "comment") + mkLineSplitResult{ + main: "VAR2=\t\\\\", + tokens: tokens(text("VAR2=\t\\\\")), + hasComment: true, + comment: "comment", + }) + + // At this stage, MkLine.split doesn't know that empty(...) takes + // a variable use. Instead it just sees ordinary characters and + // other uses of variables. + test(".if empty(${VAR.${tool}}:C/\\:.*$//:M${pattern})", + mkLineSplitResult{ + main: ".if empty(${VAR.${tool}}:C/\\:.*$//:M${pattern})", + tokens: tokens( + text(".if empty("), + varuse("VAR.${tool}"), + text(":C/\\:.*"), + text("$"), + text("//:M"), + varuse("pattern"), + text(")")), + }) } -func (s *Suite) Test_matchMkDirective(c *check.C) { +func (s *Suite) Test_MkLineParser_parseDirective(c *check.C) { + t := s.Init(c) + + test := func(input, expectedIndent, expectedDirective, expectedArgs, expectedComment string, diagnostics ...string) { + line := t.NewLine("filename.mk", 123, input) + data := MkLineParser{}.split(line, input) + mkline := MkLineParser{}.parseDirective(line, data) + if !c.Check(mkline, check.NotNil) { + return + } - test := func(input, expectedIndent, expectedDirective, expectedArgs, expectedComment string) { - m, indent, directive, args, comment := matchMkDirective(input) c.Check( - []interface{}{m, indent, directive, args, comment}, + []interface{}{mkline.Indent(), mkline.Directive(), mkline.Args(), mkline.DirectiveComment()}, deepEquals, - []interface{}{true, expectedIndent, expectedDirective, expectedArgs, expectedComment}) - } - - testFail := func(input string) { - m, indent, directive, args, comment := matchMkDirective(input) - if m { - c.Errorf("The line %q could be parsed as directive (%q, %q, %q, %q) but shouldn't.", - indent, directive, args, comment) - } + []interface{}{expectedIndent, expectedDirective, expectedArgs, expectedComment}) + t.CheckOutput(diagnostics) } test(".if ${VAR} == value", "", "if", "${VAR} == value", "") test(".\tendif # comment", - "\t", "endif", "", " comment") + "\t", "endif", "", "comment") test(".if ${VAR} == \"#\"", "", "if", "${VAR} == \"", "\"") @@ -2019,8 +2113,9 @@ func (s *Suite) Test_matchMkDirective(c *check.C) { test(".if ${VAR} == \\", "", "if", "${VAR} == \\", "") - // Unclosed variable - testFail(".if ${VAR") + test(".if ${VAR", + "", "if", "${VAR", "", + "WARN: filename.mk:123: Missing closing \"}\" for \"VAR\".") } func (s *Suite) Test_MatchMkInclude(c *check.C) { diff --git a/pkgtools/pkglint/files/mklinechecker.go b/pkgtools/pkglint/files/mklinechecker.go index edf108f47ef..a461ddf99aa 100644 --- a/pkgtools/pkglint/files/mklinechecker.go +++ b/pkgtools/pkglint/files/mklinechecker.go @@ -84,7 +84,7 @@ func (ck MkLineChecker) checkInclude() { switch { case hasSuffix(includedFile, "/Makefile"): mkline.Errorf("Other Makefiles must not be included directly.") - G.Explain( + mkline.Explain( "To include portions of another Makefile, extract the common parts", "and put them into a Makefile.common or a Makefile fragment called", "module.mk or similar.", @@ -222,7 +222,7 @@ func (ck MkLineChecker) checkDirectiveFor(forVars map[string]bool, indentation * // The guessed flag could also be determined more correctly. As of November 2018, // running pkglint over the whole pkgsrc tree did not produce any different result // whether guessed was true or false. - forLoopType := Vartype{lkShell, btForLoop, []ACLEntry{{"*", aclpAllRead}}, false} + forLoopType := Vartype{btForLoop, List, []ACLEntry{{"*", aclpAllRead}}} forLoopContext := VarUseContext{&forLoopType, vucTimeParse, VucQuotPlain, false} mkline.ForEachUsed(func(varUse *MkVarUse, time vucTime) { ck.CheckVaruse(varUse, &forLoopContext) @@ -271,7 +271,7 @@ func (ck MkLineChecker) checkDependencyRule(allowedTargets map[string]bool) { } else if !allowedTargets[target] { mkline.Warnf("Undeclared target %q.", target) - G.Explain( + mkline.Explain( "To define a custom target in a package, declare it like this:", "", "\t.PHONY: my-target", @@ -353,6 +353,8 @@ func (ck MkLineChecker) explainPermissions(varname string, vartype *Vartype, int return } + // TODO: Starting with the second explanation, omit the common part. Instead, only list the permission rules. + var expl []string if len(intro) > 0 { @@ -385,7 +387,7 @@ func (ck MkLineChecker) explainPermissions(varname string, vartype *Vartype, int "", "If these rules seem to be incorrect, please ask on the tech-pkg@NetBSD.org mailing list.") - G.Explain(expl...) + ck.MkLine.Explain(expl...) } // CheckVaruse checks a single use of a variable in a specific context. @@ -415,7 +417,7 @@ func (ck MkLineChecker) CheckVaruse(varuse *MkVarUse, vuc *VarUseContext) { func (ck MkLineChecker) checkVarUseVarname(varuse *MkVarUse) { if varuse.varname == "@" { ck.MkLine.Warnf("Please use %q instead of %q.", "${.TARGET}", "$@") - G.Explain( + ck.MkLine.Explain( "It is more readable and prevents confusion with the shell variable", "of the same name.") } @@ -453,7 +455,7 @@ func (ck MkLineChecker) checkVaruseUndefined(vartype *Vartype, varname string) { case !G.Opts.WarnExtra: return - case vartype != nil && !vartype.guessed: + case vartype != nil && !vartype.Guessed(): // Well-known variables are probably defined by the infrastructure. return @@ -492,9 +494,9 @@ func (ck MkLineChecker) checkVaruseModifiers(varuse *MkVarUse, vartype *Vartype) } func (ck MkLineChecker) checkVaruseModifiersSuffix(varuse *MkVarUse, vartype *Vartype) { - if varuse.modifiers[0].IsSuffixSubst() && vartype != nil && !vartype.IsConsideredList() { + if varuse.modifiers[0].IsSuffixSubst() && vartype != nil && !vartype.List() { ck.MkLine.Warnf("The :from=to modifier should only be used with lists, not with %s.", varuse.varname) - G.Explain( + ck.MkLine.Explain( "Instead of (for example):", "\tMASTER_SITES=\t${HOMEPAGE:=repository/}", "", @@ -570,7 +572,7 @@ func (ck MkLineChecker) checkVarusePermissions(varname string, vartype *Vartype, indirectly := !directly && vuc.vartype != nil && vuc.vartype.Union().Contains(aclpUseLoadtime) - if vartype.guessed { + if vartype.Guessed() { return } @@ -710,7 +712,7 @@ func (ck MkLineChecker) warnVaruseToolLoadTime(varname string, tool *Tool) { } ck.MkLine.Warnf("The tool ${%s} cannot be used at load time.", varname) - G.Explain( + ck.MkLine.Explain( "To use a tool at load time, it must be declared in the package", "Makefile by adding it to USE_TOOLS.", "After that, bsd.prefs.mk must be included.", @@ -758,7 +760,7 @@ func (ck MkLineChecker) checkVarUseQuoting(varUse *MkVarUse, vartype *Vartype, v modNoM := strings.TrimSuffix(modNoQ, ":M*") correctMod := modNoM + ifelseStr(needMstar, ":M*:Q", ":Q") if correctMod == mod+":Q" && vuc.IsWordPart && !vartype.IsShell() { - if vartype.IsConsideredList() { + if vartype.List() { mkline.Warnf("The list variable %s should not be embedded in a word.", varname) mkline.Explain( "When a list variable has multiple elements, this expression expands", @@ -876,7 +878,7 @@ func (ck MkLineChecker) checkVarassignDecreasingVersions() { if i > 0 && ver >= intVersions[i-1] { mkline.Warnf("The values for %s should be in decreasing order (%d before %d).", mkline.Varname(), ver, intVersions[i-1]) - G.Explain( + mkline.Explain( "If they aren't, it may be possible that needless versions of", "packages are installed.") } @@ -903,7 +905,7 @@ func (ck MkLineChecker) checkVarassignLeft() { ck.checkTextVarUse( ck.MkLine.Varname(), - &Vartype{lkNone, BtVariableName, []ACLEntry{{"*", aclpAll}}, false}, + &Vartype{BtVariableName, NoVartypeOptions, []ACLEntry{{"*", aclpAll}}}, vucTimeParse) } @@ -1023,10 +1025,15 @@ func (ck MkLineChecker) checkVarassignLeftNotUsed() { return } - // FIXME: Explain how to fix this warning. - // For files like module.mk that are used by other packages, - // documenting the variable already makes the warning disappear. ck.MkLine.Warnf("%s is defined but not used.", varname) + ck.MkLine.Explain( + "This might be a simple typo.", + "", + "If a package provides a file containing several related variables", + "(such as module.mk, app.mk, extension.mk), that file may define", + "variables that look unused since they are only used by other packages.", + "These variables should be documented at the head of the file;", + "see mk/subst.mk for an example of such a documentation comment.") } // checkVarassignRightVaruse checks that in a variable assignment, @@ -1121,7 +1128,7 @@ func (ck MkLineChecker) checkVarassignMisc() { if mkline.VarassignComment() == "# defined" && !hasSuffix(varname, "_MK") && !hasSuffix(varname, "_COMMON") { mkline.Notef("Please use \"# empty\", \"# none\" or \"# yes\" instead of \"# defined\".") - G.Explain( + mkline.Explain( "The value #defined says something about the state of the variable,", "but not what that _means_.", "In some cases a variable that is defined", @@ -1171,8 +1178,20 @@ func (ck MkLineChecker) checkVarassignLeftBsdPrefs() { return } + // Package-settable variables may use the ?= operator before including + // bsd.prefs.mk in situations like the following: + // + // Makefile: LICENSE= package-license + // .include "module.mk" + // module.mk: LICENSE?= default-license + // + vartype := G.Pkgsrc.VariableType(nil, mkline.Varname()) + if vartype != nil && vartype.PackageSettable() { + return + } + mkline.Warnf("Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".") - G.Explain( + mkline.Explain( "The ?= operator is used to provide a default value to a variable.", "In pkgsrc, many variables can be set by the pkgsrc user in the", "mk.conf file.", @@ -1212,8 +1231,8 @@ func (ck MkLineChecker) checkVartype(varname string, op MkOperator, value, comme trace.Step1("Unchecked use of !=: %q", value) } - case vartype.kindOfList == lkNone: - ck.CheckVartypeBasic(varname, vartype.basicType, op, value, comment, vartype.guessed) + case !vartype.List(): + ck.CheckVartypeBasic(varname, vartype.basicType, op, value, comment, vartype.Guessed()) case value == "": break @@ -1221,7 +1240,7 @@ func (ck MkLineChecker) checkVartype(varname string, op MkOperator, value, comme default: words := mkline.ValueFields(value) for _, word := range words { - ck.CheckVartypeBasic(varname, vartype.basicType, op, word, comment, vartype.guessed) + ck.CheckVartypeBasic(varname, vartype.basicType, op, word, comment, vartype.Guessed()) } } } @@ -1344,7 +1363,7 @@ func (ck MkLineChecker) checkDirectiveCondEmpty(varuse *MkVarUse) { varname := varuse.varname if matches(varname, `^\$.*:[MN]`) { ck.MkLine.Warnf("The empty() function takes a variable name as parameter, not a variable expression.") - G.Explain( + ck.MkLine.Explain( "Instead of empty(${VARNAME:Mpattern}), you should write either of the following:", "", "\tempty(VARNAME:Mpattern)", @@ -1362,10 +1381,10 @@ func (ck MkLineChecker) checkDirectiveCondEmpty(varuse *MkVarUse) { ck.checkVartype(varname, opUseMatch, pattern, "") vartype := G.Pkgsrc.VariableType(ck.MkLines, varname) - if matches(pattern, `^[\w-/]+$`) && vartype != nil && !vartype.IsConsideredList() { + if matches(pattern, `^[\w-/]+$`) && vartype != nil && !vartype.List() { ck.MkLine.Notef("%s should be compared using %s instead of matching against %q.", varname, ifelseStr(positive, "==", "!="), ":"+modifier.Text) - G.Explain( + ck.MkLine.Explain( "This variable has a single value, not a list of values.", "Therefore it feels strange to apply list operators like :M and :N onto it.", "A more direct approach is to use the == and != operators.", @@ -1383,7 +1402,7 @@ func (ck MkLineChecker) checkCompareVarStr(varname, op, value string) { if varname == "PKGSRC_COMPILER" { ck.MkLine.Warnf("Use ${PKGSRC_COMPILER:%s%s} instead of the %s operator.", ifelseStr(op == "==", "M", "N"), value, op) - G.Explain( + ck.MkLine.Explain( "The PKGSRC_COMPILER can be a list of chained compilers, e.g. \"ccache distcc clang\".", "Therefore, comparing it using == or != leads to wrong results in these cases.") } @@ -1406,18 +1425,12 @@ func (ck MkLineChecker) CheckRelativePkgdir(pkgdir string) { } mkline := ck.MkLine - ck.CheckRelativePath(pkgdir, true) + ck.CheckRelativePath(pkgdir+"/Makefile", true) pkgdir = mkline.ResolveVarsInRelativePath(pkgdir) - // XXX: Is the leading "./" realistic? - if m, otherpkgpath := match1(pkgdir, `^(?:\./)?\.\./\.\./([^/]+/[^/]+)$`); m { - if !fileExists(G.Pkgsrc.File(otherpkgpath + "/Makefile")) { - mkline.Errorf("There is no package in %q.", otherpkgpath) - } - - } else if !containsVarRef(pkgdir) { + if !matches(pkgdir, `^\.\./\.\./([^./][^/]*/[^./][^/]*)$`) && !containsVarRef(pkgdir) { mkline.Warnf("%q is not a valid relative package directory.", pkgdir) - G.Explain( + mkline.Explain( "A relative pathname always starts with \"../../\", followed", "by a category, a slash and a the directory name of the package.", "For example, \"../../misc/screen\" is a valid relative pathname.") diff --git a/pkgtools/pkglint/files/mklinechecker_test.go b/pkgtools/pkglint/files/mklinechecker_test.go index f13d9f675f5..64e92b0728d 100644 --- a/pkgtools/pkglint/files/mklinechecker_test.go +++ b/pkgtools/pkglint/files/mklinechecker_test.go @@ -66,7 +66,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftNotUsed__infra(c *check.C) "USED_IN_INFRASTRUCTURE=\t${SHORT_DOCUMENTATION}", "", "UNUSED_INFRA=\t${UNDOCUMENTED}") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -84,6 +84,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeft__infrastructure(c *check.C t.CreateFileLines("mk/infra.mk", MkRcsID, "_VARNAME=\tvalue") + t.FinishSetUp() G.Check(t.File("mk/infra.mk")) @@ -185,6 +186,7 @@ func (s *Suite) Test_MkLineChecker_checkInclude__Makefile_exists(c *check.C) { t.SetUpPackage("category/package", ".include \"../../other/existing/Makefile\"", ".include \"../../other/not-found/Makefile\"") + t.FinishSetUp() G.checkdirPackage(t.File("category/package")) @@ -321,12 +323,9 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveFor(c *check.C) { mklines.Check() t.CheckOutputLines( - // FIXME: PATH may actually be used at load time. - "WARN: for.mk:2: PATH should not be used at load time in any file.", - - // No warning about :Q in line 2 since the :C modifier converts the - // colon-separated list into a space-separated list, as required by - // the .for loop. + // No warning about a missing :Q in line 2 since the :C modifier + // converts the colon-separated list into a space-separated list, + // as required by the .for loop. // This warning is correct since PATH is separated by colons, not by spaces. "WARN: for.mk:5: Please use ${PATH:Q} instead of ${PATH}.", @@ -347,6 +346,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveFor__infrastructure(c *check.C) "", ".for _i_ in 1 2 3", // Underscores are only allowed in infrastructure files. ".endfor") + t.FinishSetUp() G.Check(t.File("mk/file.mk")) @@ -386,8 +386,8 @@ func (s *Suite) Test_MkLineChecker_checkVartype__simple_type(c *check.C) { c.Assert(vartype, check.NotNil) c.Check(vartype.basicType.name, equals, "Comment") - c.Check(vartype.guessed, equals, false) - c.Check(vartype.kindOfList, equals, lkNone) + c.Check(vartype.Guessed(), equals, false) + c.Check(vartype.List(), equals, false) mklines := t.NewMkLines("Makefile", MkRcsID, @@ -511,10 +511,10 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { "NOTE: filename.mk:1: MACHINE_ARCH should be compared using == instead of matching against \":Mx86\".") test(".if ${MASTER_SITES:Mftp://*} == \"ftp://netbsd.org/\"", - // FIXME: Indeed, indeed, the :M modifier ends at the colon. - // Why doesn't pkglint complain loudly about the unknown "//*" modifier? + "WARN: filename.mk:1: Invalid variable modifier \"//*\" for \"MASTER_SITES\".", "WARN: filename.mk:1: \"ftp\" is not a valid URL.", - "WARN: filename.mk:1: MASTER_SITES should not be used at load time in any file.") + "WARN: filename.mk:1: MASTER_SITES should not be used at load time in any file.", + "WARN: filename.mk:1: Invalid variable modifier \"//*\" for \"MASTER_SITES\".") // The only interesting line from the below tracing output is the one // containing "checkCompareVarStr". @@ -563,9 +563,9 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions(c *check.C) { t.SetUpVartypes() t.SetUpTool("awk", "AWK", AtRunTime) - G.Pkgsrc.vartypes.DefineParse("SET_ONLY", lkNone, BtUnknown, + G.Pkgsrc.vartypes.DefineParse("SET_ONLY", BtUnknown, NoVartypeOptions, "options.mk: set") - G.Pkgsrc.vartypes.DefineParse("SET_ONLY_DEFAULT_ELSEWHERE", lkNone, BtUnknown, + G.Pkgsrc.vartypes.DefineParse("SET_ONLY_DEFAULT_ELSEWHERE", BtUnknown, NoVartypeOptions, "options.mk: set", "*.mk: default, set") mklines := t.NewMkLines("options.mk", @@ -627,15 +627,15 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__license_defaul mklines := t.NewMkLines("filename.mk", MkRcsID, "LICENSE?=\tgnu-gpl-v2") + t.FinishSetUp() mklines.Check() - // FIXME: LICENSE is a package-settable variable. Therefore bsd.prefs.mk - // does not need to be included before setting a default for this - // variable. Including bsd.prefs.mk is only necessary when setting a - // default value for user-settable or system-defined variables. - t.CheckOutputLines( - "WARN: filename.mk:2: Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".") + // LICENSE is a package-settable variable. Therefore bsd.prefs.mk + // does not need to be included before setting a default for this + // variable. Including bsd.prefs.mk is only necessary when setting a + // default value for user-settable or system-defined variables. + t.CheckOutputEmpty() } // Don't check the permissions for infrastructure files since they have their own rules. @@ -678,13 +678,16 @@ func (s *Suite) Test_MkLineChecker_checkVarassignOpShell(c *check.C) { "show-package-vars: .PHONY", "\techo OS_NAME=${OS_NAME:Q}", "\techo MUST_BE_EARLY=${MUST_BE_EARLY:Q}") + t.FinishSetUp() G.Check(t.File("category/package/standalone.mk")) // There is no warning about any variable since no package is currently // being checked, therefore pkglint cannot decide whether the variable // is used a load time. - t.CheckOutputEmpty() + t.CheckOutputLines( + "WARN: ~/category/package/standalone.mk:14: Please use \"${ECHO}\" instead of \"echo\".", + "WARN: ~/category/package/standalone.mk:15: Please use \"${ECHO}\" instead of \"echo\".") t.SetUpCommandLine("-Wall", "--explain") G.Check(t.File("category/package")) @@ -722,7 +725,9 @@ func (s *Suite) Test_MkLineChecker_checkVarassignOpShell(c *check.C) { "\tend of the line, or force the variable to be evaluated at load time,", "\tby using it at the right-hand side of the := operator, or in an .if", "\tor .for directive.", - "") + "", + "WARN: ~/category/package/standalone.mk:14: Please use \"${ECHO}\" instead of \"echo\".", + "WARN: ~/category/package/standalone.mk:15: Please use \"${ECHO}\" instead of \"echo\".") } func (s *Suite) Test_MkLineChecker_checkVarassignRightVaruse(c *check.C) { @@ -736,11 +741,8 @@ func (s *Suite) Test_MkLineChecker_checkVarassignRightVaruse(c *check.C) { mklines.Check() - // TODO: Duplicate diagnostics mean twice the work being done. t.CheckOutputLines( "WARN: module.mk:2: Please use PREFIX instead of LOCALBASE.", - "NOTE: module.mk:2: The :Q operator isn't necessary for ${LOCALBASE} here.", - "WARN: module.mk:2: Please use PREFIX instead of LOCALBASE.", "NOTE: module.mk:2: The :Q operator isn't necessary for ${LOCALBASE} here.") } @@ -873,9 +875,9 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time(c *check.C) func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time_in_condition(c *check.C) { t := s.Init(c) - G.Pkgsrc.vartypes.DefineParse("LOAD_TIME", lkShell, BtPathmask, + G.Pkgsrc.vartypes.DefineParse("LOAD_TIME", BtPathmask, List, "special:filename.mk: use-loadtime") - G.Pkgsrc.vartypes.DefineParse("RUN_TIME", lkShell, BtPathmask, + G.Pkgsrc.vartypes.DefineParse("RUN_TIME", BtPathmask, List, "special:filename.mk: use") mklines := t.NewMkLines("filename.mk", @@ -892,9 +894,9 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time_in_conditio func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time_in_for_loop(c *check.C) { t := s.Init(c) - G.Pkgsrc.vartypes.DefineParse("LOAD_TIME", lkShell, BtPathmask, + G.Pkgsrc.vartypes.DefineParse("LOAD_TIME", BtPathmask, List, "special:filename.mk: use-loadtime") - G.Pkgsrc.vartypes.DefineParse("RUN_TIME", lkShell, BtPathmask, + G.Pkgsrc.vartypes.DefineParse("RUN_TIME", BtPathmask, List, "special:filename.mk: use") mklines := t.NewMkLines("filename.mk", @@ -940,16 +942,16 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time_guessed(c * func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time_run_time(c *check.C) { t := s.Init(c) - G.Pkgsrc.vartypes.DefineParse("LOAD_TIME", lkNone, BtUnknown, + G.Pkgsrc.vartypes.DefineParse("LOAD_TIME", BtUnknown, NoVartypeOptions, "*.mk: use, use-loadtime") - G.Pkgsrc.vartypes.DefineParse("RUN_TIME", lkNone, BtUnknown, + G.Pkgsrc.vartypes.DefineParse("RUN_TIME", BtUnknown, NoVartypeOptions, "*.mk: use") - G.Pkgsrc.vartypes.DefineParse("WRITE_ONLY", lkNone, BtUnknown, + G.Pkgsrc.vartypes.DefineParse("WRITE_ONLY", BtUnknown, NoVartypeOptions, "*.mk: set") - G.Pkgsrc.vartypes.DefineParse("LOAD_TIME_ELSEWHERE", lkNone, BtUnknown, + G.Pkgsrc.vartypes.DefineParse("LOAD_TIME_ELSEWHERE", BtUnknown, NoVartypeOptions, "Makefile: use-loadtime", "*.mk: set") - G.Pkgsrc.vartypes.DefineParse("RUN_TIME_ELSEWHERE", lkNone, BtUnknown, + G.Pkgsrc.vartypes.DefineParse("RUN_TIME_ELSEWHERE", BtUnknown, NoVartypeOptions, "Makefile: use", "*.mk: set") @@ -1041,7 +1043,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__write_only_usable_in_ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__usable_only_at_loadtime_in_other_file(c *check.C) { t := s.Init(c) - G.Pkgsrc.vartypes.DefineParse("VAR", lkNone, BtFileName, + G.Pkgsrc.vartypes.DefineParse("VAR", BtFileName, NoVartypeOptions, "*: set, use-loadtime") mklines := t.NewMkLines("Makefile", MkRcsID, @@ -1060,9 +1062,9 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__assigned_to_infrastru // This combination of BtUnknown and all permissions is typical for // otherwise unknown variables from the pkgsrc infrastructure. - G.Pkgsrc.vartypes.Define("INFRA", lkNone, BtUnknown, + G.Pkgsrc.vartypes.Define("INFRA", BtUnknown, NoVartypeOptions, ACLEntry{"*", aclpAll}) - G.Pkgsrc.vartypes.DefineParse("VAR", lkNone, BtUnknown, + G.Pkgsrc.vartypes.DefineParse("VAR", BtUnknown, NoVartypeOptions, "buildlink3.mk: none", "*: use") mklines := t.NewMkLines("buildlink3.mk", @@ -1096,10 +1098,10 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__assigned_to_load_time // to use its value in LOAD_TIME, as the latter might be evaluated later // at load time, and at that point VAR would be evaluated as well. - G.Pkgsrc.vartypes.DefineParse("LOAD_TIME", lkNone, BtMessage, + G.Pkgsrc.vartypes.DefineParse("LOAD_TIME", BtMessage, NoVartypeOptions, "buildlink3.mk: set", "*.mk: use-loadtime") - G.Pkgsrc.vartypes.DefineParse("VAR", lkNone, BtUnknown, + G.Pkgsrc.vartypes.DefineParse("VAR", BtUnknown, NoVartypeOptions, "buildlink3.mk: none", "*.mk: use") mklines := t.NewMkLines("buildlink3.mk", @@ -1247,24 +1249,30 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePkgdir(c *check.C) { t := s.Init(c) t.CreateFileLines("other/package/Makefile") - // Must be in the filesystem because of directory references. - mklines := t.SetUpFileMkLines("category/package/Makefile", - "# dummy") - mklines.ForEach(func(mkline MkLine) { - ck := MkLineChecker{mklines, mkline} + test := func(relativePkgdir string, diagnostics ...string) { + // Must be in the filesystem because of directory references. + mklines := t.SetUpFileMkLines("category/package/Makefile", + "# dummy") - ck.CheckRelativePkgdir("../pkgbase") - ck.CheckRelativePkgdir("../../other/package") - ck.CheckRelativePkgdir("../../other/does-not-exist") - }) + checkRelativePkgdir := func(mkline MkLine) { + MkLineChecker{mklines, mkline}.CheckRelativePkgdir(relativePkgdir) + } - // FIXME: The diagnostics for does-not-exist are redundant. - t.CheckOutputLines( - "ERROR: ~/category/package/Makefile:1: Relative path \"../pkgbase\" does not exist.", - "WARN: ~/category/package/Makefile:1: \"../pkgbase\" is not a valid relative package directory.", - "ERROR: ~/category/package/Makefile:1: Relative path \"../../other/does-not-exist\" does not exist.", - "ERROR: ~/category/package/Makefile:1: There is no package in \"other/does-not-exist\".") + mklines.ForEach(checkRelativePkgdir) + + t.CheckOutput(diagnostics) + } + + test("../pkgbase", + "ERROR: ~/category/package/Makefile:1: Relative path \"../pkgbase/Makefile\" does not exist.", + "WARN: ~/category/package/Makefile:1: \"../pkgbase\" is not a valid relative package directory.") + + test("../../other/package", + nil...) + + test("../../other/does-not-exist", + "ERROR: ~/category/package/Makefile:1: Relative path \"../../other/does-not-exist/Makefile\" does not exist.") } // PR pkg/46570, item 2 @@ -1278,12 +1286,14 @@ func (s *Suite) Test_MkLineChecker__unclosed_varuse(c *check.C) { mklines.Check() t.CheckOutputLines( + "WARN: Makefile:2: Missing closing \"}\" for \"EGDIR/pam.d\".", + "WARN: Makefile:2: Invalid part \"/pam.d\" after variable name \"EGDIR\".", + "WARN: Makefile:2: Missing closing \"}\" for \"EGDIR/dbus-1/system.d ${EGDIR/pam.d\".", + "WARN: Makefile:2: Invalid part \"/dbus-1/system.d ${EGDIR/pam.d\" after variable name \"EGDIR\".", + "WARN: Makefile:2: Missing closing \"}\" for \"EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".", + "WARN: Makefile:2: Invalid part \"/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\" after variable name \"EGDIR\".", "WARN: Makefile:2: EGDIRS is defined but not used.", - "WARN: Makefile:2: Unclosed Make variable starting at \"${EGDIR/apparmor.d $...\".", - - // XXX: This warning is redundant because of the "Unclosed" warning above. - "WARN: Makefile:2: Internal pkglint error in MkLine.Tokenize at "+ - "\"${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".") + "WARN: Makefile:2: EGDIR/pam.d is used but not defined.") } func (s *Suite) Test_MkLineChecker_Check__varuse_modifier_L(c *check.C) { @@ -1304,13 +1314,11 @@ func (s *Suite) Test_MkLineChecker_Check__varuse_modifier_L(c *check.C) { // In line 2 the :L modifier is missing, therefore ${XKBBASE}/xkbcomp is the // name of another variable, and that variable is not known. Only XKBBASE is known. // - // FIXME: The below warnings are wrong because the MkParser does not recognize the - // slash as part of a variable name. Because of that, parsing stops before the $. - // The warning "Unclosed Make variable" wrongly assumes that any parse error from - // a variable use is because of unclosed braces, which it isn't in this case. + // In line 2, warn about the invalid "/" as part of the variable name. t.CheckOutputLines( - "WARN: x11/xkeyboard-config/Makefile:2: Unclosed Make variable starting at \"${${XKBBASE}/xkbcomp...\".", - "WARN: x11/xkeyboard-config/Makefile:2: Unclosed Make variable starting at \"${${XKBBASE}/xkbcomp...\".") + "WARN: x11/xkeyboard-config/Makefile:2: "+ + "Invalid part \"/xkbcomp\" after variable name \"${XKBBASE}\".", + "WARN: x11/xkeyboard-config/Makefile:2: XKBBASE is used but not defined.") } func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparison_with_shell_command(c *check.C) { @@ -1362,9 +1370,6 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCondEmpty(c *check.C) { mkline := t.NewMkLine("module.mk", 123, ".if ${PKGPATH} == \"category/package\"") ck := MkLineChecker{nil, mkline} - // FIXME: checkDirectiveCondEmpty cannot know whether it is empty(...) or !empty(...). - // It must know that to generate the proper diagnostics. - ck.checkDirectiveCondEmpty(NewMkVarUse("PKGPATH", "Mpattern")) // When the pattern contains placeholders, it cannot be converted to == or !=. @@ -1546,21 +1551,21 @@ func (s *Suite) Test_MkLineChecker_checkVarUseQuoting__mstar(c *check.C) { t.SetUpVartypes() mklines := t.SetUpFileMkLines("options.mk", MkRcsID, - "CONFIGURE_ARGS+= ${CFLAGS:Q}", - "CONFIGURE_ARGS+= ${CFLAGS:M*:Q}", - "CONFIGURE_ARGS+= ${ADA_FLAGS:Q}", - "CONFIGURE_ARGS+= ${ADA_FLAGS:M*:Q}", - "CONFIGURE_ENV+= ${CFLAGS:Q}", - "CONFIGURE_ENV+= ${CFLAGS:M*:Q}", - "CONFIGURE_ENV+= ${ADA_FLAGS:Q}", - "CONFIGURE_ENV+= ${ADA_FLAGS:M*:Q}") + "CONFIGURE_ARGS+= CFLAGS=${CFLAGS:Q}", + "CONFIGURE_ARGS+= CFLAGS=${CFLAGS:M*:Q}", + "CONFIGURE_ARGS+= ADA_FLAGS=${ADA_FLAGS:Q}", + "CONFIGURE_ARGS+= ADA_FLAGS=${ADA_FLAGS:M*:Q}", + "CONFIGURE_ENV+= CFLAGS=${CFLAGS:Q}", + "CONFIGURE_ENV+= CFLAGS=${CFLAGS:M*:Q}", + "CONFIGURE_ENV+= ADA_FLAGS=${ADA_FLAGS:Q}", + "CONFIGURE_ENV+= ADA_FLAGS=${ADA_FLAGS:M*:Q}") mklines.Check() - // FIXME: There should be some notes and warnings about missing :M*; - // these are prevented by the PERL5 case in VariableNeedsQuoting. t.CheckOutputLines( - "WARN: ~/options.mk:4: ADA_FLAGS is used but not defined.") + "WARN: ~/options.mk:2: Please use ${CFLAGS:M*:Q} instead of ${CFLAGS:Q}.", + "WARN: ~/options.mk:4: ADA_FLAGS is used but not defined.", + "WARN: ~/options.mk:6: Please use ${CFLAGS:M*:Q} instead of ${CFLAGS:Q}.") } func (s *Suite) Test_MkLineChecker_checkVarUseQuoting__mstar_not_needed(c *check.C) { @@ -1570,18 +1575,14 @@ func (s *Suite) Test_MkLineChecker_checkVarUseQuoting__mstar_not_needed(c *check pkg := t.SetUpPackage("category/package", "MAKE_FLAGS+=\tCFLAGS=${CFLAGS:M*:Q}", "MAKE_FLAGS+=\tLFLAGS=${LDFLAGS:M*:Q}") - G.Pkgsrc.LoadInfrastructure() - // FIXME: It is too easy to forget this important call. + t.FinishSetUp() // This package is guaranteed to not use GNU_CONFIGURE. // Since the :M* hack is only needed for GNU_CONFIGURE, it is not necessary here. G.Check(pkg) - // FIXME: Duplicate diagnostics. t.CheckOutputLines( "NOTE: ~/category/package/Makefile:20: The :M* modifier is not needed here.", - "NOTE: ~/category/package/Makefile:20: The :M* modifier is not needed here.", - "NOTE: ~/category/package/Makefile:21: The :M* modifier is not needed here.", "NOTE: ~/category/package/Makefile:21: The :M* modifier is not needed here.") } @@ -1590,7 +1591,7 @@ func (s *Suite) Test_MkLineChecker_checkVarUseQuoting__q_not_needed(c *check.C) pkg := t.SetUpPackage("category/package", "MASTER_SITES=\t${HOMEPAGE:Q}") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(pkg) @@ -1645,16 +1646,15 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__varcanon(c *check.C) { t.CreateFileLines("mk/sys-vars.mk", MkRcsID, "CPPPATH.Linux=\t/usr/bin/cpp") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() ck := MkLineChecker{nil, t.NewMkLine("module.mk", 101, "COMMENT=\t${CPPPATH.SunOS}")} ck.CheckVaruse(NewMkVarUse("CPPPATH.SunOS"), &VarUseContext{ vartype: &Vartype{ - kindOfList: lkNone, basicType: BtPathname, + options: Guessed, aclEntries: nil, - guessed: true, }, time: vucTimeRun, quoting: VucQuotPlain, @@ -1676,7 +1676,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__defined_in_infrastructure(c *che t.CreateFileLines("mk/deeply/nested/infra.mk", MkRcsID, "INFRA_VAR?=\tvalue") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() mklines := t.SetUpFileMkLines("category/package/module.mk", MkRcsID, "do-fetch:", @@ -1696,10 +1696,9 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__build_defs(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("mk/defaults/mk.conf", "VARBASE?= /usr/pkg/var") - G.Pkgsrc.LoadInfrastructure() - t.SetUpCommandLine("-Wall,no-space") - t.SetUpVartypes() + t.FinishSetUp() + mklines := t.SetUpFileMkLines("options.mk", MkRcsID, "COMMENT= ${VARBASE} ${X11_TYPE}", @@ -1722,7 +1721,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__LOCALBASE_in_infrastructure(c *c MkRcsID, "LOCALBASE?=\t${PREFIX}", "DEFAULT_PREFIX=\t${LOCALBASE}") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("mk/infra.mk")) @@ -1743,13 +1742,13 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__user_defined_variable_and_BUILD_ t.CreateFileLines("mk/defaults/mk.conf", "VARBASE?=\t${PREFIX}/var", "PYTHON_VER?=\t36") - G.Pkgsrc.LoadInfrastructure() mklines := t.NewMkLines("file.mk", MkRcsID, "BUILD_DEFS+=\tPYTHON_VER", "\t: ${VARBASE}", "\t: ${VARBASE}", "\t: ${PYTHON_VER}") + t.FinishSetUp() mklines.Check() @@ -1784,7 +1783,6 @@ func (s *Suite) Test_MkLineChecker_checkVaruseModifiersRange(c *check.C) { MkLineChecker{nil, mkline}.Check() - // FIXME: The check is called two times, even though it only produces a single NOTE. t.CheckOutputLines( "NOTE: mk/compiler/gcc.mk:150: "+ "The modifier \":C/^/_asdf_/1:M_asdf_*:S/^_asdf_//\" can be written as \":[1]\".", @@ -1853,9 +1851,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignMisc(c *check.C) { t := s.Init(c) t.SetUpPkgsrc() - G.Pkgsrc.LoadInfrastructure() t.SetUpCommandLine("-Wall,no-space") - t.SetUpVartypes() mklines := t.SetUpFileMkLines("module.mk", MkRcsID, "EGDIR= ${PREFIX}/etc/rc.d", @@ -1864,6 +1860,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignMisc(c *check.C) { "DIST_SUBDIR= ${PKGNAME}", "WRKSRC= ${PKGNAME}", "SITES_distfile.tar.gz= ${MASTER_SITE_GITHUB:=user/}") + t.FinishSetUp() mklines.Check() @@ -1897,6 +1894,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignMisc__multiple_inclusion_guards t.CreateFileLines("other.mk", MkRcsID, "COMMENT=\t# defined") + t.FinishSetUp() G.Check(t.File("filename.mk")) G.Check(t.File("Makefile.common")) @@ -1913,20 +1911,18 @@ func (s *Suite) Test_MkLineChecker_checkText(c *check.C) { t := s.Init(c) t.SetUpPkgsrc() - G.Pkgsrc.LoadInfrastructure() t.SetUpCommandLine("-Wall,no-space") mklines := t.SetUpFileMkLines("module.mk", MkRcsID, "CFLAGS+= -Wl,--rpath,${PREFIX}/lib", "PKG_FAIL_REASON+= \"Group ${GAMEGRP} doesn't exist.\"") + t.FinishSetUp() mklines.Check() - // FIXME: Duplicate diagnostics. t.CheckOutputLines( "WARN: ~/module.mk:2: Please use ${COMPILER_RPATH_FLAG} instead of \"-Wl,--rpath,\".", - "WARN: ~/module.mk:3: Use of \"GAMEGRP\" is deprecated. Use GAMES_GROUP instead.", "WARN: ~/module.mk:3: Use of \"GAMEGRP\" is deprecated. Use GAMES_GROUP instead.") } @@ -1964,7 +1960,6 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePath(c *check.C) { t := s.Init(c) t.SetUpPkgsrc() - G.Pkgsrc.LoadInfrastructure() t.CreateFileLines("wip/package/Makefile") t.CreateFileLines("wip/package/module.mk") mklines := t.SetUpFileMkLines("category/package/module.mk", @@ -1979,6 +1974,7 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePath(c *check.C) { ".include \"../../category/../category/package/module.mk\"", // Oops ".include \"../../mk/bsd.prefs.mk\"", ".include \"../package/module.mk\"") + t.FinishSetUp() mklines.Check() @@ -1998,10 +1994,10 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePath__absolute_path(c *check.C) absPath := absDir + "0f5c2d56-8a7a-4c9d-9caa-859b52bbc8c7" t.SetUpPkgsrc() - G.Pkgsrc.LoadInfrastructure() mklines := t.SetUpFileMkLines("category/package/module.mk", MkRcsID, "DISTINFO_FILE=\t"+absPath) + t.FinishSetUp() mklines.Check() @@ -2031,7 +2027,7 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePath__wip_mk(c *check.C) { MkRcsID) t.SetUpPackage("wip/package", ".include \"../mk/git-package.mk\"") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("wip/package")) diff --git a/pkgtools/pkglint/files/mklines.go b/pkgtools/pkglint/files/mklines.go index 66ec947b861..ddd8c08fdff 100644 --- a/pkgtools/pkglint/files/mklines.go +++ b/pkgtools/pkglint/files/mklines.go @@ -28,7 +28,7 @@ type MkLinesImpl struct { func NewMkLines(lines Lines) MkLines { mklines := make([]MkLine, lines.Len()) for i, line := range lines.Lines { - mklines[i] = NewMkLine(line) + mklines[i] = MkLineParser{}.Parse(line) } tools := NewTools() @@ -459,7 +459,7 @@ func (mklines *MkLinesImpl) SaveAutofixChanges() { } func (mklines *MkLinesImpl) EOFLine() MkLine { - return NewMkLine(mklines.lines.EOFLine()) + return MkLineParser{}.Parse(mklines.lines.EOFLine()) } // VaralignBlock checks that all variable assignments from a paragraph @@ -531,7 +531,8 @@ func (va *VaralignBlock) processVarassign(mkline MkLine) { if mkline.IsMultiline() { // Parsing the continuation marker as variable value is cheating but works well. text := strings.TrimSuffix(mkline.raw[0].orignl, "\n") - m, a := MatchVarassign(text) + data := MkLineParser{}.split(nil, text) + m, a := MkLineParser{}.MatchVarassign(mkline.Line, text, data) continuation = m && a.value == "\\" } diff --git a/pkgtools/pkglint/files/mklines_test.go b/pkgtools/pkglint/files/mklines_test.go index 05184dd644f..ae974a52577 100644 --- a/pkgtools/pkglint/files/mklines_test.go +++ b/pkgtools/pkglint/files/mklines_test.go @@ -35,7 +35,6 @@ func (s *Suite) Test_MkLines__quoting_LDFLAGS_for_GNU_configure(c *check.C) { mklines.Check() t.CheckOutputLines( - "WARN: Makefile:3: Please use ${X11_LDFLAGS:M*:Q} instead of ${X11_LDFLAGS:Q}.", "WARN: Makefile:3: Please use ${X11_LDFLAGS:M*:Q} instead of ${X11_LDFLAGS:Q}.") } @@ -312,7 +311,6 @@ func (s *Suite) Test_MkLines_collectDefinedVariables(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("mk/tools/defaults.mk", "USE_TOOLS+= autoconf autoconf213") - G.Pkgsrc.LoadInfrastructure() mklines := t.NewMkLines("determine-defined-variables.mk", MkRcsID, "", @@ -330,6 +328,7 @@ func (s *Suite) Test_MkLines_collectDefinedVariables(c *check.C) { "pre-configure:", "\t${RUN} autoreconf; autoheader-2.13", "\t${ECHO} ${OSV:Q}") + t.FinishSetUp() mklines.Check() @@ -357,7 +356,7 @@ func (s *Suite) Test_MkLines_collectDefinedVariables__BUILTIN_FIND_FILES_VAR(c * "", ".if ${H_XFT2:N__nonexistent__} && ${H_UNDEF:N__nonexistent__}", ".endif") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() mklines.Check() @@ -925,7 +924,6 @@ func (s *Suite) Test_MkLines_Check__shell_command_as_word_part_in_ENV_list(c *ch mklines.Check() t.CheckOutputLines( - "WARN: x11/lablgtk1/Makefile:2: Please use ${CC:Q} instead of ${CC}.", "WARN: x11/lablgtk1/Makefile:2: Please use ${CC:Q} instead of ${CC}.") } diff --git a/pkgtools/pkglint/files/mklines_varalign_test.go b/pkgtools/pkglint/files/mklines_varalign_test.go index 248108799ed..fa7db44ae5b 100755 --- a/pkgtools/pkglint/files/mklines_varalign_test.go +++ b/pkgtools/pkglint/files/mklines_varalign_test.go @@ -1012,20 +1012,38 @@ func (s *Suite) Test_Varalign__realign_commented_multiline(c *check.C) { vt.Run() } -// FIXME: The diagnostic does not correspond to the autofix; see "if oldWidth == 8". +// The VAR2 line is a continuation line that starts in column 9, just like +// the VAR1 line. Therefore the alignment is correct. +// +// Its continuation line is indented using effectively tab-tab-space, and +// this relative indentation compared to the VAR2 line is preserved since +// it is often used for indenting AWK or shell programs. func (s *Suite) Test_Varalign__mixed_indentation(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( "VAR1=\tvalue1", "VAR2=\tvalue2 \\", " \t \t value2 continued") - vt.Diagnostics( - /*"NOTE: ~/Makefile:2--3: This line should be aligned with \"\\t\"."*/ ) - vt.Autofixes( - /*"AUTOFIX: ~/Makefile:3: Replacing indentation \" \\t \\t \" with \"\\t\\t \"."*/ ) + vt.Diagnostics() + vt.Autofixes() vt.Fixed( "VAR1= value1", "VAR2= value2 \\", " value2 continued") vt.Run() } + +func (s *Suite) Test_Varalign__eol_comment(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VAR1=\tdefined", + "VAR2=\t# defined", + "VAR3=\t#empty") + vt.Diagnostics() + vt.Autofixes() + vt.Fixed( + "VAR1= defined", + "VAR2= # defined", + "VAR3= #empty") + vt.Run() +} diff --git a/pkgtools/pkglint/files/mkparser.go b/pkgtools/pkglint/files/mkparser.go index 50e3dd05cc9..b37e4c43153 100644 --- a/pkgtools/pkglint/files/mkparser.go +++ b/pkgtools/pkglint/files/mkparser.go @@ -66,109 +66,115 @@ func (p *MkParser) MkTokens() []*MkToken { } func (p *MkParser) VarUse() *MkVarUse { - lexer := p.lexer + rest := p.lexer.Rest() + if len(rest) < 2 || rest[0] != '$' { + return nil + } + + switch rest[1] { + case '{', '(': + return p.varUseBrace(rest[1] == '(') - if lexer.PeekByte() != '$' { + case '$': + // This is an escaped dollar character and not a variable use. return nil + + case '@', '<', ' ': + // These variable names are known to exist. + // + // Many others are also possible but not used in practice. + // In particular, when parsing the :C or :S modifier, + // the $ must not be interpreted as a variable name, + // even when it looks like $/ could refer to the "/" variable. + // + // TODO: Find out whether $" is a variable use when it appears in the :M modifier. + p.lexer.Skip(2) + return &MkVarUse{rest[1:2], nil} + + default: + return p.varUseAlnum() } +} - mark := lexer.Mark() - lexer.Skip(1) +// varUseBrace parses: +// ${VAR} +// ${arbitrary text:L} +// ${variable with invalid chars} +// $(PARENTHESES) +// ${VAR:Mpattern:C,:,colon,g:Q:Q:Q} +func (p *MkParser) varUseBrace(usingRoundParen bool) *MkVarUse { + lexer := p.lexer - if lexer.SkipByte('{') || lexer.SkipByte('(') { - usingRoundParen := lexer.Since(mark)[1] == '(' + beforeDollar := lexer.Mark() + lexer.Skip(2) - closing := byte('}') - if usingRoundParen { - closing = ')' - } + closing := byte('}') + if usingRoundParen { + closing = ')' + } - varnameMark := lexer.Mark() - varname := p.Varname() + beforeVarname := lexer.Mark() + varname := p.Varname() + p.varUseText(closing) + varExpr := lexer.Since(beforeVarname) - modifiers := p.VarUseModifiers(varname, closing) - if lexer.SkipByte(closing) { - if usingRoundParen && p.EmitWarnings { - parenVaruse := lexer.Since(mark) - edit := []byte(parenVaruse) - edit[1] = '{' - edit[len(edit)-1] = '}' - bracesVaruse := string(edit) - - fix := p.Line.Autofix() - fix.Warnf("Please use curly braces {} instead of round parentheses () for %s.", varname) - fix.Replace(parenVaruse, bracesVaruse) - fix.Apply() - } + modifiers := p.VarUseModifiers(varExpr, closing) + + closed := lexer.SkipByte(closing) - return &MkVarUse{varname, modifiers} + if p.EmitWarnings { + if !closed { + p.Line.Warnf("Missing closing %q for %q.", string(rune(closing)), varExpr) } - // This code path parses ${arbitrary text :L} and ${expression :? true-branch : false-branch }. - // The text in front of the :L or :? modifier doesn't have to be a variable name. + if usingRoundParen && closed { + parenVaruse := lexer.Since(beforeDollar) + edit := []byte(parenVaruse) + edit[1] = '{' + edit[len(edit)-1] = '}' + bracesVaruse := string(edit) - re := G.res.Compile(regex.Pattern(ifelseStr(usingRoundParen, `^(?:[^$:)]|\$\$)+`, `^(?:[^$:}]|\$\$)+`))) - for p.VarUse() != nil || lexer.SkipRegexp(re) { + fix := p.Line.Autofix() + fix.Warnf("Please use curly braces {} instead of round parentheses () for %s.", varExpr) + fix.Replace(parenVaruse, bracesVaruse) + fix.Apply() } - rest := p.Rest() - if hasPrefix(rest, ":L") || hasPrefix(rest, ":?") { - varexpr := lexer.Since(varnameMark) - modifiers := p.VarUseModifiers(varexpr, closing) - if lexer.SkipByte(closing) { - return &MkVarUse{varexpr, modifiers} - } + if len(varExpr) > len(varname) && !(&MkVarUse{varExpr, modifiers}).IsExpression() { + p.Line.Warnf("Invalid part %q after variable name %q.", varExpr[len(varname):], varname) } - - lexer.Reset(mark) - return nil } - varname := lexer.NextByteSet(textproc.AlnumU) - if varname != -1 { + return &MkVarUse{varExpr, modifiers} +} - if p.EmitWarnings { - varnameRest := lexer.Copy().NextBytesSet(textproc.AlnumU) - if varnameRest != "" { - p.Line.Errorf("$%[1]s is ambiguous. Use ${%[1]s} if you mean a Make variable or $$%[1]s if you mean a shell variable.", - sprintf("%c%s", varname, varnameRest)) - p.Line.Explain( - "Only the first letter after the dollar is the variable name.", - "Everything following it is normal text, even if it looks like a variable name to human readers.") - } else { - p.Line.Warnf("$%[1]c is ambiguous. Use ${%[1]c} if you mean a Make variable or $$%[1]c if you mean a shell variable.", varname) - p.Line.Explain( - "In its current form, this variable is parsed as a Make variable.", - "For human readers though, $x looks more like a shell variable than a Make variable,", - "since Make variables are usually written using braces (BSD-style) or parentheses (GNU-style).") - } - } +func (p *MkParser) varUseAlnum() *MkVarUse { + lexer := p.lexer - return &MkVarUse{sprintf("%c", varname), nil} + apparentVarname := textproc.NewLexer(lexer.Rest()[1:]).NextBytesSet(textproc.AlnumU) + if apparentVarname == "" { + return nil } - if !lexer.EOF() { - symbol := lexer.Rest()[:1] - switch symbol { - case "$": - // This is an escaped dollar character and not a variable use. + lexer.Skip(2) - case "@", "<", " ": - // These variable names are known to exist. - // - // Many others are also possible but not used in practice. - // In particular, when parsing the :C or :S modifier, - // the $ must not be interpreted as a variable name, - // even when it looks like $/ could refer to the "/" variable. - // - // TODO: Find out whether $" is a variable use when it appears in the :M modifier. - lexer.Skip(1) - return &MkVarUse{symbol, nil} + if p.EmitWarnings { + if len(apparentVarname) > 1 { + p.Line.Errorf("$%[1]s is ambiguous. Use ${%[1]s} if you mean a Make variable or $$%[1]s if you mean a shell variable.", + apparentVarname) + p.Line.Explain( + "Only the first letter after the dollar is the variable name.", + "Everything following it is normal text, even if it looks like a variable name to human readers.") + } else { + p.Line.Warnf("$%[1]s is ambiguous. Use ${%[1]s} if you mean a Make variable or $$%[1]s if you mean a shell variable.", apparentVarname) + p.Line.Explain( + "In its current form, this variable is parsed as a Make variable.", + "For human readers though, $x looks more like a shell variable than a Make variable,", + "since Make variables are usually written using braces (BSD-style) or parentheses (GNU-style).") } } - lexer.Reset(mark) - return nil + return &MkVarUse{apparentVarname[:1], nil} } // VarUseModifiers parses the modifiers of a variable being used, such as :Q, :Mpattern. @@ -177,6 +183,8 @@ func (p *MkParser) VarUse() *MkVarUse { func (p *MkParser) VarUseModifiers(varname string, closing byte) []MkVarUseModifier { lexer := p.lexer + // TODO: Split into VarUseModifier for parsing a single modifier. + var modifiers []MkVarUseModifier appendModifier := func(s string) { modifiers = append(modifiers, MkVarUseModifier{s}) } @@ -213,16 +221,18 @@ func (p *MkParser) VarUseModifiers(varname string, closing byte) []MkVarUseModif case "ts": // See devel/bmake/files/var.c:/case 't' - rest := lexer.Rest() + sep := p.varUseText(closing) switch { - case len(rest) >= 2 && (rest[1] == closing || rest[1] == ':'): - lexer.Skip(1) - case len(rest) >= 1 && (rest[0] == closing || rest[0] == ':'): + case sep == "": + lexer.SkipString(":") + case len(sep) == 1: break - case lexer.SkipRegexp(G.res.Compile(`^\\\d+`)): + case matches(sep, `^\\\d+`): break default: - continue + if p.EmitWarnings { + p.Line.Warnf("Invalid separator %q for :ts modifier of %q.", sep, varname) + } } appendModifier(lexer.Since(modifierMark)) continue @@ -238,14 +248,14 @@ func (p *MkParser) VarUseModifiers(varname string, closing byte) []MkVarUseModif continue case 'C', 'S': - if p.varUseModifierSubst(lexer, closing) { + if ok, _, _, _, _ := p.varUseModifierSubst(closing); ok { appendModifier(lexer.Since(modifierMark)) mayOmitColon = true continue } case '@': - if p.varUseModifierAt(lexer, closing, varname) { + if p.varUseModifierAt(lexer, varname) { appendModifier(lexer.Since(modifierMark)) continue } @@ -258,12 +268,9 @@ func (p *MkParser) VarUseModifiers(varname string, closing byte) []MkVarUseModif case '?': lexer.Skip(1) - re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^$:}]|\$\$)+`, `^([^$:)]|\$\$)+`))) - for p.VarUse() != nil || lexer.SkipRegexp(re) { - } + p.varUseText(closing) if lexer.SkipByte(':') { - for p.VarUse() != nil || lexer.SkipRegexp(re) { - } + p.varUseText(closing) appendModifier(lexer.Since(modifierMark)) continue } @@ -290,52 +297,76 @@ func (p *MkParser) VarUseModifiers(varname string, closing byte) []MkVarUseModif return modifiers } +// varUseText parses any text up to the next colon or closing mark. +// Nested variable uses are parsed as well. +// +// This is used for the :L and :? modifiers since they accept arbitrary +// text as the "variable name" and effectively interpret it as the variable +// value instead. +func (p *MkParser) varUseText(closing byte) string { + lexer := p.lexer + start := lexer.Mark() + re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^$:}]|\$\$)+`, `^([^$:)]|\$\$)+`))) + for p.VarUse() != nil || lexer.SkipRegexp(re) { + } + return lexer.Since(start) +} + // varUseModifierSubst parses a :S,from,to, or a :C,from,to, modifier. -func (p *MkParser) varUseModifierSubst(lexer *textproc.Lexer, closing byte) bool { +func (p *MkParser) varUseModifierSubst(closing byte) (ok bool, regex bool, from string, to string, options string) { + lexer := p.lexer + regex = lexer.PeekByte() == 'C' lexer.Skip(1 /* the initial S or C */) sep := lexer.PeekByte() // bmake allows _any_ separator, even letters. if sep == -1 || byte(sep) == closing { - return false + return } lexer.Skip(1) separator := byte(sep) isOther := func(b byte) bool { - return b != separator && b != '$' && b != closing && b != '\\' + return b != separator && b != '$' && b != '\\' } skipOther := func() { for p.VarUse() != nil || lexer.SkipString("$$") || - (len(lexer.Rest()) >= 2 && lexer.PeekByte() == '\\' && lexer.Skip(2)) || + (len(lexer.Rest()) >= 2 && lexer.PeekByte() == '\\' && separator != '\\' && lexer.Skip(2)) || lexer.NextBytesFunc(isOther) != "" { } } + fromStart := lexer.Mark() lexer.SkipByte('^') skipOther() lexer.SkipByte('$') + from = lexer.Since(fromStart) if !lexer.SkipByte(separator) { - return false + return } + toStart := lexer.Mark() skipOther() + to = lexer.Since(toStart) if !lexer.SkipByte(separator) { - return false + return } + optionsStart := lexer.Mark() lexer.NextBytesFunc(func(b byte) bool { return b == '1' || b == 'g' || b == 'W' }) + options = lexer.Since(optionsStart) - return true + ok = true + return } // varUseModifierAt parses a variable modifier like ":@v@echo ${v};@", // which expands the variable value in a loop. -func (p *MkParser) varUseModifierAt(lexer *textproc.Lexer, closing byte, varname string) bool { +func (p *MkParser) varUseModifierAt(lexer *textproc.Lexer, varname string) bool { lexer.Skip(1 /* the initial @ */) loopVar := lexer.NextBytesSet(AlnumDot) @@ -343,7 +374,7 @@ func (p *MkParser) varUseModifierAt(lexer *textproc.Lexer, closing byte, varname return false } - re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^$:@}\\]|\\.)+`, `^([^$:@)\\]|\\.)+`))) + re := G.res.Compile(`^([^$@\\]|\\.)+`) for p.VarUse() != nil || lexer.SkipString("$$") || lexer.SkipRegexp(re) { } @@ -448,16 +479,19 @@ func (p *MkParser) mkCondAtom() MkCond { } if lhs != nil { - if m := lexer.NextRegexp(G.res.Compile(`^[\t ]*(<|<=|==|!=|>=|>)[\t ]*(0x[0-9A-Fa-f]+|\d+(?:\.\d+)?)`)); m != nil { + lexer.SkipHspace() + + if m := lexer.NextRegexp(G.res.Compile(`^(<|<=|==|!=|>=|>)[\t ]*(0x[0-9A-Fa-f]+|\d+(?:\.\d+)?)`)); m != nil { return &mkCond{CompareVarNum: &MkCondCompareVarNum{lhs, m[1], m[2]}} } - m := lexer.NextRegexp(G.res.Compile(`^[\t ]*(<|<=|==|!=|>=|>)[\t ]*`)) + m := lexer.NextRegexp(G.res.Compile(`^(?:<|<=|==|!=|>=|>)`)) if m == nil { return &mkCond{Var: lhs} // See devel/bmake/files/cond.c:/\* For \.if \$/ } + lexer.SkipHspace() - op := m[1] + op := m[0] if op == "==" || op == "!=" { if mrhs := lexer.NextRegexp(G.res.Compile(`^"([^"\$\\]*)"`)); mrhs != nil { return &mkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, mrhs[1]}} diff --git a/pkgtools/pkglint/files/mkparser_test.go b/pkgtools/pkglint/files/mkparser_test.go index 2238d9eff1a..32a456c8583 100644 --- a/pkgtools/pkglint/files/mkparser_test.go +++ b/pkgtools/pkglint/files/mkparser_test.go @@ -86,7 +86,7 @@ func (s *Suite) Test_MkParser_MkTokens(c *check.C) { func (s *Suite) Test_MkParser_VarUse(c *check.C) { t := s.Init(c) - testRest := func(input string, expectedTokens []*MkToken, expectedRest string) { + testRest := func(input string, expectedTokens []*MkToken, expectedRest string, diagnostics ...string) { line := t.NewLines("Test_MkParser_VarUse.mk", input).Lines[0] p := NewMkParser(line, input, true) actualTokens := p.MkTokens() @@ -98,9 +98,11 @@ func (s *Suite) Test_MkParser_VarUse(c *check.C) { } } c.Check(p.Rest(), equals, expectedRest) + t.CheckOutput(diagnostics) } - test := func(input string, expectedToken *MkToken) { - testRest(input, []*MkToken{expectedToken}, "") + tokens := func(tokens ...*MkToken) []*MkToken { return tokens } + test := func(input string, expectedToken *MkToken, diagnostics ...string) { + testRest(input, []*MkToken{expectedToken}, "", diagnostics...) } varuse := func(varname string, modifiers ...string) *MkToken { text := "${" + varname @@ -114,6 +116,8 @@ func (s *Suite) Test_MkParser_VarUse(c *check.C) { return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)} } + t.Use(testRest, tokens, test, varuse, varuseText) + test("${VARIABLE}", varuse("VARIABLE")) @@ -304,44 +308,63 @@ func (s *Suite) Test_MkParser_VarUse(c *check.C) { test("${VAR:Sahara}", varuse("VAR", "Sahara")) + // The separator character can be left out, which means empty. test("${VAR:ts}", - varuse("VAR", "ts")) // The separator character can be left out, which means empty. + varuse("VAR", "ts")) + // The separator character can be a long octal number. test("${VAR:ts\\000012}", - varuse("VAR", "ts\\000012")) // The separator character can be a long octal number. + varuse("VAR", "ts\\000012")) + // Or even decimal. test("${VAR:ts\\124}", - varuse("VAR", "ts\\124")) // Or even decimal. + varuse("VAR", "ts\\124")) - testRest("${VAR:ts---}", nil, "${VAR:ts---}") // The :ts modifier only takes single-character separators. + // The :ts modifier only takes single-character separators. + test("${VAR:ts---}", + varuse("VAR", "ts---"), + "WARN: Test_MkParser_VarUse.mk:1: Invalid separator \"---\" for :ts modifier of \"VAR\".") test("$<", varuseText("$<", "<")) // Same as ${.IMPSRC} test("$(GNUSTEP_USER_ROOT)", - varuseText("$(GNUSTEP_USER_ROOT)", "GNUSTEP_USER_ROOT")) - - t.CheckOutputLines( + varuseText("$(GNUSTEP_USER_ROOT)", "GNUSTEP_USER_ROOT"), "WARN: Test_MkParser_VarUse.mk:1: Please use curly braces {} instead of round parentheses () for GNUSTEP_USER_ROOT.") - testRest("${VAR)", nil, "${VAR)") // Opening brace, closing parenthesis - testRest("$(VAR}", nil, "$(VAR}") // Opening parenthesis, closing brace - t.CheckOutputEmpty() // Warnings are only printed for balanced expressions. + // Opening brace, closing parenthesis. + // Warnings are only printed for balanced expressions. + test("${VAR)", + varuseText("${VAR)", "VAR)"), + "WARN: Test_MkParser_VarUse.mk:1: Missing closing \"}\" for \"VAR)\".", + "WARN: Test_MkParser_VarUse.mk:1: Invalid part \")\" after variable name \"VAR\".") + + // Opening parenthesis, closing brace + // Warnings are only printed for balanced expressions. + test("$(VAR}", + varuseText("$(VAR}", "VAR}"), + "WARN: Test_MkParser_VarUse.mk:1: Missing closing \")\" for \"VAR}\".", + "WARN: Test_MkParser_VarUse.mk:1: Invalid part \"}\" after variable name \"VAR\".") test("${PLIST_SUBST_VARS:@var@${var}=${${var}:Q}@}", varuse("PLIST_SUBST_VARS", "@var@${var}=${${var}:Q}@")) test("${PLIST_SUBST_VARS:@var@${var}=${${var}:Q}}", - varuse("PLIST_SUBST_VARS", "@var@${var}=${${var}:Q}")) // Missing @ at the end - - t.CheckOutputLines( - "WARN: Test_MkParser_VarUse.mk:1: Modifier ${PLIST_SUBST_VARS:@var@...@} is missing the final \"@\".") + varuseText("${PLIST_SUBST_VARS:@var@${var}=${${var}:Q}}", + "PLIST_SUBST_VARS", "@var@${var}=${${var}:Q}}"), + "WARN: Test_MkParser_VarUse.mk:1: Modifier ${PLIST_SUBST_VARS:@var@...@} is missing the final \"@\".", + "WARN: Test_MkParser_VarUse.mk:1: Missing closing \"}\" for \"PLIST_SUBST_VARS\".") // Unfinished variable use - testRest("${", nil, "${") + test("${", + varuseText("${", ""), + "WARN: Test_MkParser_VarUse.mk:1: Missing closing \"}\" for \"\".") // Unfinished nested variable use - testRest("${${", nil, "${${") + test("${${", + varuseText("${${", "${"), + "WARN: Test_MkParser_VarUse.mk:1: Missing closing \"}\" for \"\".", + "WARN: Test_MkParser_VarUse.mk:1: Missing closing \"}\" for \"${\".") } func (s *Suite) Test_MkParser_VarUse__ambiguous(c *check.C) { @@ -349,8 +372,8 @@ func (s *Suite) Test_MkParser_VarUse__ambiguous(c *check.C) { t.SetUpCommandLine("--explain") - mkline := t.NewMkLine("module.mk", 123, "\t$Varname $X") - p := NewMkParser(mkline.Line, mkline.ShellCommand(), true) + line := t.NewLine("module.mk", 123, "\t$Varname $X") + p := NewMkParser(line, line.Text[1:], true) tokens := p.MkTokens() c.Check(tokens, deepEquals, []*MkToken{ @@ -375,6 +398,8 @@ func (s *Suite) Test_MkParser_VarUse__ambiguous(c *check.C) { } func (s *Suite) Test_MkParser_MkCond(c *check.C) { + t := s.Init(c) + testRest := func(input string, expectedTree MkCond, expectedRest string) { p := NewMkParser(nil, input, false) actualTree := p.MkCond() @@ -386,6 +411,8 @@ func (s *Suite) Test_MkParser_MkCond(c *check.C) { } varuse := NewMkVarUse + t.Use(testRest, test, varuse) + test("${OPSYS:MNetBSD}", &mkCond{Var: varuse("OPSYS", "MNetBSD")}) @@ -515,7 +542,15 @@ func (s *Suite) Test_MkParser_MkCond(c *check.C) { nil, "\"unfinished string literal") - // Not even the ${VAR} gets through here, although that can be expected. FIXME: Why? + // Parsing stops before the variable since the comparison between + // a variable and a string is one of the smallest building blocks. + // Letting the ${VAR} through and stopping at the == operator would + // be misleading. + // + // Another possibility would be to fix the unfinished string literal + // and continue parsing. As of April 2019, the error handling is not + // robust enough to support this approach; magically fixing parse + // errors might lead to wrong conclusions and warnings. testRest("${VAR} == \"unfinished string literal", nil, "${VAR} == \"unfinished string literal") @@ -551,14 +586,14 @@ func (s *Suite) Test_MkParser_VarUseModifiers(c *check.C) { t := s.Init(c) varUse := NewMkVarUse - test := func(text string, varUse *MkVarUse, rest string, diagnostics ...string) { + test := func(text string, varUse *MkVarUse, diagnostics ...string) { mkline := t.NewMkLine("Makefile", 20, "\t"+text) p := NewMkParser(mkline.Line, mkline.ShellCommand(), true) actual := p.VarUse() t.Check(actual, deepEquals, varUse) - t.Check(p.Rest(), equals, rest) + t.Check(p.Rest(), equals, "") t.CheckOutput(diagnostics) } @@ -566,21 +601,27 @@ func (s *Suite) Test_MkParser_VarUseModifiers(c *check.C) { // check whether the command is actually valid. // At least not while parsing the modifier since at this point it might // be still unknown which of the commands can be used and which cannot. - test("${VAR:!command!}", varUse("VAR", "!command!"), "") + test("${VAR:!command!}", varUse("VAR", "!command!")) - test("${VAR:!command}", varUse("VAR"), "", + test("${VAR:!command}", varUse("VAR"), + // FIXME: duplicate diagnostic + "WARN: Makefile:20: Invalid variable modifier \"!command\" for \"VAR\".", "WARN: Makefile:20: Invalid variable modifier \"!command\" for \"VAR\".") - test("${VAR:command!}", varUse("VAR"), "", + test("${VAR:command!}", varUse("VAR"), + // FIXME: duplicate diagnostic + "WARN: Makefile:20: Invalid variable modifier \"command!\" for \"VAR\".", "WARN: Makefile:20: Invalid variable modifier \"command!\" for \"VAR\".") // The :L modifier makes the variable value "echo hello", and the :[1] // modifier extracts the "echo". - test("${echo hello:L:[1]}", varUse("echo hello", "L", "[1]"), "") + test("${echo hello:L:[1]}", varUse("echo hello", "L", "[1]")) // bmake ignores the :[3] modifier, and the :L modifier just returns the // variable name, in this case BUILD_DIRS. - test("${BUILD_DIRS:[3]:L}", varUse("BUILD_DIRS", "[3]", "L"), "") + test("${BUILD_DIRS:[3]:L}", varUse("BUILD_DIRS", "[3]", "L")) + + test("${PATH:ts::Q}", varUse("PATH", "ts:", "Q")) } func (s *Suite) Test_MkParser_varUseModifierSubst(c *check.C) { @@ -588,8 +629,8 @@ func (s *Suite) Test_MkParser_varUseModifierSubst(c *check.C) { varUse := NewMkVarUse test := func(text string, varUse *MkVarUse, rest string, diagnostics ...string) { - mkline := t.NewMkLine("Makefile", 20, "\t"+text) - p := NewMkParser(mkline.Line, mkline.ShellCommand(), true) + line := t.NewLine("Makefile", 20, "\t"+text) + p := NewMkParser(line, text, true) actual := p.VarUse() @@ -598,8 +639,9 @@ func (s *Suite) Test_MkParser_varUseModifierSubst(c *check.C) { t.CheckOutput(diagnostics) } - test("${VAR:S", nil, "${VAR:S", - "WARN: Makefile:20: Invalid variable modifier \"S\" for \"VAR\".") + test("${VAR:S", varUse("VAR"), "", + "WARN: Makefile:20: Invalid variable modifier \"S\" for \"VAR\".", + "WARN: Makefile:20: Missing closing \"}\" for \"VAR\".") test("${VAR:S}", varUse("VAR"), "", "WARN: Makefile:20: Invalid variable modifier \"S\" for \"VAR\".") @@ -621,6 +663,23 @@ func (s *Suite) Test_MkParser_varUseModifierSubst(c *check.C) { test("${VAR:S,from,to,W}", varUse("VAR", "S,from,to,W"), "") test("${VAR:S,from,to,1gW}", varUse("VAR", "S,from,to,1gW"), "") + + // Inside the :S or :C modifiers, neither a colon nor the closing + // brace need to be escaped. Otherwise these patterns would become + // too difficult to read and write. + test("${VAR:C/[[:alnum:]]{2}/**/g}", + varUse("VAR", "C/[[:alnum:]]{2}/**/g"), + "") + + // Some pkgsrc users really explore the darkest corners of bmake by using + // the backslash as the separator in the :S modifier. Sure, it works, it + // just looks totally unexpected to the average pkgsrc reader. + // + // Using the backslash as separator means that it cannot be used for anything + // else, not even for escaping other characters. + test("${VAR:S\\.post1\\\\1}", + varUse("VAR", "S\\.post1\\\\1"), + "") } func (s *Suite) Test_MkParser_varUseModifierAt(c *check.C) { @@ -628,8 +687,8 @@ func (s *Suite) Test_MkParser_varUseModifierAt(c *check.C) { varUse := NewMkVarUse test := func(text string, varUse *MkVarUse, rest string, diagnostics ...string) { - mkline := t.NewMkLine("Makefile", 20, "\t"+text) - p := NewMkParser(mkline.Line, mkline.ShellCommand(), true) + line := t.NewLine("Makefile", 20, "\t"+text) + p := NewMkParser(line, text, true) actual := p.VarUse() @@ -638,13 +697,21 @@ func (s *Suite) Test_MkParser_varUseModifierAt(c *check.C) { t.CheckOutput(diagnostics) } - test("${VAR:@", nil, "${VAR:@", - "WARN: Makefile:20: Invalid variable modifier \"@\" for \"VAR\".") + test("${VAR:@", + varUse("VAR"), + "", + "WARN: Makefile:20: Invalid variable modifier \"@\" for \"VAR\".", + "WARN: Makefile:20: Missing closing \"}\" for \"VAR\".") - test("${VAR:@i@${i}}", varUse("VAR", "@i@${i}"), "", - "WARN: Makefile:20: Modifier ${VAR:@i@...@} is missing the final \"@\".") + test("${VAR:@i@${i}}", varUse("VAR", "@i@${i}}"), "", + "WARN: Makefile:20: Modifier ${VAR:@i@...@} is missing the final \"@\".", + "WARN: Makefile:20: Missing closing \"}\" for \"VAR\".") test("${VAR:@i@${i}@}", varUse("VAR", "@i@${i}@"), "") + + test("${PKG_GROUPS:@g@${g:Q}:${PKG_GID.${g}:Q}@:C/:*$//g}", + varUse("PKG_GROUPS", "@g@${g:Q}:${PKG_GID.${g}:Q}@", "C/:*$//g"), + "") } func (s *Suite) Test_MkParser_PkgbasePattern(c *check.C) { diff --git a/pkgtools/pkglint/files/mkshwalker.go b/pkgtools/pkglint/files/mkshwalker.go index 998c429c43c..54651018f3a 100644 --- a/pkgtools/pkglint/files/mkshwalker.go +++ b/pkgtools/pkglint/files/mkshwalker.go @@ -72,6 +72,9 @@ func (w *MkShWalker) Path() string { var path []string for _, level := range w.Context { typeName := reflect.TypeOf(level.Element).Elem().Name() + if typeName == "" && reflect.TypeOf(level.Element).Kind() == reflect.Slice { + typeName = "[]" + reflect.TypeOf(level.Element).Elem().Elem().Name() + } abbreviated := strings.TrimPrefix(typeName, "MkSh") if level.Index == -1 { // TODO: This form should also be used if index == 0 and len == 1. @@ -146,10 +149,10 @@ func (w *MkShWalker) walkCommand(index int, command *MkShCommand) { w.walkSimpleCommand(-1, command.Simple) case command.Compound != nil: w.walkCompoundCommand(-1, command.Compound) - w.walkRedirects(-1, command.Redirects) + w.walkRedirects(command.Redirects) case command.FuncDef != nil: w.walkFunctionDefinition(-1, command.FuncDef) - w.walkRedirects(-1, command.Redirects) + w.walkRedirects(command.Redirects) } w.pop() @@ -167,7 +170,7 @@ func (w *MkShWalker) walkSimpleCommand(index int, command *MkShSimpleCommand) { w.walkWord(-1, command.Name) } w.walkWords(1, command.Args) - w.walkRedirects(-1, command.Redirections) + w.walkRedirects(command.Redirections) w.pop() } @@ -290,26 +293,25 @@ func (w *MkShWalker) walkWord(index int, word *ShToken) { w.pop() } -func (w *MkShWalker) walkRedirects(index int, redirects []*MkShRedirection) { +func (w *MkShWalker) walkRedirects(redirects []*MkShRedirection) { if len(redirects) == 0 { return } - w.push(index, redirects) + w.push(-1, redirects) if callback := w.Callback.Redirects; callback != nil { callback(redirects) } for i, redirect := range redirects { - // FIXME: The w.push/w.pop is missing here. - // How does the path look like? - // Are there ambiguities? + w.push(i, redirect) if callback := w.Callback.Redirect; callback != nil { callback(redirect) } w.walkWord(i, redirect.Target) + w.pop() } w.pop() diff --git a/pkgtools/pkglint/files/mkshwalker_test.go b/pkgtools/pkglint/files/mkshwalker_test.go index 3955acc0b1a..a638dd1e347 100644 --- a/pkgtools/pkglint/files/mkshwalker_test.go +++ b/pkgtools/pkglint/files/mkshwalker_test.go @@ -3,27 +3,39 @@ package pkglint import "gopkg.in/check.v1" func (s *Suite) Test_MkShWalker_Walk(c *check.C) { - list, err := parseShellProgram(dummyLine, ""+ - "if condition; then action; else case selector in pattern) case-item-action ;; esac; fi; "+ - "set -e; "+ - "cd ${WRKSRC}/locale; "+ - "for lang in *.po; do "+ - " [ \"$${lang}\" = \"wxstd.po\" ] && continue; "+ - " ${TOOLS_PATH.msgfmt} -c -o \"$${lang%.po}.mo\" \"$${lang}\"; "+ - "done; "+ - "while :; do fun() { :; } 1>&2; done") - if c.Check(err, check.IsNil) && c.Check(list, check.NotNil) { + pathFor := map[string]bool{} + + outputPathFor := func(kinds ...string) { + for key := range pathFor { + pathFor[key] = false + } + for _, kind := range kinds { + pathFor[kind] = true + } + } + + test := func(program string, output ...string) { + list, err := parseShellProgram(dummyLine, program) + + if !c.Check(err, check.IsNil) || !c.Check(list, check.NotNil) { + return + } + + walker := NewMkShWalker() var commands []string + add := func(kind string, format string, args ...interface{}) { if format != "" && !contains(format, "%") { panic(format) } detail := sprintf(format, args...) commands = append(commands, sprintf("%16s %s", kind, detail)) + if pathFor[kind] { + commands = append(commands, sprintf("%16s %s", "Path", walker.Path())) + } } - walker := NewMkShWalker() callback := &walker.Callback callback.List = func(list *MkShList) { add("List", "with %d andOrs", len(list.AndOrs)) } callback.AndOr = func(andor *MkShAndOr) { add("AndOr", "with %d pipelines", len(andor.Pipes)) } @@ -31,7 +43,6 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { callback.Command = func(command *MkShCommand) { add("Command", "") } callback.SimpleCommand = func(command *MkShSimpleCommand) { add("SimpleCommand", "%s", NewStrCommand(command).String()) - add("Path", "%s", walker.Path()) } callback.CompoundCommand = func(command *MkShCompoundCommand) { add("CompoundCommand", "") } callback.Case = func(caseClause *MkShCase) { add("Case", "with %d items", len(caseClause.Cases)) } @@ -58,131 +69,174 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { // Case with 1 item(s) // ... - c.Check(commands, deepEquals, []string{ - " List with 5 andOrs", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " CompoundCommand ", - " If with 1 then-branches", - " List with 1 andOrs", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " SimpleCommand condition", - " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.If.List[0].AndOr[0].Pipeline[0].Command[0].SimpleCommand", - " Word condition", - " List with 1 andOrs", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " SimpleCommand action", - " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.If.List[1].AndOr[0].Pipeline[0].Command[0].SimpleCommand", - " Word action", - " List with 1 andOrs", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " CompoundCommand ", - " Case with 1 items", - " Word selector", - " CaseItem with 1 patterns", - " Words with 1 words", - " Word pattern", - " List with 1 andOrs", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " SimpleCommand case-item-action", - " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.If." + - "List[2].AndOr[0].Pipeline[0].Command[0].CompoundCommand.Case.CaseItem[0]." + - "List.AndOr[0].Pipeline[0].Command[0].SimpleCommand", - " Word case-item-action", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " SimpleCommand set -e", - " Path List.AndOr[1].Pipeline[0].Command[0].SimpleCommand", - " Word set", - " Words with 1 words", - " Word -e", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " SimpleCommand cd ${WRKSRC}/locale", - " Path List.AndOr[2].Pipeline[0].Command[0].SimpleCommand", - " Word cd", - " Words with 1 words", - " Word ${WRKSRC}/locale", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " CompoundCommand ", - " For variable lang", - " Varname lang", - " Words with 1 words", - " Word *.po", - " List with 2 andOrs", - " AndOr with 2 pipelines", - " Pipeline with 1 commands", - " Command ", - " SimpleCommand [ \"$${lang}\" = \"wxstd.po\" ]", - " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.For.List.AndOr[0].Pipeline[0].Command[0].SimpleCommand", - " Word [", - " Words with 4 words", - " Word \"$${lang}\"", - " Word =", - " Word \"wxstd.po\"", - " Word ]", - " Pipeline with 1 commands", - " Command ", - " SimpleCommand continue", - " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.For.List.AndOr[0].Pipeline[1].Command[0].SimpleCommand", - " Word continue", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " SimpleCommand ${TOOLS_PATH.msgfmt} -c -o \"$${lang%.po}.mo\" \"$${lang}\"", - " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.For.List.AndOr[1].Pipeline[0].Command[0].SimpleCommand", - " Word ${TOOLS_PATH.msgfmt}", - " Words with 4 words", - " Word -c", - " Word -o", - " Word \"$${lang%.po}.mo\"", - " Word \"$${lang}\"", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " CompoundCommand ", - " Loop ", - " List with 1 andOrs", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " SimpleCommand :", - " Path List.AndOr[4].Pipeline[0].Command[0].CompoundCommand.Loop.List[0].AndOr[0].Pipeline[0].Command[0].SimpleCommand", - " Word :", - " List with 1 andOrs", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " FunctionDef for fun", - " CompoundCommand ", - " List with 1 andOrs", - " AndOr with 1 pipelines", - " Pipeline with 1 commands", - " Command ", - " SimpleCommand :", - " Path List.AndOr[4].Pipeline[0].Command[0].CompoundCommand.Loop." + - "List[1].AndOr[0].Pipeline[0].Command[0].FunctionDefinition.CompoundCommand." + - "List.AndOr[0].Pipeline[0].Command[0].SimpleCommand", - " Word :", - " Redirects with 1 redirects", - " Redirect >&", - " Word 2"}) + c.Check(commands, deepEquals, output) // After parsing, there is not a single level of indentation, // therefore even Parent(0) returns nil. + // + // This ensures that the w.push/w.pop calls are balanced. c.Check(walker.Parent(0), equals, nil) } + + outputPathFor("SimpleCommand") + test(""+ + "if condition; then action; else case selector in pattern) case-item-action ;; esac; fi; "+ + "set -e; "+ + "cd ${WRKSRC}/locale; "+ + "for lang in *.po; do "+ + " [ \"$${lang}\" = \"wxstd.po\" ] && continue; "+ + " ${TOOLS_PATH.msgfmt} -c -o \"$${lang%.po}.mo\" \"$${lang}\"; "+ + "done; "+ + "while :; do fun() { :; } 1>&2; done", + + " List with 5 andOrs", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " CompoundCommand ", + " If with 1 then-branches", + " List with 1 andOrs", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " SimpleCommand condition", + " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.If.List[0].AndOr[0].Pipeline[0].Command[0].SimpleCommand", + " Word condition", + " List with 1 andOrs", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " SimpleCommand action", + " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.If.List[1].AndOr[0].Pipeline[0].Command[0].SimpleCommand", + " Word action", + " List with 1 andOrs", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " CompoundCommand ", + " Case with 1 items", + " Word selector", + " CaseItem with 1 patterns", + " Words with 1 words", + " Word pattern", + " List with 1 andOrs", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " SimpleCommand case-item-action", + " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.If."+ + "List[2].AndOr[0].Pipeline[0].Command[0].CompoundCommand.Case.CaseItem[0]."+ + "List.AndOr[0].Pipeline[0].Command[0].SimpleCommand", + " Word case-item-action", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " SimpleCommand set -e", + " Path List.AndOr[1].Pipeline[0].Command[0].SimpleCommand", + " Word set", + " Words with 1 words", + " Word -e", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " SimpleCommand cd ${WRKSRC}/locale", + " Path List.AndOr[2].Pipeline[0].Command[0].SimpleCommand", + " Word cd", + " Words with 1 words", + " Word ${WRKSRC}/locale", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " CompoundCommand ", + " For variable lang", + " Varname lang", + " Words with 1 words", + " Word *.po", + " List with 2 andOrs", + " AndOr with 2 pipelines", + " Pipeline with 1 commands", + " Command ", + " SimpleCommand [ \"$${lang}\" = \"wxstd.po\" ]", + " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.For.List.AndOr[0].Pipeline[0].Command[0].SimpleCommand", + " Word [", + " Words with 4 words", + " Word \"$${lang}\"", + " Word =", + " Word \"wxstd.po\"", + " Word ]", + " Pipeline with 1 commands", + " Command ", + " SimpleCommand continue", + " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.For.List.AndOr[0].Pipeline[1].Command[0].SimpleCommand", + " Word continue", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " SimpleCommand ${TOOLS_PATH.msgfmt} -c -o \"$${lang%.po}.mo\" \"$${lang}\"", + " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.For.List.AndOr[1].Pipeline[0].Command[0].SimpleCommand", + " Word ${TOOLS_PATH.msgfmt}", + " Words with 4 words", + " Word -c", + " Word -o", + " Word \"$${lang%.po}.mo\"", + " Word \"$${lang}\"", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " CompoundCommand ", + " Loop ", + " List with 1 andOrs", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " SimpleCommand :", + " Path List.AndOr[4].Pipeline[0].Command[0].CompoundCommand.Loop.List[0].AndOr[0].Pipeline[0].Command[0].SimpleCommand", + " Word :", + " List with 1 andOrs", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " FunctionDef for fun", + " CompoundCommand ", + " List with 1 andOrs", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " SimpleCommand :", + " Path List.AndOr[4].Pipeline[0].Command[0].CompoundCommand.Loop."+ + "List[1].AndOr[0].Pipeline[0].Command[0].FunctionDefinition.CompoundCommand."+ + "List.AndOr[0].Pipeline[0].Command[0].SimpleCommand", + " Word :", + " Redirects with 1 redirects", + " Redirect >&", + " Word 2") + + outputPathFor("Redirects", "Redirect", "Word") + test(""+ + "echo 'hello world' 1>/dev/null 2>&1 0</dev/random", + + " List with 1 andOrs", + " AndOr with 1 pipelines", + " Pipeline with 1 commands", + " Command ", + " SimpleCommand echo 'hello world'", + " Word echo", + " Path List.AndOr[0].Pipeline[0].Command[0].SimpleCommand.ShToken", + " Words with 1 words", + " Word 'hello world'", + " Path List.AndOr[0].Pipeline[0].Command[0].SimpleCommand.[]ShToken[1].ShToken[0]", + " Redirects with 3 redirects", + " Path List.AndOr[0].Pipeline[0].Command[0].SimpleCommand.[]MkShRedirection", + " Redirect >", + " Path List.AndOr[0].Pipeline[0].Command[0].SimpleCommand.[]MkShRedirection.Redirection[0]", + " Word /dev/null", + " Path List.AndOr[0].Pipeline[0].Command[0].SimpleCommand.[]MkShRedirection.Redirection[0].ShToken[0]", + " Redirect >&", + " Path List.AndOr[0].Pipeline[0].Command[0].SimpleCommand.[]MkShRedirection.Redirection[1]", + " Word 1", + " Path List.AndOr[0].Pipeline[0].Command[0].SimpleCommand.[]MkShRedirection.Redirection[1].ShToken[1]", + " Redirect <", + " Path List.AndOr[0].Pipeline[0].Command[0].SimpleCommand.[]MkShRedirection.Redirection[2]", + " Word /dev/random", + " Path List.AndOr[0].Pipeline[0].Command[0].SimpleCommand.[]MkShRedirection.Redirection[2].ShToken[2]") } diff --git a/pkgtools/pkglint/files/mktypes.go b/pkgtools/pkglint/files/mktypes.go index 60d1a12718e..7e1658d1501 100644 --- a/pkgtools/pkglint/files/mktypes.go +++ b/pkgtools/pkglint/files/mktypes.go @@ -1,10 +1,6 @@ package pkglint -import ( - "netbsd.org/pkglint/textproc" - "strings" - "unicode" -) +import "strings" // MkToken represents a contiguous string from a Makefile. // It is either a literal string or a variable use. @@ -47,42 +43,8 @@ func (m MkVarUseModifier) IsSuffixSubst() bool { } func (m MkVarUseModifier) MatchSubst() (ok bool, regex bool, from string, to string, options string) { - l := textproc.NewLexer(m.Text) - regex = l.PeekByte() == 'C' - if l.SkipByte('S') || l.SkipByte('C') { - separator := l.PeekByte() - l.Skip(1) - if unicode.IsPunct(rune(separator)) || separator == '|' { - noSeparator := func(b byte) bool { return int(b) != separator && b != '\\' } - nextToken := func() string { - start := l.Mark() - for { - switch { - case l.NextBytesFunc(noSeparator) != "": - continue - case l.PeekByte() == '\\' && len(l.Rest()) >= 2: - // TODO: Compare with devel/bmake for the exact behavior - l.Skip(2) - default: - return l.Since(start) - } - } - } - - from = nextToken() - if from != "" && l.SkipByte(byte(separator)) { - to = nextToken() - if l.SkipByte(byte(separator)) { - options = l.NextBytesFunc(func(b byte) bool { - return b == '1' || b == 'g' || b == 'W' - }) - ok = l.EOF() - return - } - } - } - } - return + p := NewMkParser(nil, m.Text, false) + return p.varUseModifierSubst('}') } // Subst evaluates an S/from/to/ modifier. diff --git a/pkgtools/pkglint/files/mktypes_test.go b/pkgtools/pkglint/files/mktypes_test.go index 468095bd518..f13c6c3676e 100644 --- a/pkgtools/pkglint/files/mktypes_test.go +++ b/pkgtools/pkglint/files/mktypes_test.go @@ -13,9 +13,17 @@ func NewMkVarUse(varname string, modifiers ...string) *MkVarUse { } func (s *Suite) Test_MkVarUse_Mod(c *check.C) { - varuse := NewMkVarUse("varname", "Q") + t := s.Init(c) + + test := func(varUseText string, mod string) { + line := t.NewLine("filename.mk", 123, "") + varUse := NewMkParser(line, varUseText, true).VarUse() + t.CheckOutputEmpty() + c.Check(varUse.Mod(), equals, mod) + } - c.Check(varuse.Mod(), equals, ":Q") + test("${varname:Q}", ":Q") + test("${PATH:ts::Q}", ":ts::Q") } // AddCommand adds a command directly to a list of commands, @@ -92,6 +100,24 @@ func (s *Suite) Test_MkVarUseModifier_MatchSubst__backslash(c *check.C) { c.Check(options, equals, "") } +// Some pkgsrc users really explore the darkest corners of bmake by using +// the backslash as the separator in the :S modifier. Sure, it works, it +// just looks totally unexpected to the average pkgsrc reader. +// +// Using the backslash as separator means that it cannot be used for anything +// else, not even for escaping other characters. +func (s *Suite) Test_MkVarUseModifier_MatchSubst__backslash_as_separator(c *check.C) { + mod := MkVarUseModifier{"S\\.post1\\\\1"} + + ok, regex, from, to, options := mod.MatchSubst() + + c.Check(ok, equals, true) + c.Check(regex, equals, false) + c.Check(from, equals, ".post1") + c.Check(to, equals, "") + c.Check(options, equals, "1") +} + // As of 2019-03-24, pkglint doesn't know how to handle complicated // :C modifiers. func (s *Suite) Test_MkVarUseModifier_Subst__regexp(c *check.C) { diff --git a/pkgtools/pkglint/files/options.go b/pkgtools/pkglint/files/options.go index 1a0e56f2c7b..e25d4357888 100755 --- a/pkgtools/pkglint/files/options.go +++ b/pkgtools/pkglint/files/options.go @@ -1,134 +1,180 @@ package pkglint func CheckLinesOptionsMk(mklines MkLines) { - if trace.Tracing { - defer trace.Call1(mklines.lines.FileName)() - } + ck := OptionsLinesChecker{ + mklines, + make(map[string]MkLine), + make(map[string]MkLine), + nil} + + ck.Check() +} + +// OptionsLinesChecker checks an options.mk file of a pkgsrc package. +// +// See mk/bsd.options.mk for a detailed description. +type OptionsLinesChecker struct { + mklines MkLines + + declaredOptions map[string]MkLine + handledOptions map[string]MkLine + optionsInDeclarationOrder []string +} + +func (ck *OptionsLinesChecker) Check() { + mklines := ck.mklines mklines.Check() mlex := NewMkLinesLexer(mklines) mlex.SkipWhile(func(mkline MkLine) bool { return mkline.IsComment() || mkline.IsEmpty() }) - if mlex.EOF() || !(mlex.CurrentMkLine().IsVarassign() && mlex.CurrentMkLine().Varname() == "PKG_OPTIONS_VAR") { - mlex.CurrentLine().Warnf("Expected definition of PKG_OPTIONS_VAR.") - G.Explain( - "The input variables in an options.mk file should always be", - "mentioned in the same order: PKG_OPTIONS_VAR,", - "PKG_SUPPORTED_OPTIONS, PKG_SUGGESTED_OPTIONS.", - "This way, the options.mk files have the same structure and are easy to understand.") + if !ck.lookingAtPkgOptionsVar(mlex) { return } mlex.Skip() - declaredOptions := make(map[string]MkLine) - handledOptions := make(map[string]MkLine) - var optionsInDeclarationOrder []string + upper := true + for !mlex.EOF() && upper { + upper = ck.handleUpperLine(mlex.CurrentMkLine()) + mlex.Skip() + } + + for !mlex.EOF() { + ck.handleLowerLine(mlex.CurrentMkLine()) + mlex.Skip() + } -loop: - for ; !mlex.EOF(); mlex.Skip() { + ck.checkOptionsMismatch() + + mklines.SaveAutofixChanges() +} + +func (ck *OptionsLinesChecker) lookingAtPkgOptionsVar(mlex *MkLinesLexer) bool { + if !mlex.EOF() { mkline := mlex.CurrentMkLine() - switch { - case mkline.IsComment(): - break - case mkline.IsEmpty(): - break - - case mkline.IsVarassign(): - switch mkline.Varcanon() { - case "PKG_SUPPORTED_OPTIONS", "PKG_OPTIONS_GROUP.*", "PKG_OPTIONS_SET.*": - for _, option := range mkline.ValueFields(mkline.Value()) { - if !containsVarRef(option) { - declaredOptions[option] = mkline - optionsInDeclarationOrder = append(optionsInDeclarationOrder, option) - } - } - } + if mkline.IsVarassign() && mkline.Varname() == "PKG_OPTIONS_VAR" { + return true + } + } - case mkline.IsDirective(): - // The conditionals are typically for OPSYS and MACHINE_ARCH. + line := mlex.CurrentLine() + line.Warnf("Expected definition of PKG_OPTIONS_VAR.") + line.Explain( + "The input variables in an options.mk file should always be", + "mentioned in the same order: PKG_OPTIONS_VAR,", + "PKG_SUPPORTED_OPTIONS, PKG_SUGGESTED_OPTIONS.", + "This way, the options.mk files have the same structure and are easy to understand.") + return false +} - case mkline.IsInclude(): - if mkline.IncludedFile() == "../../mk/bsd.options.mk" { - mlex.Skip() - break loop +// checkLineUpper checks a line from the upper part of an options.mk file, +// before bsd.options.mk is included. +func (ck *OptionsLinesChecker) handleUpperLine(mkline MkLine) bool { + switch { + case mkline.IsComment(): + break + case mkline.IsEmpty(): + break + + case mkline.IsVarassign(): + switch mkline.Varcanon() { + case "PKG_SUPPORTED_OPTIONS", "PKG_OPTIONS_GROUP.*", "PKG_OPTIONS_SET.*": + for _, option := range mkline.ValueFields(mkline.Value()) { + if !containsVarRef(option) { + ck.declaredOptions[option] = mkline + ck.optionsInDeclarationOrder = append(ck.optionsInDeclarationOrder, option) + } } + } - default: - mlex.CurrentLine().Warnf("Expected inclusion of \"../../mk/bsd.options.mk\".") - G.Explain( - "After defining the input variables (PKG_OPTIONS_VAR, etc.),", - "bsd.options.mk should be included to do the actual processing.", - "No other actions should take place in this part of the file", - "in order to have the same structure in all options.mk files.") - return + case mkline.IsDirective(): + // The conditionals are typically for OPSYS and MACHINE_ARCH. + + case mkline.IsInclude(): + if mkline.IncludedFile() == "../../mk/bsd.options.mk" { + return false } + + default: + line := mkline + line.Warnf("Expected inclusion of \"../../mk/bsd.options.mk\".") + line.Explain( + "After defining the input variables (PKG_OPTIONS_VAR, etc.),", + "bsd.options.mk should be included to do the actual processing.", + "No other actions should take place in this part of the file", + "in order to have the same structure in all options.mk files.") + return false } - for ; !mlex.EOF(); mlex.Skip() { - mkline := mlex.CurrentMkLine() - if mkline.IsDirective() && (mkline.Directive() == "if" || mkline.Directive() == "elif") { + return true +} + +func (ck *OptionsLinesChecker) handleLowerLine(mkline MkLine) { + if mkline.IsDirective() { + directive := mkline.Directive() + if directive == "if" || directive == "elif" { cond := mkline.Cond() - if cond == nil { - continue + if cond != nil { + ck.handleLowerCondition(mkline, cond) } + } + } +} + +func (ck *OptionsLinesChecker) handleLowerCondition(mkline MkLine, cond MkCond) { - recordUsedOption := func(varuse *MkVarUse) { - if varuse.varname == "PKG_OPTIONS" && len(varuse.modifiers) == 1 { - if m, positive, pattern := varuse.modifiers[0].MatchMatch(); m && positive { - option := pattern - if !containsVarRef(option) { - handledOptions[option] = mkline - optionsInDeclarationOrder = append(optionsInDeclarationOrder, option) - } - } + recordUsedOption := func(varuse *MkVarUse) { + if varuse.varname == "PKG_OPTIONS" && len(varuse.modifiers) == 1 { + if m, positive, pattern := varuse.modifiers[0].MatchMatch(); m && positive { + option := pattern + if !containsVarRef(option) { + ck.handledOptions[option] = mkline + ck.optionsInDeclarationOrder = append(ck.optionsInDeclarationOrder, option) } } - cond.Walk(&MkCondCallback{ - Empty: recordUsedOption, - Var: recordUsedOption}) - - // FIXME: Is this note also issued for the following lines? - // .if empty(ANY_OTHER_VARIABLE) - // .else - // .endif - if cond.Empty != nil && mkline.HasElseBranch() { - mkline.Notef("The positive branch of the .if/.else should be the one where the option is set.") - G.Explain( - "For consistency among packages, the upper branch of this", - ".if/.else statement should always handle the case where the", - "option is activated.", - "A missing exclamation mark at this point can easily be overlooked.", - "", - "If that seems too much to type and the exclamation mark", - "seems wrong for a positive test, switch the blocks nevertheless", - "and write the condition like this, which has the same effect", - "as the !empty(...).", - "", - "\t.if ${PKG_OPTIONS.packagename:Moption}") - } } } - for _, option := range optionsInDeclarationOrder { - declared := declaredOptions[option] - handled := handledOptions[option] + cond.Walk(&MkCondCallback{ + Empty: recordUsedOption, + Var: recordUsedOption}) + + if cond.Empty != nil && cond.Empty.varname == "PKG_OPTIONS" && mkline.HasElseBranch() { + mkline.Notef("The positive branch of the .if/.else should be the one where the option is set.") + mkline.Explain( + "For consistency among packages, the upper branch of this", + ".if/.else statement should always handle the case where the", + "option is activated.", + "A missing exclamation mark at this point can easily be overlooked.", + "", + "If that seems too much to type and the exclamation mark", + "seems wrong for a positive test, switch the blocks nevertheless", + "and write the condition like this, which has the same effect", + "as the !empty(...).", + "", + "\t.if ${PKG_OPTIONS.packagename:Moption}") + } +} + +func (ck *OptionsLinesChecker) checkOptionsMismatch() { + for _, option := range ck.optionsInDeclarationOrder { + declared := ck.declaredOptions[option] + handled := ck.handledOptions[option] switch { case handled == nil: declared.Warnf("Option %q should be handled below in an .if block.", option) - G.Explain( + declared.Explain( "If an option is not processed in this file, it may either be a", "typo, or the option does not have any effect.") case declared == nil: handled.Warnf("Option %q is handled but not added to PKG_SUPPORTED_OPTIONS.", option) - G.Explain( + handled.Explain( "This block of code will never be run since PKG_OPTIONS cannot", "contain this value.", "This is most probably a typo.") } } - - mklines.SaveAutofixChanges() } diff --git a/pkgtools/pkglint/files/options_test.go b/pkgtools/pkglint/files/options_test.go index 1f98bbcecdf..fe75009e0cb 100755 --- a/pkgtools/pkglint/files/options_test.go +++ b/pkgtools/pkglint/files/options_test.go @@ -221,6 +221,7 @@ func (s *Suite) Test_CheckLinesOptionsMk__PLIST_VARS_based_on_PKG_SUPPORTED_OPTI "PLIST.three=\tyes", ".endif") t.Chdir("category/package") + t.FinishSetUp() G.Check(".") @@ -233,3 +234,33 @@ func (s *Suite) Test_CheckLinesOptionsMk__PLIST_VARS_based_on_PKG_SUPPORTED_OPTI "\"two\" is added to PLIST_VARS, but PLIST.two is not defined in this file.", "WARN: options.mk:5: Option \"two\" should be handled below in an .if block.") } + +// Up to April 2019, pkglint logged a wrong note saying that OTHER_VARIABLE +// should have the positive branch first. That note was only ever intended +// for PKG_OPTIONS. +func (s *Suite) Test_OptionsLinesChecker_handleLowerCondition__foreign_variable(c *check.C) { + t := s.Init(c) + + t.SetUpOption("opt", "") + t.CreateFileLines("mk/bsd.options.mk") + t.SetUpPackage("category/package", + ".include \"options.mk\"") + t.CreateFileLines("category/package/options.mk", + MkRcsID, + "", + "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", + "PKG_SUPPORTED_OPTIONS=\topt", + "", + ".include \"../../mk/bsd.options.mk\"", + "", + ".if empty(OTHER_VARIABLE)", + ".else", + ".endif") + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/package/options.mk:8: OTHER_VARIABLE is used but not defined.", + "WARN: ~/category/package/options.mk:4: Option \"opt\" should be handled below in an .if block.") +} diff --git a/pkgtools/pkglint/files/package.go b/pkgtools/pkglint/files/package.go index 5af20015b2e..ed737c4b6c6 100644 --- a/pkgtools/pkglint/files/package.go +++ b/pkgtools/pkglint/files/package.go @@ -85,7 +85,11 @@ func NewPackage(dir string) *Package { pkg.vars.Fallback("PATCHDIR", "patches") pkg.vars.Fallback("KRB5_TYPE", "heimdal") pkg.vars.Fallback("PGSQL_VERSION", "95") - pkg.vars.Fallback(".CURDIR", ".") // FIXME: In reality, this is an absolute pathname. + + // In reality, this is an absolute pathname. Since this variable is + // typically used in the form ${.CURDIR}/../../somewhere, this doesn't + // matter much. + pkg.vars.Fallback(".CURDIR", ".") return &pkg } @@ -118,17 +122,27 @@ func (pkg *Package) checkPossibleDowngrade() { } if change.Action == "Updated" { - changeVersion := replaceAll(change.Version, `nb\d+$`, "") - if pkgver.Compare(pkgversion, changeVersion) < 0 { + pkgversionNorev := replaceAll(pkgversion, `nb\d+$`, "") + changeNorev := replaceAll(change.Version, `nb\d+$`, "") + cmp := pkgver.Compare(pkgversionNorev, changeNorev) + switch { + case cmp < 0: mkline.Warnf("The package is being downgraded from %s (see %s) to %s.", change.Version, mkline.Line.RefToLocation(change.Location), pkgversion) - G.Explain( + mkline.Explain( "The files in doc/CHANGES-*, in which all version changes are", "recorded, have a higher version number than what the package says.", "This is unusual, since packages are typically upgraded instead of", "downgraded.") - // TODO: Check whether the current version is mentioned in doc/CHANGES. + case cmp > 0: + mkline.Notef("Package version %q is greater than the latest %q from %s.", + pkgversion, change.Version, mkline.Line.RefToLocation(change.Location)) + mkline.Explain( + "Each update to a package should be mentioned in the doc/CHANGES file.", + "To do this after updating a package, run", + sprintf("%q,", bmake("cce")), + "which is the abbreviation for commit-changes-entry.") } } } @@ -567,6 +581,7 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines, a scope := NewRedundantScope() scope.Check(allLines) // Updates the variables in the scope pkg.checkGnuConfigureUseLanguages(scope) + pkg.checkUseLanguagesCompilerMk(allLines) pkg.determineEffectivePkgVars() pkg.checkPossibleDowngrade() @@ -761,7 +776,7 @@ func (pkg *Package) checkUpdate() { case cmp < 0: pkgnameLine.Warnf("This package should be updated to %s%s.", sugg.Version, comment) - G.Explain( + pkgnameLine.Explain( "The wishlist for package updates in doc/TODO mentions that a newer", "version of this package is available.") @@ -989,7 +1004,7 @@ func (pkg *Package) CheckVarorder(mklines MkLines) { // except if they are helpful for locating the mistakes. mkline := relevantLines[0] mkline.Warnf("The canonical order of the variables is %s.", strings.Join(canonical, ", ")) - G.Explain( + mkline.Explain( "In simple package Makefiles, some common variables should be", "arranged in a specific order.", "", @@ -1026,19 +1041,21 @@ func (pkg *Package) checkLocallyModified(filename string) { return } - if !isLocallyModified(filename) { + if !isLocallyModified(filename) || !fileExists(filename) { return } if owner != "" { - NewLineWhole(filename).Warnf("Don't commit changes to this file without asking the OWNER, %s.", owner) - G.Explain( + line := NewLineWhole(filename) + line.Warnf("Don't commit changes to this file without asking the OWNER, %s.", owner) + line.Explain( seeGuide("Package components, Makefile", "components.Makefile")) } if maintainer != "" { - NewLineWhole(filename).Notef("Please only commit changes that %s would approve.", maintainer) - G.Explain( + line := NewLineWhole(filename) + line.Notef("Please only commit changes that %s would approve.", maintainer) + line.Explain( "See the pkgsrc guide, section \"Package components\",", "keyword \"maintainer\", for more information.") } @@ -1107,6 +1124,53 @@ func (pkg *Package) AutofixDistinfo(oldSha1, newSha1 string) { } } +// checkUseLanguagesCompilerMk checks that after including mk/compiler.mk +// or mk/endian.mk for the first time, there are no more changes to +// USE_LANGUAGES, as these would be ignored by the pkgsrc infrastructure. +func (pkg *Package) checkUseLanguagesCompilerMk(mklines MkLines) { + + var seen Once + + handleVarassign := func(mkline MkLine) { + if mkline.Varname() != "USE_LANGUAGES" { + return + } + + if !seen.Seen("../../mk/compiler.mk") && !seen.Seen("../../mk/endian.mk") { + return + } + + if mkline.Basename == "compiler.mk" { + if relpath(pkg.dir, mkline.Filename) == "../../mk/compiler.mk" { + return + } + } + + mkline.Warnf("Modifying USE_LANGUAGES after including ../../mk/compiler.mk has no effect.") + mkline.Explain( + "The file compiler.mk guards itself against multiple inclusion.") + } + + handleInclude := func(mkline MkLine) { + dirname, _ := path.Split(mkline.Filename) + dirname = cleanpath(dirname) + fullIncluded := dirname + "/" + mkline.IncludedFile() + relIncludedFile := relpath(pkg.dir, fullIncluded) + + seen.FirstTime(relIncludedFile) + } + + mklines.ForEach(func(mkline MkLine) { + switch { + case mkline.IsVarassign(): + handleVarassign(mkline) + + case mkline.IsInclude(): + handleInclude(mkline) + } + }) +} + type PlistContent struct { Dirs map[string]bool Files map[string]bool diff --git a/pkgtools/pkglint/files/package_test.go b/pkgtools/pkglint/files/package_test.go index 4bc10548e75..68c4d590202 100644 --- a/pkgtools/pkglint/files/package_test.go +++ b/pkgtools/pkglint/files/package_test.go @@ -49,31 +49,51 @@ func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__package_but_not_file func (s *Suite) Test_Package_pkgnameFromDistname(c *check.C) { t := s.Init(c) - pkg := NewPackage(t.File("category/package")) - pkg.vars.Define("PKGNAME", t.NewMkLine("Makefile", 5, "PKGNAME=dummy")) - - test := func(pkgname, distname, expectedPkgname string) { - merged, ok := pkg.pkgnameFromDistname(pkgname, distname) - if !ok { - merged = "" + var once Once + test := func(pkgname, distname, expectedPkgname string, diagnostics ...string) { + t.SetUpPackage("category/package", + "PKGNAME=\t"+pkgname, + "DISTNAME=\t"+distname) + if once.FirstTime("called") { + t.FinishSetUp() } - c.Check(merged, equals, expectedPkgname) + + pkg := NewPackage(t.File("category/package")) + pkg.loadPackageMakefile() + pkg.determineEffectivePkgVars() + t.Check(pkg.EffectivePkgname, equals, expectedPkgname) + t.CheckOutput(diagnostics) } test("pkgname-1.0", "whatever", "pkgname-1.0") - test("${DISTNAME}", "distname-1.0", "distname-1.0") + + test("${DISTNAME}", "distname-1.0", "distname-1.0", + "NOTE: ~/category/package/Makefile:4: This assignment is probably redundant since PKGNAME is ${DISTNAME} by default.") + test("${DISTNAME:S/dist/pkg/}", "distname-1.0", "pkgname-1.0") + test("${DISTNAME:S|a|b|g}", "panama-0.13", "pbnbmb-0.13") - test("${DISTNAME:S|^lib||}", "libncurses", "ncurses") - test("${DISTNAME:S|^lib||}", "mylib", "mylib") + + // The substitution succeeds, but the substituted value is missing + // the package version. Therefore it is discarded completely. + test("${DISTNAME:S|^lib||}", "libncurses", "") + + // The substitution succeeds, but the substituted value is missing + // the package version. Therefore it is discarded completely. + test("${DISTNAME:S|^lib||}", "mylib", "") + test("${DISTNAME:tl:S/-/./g:S/he/-/1}", "SaxonHE9-5-0-1J", "saxon-9.5.0.1j") + test("${DISTNAME:C/beta/.0./}", "fspanel-0.8beta1", "fspanel-0.8.0.1") + test("${DISTNAME:C/Gtk2/p5-gtk2/}", "Gtk2-1.0", "p5-gtk2-1.0") + test("${DISTNAME:S/-0$/.0/1}", "aspell-af-0.50-0", "aspell-af-0.50.0") + test("${DISTNAME:M*.tar.gz:C,\\..*,,}", "aspell-af-0.50-0", "") - // FIXME: Should produce a parse error since the :S modifier is malformed; see Test_MkParser_MkTokens. - test("${DISTNAME:S,a,b,c,d}", "aspell-af-0.50-0", "bspell-af-0.50-0") + test("${DISTNAME:S,a,b,c,d}", "aspell-af-0.50-0", "bspell-af-0.50-0", + "WARN: ~/category/package/Makefile:4: Invalid variable modifier \"c,d\" for \"DISTNAME\".") test("${DISTFILE:C,\\..*,,}", "aspell-af-0.50-0", "") } @@ -356,6 +376,7 @@ func (s *Suite) Test_Package_determineEffectivePkgVars__same(c *check.C) { pkg := t.SetUpPackage("category/package", "DISTNAME=\tdistname-1.0", "PKGNAME=\tdistname-1.0") + t.FinishSetUp() G.Check(pkg) @@ -370,6 +391,7 @@ func (s *Suite) Test_Package_determineEffectivePkgVars__simple_reference(c *chec pkg := t.SetUpPackage("category/package", "DISTNAME=\tdistname-1.0", "PKGNAME=\t${DISTNAME}") + t.FinishSetUp() G.Check(pkg) @@ -383,6 +405,7 @@ func (s *Suite) Test_Package_determineEffectivePkgVars__invalid_DISTNAME(c *chec pkg := t.SetUpPackage("category/package", "DISTNAME=\tpkgname-version") + t.FinishSetUp() G.Check(pkg) @@ -397,9 +420,10 @@ func (s *Suite) Test_Package_determineEffectivePkgVars__C_modifier(c *check.C) { t.SetUpPackage("x11/p5-gtk2", "DISTNAME=\tGtk2-1.0", "PKGNAME=\t${DISTNAME:C:Gtk2:p5-gtk2:}") + t.FinishSetUp() pkg := NewPackage(t.File("x11/p5-gtk2")) - files, mklines, allLines := pkg.load() + pkg.check(files, mklines, allLines) t.Check(pkg.EffectivePkgname, equals, "p5-gtk2-1.0") @@ -415,9 +439,10 @@ func (s *Suite) Test_Package_determineEffectivePkgVars__ineffective_C_modifier(c t.SetUpPackage("category/package", "DISTNAME=\tdistname-1.0", "PKGNAME=\t${DISTNAME:C:does_not_match:replacement:}") + t.FinishSetUp() pkg := NewPackage(t.File("category/package")) - files, mklines, allLines := pkg.load() + pkg.check(files, mklines, allLines) t.Check(pkg.EffectivePkgname, equals, "distname-1.0") @@ -474,6 +499,7 @@ func (s *Suite) Test_Package_loadPackageMakefile__dump(c *check.C) { "LICENSE=\t2-clause-bsd") // TODO: There is no .include line at the end of the Makefile. // This should always be checked though. + t.FinishSetUp() G.checkdirPackage(t.File("category/package")) @@ -556,7 +582,8 @@ func (s *Suite) Test_Package__varuse_at_load_time(c *check.C) { ".include \"../../mk/bsd.pkg.mk\"") t.SetUpCommandLine("-q", "-Wall,no-space") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() + G.Check(t.File("category/pkgbase")) t.CheckOutputLines( @@ -608,7 +635,7 @@ func (s *Suite) Test_Package__relative_included_filenames_in_same_directory(c *c "PKGNAME=\tpkgname-1.67", "DISTNAME=\tdistfile_1_67", ".include \"../../category/package/other.mk\"") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -647,6 +674,7 @@ func (s *Suite) Test_Package_loadPackageMakefile__PECL_VERSION(c *check.C) { pkg := t.SetUpPackage("category/package", "PECL_VERSION=\t1.1.2", ".include \"../../lang/php/ext.mk\"") + t.FinishSetUp() G.Check(pkg) } @@ -672,6 +700,7 @@ func (s *Suite) Test_Package_checkIncludeConditionally__conditional_and_uncondit ".endif", ".include \"../../sysutils/coreutils/buildlink3.mk\"") t.Chdir("category/package") + t.FinishSetUp() G.checkdirPackage(".") @@ -692,6 +721,7 @@ func (s *Suite) Test_Package__include_without_exists(c *check.C) { t.SetUpVartypes() t.SetUpPackage("category/package", ".include \"options.mk\"") + t.FinishSetUp() G.checkdirPackage(t.File("category/package")) @@ -708,6 +738,7 @@ func (s *Suite) Test_Package__include_after_exists(c *check.C) { ".if exists(options.mk)", ". include \"options.mk\"", ".endif") + t.FinishSetUp() G.checkdirPackage(t.File("category/package")) @@ -724,6 +755,7 @@ func (s *Suite) Test_Package_readMakefile__include_other_after_exists(c *check.C ".if exists(options.mk)", ". include \"another.mk\"", ".endif") + t.FinishSetUp() G.checkdirPackage(t.File("category/package")) @@ -755,7 +787,7 @@ func (s *Suite) Test_Package__redundant_master_sites(c *check.C) { "", ".include \"../../math/R/Makefile.extension\"", ".include \"../../mk/bsd.pkg.mk\"") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() // See Package.checkfilePackageMakefile G.checkdirPackage(t.File("math/R-date")) @@ -791,9 +823,9 @@ func (s *Suite) Test_Package_checkUpdate(c *check.C) { "\t"+"o package1-1.0", "\t"+"o package2-2.0 [nice new features]", "\t"+"o package3-3.0 [security update]") - t.Chdir(".") - G.Main("pkglint", "-Wall,no-space", "category/pkg1", "category/pkg2", "category/pkg3") + + t.Main("-Wall,no-space", "category/pkg1", "category/pkg2", "category/pkg3") t.CheckOutputLines( "WARN: category/pkg1/../../doc/TODO:3: Invalid line format \"\".", @@ -812,6 +844,7 @@ func (s *Suite) Test_NewPackage(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("category/Makefile", MkRcsID) + t.FinishSetUp() c.Check( func() { NewPackage("category") }, @@ -844,6 +877,7 @@ func (s *Suite) Test__distinfo_from_other_package(c *check.C) { RcsID, "", "SHA1 (patch-aa) = 1234") + t.FinishSetUp() G.Check("x11/gst-x11") @@ -861,6 +895,7 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__GNU_CONFIGURE(c *check.C) pkg := t.SetUpPackage("category/package", "GNU_CONFIGURE=\tyes", "USE_LANGUAGES=\t#") + t.FinishSetUp() G.Check(pkg) @@ -877,6 +912,7 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__GNU_CONFIGURE_ok(c *check pkg := t.SetUpPackage("category/package", "GNU_CONFIGURE=\tyes", "USE_LANGUAGES=\t# none, really") + t.FinishSetUp() G.Check(pkg) @@ -890,6 +926,7 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__REPLACE_PERL(c *check.C) pkg := t.SetUpPackage("category/package", "REPLACE_PERL=\t*.pl", "NO_CONFIGURE=\tyes") + t.FinishSetUp() G.Check(pkg) @@ -902,6 +939,7 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__META_PACKAGE_with_distinf pkg := t.SetUpPackage("category/package", "META_PACKAGE=\tyes") + t.FinishSetUp() G.Check(pkg) @@ -916,6 +954,7 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__USE_IMAKE_and_USE_X11(c * pkg := t.SetUpPackage("category/package", "USE_X11=\tyes", "USE_IMAKE=\tyes") + t.FinishSetUp() G.Check(pkg) @@ -931,7 +970,7 @@ func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__no_C(c *check.C) { "USE_LANGUAGES+=\tc++14", "USE_LANGUAGES+=\tada", "GNU_CONFIGURE=\tyes") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -955,7 +994,7 @@ func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__C_in_the_middle(c *c "USE_LANGUAGES+=\tc99", "USE_LANGUAGES+=\tada", "GNU_CONFIGURE=\tyes") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -981,7 +1020,7 @@ func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__realistic_compiler_m "USE_LANGUAGES?=\tc", "USE_LANGUAGES+=\tc", "USE_LANGUAGES+=\tc++") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -1012,7 +1051,7 @@ func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__only_GNU_CONFIGURE(c t.SetUpPackage("category/package", "GNU_CONFIGURE=\tyes") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -1025,14 +1064,14 @@ func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__ok(c *check.C) { t.SetUpPackage("category/package", "GNU_CONFIGURE=\tyes", "USE_LANGUAGES=\tc++ objc") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) t.CheckOutputEmpty() } -func (s *Suite) Test_Package__USE_LANGUAGES_too_late(c *check.C) { +func (s *Suite) Test_Package_checkUseLanguagesCompilerMk__too_late(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package", @@ -1040,12 +1079,36 @@ func (s *Suite) Test_Package__USE_LANGUAGES_too_late(c *check.C) { "USE_LANGUAGES=\tc c99 fortran ada c++14") t.CreateFileLines("mk/compiler.mk", MkRcsID) - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) - // FIXME: There must be a warning "USE_LANGUAGES must be added before including compiler.mk." - t.CheckOutputEmpty() + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:21: " + + "Modifying USE_LANGUAGES after including ../../mk/compiler.mk has no effect.") +} + +func (s *Suite) Test_Package_checkUseLanguagesCompilerMk__compiler_mk(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"compiler.mk\"", + "USE_LANGUAGES=\tc c99 fortran ada c++14", + ".include \"../../mk/compiler.mk\"", + "USE_LANGUAGES=\tc c99 fortran ada c++14") + t.CreateFileLines("category/package/compiler.mk", + MkRcsID) + t.CreateFileLines("mk/compiler.mk", + MkRcsID) + t.FinishSetUp() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "NOTE: ~/category/package/Makefile:23: "+ + "Definition of USE_LANGUAGES is redundant because of line 21.", + "WARN: ~/category/package/Makefile:23: "+ + "Modifying USE_LANGUAGES after including ../../mk/compiler.mk has no effect.") } func (s *Suite) Test_Package_readMakefile__skipping(c *check.C) { @@ -1054,6 +1117,7 @@ func (s *Suite) Test_Package_readMakefile__skipping(c *check.C) { t.SetUpCommandLine("-Wall,no-space") pkg := t.SetUpPackage("category/package", ".include \"${MYSQL_PKGSRCDIR:S/-client$/-server/}/buildlink3.mk\"") + t.FinishSetUp() t.EnableTracingToLog() G.Check(pkg) @@ -1086,6 +1150,7 @@ func (s *Suite) Test_Package_readMakefile__not_found(c *check.C) { ".include \"../../devel/zlib/buildlink3.mk\"") t.CreateFileLines("devel/zlib/buildlink3.mk", ".include \"../../enoent/enoent/buildlink3.mk\"") + t.FinishSetUp() G.checkdirPackage(pkg) @@ -1100,6 +1165,7 @@ func (s *Suite) Test_Package_readMakefile__relative(c *check.C) { MkRcsID) pkg := t.SetUpPackage("category/package", ".include \"../package/extra.mk\"") + t.FinishSetUp() G.Check(pkg) @@ -1127,10 +1193,12 @@ func (s *Suite) Test_Package_readMakefile__builtin_mk(c *check.C) { t.CreateFileLines("category/lib1/builtin.mk", MkRcsID, "VAR_FROM_BUILTIN=\t# defined") + t.FinishSetUp() G.Check(t.File("category/package")) t.CheckOutputLines( + "WARN: ~/category/package/Makefile:23: Please use \"${ECHO}\" instead of \"echo\".", "WARN: ~/category/package/Makefile:23: OTHER_VAR is used but not defined.") } @@ -1151,6 +1219,7 @@ func (s *Suite) Test_Package_readMakefile__included(c *check.C) { ".include \"version.mk\"") t.CreateFileLines("lang/language/version.mk", MkRcsID) + t.FinishSetUp() pkg := NewPackage(t.File("category/package")) pkg.loadPackageMakefile() @@ -1182,6 +1251,7 @@ func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { pkg := t.SetUpPackage("category/package", "MAINTAINER=\tpkgsrc-users@NetBSD.org") + t.FinishSetUp() G.Check(pkg) @@ -1248,16 +1318,13 @@ func (s *Suite) Test_Package_checkLocallyModified__directory(c *check.C) { RcsID, "", "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") + t.FinishSetUp() G.Check(pkg) t.CheckOutputLines( - "NOTE: ~/category/package/Makefile: "+ - "Please only commit changes that "+ - "maintainer@example.org would approve.", - // FIXME: There must be no warning for directories. - "NOTE: ~/category/package/patches: "+ - "Please only commit changes that "+ + "NOTE: ~/category/package/Makefile: " + + "Please only commit changes that " + "maintainer@example.org would approve.") } @@ -1269,6 +1336,7 @@ func (s *Suite) Test_Package_AutofixDistinfo__missing_file(c *check.C) { t.SetUpPkgsrc() G.Pkg = NewPackage(t.File("category/package")) + t.FinishSetUp() G.Pkg.AutofixDistinfo("old", "new") @@ -1291,7 +1359,7 @@ func (s *Suite) Test_Package__using_common_Makefile_overriding_DISTINFO_FILE(c * RcsID, "", "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("security/pinentry")) @@ -1317,7 +1385,7 @@ func (s *Suite) Test_Package__redundant_variable_in_unrelated_files(c *check.C) t.CreateFileLines("lang/python/egg.mk", MkRcsID, "PY_PATCHPLIST=\tyes") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("databases/py-trytond-ldap-authentication")) @@ -1349,6 +1417,7 @@ func (s *Suite) Test_Package_readMakefile__include_infrastructure(c *check.C) { ".include \"pthread.builtin.mk\"") t.CreateFileLines("mk/pthread.builtin.mk", "# This should be included by pthread.buildlink3.mk") + t.FinishSetUp() G.Check(t.File("category/package")) diff --git a/pkgtools/pkglint/files/patches.go b/pkgtools/pkglint/files/patches.go index b9a0a529c9c..db9b760c5ff 100644 --- a/pkgtools/pkglint/files/patches.go +++ b/pkgtools/pkglint/files/patches.go @@ -187,7 +187,7 @@ func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { line := ck.llex.CurrentLine() if !ck.isEmptyLine(line.Text) && !matches(line.Text, rePatchUniFileDel) { line.Warnf("Empty line or end of file expected.") - G.Explain( + line.Explain( "This line is not part of the patch anymore, although it may look so.", "To make this situation clear, there should be an", "empty line before this line.", @@ -203,7 +203,7 @@ func (ck *PatchChecker) checkBeginDiff(line Line, patchedFiles int) { if !ck.seenDocumentation && patchedFiles == 0 { line.Errorf("Each patch must be documented.") - G.Explain( + line.Explain( "Pkgsrc tries to have as few patches as possible.", "Therefore, each patch must document why it is necessary.", "Typical reasons are portability or security.", @@ -252,7 +252,7 @@ func (ck *PatchChecker) checklineAdded(addedText string, patchedFileType FileTyp case ftConfigure: if hasSuffix(addedText, ": Avoid regenerating within pkgsrc") { line.Errorf("This code must not be included in patches.") - G.Explain( + line.Explain( "It is generated automatically by pkgsrc after the patch phase.", "", "For more details, look for \"configure-scripts-override\" in", diff --git a/pkgtools/pkglint/files/pkglint.go b/pkgtools/pkglint/files/pkglint.go index 2719ac714aa..a0d24c9e6b0 100644 --- a/pkgtools/pkglint/files/pkglint.go +++ b/pkgtools/pkglint/files/pkglint.go @@ -521,8 +521,9 @@ func CheckLinesMessage(lines Lines) { } if lines.Len() < 3 { - lines.LastLine().Warnf("File too short.") - G.Explain(explanation()...) + line := lines.LastLine() + line.Warnf("File too short.") + line.Explain(explanation()...) return } diff --git a/pkgtools/pkglint/files/pkglint_test.go b/pkgtools/pkglint/files/pkglint_test.go index 8dee31aeea5..7cfc3d273a5 100644 --- a/pkgtools/pkglint/files/pkglint_test.go +++ b/pkgtools/pkglint/files/pkglint_test.go @@ -13,7 +13,7 @@ import ( func (s *Suite) Test_Pkglint_Main__help(c *check.C) { t := s.Init(c) - exitCode := G.Main("pkglint", "-h") + exitCode := t.Main("-h") c.Check(exitCode, equals, 0) t.CheckOutputLines( @@ -58,7 +58,7 @@ func (s *Suite) Test_Pkglint_Main__help(c *check.C) { func (s *Suite) Test_Pkglint_Main__version(c *check.C) { t := s.Init(c) - exitcode := G.Main("pkglint", "--version") + exitcode := t.Main("--version") c.Check(exitcode, equals, 0) t.CheckOutputLines( @@ -68,7 +68,7 @@ func (s *Suite) Test_Pkglint_Main__version(c *check.C) { func (s *Suite) Test_Pkglint_Main__no_args(c *check.C) { t := s.Init(c) - exitcode := G.Main("pkglint") + exitcode := t.Main() // The "." from the error message is the implicit argument added in Pkglint.Main. c.Check(exitcode, equals, 1) @@ -76,7 +76,7 @@ func (s *Suite) Test_Pkglint_Main__no_args(c *check.C) { "FATAL: \".\" must be inside a pkgsrc tree.") } -func (s *Suite) Test_Pkglint_Main__only(c *check.C) { +func (s *Suite) Test_Pkglint_ParseCommandLine__only(c *check.C) { t := s.Init(c) exitcode := G.ParseCommandLine([]string{"pkglint", "-Wall", "--only", ":Q", "--version"}) @@ -92,7 +92,7 @@ func (s *Suite) Test_Pkglint_Main__only(c *check.C) { func (s *Suite) Test_Pkglint_Main__unknown_option(c *check.C) { t := s.Init(c) - exitcode := G.Main("pkglint", "--unknown-option") + exitcode := t.Main("--unknown-option") c.Check(exitcode, equals, 1) c.Check(t.Output(), check.Matches, @@ -112,7 +112,7 @@ func (s *Suite) Test_Pkglint_Main__panic(c *check.C) { G.out = nil // Force an error that cannot happen in practice. c.Check( - func() { G.Main("pkglint", pkg) }, + func() { t.Main(pkg) }, check.PanicMatches, `(?s).*\bnil pointer\b.*`) } @@ -131,8 +131,6 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { // This is typical of the pkglint tests. t.SetUpPkgsrc() - // FIXME: pkglint should warn that the latest version in this file - // (1.10) doesn't match the current version in the package (1.11). t.CreateFileLines("doc/CHANGES-2018", RcsID, "", @@ -230,9 +228,12 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { "Size (checkperms-1.12.tar.gz) = 6621 bytes", "SHA1 (patch-checkperms.c) = asdfasdf") // Invalid SHA-1 checksum - G.Main("pkglint", "-Wall", "-Call", t.File("sysutils/checkperms")) + t.Main("-Wall", "-Call", t.File("sysutils/checkperms")) t.CheckOutputLines( + "NOTE: ~/sysutils/checkperms/Makefile:3: "+ + "Package version \"1.11\" is greater than the latest \"1.10\" "+ + "from ../../doc/CHANGES-2018:5.", "WARN: ~/sysutils/checkperms/Makefile:3: "+ "This package should be updated to 1.13 ([supports more file formats]).", "ERROR: ~/sysutils/checkperms/Makefile:4: Invalid category \"tools\".", @@ -270,6 +271,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { // // See https://github.com/rillig/gobco for the tool to measure the branch coverage. func (s *Suite) Test_Pkglint__realistic(c *check.C) { + t := s.Init(c) if cwd := os.Getenv("PKGLINT_TESTDIR"); cwd != "" { err := os.Chdir(cwd) @@ -281,7 +283,7 @@ func (s *Suite) Test_Pkglint__realistic(c *check.C) { G.out = NewSeparatorWriter(os.Stdout) G.err = NewSeparatorWriter(os.Stderr) trace.Out = os.Stdout - G.Main(append([]string{"pkglint"}, strings.Fields(cmdline)...)...) + t.Main(strings.Fields(cmdline)...) } } @@ -308,6 +310,7 @@ func (s *Suite) Test_Pkglint_Check__empty_directory(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("category/package/CVS/Entries") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -320,6 +323,7 @@ func (s *Suite) Test_Pkglint_Check__files_directory(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("category/package/files/README.md") + t.FinishSetUp() G.Check(t.File("category/package/files")) @@ -333,6 +337,7 @@ func (s *Suite) Test_Pkglint_Check__patches_directory(c *check.C) { t.SetUpPkgsrc() t.CreateFileDummyPatch("category/package/patches/patch-README.md") + t.FinishSetUp() G.Check(t.File("category/package/patches")) @@ -348,6 +353,7 @@ func (s *Suite) Test_Pkglint_Check__manual_patch(c *check.C) { t.SetUpPackage("category/package") t.CreateFileLines("category/package/patches/unknown-file") t.CreateFileLines("category/package/patches/manual-configure") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -361,6 +367,7 @@ func (s *Suite) Test_Pkglint_Check__doc_TODO(c *check.C) { t := s.Init(c) t.SetUpPkgsrc() + t.FinishSetUp() G.Check(G.Pkgsrc.File("doc/TODO")) @@ -569,7 +576,7 @@ func (s *Suite) Test_Pkglint_checkReg__alternatives(c *check.C) { lines := t.SetUpFileLines("category/package/ALTERNATIVES", "bin/tar bin/gnu-tar") - G.Main("pkglint", lines.FileName) + t.Main(lines.FileName) t.CheckOutputLines( "ERROR: ~/category/package/ALTERNATIVES:1: Alternative implementation \"bin/gnu-tar\" must be an absolute path.", @@ -583,7 +590,7 @@ func (s *Suite) Test_Pkglint__profiling(c *check.C) { t.SetUpPkgsrc() t.Chdir(".") - G.Main("pkglint", "--profiling") + t.Main("--profiling") // Pkglint always writes the profiling data into the current directory. // TODO: Make the location of the profiling log a mandatory parameter. @@ -602,11 +609,10 @@ func (s *Suite) Test_Pkglint__profiling(c *check.C) { func (s *Suite) Test_Pkglint__profiling_error(c *check.C) { t := s.Init(c) - t.SetUpPkgsrc() t.Chdir(".") t.CreateFileLines("pkglint.pprof/file") - exitcode := G.Main("pkglint", "--profiling") + exitcode := t.Main("--profiling") c.Check(exitcode, equals, 1) c.Check(t.Output(), check.Matches, @@ -620,7 +626,7 @@ func (s *Suite) Test_Pkglint_checkReg__in_current_working_directory(c *check.C) t.Chdir("category/package") t.CreateFileLines("log") - G.Main("pkglint") + t.Main() t.CheckOutputLines( "WARN: log: Unexpected file found.", @@ -750,6 +756,7 @@ func (s *Suite) Test_Pkglint_checkReg__other(c *check.C) { "#! /bin/sh") t.CreateFileLines("category/package/DEINSTALL", "#! /bin/sh") + t.FinishSetUp() G.Check(pkg) @@ -765,6 +772,7 @@ func (s *Suite) Test_Pkglint_Check__invalid_files_before_import(c *check.C) { t.CreateFileLines("category/package/Makefile~") t.CreateFileLines("category/package/Makefile.orig") t.CreateFileLines("category/package/Makefile.rej") + t.FinishSetUp() G.Check(pkg) @@ -782,7 +790,7 @@ func (s *Suite) Test_Pkglint_checkDirent__errors(c *check.C) { t.SetUpPkgsrc() t.CreateFileLines("category/package/files/subdir/file") t.CreateFileLines("category/package/files/subdir/subsub/file") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.checkDirent(t.File("category/package/options.mk"), 0444) G.checkDirent(t.File("category/package/files/subdir"), 0555|os.ModeDir) @@ -805,7 +813,7 @@ func (s *Suite) Test_Pkglint_checkDirent__file_selection(c *check.C) { MkRcsID) t.CreateFileLines("category/package/unexpected.txt", RcsID) - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.checkDirent(t.File("doc/CHANGES-2018"), 0444) G.checkDirent(t.File("category/package/buildlink3.mk"), 0444) @@ -861,21 +869,16 @@ func (s *Suite) Test_Pkglint_checkReg__readme_and_todo(c *check.C) { c.Check(err, check.IsNil) t.SetUpPkgsrc() - G.Pkgsrc.LoadInfrastructure() t.Chdir(".") - G.Main("pkglint", "category/package", "wip/package") + t.Main("category/package", "wip/package") t.CheckOutputLines( "ERROR: category/package/README: Packages in main pkgsrc must not have a README file.", "ERROR: category/package/TODO: Packages in main pkgsrc must not have a TODO file.", "2 errors and 0 warnings found.") - // FIXME: Do this resetting properly - G.errors = 0 - G.warnings = 0 - G.logged = Once{} - G.Main("pkglint", "--import", "category/package", "wip/package") + t.Main("--import", "category/package", "wip/package") t.CheckOutputLines( "ERROR: category/package/README: Packages in main pkgsrc must not have a README file.", @@ -983,6 +986,7 @@ func (s *Suite) Test_Pkglint_checkdirPackage__PKGDIR(c *check.C) { "COMMENT=\tComment", "LICENSE=\t2-clause-bsd", "PKGDIR=\t\t../../other/package") + t.FinishSetUp() // DISTINFO_FILE is resolved relative to PKGDIR, // the other locations are resolved relative to the package base directory. @@ -998,6 +1002,7 @@ func (s *Suite) Test_Pkglint_checkdirPackage__patch_without_distinfo(c *check.C) pkg := t.SetUpPackage("category/package") t.CreateFileDummyPatch("category/package/patches/patch-aa") t.Remove("category/package/distinfo") + t.FinishSetUp() G.Check(pkg) @@ -1034,13 +1039,14 @@ func (s *Suite) Test_Pkglint_checkdirPackage__filename_with_variable(c *check.C) pkg := t.SetUpPackage("category/package", ".include \"../../mk/bsd.prefs.mk\"", "", - "RUBY_VERSIONS_ACCEPTED=\t22 24 25 26", // As of 2018. + "RUBY_VERSIONS_ACCEPTED=\t22 23 24 25", // As of 2018. ".for rv in ${RUBY_VERSIONS_ACCEPTED}", "RUBY_VER?=\t\t${rv}", ".endfor", "", "RUBY_PKGDIR=\t../../lang/ruby-${RUBY_VER}-base", "DISTINFO_FILE=\t${RUBY_PKGDIR}/distinfo") + t.FinishSetUp() // As of January 2019, pkglint cannot resolve the location of DISTINFO_FILE completely // because the variable \"rv\" comes from a .for loop. @@ -1059,6 +1065,7 @@ func (s *Suite) Test_Pkglint_checkdirPackage__ALTERNATIVES(c *check.C) { pkg := t.SetUpPackage("category/package") t.CreateFileLines("category/package/ALTERNATIVES", "bin/wrapper bin/wrapper-impl") + t.FinishSetUp() G.Check(pkg) @@ -1074,7 +1081,7 @@ func (s *Suite) Test_Pkglint_checkdirPackage__nonexistent_DISTINFO_FILE(c *check t.SetUpPackage("category/package", "DISTINFO_FILE=\tnonexistent") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -1146,6 +1153,7 @@ func (s *Suite) Test_Main(c *check.C) { t.SetUpPackage("category/package") t.Chdir("category/package") + t.FinishSetUp() runMain := func(out *os.File, commandLine ...string) { args := os.Args diff --git a/pkgtools/pkglint/files/pkgsrc.go b/pkgtools/pkglint/files/pkgsrc.go index 715cd8d47f9..dec8efa4014 100644 --- a/pkgtools/pkglint/files/pkgsrc.go +++ b/pkgtools/pkglint/files/pkgsrc.go @@ -344,7 +344,7 @@ func (src *Pkgsrc) loadTools() { func (src *Pkgsrc) loadUntypedVars() { // Setting guessed to false prevents the vartype.guessed case in MkLineChecker.CheckVaruse. - unknownType := Vartype{lkNone, BtUnknown, []ACLEntry{{"*", aclpAll}}, false} + unknownType := Vartype{BtUnknown, NoVartypeOptions, []ACLEntry{{"*", aclpAll}}} define := func(varcanon string, mkline MkLine) { switch { @@ -909,12 +909,12 @@ func (src *Pkgsrc) VariableType(mklines MkLines, varname string) (vartype *Varty if tool.Validity == AfterPrefsMk && mklines.Tools.SeenPrefs { perms |= aclpUseLoadtime } - return &Vartype{lkNone, BtShellCommand, []ACLEntry{{"*", perms}}, false} + return &Vartype{BtShellCommand, NoVartypeOptions, []ACLEntry{{"*", perms}}} } if m, toolVarname := match1(varname, `^TOOLS_(.*)`); m { if tool := G.ToolByVarname(mklines, toolVarname); tool != nil { - return &Vartype{lkNone, BtPathname, []ACLEntry{{"*", aclpUse}}, false} + return &Vartype{BtPathname, NoVartypeOptions, []ACLEntry{{"*", aclpUse}}} } } @@ -930,37 +930,37 @@ func (src *Pkgsrc) guessVariableType(varname string) (vartype *Vartype) { var gtype *Vartype switch { case hasSuffix(varbase, "DIRS"): - gtype = &Vartype{lkShell, BtPathmask, allowRuntime, true} + gtype = &Vartype{BtPathmask, List | Guessed, allowRuntime} case hasSuffix(varbase, "DIR") && !hasSuffix(varbase, "DESTDIR"), hasSuffix(varname, "_HOME"): // TODO: hasSuffix(varbase, "BASE") - gtype = &Vartype{lkNone, BtPathname, allowRuntime, true} + gtype = &Vartype{BtPathname, Guessed, allowRuntime} case hasSuffix(varbase, "FILES"): - gtype = &Vartype{lkShell, BtPathmask, allowRuntime, true} + gtype = &Vartype{BtPathmask, List | Guessed, allowRuntime} case hasSuffix(varbase, "FILE"): - gtype = &Vartype{lkNone, BtPathname, allowRuntime, true} + gtype = &Vartype{BtPathname, Guessed, allowRuntime} case hasSuffix(varbase, "PATH"): - gtype = &Vartype{lkNone, BtPathlist, allowRuntime, true} + gtype = &Vartype{BtPathlist, Guessed, allowRuntime} case hasSuffix(varbase, "PATHS"): - gtype = &Vartype{lkShell, BtPathname, allowRuntime, true} + gtype = &Vartype{BtPathname, List | Guessed, allowRuntime} case hasSuffix(varbase, "_USER"): - gtype = &Vartype{lkNone, BtUserGroupName, allowAll, true} + gtype = &Vartype{BtUserGroupName, Guessed, allowAll} case hasSuffix(varbase, "_GROUP"): - gtype = &Vartype{lkNone, BtUserGroupName, allowAll, true} + gtype = &Vartype{BtUserGroupName, Guessed, allowAll} case hasSuffix(varbase, "_ENV"): - gtype = &Vartype{lkShell, BtShellWord, allowRuntime, true} + gtype = &Vartype{BtShellWord, List | Guessed, allowRuntime} case hasSuffix(varbase, "_CMD"): - gtype = &Vartype{lkNone, BtShellCommand, allowRuntime, true} + gtype = &Vartype{BtShellCommand, Guessed, allowRuntime} case hasSuffix(varbase, "_ARGS"): - gtype = &Vartype{lkShell, BtShellWord, allowRuntime, true} + gtype = &Vartype{BtShellWord, List | Guessed, allowRuntime} case hasSuffix(varbase, "_CFLAGS"), hasSuffix(varname, "_CPPFLAGS"), hasSuffix(varname, "_CXXFLAGS"): - gtype = &Vartype{lkShell, BtCFlag, allowRuntime, true} + gtype = &Vartype{BtCFlag, List | Guessed, allowRuntime} case hasSuffix(varname, "_LDFLAGS"): - gtype = &Vartype{lkShell, BtLdFlag, allowRuntime, true} + gtype = &Vartype{BtLdFlag, List | Guessed, allowRuntime} case hasSuffix(varbase, "_MK"): // TODO: Add BtGuard for inclusion guards, since these variables may only be checked using defined(). - gtype = &Vartype{lkNone, BtUnknown, allowAll, true} + gtype = &Vartype{BtUnknown, Guessed, allowAll} case hasSuffix(varbase, "_SKIP"): - gtype = &Vartype{lkShell, BtPathmask, allowRuntime, true} + gtype = &Vartype{BtPathmask, List | Guessed, allowRuntime} } if gtype == nil { diff --git a/pkgtools/pkglint/files/pkgsrc_test.go b/pkgtools/pkglint/files/pkgsrc_test.go index 9e968906c0d..499c0f62ab9 100644 --- a/pkgtools/pkglint/files/pkgsrc_test.go +++ b/pkgtools/pkglint/files/pkgsrc_test.go @@ -71,7 +71,7 @@ func (s *Suite) Test_Pkgsrc_checkToplevelUnusedLicenses(c *check.C) { t.SetUpPackage("category/package", "LICENSE=\t2-clause-bsd") - G.Main("pkglint", "-r", "-Cglobal", t.File(".")) + t.Main("-r", "-Cglobal", t.File(".")) t.CheckOutputLines( "WARN: ~/licenses/gnu-gpl-v2: This license seems to be unused.", // Added by Tester.SetUpPkgsrc @@ -153,7 +153,7 @@ func (s *Suite) Test_Pkgsrc_loadTools__BUILD_DEFS(c *check.C) { t.CreateFileLines("mk/bsd.pkg.mk", MkRcsID, "_BUILD_DEFS+=\tPKG_SYSCONFBASEDIR PKG_SYSCONFDIR") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(pkg) @@ -173,7 +173,7 @@ func (s *Suite) Test_Pkgsrc_loadDocChanges__not_found(c *check.C) { t.Remove("doc") t.ExpectFatal( - G.Pkgsrc.loadDocChanges, + t.FinishSetUp, "FATAL: ~/doc: Cannot be read for loading the package changes.") } @@ -241,7 +241,7 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__wip_suppresses_warnings(c *c "\tUpdated sysutils/checkperms to 1.10 [rillig 2018-01-05]", "\tUpdated sysutils/checkperms to 1.11 [rillig 2018-01-01]") - G.Main("pkglint", t.File("wip/package")) + t.Main(t.File("wip/package")) t.CheckOutputLines( "Looks fine.") @@ -259,7 +259,7 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__wrong_indentation(c *check.C " Updated sysutils/checkperms to 1.10 [rillig 2018-01-05]", " \tUpdated sysutils/checkperms to 1.11 [rillig 2018-01-01]") - G.Main("pkglint", t.File("category/package")) + t.Main(t.File("category/package")) t.CheckOutputLines( "WARN: ~/doc/CHANGES-2018:5: Package changes should be indented using a single tab, not \" \".", @@ -284,7 +284,7 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__infrastructure(c *check.C) { "\t\tdistfile directly from GitHub [rillig 2018-01-01]", "\tmk/bsd.pkg.mk: Another infrastructure change [rillig 2018-01-02]") - G.Main("pkglint", t.File("category/package")) + t.Main(t.File("category/package")) // For pkglint's purpose, the infrastructure entries are simply ignored // since they do not belong to a single package. @@ -303,7 +303,7 @@ func (s *Suite) Test_Pkgsrc_parseSuggestedUpdates__wip(c *check.C) { "Suggested package updates", "", "\to package-1.13 [cool new features]") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(pkg) @@ -586,14 +586,14 @@ func (s *Suite) Test_Pkgsrc_VariableType(c *check.C) { test("_PERL5_PACKLIST_AWK_STRIP_DESTDIR", "") test("SOME_DIR", "Pathname (guessed)") test("SOMEDIR", "Pathname (guessed)") - test("SEARCHPATHS", "List of Pathname (guessed)") + test("SEARCHPATHS", "Pathname (list, guessed)") test("MYPACKAGE_USER", "UserGroupName (guessed)") test("MYPACKAGE_GROUP", "UserGroupName (guessed)") - test("MY_CMD_ENV", "List of ShellWord (guessed)") - test("MY_CMD_ARGS", "List of ShellWord (guessed)") - test("MY_CMD_CFLAGS", "List of CFlag (guessed)") - test("MY_CMD_LDFLAGS", "List of LdFlag (guessed)") - test("PLIST.abcde", "Yes") + test("MY_CMD_ENV", "ShellWord (list, guessed)") + test("MY_CMD_ARGS", "ShellWord (list, guessed)") + test("MY_CMD_CFLAGS", "CFlag (list, guessed)") + test("MY_CMD_LDFLAGS", "LdFlag (list, guessed)") + test("PLIST.abcde", "Yes (package-settable)") } // Guessing the variable type works for both plain and parameterized variable names. @@ -605,12 +605,12 @@ func (s *Suite) Test_Pkgsrc_VariableType__varparam(c *check.C) { t1 := G.Pkgsrc.VariableType(nil, "FONT_DIRS") c.Assert(t1, check.NotNil) - c.Check(t1.String(), equals, "List of PathMask (guessed)") + c.Check(t1.String(), equals, "PathMask (list, guessed)") t2 := G.Pkgsrc.VariableType(nil, "FONT_DIRS.ttf") c.Assert(t2, check.NotNil) - c.Check(t2.String(), equals, "List of PathMask (guessed)") + c.Check(t2.String(), equals, "PathMask (list, guessed)") } // Guessing the variable type also works for variables that are @@ -630,16 +630,15 @@ func (s *Suite) Test_Pkgsrc_VariableType__from_mk(c *check.C) { "PKGSRC_MAKE_ENV?=\t# none", "CPPPATH?=\tcpp", "OSNAME.Linux?=\tLinux") - pkg := t.SetUpPackage("category/package", "PKGSRC_MAKE_ENV+=\tCPP=${CPPPATH:Q}", "PKGSRC_UNKNOWN_ENV+=\tCPP=${ABCPATH:Q}", "OSNAME.SunOS=\t\t${OSNAME.Other}") - G.Main("pkglint", "-Wall", pkg) + t.Main("-Wall", pkg) if typ := G.Pkgsrc.VariableType(nil, "PKGSRC_MAKE_ENV"); c.Check(typ, check.NotNil) { - c.Check(typ.String(), equals, "List of ShellWord (guessed)") + c.Check(typ.String(), equals, "ShellWord (list, guessed)") } if typ := G.Pkgsrc.VariableType(nil, "CPPPATH"); c.Check(typ, check.NotNil) { @@ -657,7 +656,8 @@ func (s *Suite) Test_Pkgsrc_VariableType__from_mk(c *check.C) { t.CheckOutputLines( "WARN: ~/category/package/Makefile:21: PKGSRC_UNKNOWN_ENV is defined but not used.", "WARN: ~/category/package/Makefile:21: ABCPATH is used but not defined.", - "0 errors and 2 warnings found.") + "0 errors and 2 warnings found.", + "(Run \"pkglint -e\" to show explanations.)") } func (s *Suite) Test_Pkgsrc_guessVariableType__SKIP(c *check.C) { @@ -673,7 +673,7 @@ func (s *Suite) Test_Pkgsrc_guessVariableType__SKIP(c *check.C) { mklines.Check() vartype := G.Pkgsrc.VariableType(mklines, "MY_CHECK_SKIP") - t.Check(vartype.guessed, equals, true) + t.Check(vartype.Guessed(), equals, true) t.Check(vartype.EffectivePermissions("filename.mk"), equals, aclpAllRuntime) // The permissions for MY_CHECK_SKIP say aclpAllRuntime, which excludes diff --git a/pkgtools/pkglint/files/plist.go b/pkgtools/pkglint/files/plist.go index 27e24dc7b2a..1654079e9fb 100644 --- a/pkgtools/pkglint/files/plist.go +++ b/pkgtools/pkglint/files/plist.go @@ -15,8 +15,9 @@ func CheckLinesPlist(pkg *Package, lines Lines) { lines.CheckRcsID(0, `@comment `, "@comment ") if lines.Len() == 1 { - lines.Lines[0].Warnf("PLIST files shouldn't be empty.") - G.Explain( + line := lines.Lines[0] + line.Warnf("PLIST files shouldn't be empty.") + line.Explain( "One reason for empty PLISTs is that this is a newly created package", sprintf("and that the author didn't run %q after installing the files.", bmake("print-PLIST")), "", @@ -200,7 +201,7 @@ func (ck *PlistChecker) checkPath(pline *PlistLine) { } if hasSuffix(text, "/perllocal.pod") { pline.Warnf("The perllocal.pod file should not be in the PLIST.") - G.Explain( + pline.Explain( "This file is handled automatically by the INSTALL/DEINSTALL scripts", "since its contents depends on more than one package.") } @@ -242,7 +243,7 @@ func (ck *PlistChecker) checkSorted(pline *PlistLine) { if ck.lastFname != "" { if ck.lastFname > text && !G.Logger.Opts.Autofix { pline.Warnf("%q should be sorted before %q.", text, ck.lastFname) - G.Explain( + pline.Explain( "The files in the PLIST should be sorted alphabetically.", "This allows human readers to quickly see whether a file is included or not.") } @@ -271,7 +272,7 @@ func (ck *PlistChecker) checkDuplicate(pline *PlistLine) { func (ck *PlistChecker) checkPathBin(pline *PlistLine, dirname, basename string) { if contains(dirname, "/") { pline.Warnf("The bin/ directory should not have subdirectories.") - G.Explain( + pline.Explain( "The programs in bin/ are collected there to be executable by the", "user without having to type an absolute path.", "This advantage does not apply to programs in subdirectories of bin/.", @@ -389,7 +390,7 @@ func (ck *PlistChecker) checkPathShare(pline *PlistLine) { if text == "share/icons/hicolor/icon-theme.cache" && pkg.Pkgpath != "graphics/hicolor-icon-theme" { pline.Errorf("The file icon-theme.cache must not appear in any PLIST file.") - G.Explain( + pline.Explain( "Remove this line and add the following line to the package Makefile.", "", ".include \"../../graphics/hicolor-icon-theme/buildlink3.mk\"") @@ -399,7 +400,7 @@ func (ck *PlistChecker) checkPathShare(pline *PlistLine) { f := "../../graphics/gnome-icon-theme/buildlink3.mk" if !pkg.included.Seen(f) { pline.Errorf("The package Makefile must include %q.", f) - G.Explain( + pline.Explain( "Packages that install GNOME icons must maintain the icon theme", "cache.") } @@ -418,7 +419,7 @@ func (ck *PlistChecker) checkPathShare(pline *PlistLine) { case hasPrefix(text, "share/info/"): pline.Warnf("Info pages should be installed into info/, not share/info/.") - G.Explain( + pline.Explain( "To fix this, add INFO_FILES=yes to the package Makefile.") case hasPrefix(text, "share/man/"): @@ -429,7 +430,7 @@ func (ck *PlistChecker) checkPathShare(pline *PlistLine) { func (pline *PlistLine) CheckTrailingWhitespace() { if hasSuffix(pline.text, " ") || hasSuffix(pline.text, "\t") { pline.Errorf("Pkgsrc does not support filenames ending in whitespace.") - G.Explain( + pline.Explain( "Each character in the PLIST is relevant, even trailing whitespace.") } } @@ -458,7 +459,7 @@ func (pline *PlistLine) CheckDirective(cmd, arg string) { case "dirrm": pline.Warnf("@dirrm is obsolete. Please remove this line.") - G.Explain( + pline.Explain( "Directories are removed automatically when they are empty.", "When a package needs an empty directory, it can use the @pkgdir", "command in the PLIST.") @@ -482,7 +483,7 @@ func (pline *PlistLine) CheckDirective(cmd, arg string) { func (pline *PlistLine) warnImakeMannewsuffix() { pline.Warnf("IMAKE_MANNEWSUFFIX is not meant to appear in PLISTs.") - G.Explain( + pline.Explain( "This is the result of a print-PLIST call that has not been edited", "manually by the package maintainer.", "Please replace the IMAKE_MANNEWSUFFIX with:", diff --git a/pkgtools/pkglint/files/plist_test.go b/pkgtools/pkglint/files/plist_test.go index f3194959ec2..7340f3e038f 100644 --- a/pkgtools/pkglint/files/plist_test.go +++ b/pkgtools/pkglint/files/plist_test.go @@ -631,7 +631,7 @@ func (s *Suite) Test_PlistChecker_checkPathShare__gnome_icon_theme(c *check.C) { PlistRcsID, "share/icons/gnome/16x16/devices/media-optical-cd-audio.png", "share/icons/gnome/16x16/devices/media-optical-dvd.png") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() t.Chdir(".") // This variant is typically run interactively. diff --git a/pkgtools/pkglint/files/redundantscope.go b/pkgtools/pkglint/files/redundantscope.go index b7e84ffcb2d..2583ae01b98 100644 --- a/pkgtools/pkglint/files/redundantscope.go +++ b/pkgtools/pkglint/files/redundantscope.go @@ -204,7 +204,7 @@ func (s *RedundantScope) onRedundant(redundant MkLine, because MkLine) { func (s *RedundantScope) onOverwrite(overwritten MkLine, by MkLine) { overwritten.Warnf("Variable %s is overwritten in %s.", overwritten.Varname(), overwritten.RefTo(by)) - G.Explain( + overwritten.Explain( "The variable definition in this line does not have an effect since", "it is overwritten elsewhere.", "This typically happens because of a typo (writing = instead of +=)", diff --git a/pkgtools/pkglint/files/redundantscope_test.go b/pkgtools/pkglint/files/redundantscope_test.go index 6cacc14719a..0eefb801036 100644 --- a/pkgtools/pkglint/files/redundantscope_test.go +++ b/pkgtools/pkglint/files/redundantscope_test.go @@ -1022,7 +1022,7 @@ func (s *Suite) Test_RedundantScope__procedure_call_implemented_package(c *check "CHECK_BUILTIN.gettext:= yes", ".include \"builtin.mk\"", "CHECK_BUILTIN.gettext:= no") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() // Checking x11/Xaos instead of devel/gettext-lib avoids warnings // about the minimal buildlink3.mk file. @@ -1049,7 +1049,7 @@ func (s *Suite) Test_RedundantScope__procedure_call_infrastructure(c *check.C) { "CHECK_BUILTIN.gettext?= no", ".if !empty(CHECK_BUILTIN.gettext:M[nN][oO])", ".endif") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("x11/alacarte")) @@ -1142,6 +1142,7 @@ func (s *Suite) Test_RedundantScope__included_OPSYS_variable(c *check.C) { t.CreateFileLines("category/dependency/builtin.mk", MkRcsID, "CONFIGURE_ARGS.Darwin+= darwin") + t.FinishSetUp() G.Check(t.File("category/package")) diff --git a/pkgtools/pkglint/files/shell.go b/pkgtools/pkglint/files/shell.go index eb11cfaac8f..b7cb7a3d085 100644 --- a/pkgtools/pkglint/files/shell.go +++ b/pkgtools/pkglint/files/shell.go @@ -18,13 +18,24 @@ import ( type ShellLineChecker struct { MkLines MkLines mkline MkLine + + // checkVarUse is set to false when checking a single shell word + // in order to skip duplicate warnings in variable assignments. + checkVarUse bool } func NewShellLineChecker(mklines MkLines, mkline MkLine) *ShellLineChecker { - return &ShellLineChecker{mklines, mkline} + return &ShellLineChecker{mklines, mkline, true} +} + +func (ck *ShellLineChecker) Warnf(format string, args ...interface{}) { + ck.mkline.Warnf(format, args...) +} +func (ck *ShellLineChecker) Explain(explanation ...string) { + ck.mkline.Explain(explanation...) } -var shellCommandsType = &Vartype{lkNone, BtShellCommands, []ACLEntry{{"*", aclpAllRuntime}}, false} +var shellCommandsType = &Vartype{BtShellCommands, NoVartypeOptions, []ACLEntry{{"*", aclpAllRuntime}}} var shellWordVuc = &VarUseContext{shellCommandsType, vucTimeUnknown, VucQuotPlain, false} func (ck *ShellLineChecker) CheckWord(token string, checkQuoting bool, time ToolTime) { @@ -42,7 +53,9 @@ func (ck *ShellLineChecker) CheckWord(token string, checkQuoting bool, time Tool // to the MkLineChecker. Examples for these are ${VAR:Mpattern} or $@. p := NewMkParser(nil, token, false) if varuse := p.VarUse(); varuse != nil && p.EOF() { - MkLineChecker{ck.MkLines, ck.mkline}.CheckVaruse(varuse, shellWordVuc) + if ck.checkVarUse { + MkLineChecker{ck.MkLines, ck.mkline}.CheckVaruse(varuse, shellWordVuc) + } return } @@ -58,8 +71,7 @@ func (ck *ShellLineChecker) CheckWord(token string, checkQuoting bool, time Tool } func (ck *ShellLineChecker) checkWordQuoting(token string, checkQuoting bool, time ToolTime) { - line := ck.mkline.Line - tok := NewShTokenizer(line, token, false) + tok := NewShTokenizer(ck.mkline.Line, token, false) atoms := tok.ShAtoms() quoting := shqPlain @@ -93,8 +105,8 @@ outer: ck.checkShVarUsePlain(atom, checkQuoting) case atom.Type == shtSubshell: - line.Warnf("Invoking subshells via $(...) is not portable enough.") - G.Explain( + ck.Warnf("Invoking subshells via $(...) is not portable enough.") + ck.Explain( "The Solaris /bin/sh does not know this way to execute a command in a subshell.", "Please use backticks (`...`) as a replacement.") @@ -116,21 +128,20 @@ outer: } if trimHspace(tok.Rest()) != "" { - line.Warnf("Internal pkglint error in ShellLine.CheckWord at %q (quoting=%s), rest: %s", + ck.Warnf("Internal pkglint error in ShellLine.CheckWord at %q (quoting=%s), rest: %s", token, quoting, tok.Rest()) } } func (ck *ShellLineChecker) checkShVarUsePlain(atom *ShAtom, checkQuoting bool) { - line := ck.mkline.Line shVarname := atom.ShVarname() if shVarname == "@" { - line.Warnf("The $@ shell variable should only be used in double quotes.") + ck.Warnf("The $@ shell variable should only be used in double quotes.") } else if G.Opts.WarnQuoting && checkQuoting && ck.variableNeedsQuoting(shVarname) { - line.Warnf("Unquoted shell variable %q.", shVarname) - G.Explain( + ck.Warnf("Unquoted shell variable %q.", shVarname) + ck.Explain( "When a shell variable contains whitespace, it is expanded (split into multiple words)", "when it is written as $variable in a shell script.", "If that is not intended, it should be surrounded by quotation marks, like \"$variable\".", @@ -146,7 +157,7 @@ func (ck *ShellLineChecker) checkShVarUsePlain(atom *ShAtom, checkQuoting bool) } if shVarname == "?" { - line.Warnf("The $? shell variable is often not available in \"set -e\" mode.") + ck.Warnf("The $? shell variable is often not available in \"set -e\" mode.") // TODO: Explain how to properly fix this warning. // TODO: Make sure the warning is only shown when applicable. } @@ -162,10 +173,11 @@ func (ck *ShellLineChecker) checkVaruseToken(atoms *[]*ShAtom, quoting ShQuoting varname := varuse.varname if varname == "@" { - ck.mkline.Warnf("Please use \"${.TARGET}\" instead of \"$@\".") - G.Explain( + ck.Warnf("Please use \"${.TARGET}\" instead of \"$@\".") + ck.Explain( "The variable $@ can easily be confused with the shell variable of", "the same name, which has a completely different meaning.") + varname = ".TARGET" varuse = &MkVarUse{varname, varuse.modifiers} } @@ -179,13 +191,15 @@ func (ck *ShellLineChecker) checkVaruseToken(atoms *[]*ShAtom, quoting ShQuoting case quoting == shqDquot && varuse.IsQ(): ck.mkline.Warnf("The :Q modifier should not be used inside double quotes.") - G.Explain( + ck.mkline.Explain( "To fix this warning, either remove the :Q or the double quotes.", "In most cases, it is more appropriate to remove the double quotes.") } - vuc := VarUseContext{shellCommandsType, vucTimeUnknown, quoting.ToVarUseContext(), true} - MkLineChecker{ck.MkLines, ck.mkline}.CheckVaruse(varuse, &vuc) + if ck.checkVarUse { + vuc := VarUseContext{shellCommandsType, vucTimeUnknown, quoting.ToVarUseContext(), true} + MkLineChecker{ck.MkLines, ck.mkline}.CheckVaruse(varuse, &vuc) + } return true } @@ -237,7 +251,7 @@ func (ck *ShellLineChecker) unescapeBackticks(atoms *[]*ShAtom, quoting ShQuotin // pkglint has a real parser for all shell constructs. if atom.Quoting == shqDquotBackt && matches(atom.MkText, `(^|[^\\])"`) { line.Warnf("Double quotes inside backticks inside double quotes are error prone.") - G.Explain( + line.Explain( "According to the SUSv3, they produce undefined results.", "", "See the paragraph starting \"Within the backquoted ...\" in", @@ -275,7 +289,7 @@ func (ck *ShellLineChecker) CheckShellCommandLine(shelltext string) { // TODO: Now that a shell command parser is available, be more precise in the condition. if contains(shelltext, "${SED}") && contains(shelltext, "${MV}") { line.Notef("Please use the SUBST framework instead of ${SED} and ${MV}.") - G.Explain( + line.Explain( "Using the SUBST framework instead of explicit commands is easier", "to understand, since all the complexity of using sed and mv is", "hidden behind the scenes.", @@ -283,7 +297,7 @@ func (ck *ShellLineChecker) CheckShellCommandLine(shelltext string) { // TODO: Provide a copy-and-paste example. sprintf("Run %q for more information.", makeHelp("subst"))) if contains(shelltext, "#") { - G.Explain( + line.Explain( "When migrating to the SUBST framework, pay attention to \"#\" characters.", "In shell commands, make(1) does not interpret them as", "comment character, but in variable assignments it does.", @@ -348,7 +362,7 @@ func (ck *ShellLineChecker) CheckShellCommand(shellcmd string, pSetE *bool, time } } walker.Callback.Pipeline = func(pipeline *MkShPipeline) { - spc.checkPipeExitcode(line, pipeline) + spc.checkPipeExitcode(pipeline) } walker.Callback.Word = func(word *ShToken) { // TODO: Try to replace false with true here; it had been set to false @@ -397,7 +411,7 @@ func (ck *ShellLineChecker) checkHiddenAndSuppress(hiddenAndSuppress, rest strin break default: ck.mkline.Warnf("The shell command %q should not be hidden.", cmd) - G.Explain( + ck.mkline.Explain( "Hidden shell commands do not appear on the terminal", "or in the log file when they are executed.", "When they fail, the error message cannot be related to the command,", @@ -411,7 +425,7 @@ func (ck *ShellLineChecker) checkHiddenAndSuppress(hiddenAndSuppress, rest strin if contains(hiddenAndSuppress, "-") { ck.mkline.Warnf("Using a leading \"-\" to suppress errors is deprecated.") - G.Explain( + ck.mkline.Explain( "If you really want to ignore any errors from this command, append \"|| ${TRUE}\" to the command.", "This is more visible than a single hyphen, and it should be.") } @@ -473,7 +487,7 @@ func (scc *SimpleCommandChecker) checkCommandStart() { default: if G.Opts.WarnExtra && !(scc.MkLines != nil && scc.MkLines.indentation.DependsOn("OPSYS")) { scc.mkline.Warnf("Unknown shell command %q.", shellword) - G.Explain( + scc.mkline.Explain( "To make the package portable to all platforms that pkgsrc supports,", "it should only use shell commands that are covered by the tools framework.", "", @@ -512,8 +526,8 @@ func (scc *SimpleCommandChecker) handleForbiddenCommand() bool { shellword := scc.strcmd.Name switch path.Base(shellword) { case "mktexlsr", "texconfig": - scc.mkline.Errorf("%q must not be used in Makefiles.", shellword) - G.Explain( + scc.Errorf("%q must not be used in Makefiles.", shellword) + scc.Explain( "This command may only appear in INSTALL scripts, not in the package Makefile,", "so that the package also works if it is installed as a binary package.") return true @@ -587,7 +601,7 @@ func (scc *SimpleCommandChecker) handleComment() bool { } if semicolon || multiline { - G.Explain( + scc.Explain( "When a shell command is split into multiple lines that are", "continued with a backslash, they will nevertheless be converted to", "a single line before the shell sees them.", @@ -615,8 +629,8 @@ func (scc *SimpleCommandChecker) checkRegexReplace() { isSubst := false for _, arg := range scc.strcmd.Args { if G.Testing && isSubst && !matches(arg, `"^[\"\'].*[\"\']$`) { - scc.mkline.Warnf("Substitution commands like %q should always be quoted.", arg) - G.Explain( + scc.Warnf("Substitution commands like %q should always be quoted.", arg) + scc.Explain( "Usually these substitution commands contain characters like '*' or", "other shell metacharacters that might lead to lookup of matching", "filenames and then expand to more than one word.") @@ -648,8 +662,8 @@ func (scc *SimpleCommandChecker) checkAutoMkdirs() { if !contains(arg, "$$") && !matches(arg, `\$\{[_.]*[a-z]`) { if m, dirname := match1(arg, `^(?:\$\{DESTDIR\})?\$\{PREFIX(?:|:Q)\}/(.*)`); m { if G.Pkg != nil && G.Pkg.Plist.Dirs[dirname] { - scc.mkline.Notef("You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= %s\" instead of %q.", dirname, cmdname) - G.Explain( + scc.Notef("You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= %s\" instead of %q.", dirname, cmdname) + scc.Explain( "Many packages include a list of all needed directories in their", "PLIST file.", "In such a case, you can just set AUTO_MKDIRS=yes and be done.", @@ -662,8 +676,8 @@ func (scc *SimpleCommandChecker) checkAutoMkdirs() { "of the many INSTALL_*_DIR variables is appropriate, since", "INSTALLATION_DIRS takes care of that.") } else { - scc.mkline.Notef("You can use \"INSTALLATION_DIRS+= %s\" instead of %q.", dirname, cmdname) - G.Explain( + scc.Notef("You can use \"INSTALLATION_DIRS+= %s\" instead of %q.", dirname, cmdname) + scc.Explain( "To create directories during installation, it is easier to just", "list them in INSTALLATION_DIRS than to execute the commands", "explicitly.", @@ -694,7 +708,7 @@ func (scc *SimpleCommandChecker) checkInstallMulti() { default: if prevdir != "" { scc.mkline.Warnf("The INSTALL_*_DIR commands can only handle one directory at a time.") - G.Explain( + scc.mkline.Explain( "Many implementations of install(1) can handle more, but pkgsrc aims", "at maximum portability.") return @@ -711,8 +725,8 @@ func (scc *SimpleCommandChecker) checkPaxPe() { } if (scc.strcmd.Name == "${PAX}" || scc.strcmd.Name == "pax") && scc.strcmd.HasOption("-pe") { - scc.mkline.Warnf("Please use the -pp option to pax(1) instead of -pe.") - G.Explain( + scc.Warnf("Please use the -pp option to pax(1) instead of -pe.") + scc.mkline.Explain( "The -pe option tells pax to preserve the ownership of the files.", "", "When extracting distfiles as root user, this means that whatever numeric uid was", @@ -734,6 +748,19 @@ func (scc *SimpleCommandChecker) checkEchoN() { } } +func (scc *SimpleCommandChecker) Errorf(format string, args ...interface{}) { + scc.mkline.Errorf(format, args...) +} +func (scc *SimpleCommandChecker) Warnf(format string, args ...interface{}) { + scc.mkline.Warnf(format, args...) +} +func (scc *SimpleCommandChecker) Notef(format string, args ...interface{}) { + scc.mkline.Notef(format, args...) +} +func (scc *SimpleCommandChecker) Explain(explanation ...string) { + scc.mkline.Explain(explanation...) +} + type ShellProgramChecker struct { *ShellLineChecker } @@ -756,8 +783,8 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { checkConditionalCd := func(cmd *MkShSimpleCommand) { if NewStrCommand(cmd).Name == "cd" { - spc.mkline.Errorf("The Solaris /bin/sh cannot handle \"cd\" inside conditionals.") - G.Explain( + spc.Errorf("The Solaris /bin/sh cannot handle \"cd\" inside conditionals.") + spc.Explain( "When the Solaris shell is in \"set -e\" mode and \"cd\" fails, the", "shell will exit, no matter if it is protected by an \"if\" or the", "\"||\" operator.") @@ -781,8 +808,8 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { } walker.Callback.Pipeline = func(pipeline *MkShPipeline) { if pipeline.Negated { - spc.mkline.Warnf("The Solaris /bin/sh does not support negation of shell commands.") - G.Explain( + spc.Warnf("The Solaris /bin/sh does not support negation of shell commands.") + spc.Explain( "The GNU Autoconf manual has many more details of what shell", "features to avoid for portable programs.", "It can be read at:", @@ -792,7 +819,7 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { walker.Walk(list) } -func (spc *ShellProgramChecker) checkPipeExitcode(line Line, pipeline *MkShPipeline) { +func (spc *ShellProgramChecker) checkPipeExitcode(pipeline *MkShPipeline) { if trace.Tracing { defer trace.Call0()() } @@ -812,11 +839,11 @@ func (spc *ShellProgramChecker) checkPipeExitcode(line Line, pipeline *MkShPipel if G.Opts.WarnExtra && len(pipeline.Cmds) > 1 { if canFail, cmd := canFail(); canFail { if cmd != "" { - line.Warnf("The exitcode of %q at the left of the | operator is ignored.", cmd) + spc.Warnf("The exitcode of %q at the left of the | operator is ignored.", cmd) } else { - line.Warnf("The exitcode of the command at the left of the | operator is ignored.") + spc.Warnf("The exitcode of the command at the left of the | operator is ignored.") } - G.Explain( + spc.Explain( "In a shell command like \"cat *.txt | grep keyword\", if the command", "on the left side of the \"|\" fails, this failure is ignored.", "", @@ -916,7 +943,7 @@ func (spc *ShellProgramChecker) checkSetE(list *MkShList, index int, andor *MkSh line.Warnf("Please switch to \"set -e\" mode before using a semicolon (after %q) to separate commands.", NewStrCommand(command.Simple).String()) - G.Explain( + line.Explain( "Normally, when a shell command fails (returns non-zero),", "the remaining commands are still executed.", "For example, the following commands would remove", @@ -934,6 +961,16 @@ func (spc *ShellProgramChecker) checkSetE(list *MkShList, index int, andor *MkSh "* use \"&&\" instead of \";\" to separate the commands") } +func (spc *ShellProgramChecker) Errorf(format string, args ...interface{}) { + spc.mkline.Errorf(format, args...) +} +func (spc *ShellProgramChecker) Warnf(format string, args ...interface{}) { + spc.mkline.Warnf(format, args...) +} +func (spc *ShellProgramChecker) Explain(explanation ...string) { + spc.mkline.Explain(explanation...) +} + // Some shell commands should not be used in the install phase. func (ck *ShellLineChecker) checkInstallCommand(shellcmd string) { if trace.Tracing { @@ -961,14 +998,14 @@ func (ck *ShellLineChecker) checkInstallCommand(shellcmd string) { "tr", "${TR}": // TODO: Pkglint should not complain when sed and tr are used to transform filenames. line.Warnf("The shell command %q should not be used in the install phase.", shellcmd) - G.Explain( + line.Explain( "In the install phase, the only thing that should be done is to", "install the prepared files to their final location.", "The file's contents should not be changed anymore.") case "cp", "${CP}": line.Warnf("${CP} should not be used to install files.") - G.Explain( + line.Explain( "The ${CP} command is highly platform dependent and cannot overwrite read-only files.", "Please use ${PAX} instead.", "", @@ -990,39 +1027,16 @@ func splitIntoShellTokens(line Line, text string) (tokens []string, rest string) // TODO: Check whether this function is used correctly by all callers. // It may be better to use a proper shell parser instead of this tokenizer. - word := "" - rest = text p := NewShTokenizer(line, text, false) - emit := func() { - if word != "" { - tokens = append(tokens, word) - word = "" - } - rest = p.parser.Rest() - } - - q := shqPlain - var prevAtom *ShAtom for { - atom := p.ShAtom(q) - if atom == nil { - if prevAtom == nil || prevAtom.Quoting == shqPlain { - emit() - } + token := p.ShToken() + if token == nil { break } - - q = atom.Quoting - prevAtom = atom - if atom.Type == shtSpace && q == shqPlain { - emit() - } else if atom.Type.IsWord() || atom.Quoting != shqPlain { - word += atom.MkText - } else { - emit() - tokens = append(tokens, atom.MkText) - } + tokens = append(tokens, token.MkText) } + rest = p.parser.Rest() + return } diff --git a/pkgtools/pkglint/files/shell_test.go b/pkgtools/pkglint/files/shell_test.go index d3c250637b6..671e0025a7f 100644 --- a/pkgtools/pkglint/files/shell_test.go +++ b/pkgtools/pkglint/files/shell_test.go @@ -680,10 +680,10 @@ func (s *Suite) Test_ShellLineChecker_CheckShellCommandLine__shell_variables(c * t.SetUpTool("cp", "CP", AtRunTime) t.SetUpTool("mv", "MV", AtRunTime) t.SetUpTool("sed", "SED", AtRunTime) - text := "\tfor f in *.pl; do ${SED} s,@PREFIX@,${PREFIX}, < $f > $f.tmp && ${MV} $f.tmp $f; done" + text := "for f in *.pl; do ${SED} s,@PREFIX@,${PREFIX}, < $f > $f.tmp && ${MV} $f.tmp $f; done" - ck := t.NewShellLineChecker(nil, "Makefile", 3, text) - ck.mkline.Tokenize(ck.mkline.ShellCommand(), true) + ck := t.NewShellLineChecker(nil, "Makefile", 3, "\t# dummy") + ck.mkline.Tokenize(text, true) ck.CheckShellCommandLine(text) t.CheckOutputLines( @@ -1091,6 +1091,7 @@ func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__from_package(c t.CreateFileLines("category/package/extra.mk", MkRcsID, "PYTHON_BIN=\tmy_cmd") + t.FinishSetUp() G.Check(pkg) diff --git a/pkgtools/pkglint/files/shtokenizer.go b/pkgtools/pkglint/files/shtokenizer.go index 6b6a161f7e6..1f203c56daa 100644 --- a/pkgtools/pkglint/files/shtokenizer.go +++ b/pkgtools/pkglint/files/shtokenizer.go @@ -53,6 +53,8 @@ func (p *ShTokenizer) ShAtom(quoting ShQuoting) *ShAtom { atom = p.shAtomSubshDquot() case shqSubshSquot: atom = p.shAtomSubshSquot() + case shqSubshBackt: + atom = p.shAtomSubshBackt() case shqDquotBacktDquot: atom = p.shAtomDquotBacktDquot() case shqDquotBacktSquot: @@ -61,12 +63,9 @@ func (p *ShTokenizer) ShAtom(quoting ShQuoting) *ShAtom { if atom == nil { lexer.Reset(mark) - switch { - case hasPrefix(lexer.Rest(), "${"): - p.parser.Line.Warnf("Unclosed Make variable starting at %q.", shorten(lexer.Rest(), 20)) - case hasPrefix(lexer.Rest(), "$${"): + if hasPrefix(lexer.Rest(), "$${") { p.parser.Line.Warnf("Unclosed shell variable starting at %q.", shorten(lexer.Rest(), 20)) - default: + } else { p.parser.Line.Warnf("Internal pkglint error in ShTokenizer.ShAtom at %q (quoting=%s).", lexer.Rest(), quoting) } } @@ -158,12 +157,19 @@ func (p *ShTokenizer) shAtomSubsh() *ShAtom { case lexer.SkipByte('\''): return &ShAtom{shtText, lexer.Since(mark), shqSubshSquot, nil} case lexer.SkipByte('`'): - // FIXME: return &ShAtom{shtText, lexer.Since(mark), shqBackt, nil} + return &ShAtom{shtText, lexer.Since(mark), shqSubshBackt, nil} case lexer.SkipRegexp(G.res.Compile(`^#[^)]*`)): return &ShAtom{shtComment, lexer.Since(mark), q, nil} case lexer.SkipByte(')'): - // shtText instead of shtOperator because this atom belongs to a shtText token. - return &ShAtom{shtText, lexer.Since(mark), shqPlain, nil} + // The closing parenthesis can have multiple meanings: + // - end of a subshell, such as (echo "in a subshell") + // - end of a subshell variable expression, such as var=$$(echo "from a subshell") + // - end of a case pattern + // In the "subshell variable expression" case, the atom type + // could be shtText since it is part of a text node. On the + // other hand, pkglint doesn't tokenize shell programs correctly + // anyway. This needs to be fixed someday. + return &ShAtom{shtOperator, lexer.Since(mark), shqPlain, nil} } if op := p.shOperator(q); op != nil { return op @@ -237,6 +243,19 @@ func (p *ShTokenizer) shAtomSubshSquot() *ShAtom { return p.shAtomInternal(q, false, true) } +func (p *ShTokenizer) shAtomSubshBackt() *ShAtom { + const q = shqSubshBackt + lexer := p.parser.lexer + mark := lexer.Mark() + switch { + case lexer.SkipByte('`'): + return &ShAtom{shtOperator, lexer.Since(mark), shqSubsh, nil} + case lexer.SkipHspace(): + return &ShAtom{shtSpace, lexer.Since(mark), q, nil} + } + return p.shAtomInternal(q, false, false) +} + func (p *ShTokenizer) shAtomDquotBacktDquot() *ShAtom { const q = shqDquotBacktDquot lexer := p.parser.lexer @@ -391,11 +410,13 @@ func (p *ShTokenizer) ShAtoms() []*ShAtom { func (p *ShTokenizer) ShToken() *ShToken { var curr *ShAtom q := shqPlain + prevQ := q peek := func() *ShAtom { if curr == nil { curr = p.ShAtom(q) if curr != nil { + prevQ = q q = curr.Quoting } } @@ -414,17 +435,18 @@ func (p *ShTokenizer) ShToken() *ShToken { initialMark = lexer.Mark() } - if peek() == nil { + if curr == nil { return nil } - if atom := peek(); !atom.Type.IsWord() { + + if atom := peek(); !atom.Type.IsWord() && atom.Quoting != shqSubsh { return NewShToken(atom.MkText, atom) } for { mark := lexer.Mark() atom := peek() - if atom != nil && (atom.Type.IsWord() || atom.Quoting != shqPlain) { + if atom != nil && (atom.Type.IsWord() || q != shqPlain || prevQ == shqSubsh) { skip() atoms = append(atoms, atom) continue @@ -433,6 +455,11 @@ func (p *ShTokenizer) ShToken() *ShToken { break } + if q != shqPlain { + lexer.Reset(initialMark) + return nil + } + G.Assertf(len(atoms) > 0, "ShTokenizer.ShToken") return NewShToken(lexer.Since(initialMark), atoms...) } diff --git a/pkgtools/pkglint/files/shtokenizer_test.go b/pkgtools/pkglint/files/shtokenizer_test.go index 8b81ace58be..38c7e44847a 100644 --- a/pkgtools/pkglint/files/shtokenizer_test.go +++ b/pkgtools/pkglint/files/shtokenizer_test.go @@ -10,21 +10,28 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { // testRest ensures that the given string is parsed to the expected // atoms, and returns the remaining text. - testRest := func(s string, expectedAtoms ...*ShAtom) string { + testRest := func(s string, expectedAtoms []*ShAtom, expectedRest string) { p := NewShTokenizer(dummyLine, s, false) - q := shqPlain - for _, expectedAtom := range expectedAtoms { - c.Check(p.ShAtom(q), deepEquals, expectedAtom) - q = expectedAtom.Quoting + + actualAtoms := p.ShAtoms() + + t.Check(p.Rest(), equals, expectedRest) + c.Check(len(actualAtoms), equals, len(expectedAtoms)) + + for i, actualAtom := range actualAtoms { + if i < len(expectedAtoms) { + c.Check(actualAtom, deepEquals, expectedAtoms[i]) + } else { + c.Check(actualAtom, deepEquals, nil) + } } - return p.Rest() } + atoms := func(atoms ...*ShAtom) []*ShAtom { return atoms } // test ensures that the given string is parsed to the expected // atoms, and that the text is completely consumed by the parser. test := func(str string, expected ...*ShAtom) { - rest := testRest(str, expected...) - c.Check(rest, equals, "") + testRest(str, expected, "") t.CheckOutputEmpty() } @@ -52,31 +59,35 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { pipe := operator("|") subshell := atom(shtSubshell, "$$(") - q := func(q ShQuoting, atom *ShAtom) *ShAtom { - return &ShAtom{atom.Type, atom.MkText, q, atom.data} + q := func(q ShQuoting) func(atom *ShAtom) *ShAtom { + return func(atom *ShAtom) *ShAtom { + return &ShAtom{atom.Type, atom.MkText, q, atom.data} + } } - backt := func(atom *ShAtom) *ShAtom { return q(shqBackt, atom) } - dquot := func(atom *ShAtom) *ShAtom { return q(shqDquot, atom) } - squot := func(atom *ShAtom) *ShAtom { return q(shqSquot, atom) } - subsh := func(atom *ShAtom) *ShAtom { return q(shqSubsh, atom) } - backtDquot := func(atom *ShAtom) *ShAtom { return q(shqBacktDquot, atom) } - backtSquot := func(atom *ShAtom) *ShAtom { return q(shqBacktSquot, atom) } - dquotBackt := func(atom *ShAtom) *ShAtom { return q(shqDquotBackt, atom) } - subshDquot := func(atom *ShAtom) *ShAtom { return q(shqSubshDquot, atom) } - subshSquot := func(atom *ShAtom) *ShAtom { return q(shqSubshSquot, atom) } - dquotBacktDquot := func(atom *ShAtom) *ShAtom { return q(shqDquotBacktDquot, atom) } - dquotBacktSquot := func(atom *ShAtom) *ShAtom { return q(shqDquotBacktSquot, atom) } + backt := q(shqBackt) + dquot := q(shqDquot) + squot := q(shqSquot) + subsh := q(shqSubsh) + backtDquot := q(shqBacktDquot) + backtSquot := q(shqBacktSquot) + dquotBackt := q(shqDquotBackt) + subshDquot := q(shqSubshDquot) + subshSquot := q(shqSubshSquot) + subshBackt := q(shqSubshBackt) + dquotBacktDquot := q(shqDquotBacktDquot) + dquotBacktSquot := q(shqDquotBacktSquot) // Ignore unused functions; useful for deleting some of the tests during debugging. use := func(args ...interface{}) {} - use(testRest, test) + use(testRest, test, atoms) use(operator, comment, mkvar, text, whitespace) - use(space, semicolon, pipe, subshell) + use(space, semicolon, pipe, subshell, shvar) use(backt, dquot, squot, subsh) use(backtDquot, backtSquot, dquotBackt, subshDquot, subshSquot) use(dquotBacktDquot, dquotBacktSquot) - test("" /* none */) + test("", + nil...) test("$$var", shvar("$$var", "var")) @@ -94,8 +105,8 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { squot(text("single-quoted")), text("'")) - rest := testRest("\"" /* none */) - c.Check(rest, equals, "\"") + test("\"", + dquot(text("\""))) test("$${file%.c}.o", shvar("$${file%.c}", "file"), @@ -279,15 +290,16 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { semicolon, shvar("$$-", "-")) - rest = testRest("COMMENT=\t\\Make $$$$ fast\"", + test("COMMENT=\t\\Make $$$$ fast\"", + text("COMMENT="), whitespace("\t"), text("\\Make"), space, shvar("$$$$", "$"), space, - text("fast")) - c.Check(rest, equals, "\"") + text("fast"), + dquot(text("\""))) test("var=`echo;echo|echo&echo||echo&&echo>echo`", text("var="), @@ -359,7 +371,7 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { test("$$(cat)", subsh(subshell), subsh(text("cat")), - text(")")) + operator(")")) test("$$(cat 'file')", subsh(subshell), @@ -368,12 +380,12 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { subshSquot(text("'")), subshSquot(text("file")), subsh(text("'")), - text(")")) + operator(")")) test("$$(# comment) arg", subsh(subshell), subsh(comment("# comment")), - text(")"), + operator(")"), space, text("arg")) @@ -388,7 +400,18 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { subshSquot(text("'")), subshSquot(text("second")), subsh(text("'")), - text(")")) + operator(")")) + + test("$$(echo `echo nested-subshell`)", + subsh(subshell), + subsh(text("echo")), + subsh(space), + subshBackt(text("`")), + subshBackt(text("echo")), + subshBackt(space), + subshBackt(text("nested-subshell")), + subsh(operator("`")), + operator(")")) } func (s *Suite) Test_ShTokenizer_ShAtom__quoting(c *check.C) { @@ -488,6 +511,9 @@ func (s *Suite) Test_ShTokenizer_ShToken(c *check.C) { "PATH=${PATH:Q}", "true") + test("id=$$(id)", + "id=$$(id)") + test("id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)", "id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)") @@ -552,72 +578,70 @@ func (s *Suite) Test_ShTokenizer_shVarUse(c *check.C) { func (s *Suite) Test_ShTokenizer__examples_from_fuzzing(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("fuzzing.mk", - MkRcsID, - "", - "pre-configure:", - - // Covers shAtomBacktDquot: return nil. - // These are nested backticks with double quotes, - // which should be avoided since POSIX marks them as unspecified. - "\t"+"`\"`", - - // Covers shAtomBacktSquot: return nil - "\t"+"`'$`", - - // Covers shAtomDquotBacktSquot: return nil - "\t"+"\"`'`y", - - // Covers shAtomDquotBackt: return nil - // FIXME: Pkglint must parse unescaped dollar in the same way, everywhere. - "\t"+"\"`$|", - - // Covers shAtomDquotBacktDquot: return nil - // FIXME: Pkglint must support unlimited nesting. - "\t"+"\"`\"`", - - // Covers shAtomSubshDquot: return nil - "\t"+"$$(\"'", - - // Covers shAtomSubsh: case lexer.AdvanceStr("`") - "\t"+"$$(`", - - // Covers shAtomSubshSquot: return nil - "\t"+"$$('$)", - - // Covers shAtomDquotBackt: case lexer.AdvanceRegexp("^#[^`]*") - "\t"+"\"`# comment") - - mklines.Check() - - // Just good that these redundant error messages don't occur every day. - t.CheckOutputLines( - "WARN: fuzzing.mk:4: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=bd).", - "WARN: fuzzing.mk:4: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`\\\"`\"", - - "WARN: fuzzing.mk:5: Internal pkglint error in ShTokenizer.ShAtom at \"$`\" (quoting=bs).", - "WARN: fuzzing.mk:5: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`'$`\"", - "WARN: fuzzing.mk:5: Internal pkglint error in MkLine.Tokenize at \"$`\".", - - "WARN: fuzzing.mk:6: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`'`y\"", - - "WARN: fuzzing.mk:7: Internal pkglint error in ShTokenizer.ShAtom at \"$|\" (quoting=db).", - "WARN: fuzzing.mk:7: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`$|\"", - "WARN: fuzzing.mk:7: Internal pkglint error in MkLine.Tokenize at \"$|\".", - - "WARN: fuzzing.mk:8: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=dbd).", - "WARN: fuzzing.mk:8: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`\\\"`\"", - - "WARN: fuzzing.mk:9: Invoking subshells via $(...) is not portable enough.", - - "WARN: fuzzing.mk:10: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=S).", - "WARN: fuzzing.mk:10: Invoking subshells via $(...) is not portable enough.", - - "WARN: fuzzing.mk:11: Internal pkglint error in ShTokenizer.ShAtom at \"$)\" (quoting=Ss).", - "WARN: fuzzing.mk:11: Invoking subshells via $(...) is not portable enough.", - "WARN: fuzzing.mk:11: Internal pkglint error in MkLine.Tokenize at \"$)\".", + test := func(input string, diagnostics ...string) { + mklines := t.NewMkLines("filename.mk", + MkRcsID, + "\t"+input) + mklines.Check() + t.CheckOutput(diagnostics) + } - "WARN: fuzzing.mk:12: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`# comment\"") + // Covers shAtomBacktDquot: return nil. + // These are nested backticks with double quotes, + // which should be avoided since POSIX marks them as unspecified. + test( + "`\"`", + "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=bd).", + "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`\\\"`\"") + + // Covers shAtomBacktSquot: return nil + test( + "`'$`", + "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"$`\" (quoting=bs).", + "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`'$`\"", + "WARN: filename.mk:2: Internal pkglint error in MkLine.Tokenize at \"$`\".") + + // Covers shAtomDquotBacktSquot: return nil + test( + "\"`'`y", + "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`'`y\"") + + // Covers shAtomDquotBackt: return nil + // FIXME: Pkglint must parse unescaped dollar in the same way, everywhere. + test( + "\"`$|", + "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"$|\" (quoting=db).", + "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`$|\"", + "WARN: filename.mk:2: Internal pkglint error in MkLine.Tokenize at \"$|\".") + + // Covers shAtomDquotBacktDquot: return nil + // FIXME: Pkglint must support unlimited nesting. + test( + "\"`\"`", + "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=dbd).", + "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`\\\"`\"") + + // Covers shAtomSubshDquot: return nil + test( + "$$(\"'", + "WARN: filename.mk:2: Invoking subshells via $(...) is not portable enough.") + + // Covers shAtomSubsh: case lexer.AdvanceStr("`") + test( + "$$(`", + "WARN: filename.mk:2: Invoking subshells via $(...) is not portable enough.") + + // Covers shAtomSubshSquot: return nil + test( + "$$('$)", + "WARN: filename.mk:2: Internal pkglint error in ShTokenizer.ShAtom at \"$)\" (quoting=Ss).", + "WARN: filename.mk:2: Invoking subshells via $(...) is not portable enough.", + "WARN: filename.mk:2: Internal pkglint error in MkLine.Tokenize at \"$)\".") + + // Covers shAtomDquotBackt: case lexer.AdvanceRegexp("^#[^`]*") + test( + "\"`# comment", + "WARN: filename.mk:2: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`# comment\"") } // In order to get 100% code coverage for the shell tokenizer, a panic() statement has been diff --git a/pkgtools/pkglint/files/shtypes.go b/pkgtools/pkglint/files/shtypes.go index 4acae3fedcb..4eb28dc1fc9 100644 --- a/pkgtools/pkglint/files/shtypes.go +++ b/pkgtools/pkglint/files/shtypes.go @@ -84,6 +84,7 @@ const ( shqBacktSquot // e.g. `'word'` shqSubshDquot // e.g. $("word") shqSubshSquot // e.g. $('word') + shqSubshBackt // e.g. $(`word`) shqDquotBacktDquot // e.g. "`"word"`" shqDquotBacktSquot // e.g. "`'word'`" ) @@ -95,7 +96,7 @@ func (q ShQuoting) String() string { return [...]string{ "plain", "d", "s", "b", "S", - "db", "bd", "bs", "Sd", "Ss", + "db", "bd", "bs", "Sd", "Ss", "Sb", "dbd", "dbs", }[q] } diff --git a/pkgtools/pkglint/files/substcontext.go b/pkgtools/pkglint/files/substcontext.go index 5f30908c4e6..30e8b262113 100644 --- a/pkgtools/pkglint/files/substcontext.go +++ b/pkgtools/pkglint/files/substcontext.go @@ -140,7 +140,7 @@ func (ctx *SubstContext) Varassign(mkline MkLine) { if noConfigureLine := G.Pkg.vars.FirstDefinition("NO_CONFIGURE"); noConfigureLine != nil { mkline.Warnf("SUBST_STAGE %s has no effect when NO_CONFIGURE is set (in %s).", value, mkline.RefTo(noConfigureLine)) - G.Explain( + mkline.Explain( "To fix this properly, remove the definition of NO_CONFIGURE.") } } diff --git a/pkgtools/pkglint/files/substcontext_test.go b/pkgtools/pkglint/files/substcontext_test.go index 3fb53eb3ea1..736835f4e6e 100644 --- a/pkgtools/pkglint/files/substcontext_test.go +++ b/pkgtools/pkglint/files/substcontext_test.go @@ -255,6 +255,7 @@ func (s *Suite) Test_SubstContext__pre_configure_with_NO_CONFIGURE(c *check.C) { "SUBST_SED.os= -e s,@OPSYS@,Darwin,", "", "NO_CONFIGURE= yes") + t.FinishSetUp() G.Check(pkg) diff --git a/pkgtools/pkglint/files/testnames_test.go b/pkgtools/pkglint/files/testnames_test.go index 8b2950bd187..25a49724ef8 100644 --- a/pkgtools/pkglint/files/testnames_test.go +++ b/pkgtools/pkglint/files/testnames_test.go @@ -18,6 +18,7 @@ func (s *Suite) Test__test_names(c *check.C) { "comparing_YesNo_variable_to_string", "enumFrom", "enumFromDirs", + "enumFromFiles", "dquotBacktDquot", "and_getSubdirs", "SilentAutofixFormat") diff --git a/pkgtools/pkglint/files/tools_test.go b/pkgtools/pkglint/files/tools_test.go index 2f246eeb88e..71ea6451a65 100644 --- a/pkgtools/pkglint/files/tools_test.go +++ b/pkgtools/pkglint/files/tools_test.go @@ -94,7 +94,7 @@ func (s *Suite) Test_Tools__USE_TOOLS_predefined_sed(c *check.C) { "\t${SED} < input > output", "\t${AWK} < input > output") - G.Main("pkglint", "-Wall", t.File("module.mk")) + t.Main("-Wall", t.File("module.mk")) // Since this test doesn't load the usual tool definitions via // G.Pkgsrc.loadTools, AWK is not known at all. @@ -200,7 +200,7 @@ func (s *Suite) Test_Tools__package_Makefile(c *check.C) { "USE_TOOLS+= load") t.CreateFileLines("mk/bsd.pkg.mk", "USE_TOOLS+= run") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() tools := NewTools() tools.Fallback(G.Pkgsrc.Tools) @@ -250,7 +250,7 @@ func (s *Suite) Test_Tools__builtin_mk(c *check.C) { t.CreateFileLines("mk/bsd.pkg.mk", "USE_TOOLS+= run") t.CreateFileLines("mk/buildlink3/bsd.builtin.mk") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() // Tools that are defined by pkgsrc as load-time tools // may be used in any file at load time. @@ -299,7 +299,7 @@ func (s *Suite) Test_Tools__implicit_definition_in_bsd_pkg_mk(c *check.C) { // bsd.pkg.mk and not defined earlier in mk/tools/defaults.mk, but // the pkglint code is even prepared for these rare cases. // In other words, this test is only there for the code coverage. - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() c.Check(G.Pkgsrc.Tools.ByName("run").String(), equals, "run:::AtRunTime") } @@ -318,7 +318,7 @@ func (s *Suite) Test_Tools__both_prefs_and_pkg_mk(c *check.C) { // The echo tool is mentioned in both files. The file bsd.prefs.mk // grants more use cases (load time + run time), therefore it wins. - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() c.Check(G.Pkgsrc.Tools.ByName("both").Validity, equals, AfterPrefsMk) } @@ -336,7 +336,7 @@ func (s *Suite) Test_Tools__tools_having_the_same_variable_name(c *check.C) { t.CreateFileLines("mk/bsd.prefs.mk", "USE_TOOLS+= awk sed") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() c.Check(G.Pkgsrc.Tools.ByName("awk").Validity, equals, AfterPrefsMk) c.Check(G.Pkgsrc.Tools.ByName("sed").Validity, equals, AfterPrefsMk) @@ -397,7 +397,7 @@ func (s *Suite) Test_Tools__var(c *check.C) { "_TOOLS_VARNAME.ln= LN") t.CreateFileLines("mk/bsd.pkg.mk", "USE_TOOLS+= ln") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() mklines := t.NewMkLines("module.mk", MkRcsID, @@ -498,7 +498,7 @@ func (s *Suite) Test_Tools__cmake(c *check.C) { ".if defined(USE_CMAKE)", "USE_TOOLS+=\tcmake cpack", ".endif") - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) @@ -523,8 +523,7 @@ func (s *Suite) Test_Tools__gmake(c *check.C) { t.CreateFileLines("mk/tools/replace.mk", "TOOLS_CREATE+=\tgmake", "TOOLS_PATH.gmake=\t/usr/bin/gnu-make") - - G.Pkgsrc.LoadInfrastructure() + t.FinishSetUp() G.Check(t.File("category/package")) diff --git a/pkgtools/pkglint/files/trace/tracing.go b/pkgtools/pkglint/files/trace/tracing.go index 28b678d6459..355a887facb 100644 --- a/pkgtools/pkglint/files/trace/tracing.go +++ b/pkgtools/pkglint/files/trace/tracing.go @@ -87,18 +87,18 @@ func isNil(a interface{}) bool { } func argsStr(args []interface{}) string { - rv := "" + var rv strings.Builder for _, arg := range args { - if rv != "" { - rv += ", " + if rv.Len() > 0 { + rv.WriteString(", ") } if str, ok := arg.(fmt.Stringer); ok && !isNil(str) { - rv += str.String() + rv.WriteString(str.String()) } else { - rv += fmt.Sprintf("%#v", arg) + _, _ = fmt.Fprintf(&rv, "%#v", arg) } } - return rv + return rv.String() } func (t *Tracer) traceIndent() string { diff --git a/pkgtools/pkglint/files/util.go b/pkgtools/pkglint/files/util.go index 8f3c5d731d6..0d5890d0df3 100644 --- a/pkgtools/pkglint/files/util.go +++ b/pkgtools/pkglint/files/util.go @@ -244,15 +244,15 @@ func tabWidth(s string) int { } func detab(s string) string { - detabbed := "" + var detabbed strings.Builder for _, r := range s { if r == '\t' { - detabbed += " "[:8-len(detabbed)%8] + detabbed.WriteString(" "[:8-detabbed.Len()%8]) } else { - detabbed += string(r) + detabbed.WriteString(string(r)) } } - return detabbed + return detabbed.String() } func shorten(s string, maxChars int) string { diff --git a/pkgtools/pkglint/files/util_test.go b/pkgtools/pkglint/files/util_test.go index 6b0199c341f..478a37b8d62 100644 --- a/pkgtools/pkglint/files/util_test.go +++ b/pkgtools/pkglint/files/util_test.go @@ -302,6 +302,15 @@ func emptyToNil(slice []string) []string { return slice } +func (s *Suite) Test_trimHspace(c *check.C) { + t := s.Init(c) + + t.Check(trimHspace("a b"), equals, "a b") + t.Check(trimHspace(" a b "), equals, "a b") + t.Check(trimHspace("\ta b\t"), equals, "a b") + t.Check(trimHspace(" \t a b\t \t"), equals, "a b") +} + func (s *Suite) Test_isLocallyModified(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go index fd36cc4f231..e32ffe96eb0 100644 --- a/pkgtools/pkglint/files/vardefs.go +++ b/pkgtools/pkglint/files/vardefs.go @@ -56,11 +56,11 @@ func (reg *VarTypeRegistry) DefineType(varcanon string, vartype *Vartype) { reg.types[varcanon] = vartype } -func (reg *VarTypeRegistry) Define(varname string, kindOfList KindOfList, basicType *BasicType, aclEntries ...ACLEntry) { +func (reg *VarTypeRegistry) Define(varname string, basicType *BasicType, options vartypeOptions, aclEntries ...ACLEntry) { m, varbase, varparam := match2(varname, `^([A-Z_.][A-Z0-9_]*|@)(|\*|\.\*)$`) G.Assertf(m, "invalid variable name") - vartype := Vartype{kindOfList, basicType, aclEntries, false} + vartype := Vartype{basicType, options, aclEntries} if varparam == "" || varparam == "*" { reg.types[varbase] = &vartype @@ -81,15 +81,12 @@ func (reg *VarTypeRegistry) Define(varname string, kindOfList KindOfList, basicT // TODO: To be implemented: when prefixed with "infra:", the entry only // applies to files within the pkgsrc infrastructure. Without this prefix, // the pattern only applies to files outside the pkgsrc infrastructure. -// -// FIXME: Force the permissions to always be in the same order: -// default, set, append, use, use-loadtime. -func (reg *VarTypeRegistry) DefineParse(varname string, kindOfList KindOfList, basicType *BasicType, aclEntries ...string) { +func (reg *VarTypeRegistry) DefineParse(varname string, basicType *BasicType, options vartypeOptions, aclEntries ...string) { parsedEntries := reg.parseACLEntries(varname, aclEntries...) - reg.Define(varname, kindOfList, basicType, parsedEntries...) + reg.Define(varname, basicType, options, parsedEntries...) } -// InitVartypes initializes the long list of predefined pkgsrc variables. +// Init initializes the long list of predefined pkgsrc variables. // After this is done, PKGNAME, MAKE_ENV and all the other variables // can be used in Makefiles without triggering warnings about typos. func (reg *VarTypeRegistry) Init(src *Pkgsrc) { @@ -102,9 +99,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // - how this individual permission set differs // - why the predefined permission set is not good enough // - which packages need this custom permission set. - acl := func(varname string, basicType *BasicType, aclEntries ...string) { - reg.DefineParse(varname, lkNone, basicType, aclEntries...) - } + acl := reg.DefineParse // acllist defines the permissions of a list variable by listing // the permissions individually. @@ -114,20 +109,22 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // - how this individual permission set differs // - why the predefined permission set is not good enough // - which packages need this custom permission set. - acllist := func(varname string, basicType *BasicType, aclEntries ...string) { - reg.DefineParse(varname, lkShell, basicType, aclEntries...) + acllist := func(varname string, basicType *BasicType, options vartypeOptions, aclEntries ...string) { + reg.DefineParse(varname, basicType, options|List, aclEntries...) } // A package-settable variable may be set in all Makefiles except buildlink3.mk and builtin.mk. pkg := func(varname string, basicType *BasicType) { acl(varname, basicType, + PackageSettable, "buildlink3.mk, builtin.mk: none", "Makefile, Makefile.*, *.mk: default, set, use") } // pkgload is the same as pkg, except that the variable may be accessed at load time. pkgload := func(varname string, basicType *BasicType) { - reg.DefineParse(varname, lkNone, basicType, + reg.DefineParse(varname, basicType, + PackageSettable, "buildlink3.mk: none", "builtin.mk: use, use-loadtime", "Makefile, Makefile.*, *.mk: default, set, use, use-loadtime") @@ -140,6 +137,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // catch it. pkglist := func(varname string, basicType *BasicType) { acllist(varname, basicType, + List|PackageSettable, "buildlink3.mk, builtin.mk: none", "Makefile, Makefile.*, *.mk: default, set, append, use") } @@ -157,11 +155,13 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // suffix. pkgappend := func(varname string, basicType *BasicType) { acl(varname, basicType, + PackageSettable, "buildlink3.mk, builtin.mk: none", "Makefile, Makefile.*, *.mk: default, set, append, use") } pkgappendbl3 := func(varname string, basicType *BasicType) { acl(varname, basicType, + PackageSettable, "Makefile, Makefile.*, *.mk: default, set, append, use") } @@ -169,13 +169,15 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // These variables are typically related to compiling and linking files // from C and related languages. pkgbl3 := func(varname string, basicType *BasicType) { - reg.DefineParse(varname, lkNone, basicType, + reg.DefineParse(varname, basicType, + PackageSettable, "Makefile, Makefile.*, *.mk: default, set, use") } // Some package-defined lists may also be modified in buildlink3.mk files, // for example platform-specific CFLAGS and LDFLAGS. pkglistbl3 := func(varname string, basicType *BasicType) { - reg.DefineParse(varname, lkShell, basicType, + reg.DefineParse(varname, basicType, + List|PackageSettable, "Makefile, Makefile.*, *.mk: default, set, append, use") } @@ -190,17 +192,20 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // They can be made more precise. sys := func(varname string, basicType *BasicType) { acl(varname, basicType, + SystemProvided, "buildlink3.mk: none", "*: use") } sysbl3 := func(varname string, basicType *BasicType) { acl(varname, basicType, + SystemProvided, "*: use") } syslist := func(varname string, basicType *BasicType) { acllist(varname, basicType, + List|SystemProvided, "buildlink3.mk: none", "*: use") } @@ -209,6 +214,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { usr := func(varname string, basicType *BasicType) { acl(varname, basicType, // TODO: why is builtin.mk missing here? + UserSettable, "buildlink3.mk: none", "*: use, use-loadtime") } @@ -217,25 +223,29 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { usrlist := func(varname string, basicType *BasicType) { acllist(varname, basicType, // TODO: why is builtin.mk missing here? + List|UserSettable, "buildlink3.mk: none", "*: use, use-loadtime") } // sysload declares a system-provided variable that may already be used at load time. sysload := func(varname string, basicType *BasicType) { - reg.DefineParse(varname, lkNone, basicType, + reg.DefineParse(varname, basicType, + SystemProvided, "*: use, use-loadtime") } sysloadlist := func(varname string, basicType *BasicType) { - reg.DefineParse(varname, lkShell, basicType, + reg.DefineParse(varname, basicType, + List|SystemProvided, "*: use, use-loadtime") } // bl3list declares a list variable that is defined by buildlink3.mk and // builtin.mk and can later be used by the package. bl3list := func(varname string, basicType *BasicType) { - reg.DefineParse(varname, lkShell, basicType, + reg.DefineParse(varname, basicType, + List, // not PackageSettable since the package uses it more than setting it. "buildlink3.mk, builtin.mk: append", "*: use") } @@ -243,7 +253,8 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // cmdline declares a variable that is defined on the command line. There // are only few variables of this type, such as PKG_DEBUG_LEVEL. cmdline := func(varname string, basicType *BasicType) { - reg.DefineParse(varname, lkNone, basicType, + reg.DefineParse(varname, basicType, + CommandLineProvided, "buildlink3.mk, builtin.mk: none", "*: use, use-loadtime") } @@ -251,6 +262,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // Only for infrastructure files; see mk/misc/show.mk infralist := func(varname string, basicType *BasicType) { acllist(varname, basicType, + List, "*: append") } @@ -348,6 +360,25 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { return enum(strings.Join(versions, " ")) } + // enumFromFiles reads the files from the given base directory, + // filtering it through the regular expression and the replacement. + // + // If no files are found, the allowed values are taken + // from defval. This should only happen in the pkglint tests. + enumFromFiles := func(basedir string, re regex.Pattern, repl string, defval string) *BasicType { + var relevant []string + for _, filename := range dirglob(G.Pkgsrc.File(basedir)) { + basename := path.Base(filename) + if matches(basename, re) { + relevant = append(relevant, replaceAll(basename, re, repl)) + } + } + if len(relevant) == 0 { + return enum(defval) + } + return enum(strings.Join(relevant, " ")) + } + compilers := enumFrom( "mk/compiler.mk", "ccache ccc clang distcc f2c gcc hp icc ido mipspro mipspro-ucode pcc sunpro xlc", @@ -419,8 +450,10 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // X11_TYPE and X11BASE may be used in buildlink3.mk as well, which the // standard sysload doesn't allow. acl("X11_TYPE", enum("modular native"), + UserSettable, "*: use, use-loadtime") acl("X11BASE", BtPathname, + UserSettable, "*: use, use-loadtime") usr("MOTIFBASE", BtPathname) @@ -483,6 +516,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // TODO: parse all the below information directly from mk/defaults/mk.conf. usrpkg := func(varname string, basicType *BasicType) { acl(varname, basicType, + PackageSettable|UserSettable, "Makefile: default, set, use, use-loadtime", "buildlink3.mk, builtin.mk: none", "Makefile.*, *.mk: default, set, use, use-loadtime", @@ -490,6 +524,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { } usrpkglist := func(varname string, basicType *BasicType) { acllist(varname, basicType, + List|PackageSettable|UserSettable, "Makefile: default, set, use, use-loadtime", "buildlink3.mk, builtin.mk: none", "Makefile.*, *.mk: default, set, use, use-loadtime", @@ -754,75 +789,99 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { syslist("BSD_MAKE_ENV", BtShellWord) // TODO: Align the permissions of the various BUILDLINK_*.* variables with each other. acllist("BUILDLINK_ABI_DEPENDS.*", BtDependency, + PackageSettable, "buildlink3.mk, builtin.mk: append, use-loadtime", "*: append") acllist("BUILDLINK_API_DEPENDS.*", BtDependency, + PackageSettable, "buildlink3.mk, builtin.mk: append, use-loadtime", "*: append") acl("BUILDLINK_AUTO_DIRS.*", BtYesNo, + PackageSettable, "buildlink3.mk: append", "Makefile: set") syslist("BUILDLINK_CFLAGS", BtCFlag) bl3list("BUILDLINK_CFLAGS.*", BtCFlag) acl("BUILDLINK_CONTENTS_FILTER.*", BtShellCommand, + PackageSettable, "buildlink3.mk: set") syslist("BUILDLINK_CPPFLAGS", BtCFlag) bl3list("BUILDLINK_CPPFLAGS.*", BtCFlag) acllist("BUILDLINK_DEPENDS", BtIdentifier, + PackageSettable, "buildlink3.mk: append") acllist("BUILDLINK_DEPMETHOD.*", BtBuildlinkDepmethod, + PackageSettable, "buildlink3.mk: default, append, use", "Makefile, Makefile.*, *.mk: default, set, append") acl("BUILDLINK_DIR", BtPathname, + PackageSettable, "*: use") bl3list("BUILDLINK_FILES.*", BtPathmask) pkgbl3("BUILDLINK_FILES_CMD.*", BtShellCommand) acllist("BUILDLINK_INCDIRS.*", BtPathname, + PackageSettable, "buildlink3.mk: default, append", "Makefile, Makefile.*, *.mk: use") acl("BUILDLINK_JAVA_PREFIX.*", BtPathname, + PackageSettable, "buildlink3.mk: set, use") acllist("BUILDLINK_LDADD.*", BtLdFlag, + PackageSettable, "builtin.mk: default, set, append, use", "buildlink3.mk: append, use", "Makefile, Makefile.*, *.mk: use") acllist("BUILDLINK_LDFLAGS", BtLdFlag, + PackageSettable, "*: use") bl3list("BUILDLINK_LDFLAGS.*", BtLdFlag) acllist("BUILDLINK_LIBDIRS.*", BtPathname, + PackageSettable, "buildlink3.mk, builtin.mk: append", "Makefile, Makefile.*, *.mk: use") acllist("BUILDLINK_LIBS.*", BtLdFlag, + PackageSettable, "buildlink3.mk: append", "Makefile, Makefile.*, *.mk: set, append, use") acllist("BUILDLINK_PASSTHRU_DIRS", BtPathname, + PackageSettable, "Makefile, Makefile.*, *.mk: append") acllist("BUILDLINK_PASSTHRU_RPATHDIRS", BtPathname, + PackageSettable, "Makefile, Makefile.*, *.mk: append") acl("BUILDLINK_PKGSRCDIR.*", BtRelativePkgDir, + PackageSettable, "buildlink3.mk: default, use-loadtime") acl("BUILDLINK_PREFIX.*", BtPathname, + PackageSettable, "builtin.mk: set, use", "Makefile, Makefile.*, *.mk: use") acllist("BUILDLINK_RPATHDIRS.*", BtPathname, + PackageSettable, "buildlink3.mk: append") acllist("BUILDLINK_TARGETS", BtIdentifier, + PackageSettable, "Makefile, Makefile.*, *.mk: append") acl("BUILDLINK_FNAME_TRANSFORM.*", BtSedCommands, + PackageSettable, "Makefile, buildlink3.mk, builtin.mk, hacks.mk, options.mk: append") acllist("BUILDLINK_TRANSFORM", BtWrapperTransform, + PackageSettable, "*: append") acllist("BUILDLINK_TRANSFORM.*", BtWrapperTransform, + PackageSettable, "*: append") acllist("BUILDLINK_TREE", BtIdentifier, + PackageSettable, "buildlink3.mk: append") acl("BUILDLINK_X11_DIR", BtPathname, + PackageSettable, "*: use") acllist("BUILD_DEFS", BtVariableName, + PackageSettable, "Makefile, Makefile.*, *.mk: append") pkglist("BUILD_DEFS_EFFECTS", BtVariableName) - acllist("BUILD_DEPENDS", BtDependencyWithPath, - "Makefile, Makefile.*, *.mk: append") + pkglistbl3("BUILD_DEPENDS", BtDependencyWithPath) pkglist("BUILD_DIRS", BtWrksrcSubdirectory) pkglist("BUILD_ENV", BtShellWord) sys("BUILD_MAKE_CMD", BtShellCommand) @@ -831,19 +890,25 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkglist("BUILD_TARGET.*", BtIdentifier) pkg("BUILD_USES_MSGFMT", BtYes) acl("BUILTIN_PKG", BtIdentifier, + PackageSettable, "builtin.mk: set, use, use-loadtime", "Makefile, Makefile.*, *.mk: use, use-loadtime") acl("BUILTIN_PKG.*", BtPkgName, + PackageSettable, "builtin.mk: set, use, use-loadtime") pkglistbl3("BUILTIN_FIND_FILES_VAR", BtVariableName) pkglistbl3("BUILTIN_FIND_FILES.*", BtPathname) acl("BUILTIN_FIND_GREP.*", BtUnknown, + PackageSettable, "builtin.mk: set") acllist("BUILTIN_FIND_HEADERS_VAR", BtVariableName, + PackageSettable, "builtin.mk: set") acllist("BUILTIN_FIND_HEADERS.*", BtPathname, + PackageSettable, "builtin.mk: set") acllist("BUILTIN_FIND_LIBS", BtPathname, + PackageSettable, "builtin.mk: set") sys("BUILTIN_X11_TYPE", BtUnknown) sys("BUILTIN_X11_VERSION", BtUnknown) @@ -853,9 +918,11 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkglistbl3("CFLAGS", BtCFlag) // may also be changed by the user pkglistbl3("CFLAGS.*", BtCFlag) // may also be changed by the user acl("CHECK_BUILTIN", BtYesNo, + PackageSettable, "builtin.mk: default", "Makefile: set") acl("CHECK_BUILTIN.*", BtYesNo, + PackageSettable, "Makefile, options.mk, buildlink3.mk: set", "builtin.mk: default, use-loadtime", "*: use-loadtime") @@ -942,6 +1009,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkg("DLOPEN_REQUIRE_PTHREADS", BtYesNo) pkg("DL_AUTO_VARS", BtYes) acllist("DL_LIBS", BtLdFlag, + PackageSettable, "*: append, use") sys("DOCOWN", BtUserGroupName) sys("DOCGRP", BtUserGroupName) @@ -1031,6 +1099,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // GNU_CONFIGURE needs to be tested in some buildlink3.mk files, // such as lang/vala. acl("GNU_CONFIGURE", BtYes, + PackageSettable, "buildlink3.mk: none", "builtin.mk: use, use-loadtime", "Makefile, Makefile.*, *.mk: default, set, use, use-loadtime") @@ -1045,6 +1114,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkg("HOMEPAGE", BtHomepage) pkg("ICON_THEMES", BtYes) acl("IGNORE_PKG.*", BtYes, + PackageSettable, "*: set, use-loadtime") sys("IMAKE", BtShellCommand) pkglistbl3("INCOMPAT_CURSES", BtMachinePlatformPattern) @@ -1076,6 +1146,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkgload("INSTALL_UNSTRIPPED", BtYesNo) pkglist("INTERACTIVE_STAGE", enum("fetch extract configure build test install")) acl("IS_BUILTIN.*", BtYesNoIndirectly, + PackageSettable, // These two differ from the standard, // they are needed for devel/ncursesw. "buildlink3.mk: use, use-loadtime", @@ -1208,7 +1279,8 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkglist("ONLY_FOR_COMPILER", compilers) pkglist("ONLY_FOR_PLATFORM", BtMachinePlatformPattern) pkg("ONLY_FOR_UNPRIVILEGED", BtYesNo) - sysload("OPSYS", BtIdentifier) + sysload("OPSYS", enumFromFiles("mk/platform", `(.*)\.mk$`, "$1", + "Cygwin DragonFly FreeBSD Linux NetBSD SunOS")) pkglistbl3("OPSYSVARS", BtVariableName) pkg("OSVERSION_SPECIFIC", BtYes) sysload("OS_VERSION", BtVersion) @@ -1228,7 +1300,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkg("PATCH_DIST_STRIP*", BtShellWord) pkglist("PATCH_SITES", BtFetchURL) pkg("PATCH_STRIP", BtShellWord) - sys("PATH", BtPathlist) // From the PATH environment variable. + sysload("PATH", BtPathlist) // From the PATH environment variable. sys("PAXCTL", BtShellCommand) // See mk/pax.mk. pkglist("PERL5_PACKLIST", BtPerl5Packlist) pkg("PERL5_PACKLIST_DIR", BtPathname) @@ -1252,6 +1324,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkg("PERL5_USE_PACKLIST", BtYesNo) sys("PGSQL_PREFIX", BtPathname) acllist("PGSQL_VERSIONS_ACCEPTED", pgsqlVersions, + PackageSettable, // The "set" is necessary for databases/postgresql-postgis2. "Makefile, Makefile.*, *.mk: default, set, append, use") usr("PGSQL_VERSION_DEFAULT", BtVersion) @@ -1263,11 +1336,13 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { usr("PHP_VERSION_REQD", BtVersion) acl("PHP_PKG_PREFIX", enumFromDirs("lang", `^php(\d+)$`, "php$1", "php56 php71 php72 php73"), + SystemProvided, "special:phpversion.mk: set", "*: use, use-loadtime") sys("PKGBASE", BtIdentifier) // Despite its name, this is actually a list of filenames. acllist("PKGCONFIG_FILE.*", BtPathname, + PackageSettable, "builtin.mk: set, append", "special:pkgconfig-builtin.mk: use-loadtime") pkglist("PKGCONFIG_OVERRIDE", BtPathmask) @@ -1283,18 +1358,20 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // be set in a package Makefile. // See VartypeCheck.PkgRevision for details. acl("PKGREVISION", BtPkgRevision, + PackageSettable, "Makefile: set") sys("PKGSRCDIR", BtPathname) // This definition is only valid in the top-level Makefile, // not in category or package Makefiles. acl("PKGSRCTOP", BtYes, + PackageSettable, "Makefile: set") sys("PKGSRC_SETENV", BtShellCommand) syslist("PKGTOOLS_ENV", BtShellWord) sys("PKGVERSION", BtVersion) sys("PKGVERSION_NOREV", BtVersion) // Without the nb* part. sys("PKGWILDCARD", BtFileMask) - sys("PKG_ADMIN", BtShellCommand) + sysload("PKG_ADMIN", BtShellCommand) sys("PKG_APACHE", enum("apache24")) pkglist("PKG_APACHE_ACCEPTED", enum("apache24")) usr("PKG_APACHE_DEFAULT", enum("apache24")) @@ -1322,6 +1399,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // // TODO: Is it possible to include hacks.mk files from the dependencies? acllist("PKG_HACKS", BtIdentifier, + PackageSettable, "hacks.mk: append") sys("PKG_INFO", BtShellCommand) sys("PKG_JAVA_HOME", BtPathname) @@ -1362,6 +1440,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // The special exception for buildlink3.mk is only here because // of textproc/xmlcatmgr. acl("PKG_SYSCONFDIR*", BtPathname, + PackageSettable, "Makefile: set, use, use-loadtime", "buildlink3.mk, builtin.mk: use-loadtime", "Makefile.*, *.mk: default, set, use, use-loadtime") @@ -1381,11 +1460,13 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkglistbl3("PREPEND_PATH", BtPathname) acl("PREFIX", BtPathname, + UserSettable, "*: use") // BtPathname instead of BtPkgPath since the original package doesn't exist anymore. // It would be more precise to check for a PkgPath that doesn't exist anymore. pkg("PREV_PKGPATH", BtPathname) acl("PRINT_PLIST_AWK", BtAwkCommand, + PackageSettable, "*: append") pkglist("PRIVILEGED_STAGES", enum("build install package clean")) pkgbl3("PTHREAD_AUTO_VARS", BtYesNo) @@ -1397,6 +1478,7 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkg("PY_PATCHPLIST", BtYes) acl("PYPKGPREFIX", enumFromDirs("lang", `^python(\d+)$`, "py$1", "py27 py36"), + SystemProvided, "special:pyversion.mk: set", "*: use, use-loadtime") // See lang/python/pyversion.mk @@ -1446,11 +1528,13 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkglist("RPMIGNOREPATH", BtPathmask) acl("RUBY_BASE", enumFromDirs("lang", `^ruby(\d+)$`, "ruby$1", "ruby22 ruby23 ruby24 ruby25"), + SystemProvided, "special:rubyversion.mk: set", "*: use, use-loadtime") usr("RUBY_VERSION_REQD", BtVersion) acl("RUBY_PKGPREFIX", enumFromDirs("lang", `^ruby(\d+)$`, "ruby$1", "ruby22 ruby23 ruby24 ruby25"), + SystemProvided, "special:rubyversion.mk: default, set, use", "*: use, use-loadtime") sys("RUN", BtShellCommand) @@ -1479,8 +1563,10 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkglist("SPECIAL_PERMS", BtPerms) sys("STEP_MSG", BtShellCommand) sys("STRIP", BtShellCommand) // see mk/tools/strip.mk + // Only valid in the top-level and the category Makefiles. acllist("SUBDIR", BtFileName, + PackageSettable, "Makefile: append") pkglistbl3("SUBST_CLASSES", BtIdentifier) @@ -1519,12 +1605,15 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { pkg("USERGROUP_PHASE", enum("configure build pre-install")) usrlist("USER_ADDITIONAL_PKGS", BtPkgPath) pkg("USE_BSD_MAKEFILE", BtYes) + // USE_BUILTIN.* is usually set by the builtin.mk file, after checking // whether the package is available in the base system. To override // this check, a package may set this variable before including the // corresponding buildlink3.mk file. acl("USE_BUILTIN.*", BtYesNoIndirectly, + PackageSettable, "Makefile, Makefile.*, *.mk: set, use, use-loadtime") + pkg("USE_CMAKE", BtYes) usr("USE_DESTDIR", BtYes) pkglist("USE_FEATURES", BtIdentifier) @@ -1561,11 +1650,13 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { // The use-loadtime is only for devel/ncurses/Makefile.common, which // removes tbl from USE_TOOLS. acllist("USE_TOOLS", BtTool, + PackageSettable, "special:Makefile.common: set, append, use, use-loadtime", "buildlink3.mk: append", "builtin.mk: append, use-loadtime", "*: set, append, use") acllist("USE_TOOLS.*", BtTool, // OPSYS-specific + PackageSettable, "buildlink3.mk, builtin.mk: append", "*: set, append, use") diff --git a/pkgtools/pkglint/files/vardefs_test.go b/pkgtools/pkglint/files/vardefs_test.go index 2613d57767a..8a77d0b8fae 100644 --- a/pkgtools/pkglint/files/vardefs_test.go +++ b/pkgtools/pkglint/files/vardefs_test.go @@ -52,11 +52,11 @@ func (s *Suite) Test_VarTypeRegistry_Init__enumFrom(c *check.C) { c.Check(vartype, equals, values) } - test("EMACS_VERSIONS_ACCEPTED", "List of enum: emacs29 emacs31 ") - test("PKG_JVM", "enum: jdk16 openjdk7 openjdk8 oracle-jdk8 sun-jdk6 sun-jdk7 ") - test("USE_LANGUAGES", "List of enum: ada c c++ c++03 c++0x c++11 c++14 c99 "+ - "fortran fortran77 gnu++03 gnu++0x gnu++11 gnu++14 java obj-c++ objc ") - test("PKGSRC_COMPILER", "List of enum: ccache distcc f2c g95 gcc ido mipspro-ucode sunpro ") + test("EMACS_VERSIONS_ACCEPTED", "enum: emacs29 emacs31 (list, package-settable)") + test("PKG_JVM", "enum: jdk16 openjdk7 openjdk8 oracle-jdk8 sun-jdk6 sun-jdk7 (system-provided)") + test("USE_LANGUAGES", "enum: ada c c++ c++03 c++0x c++11 c++14 c99 "+ + "fortran fortran77 gnu++03 gnu++0x gnu++11 gnu++14 java obj-c++ objc (list, package-settable)") + test("PKGSRC_COMPILER", "enum: ccache distcc f2c g95 gcc ido mipspro-ucode sunpro (list, user-settable)") } func (s *Suite) Test_VarTypeRegistry_Init__enumFromDirs(c *check.C) { @@ -74,7 +74,25 @@ func (s *Suite) Test_VarTypeRegistry_Init__enumFromDirs(c *check.C) { c.Check(vartype, equals, values) } - test("PYPKGPREFIX", "enum: py28 py33 ") + test("PYPKGPREFIX", "enum: py28 py33 (system-provided)") +} + +func (s *Suite) Test_VarTypeRegistry_Init__enumFromFiles(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("mk/platform/NetBSD.mk") + t.CreateFileLines("mk/platform/README") + t.CreateFileLines("mk/platform/SunOS.mk") + t.CreateFileLines("mk/platform/SunOS.mk~") + + t.SetUpVartypes() + + test := func(varname, values string) { + vartype := G.Pkgsrc.VariableType(nil, varname).String() + c.Check(vartype, equals, values) + } + + test("OPSYS", "enum: NetBSD SunOS (system-provided)") } func (s *Suite) Test_VarTypeRegistry_parseACLEntries__invalid_arguments(c *check.C) { @@ -120,6 +138,7 @@ func (s *Suite) Test_VarTypeRegistry_Init__LP64PLATFORMS(c *check.C) { pkg := t.SetUpPackage("category/package", "BROKEN_ON_PLATFORM=\t${LP64PLATFORMS}") + t.FinishSetUp() G.Check(pkg) diff --git a/pkgtools/pkglint/files/vartype.go b/pkgtools/pkglint/files/vartype.go index a6b2c923c2c..19f304ffa23 100644 --- a/pkgtools/pkglint/files/vartype.go +++ b/pkgtools/pkglint/files/vartype.go @@ -1,28 +1,35 @@ package pkglint -import "path" +import ( + "path" + "strings" +) // Vartype is a combination of a data type and a permission specification. // See vardefs.go for examples, and vartypecheck.go for the implementation. type Vartype struct { - kindOfList KindOfList basicType *BasicType + options vartypeOptions aclEntries []ACLEntry - guessed bool } -type KindOfList uint8 +type vartypeOptions uint8 const ( - // lkNone is a plain data type, no list at all. - lkNone KindOfList = iota - - // lkShell is a compound type, consisting of several space-separated elements. - // Elements can have embedded spaces by enclosing them in quotes, like in the shell. + // List is a compound type, consisting of several space-separated elements. + // Elements can have embedded spaces by enclosing them in double or single + // quotes, like in the shell. // // These lists are used in the :M, :S modifiers, in .for loops, // and as lists of arbitrary things. - lkShell + List vartypeOptions = 1 << iota + + Guessed + PackageSettable + UserSettable + SystemProvided + CommandLineProvided + NoVartypeOptions = 0 ) type ACLEntry struct { @@ -74,6 +81,13 @@ func (perms ACLPermissions) HumanString() string { ifelseStr(perms.Contains(aclpUse), "used", "")) } +func (vt *Vartype) List() bool { return vt.options&List != 0 } +func (vt *Vartype) Guessed() bool { return vt.options&Guessed != 0 } +func (vt *Vartype) PackageSettable() bool { return vt.options&PackageSettable != 0 } +func (vt *Vartype) UserSettable() bool { return vt.options&UserSettable != 0 } +func (vt *Vartype) SystemProvided() bool { return vt.options&SystemProvided != 0 } +func (vt *Vartype) CommandLineProvided() bool { return vt.options&CommandLineProvided != 0 } + func (vt *Vartype) EffectivePermissions(basename string) ACLPermissions { for _, aclEntry := range vt.aclEntries { if m, _ := path.Match(aclEntry.glob, basename); m { @@ -153,26 +167,16 @@ func (vt *Vartype) AlternativeFiles(perms ACLPermissions) string { return positive + ", but not " + negative } -// IsConsideredList returns whether the type is considered a list. -// -// FIXME: Explain why this method is necessary. IsList is clear, and MayBeAppendedTo also, -// but this in-between state needs a decent explanation. -// Probably MkLineChecker.checkVartype needs to be revisited completely. -func (vt *Vartype) IsConsideredList() bool { - if vt.kindOfList == lkShell { +func (vt *Vartype) MayBeAppendedTo() bool { + if vt.List() { return true } + switch vt.basicType { case BtAwkCommand, BtSedCommands, BtShellCommand, BtShellCommands, BtConfFiles: return true } - return false -} -func (vt *Vartype) MayBeAppendedTo() bool { - if vt.kindOfList != lkNone || vt.IsConsideredList() { - return true - } switch vt.basicType { case BtComment, BtLicense: return true @@ -181,9 +185,32 @@ func (vt *Vartype) MayBeAppendedTo() bool { } func (vt *Vartype) String() string { - listPrefix := [...]string{"", "List of "}[vt.kindOfList] - guessedSuffix := ifelseStr(vt.guessed, " (guessed)", "") - return listPrefix + vt.basicType.name + guessedSuffix + var opts []string + if vt.List() { + opts = append(opts, "list") + } + if vt.Guessed() { + opts = append(opts, "guessed") + } + if vt.PackageSettable() { + opts = append(opts, "package-settable") + } + if vt.UserSettable() { + opts = append(opts, "user-settable") + } + if vt.SystemProvided() { + opts = append(opts, "system-provided") + } + if vt.CommandLineProvided() { + opts = append(opts, "command-line-provided") + } + + optsSuffix := "" + if len(opts) > 0 { + optsSuffix = " (" + strings.Join(opts, ", ") + ")" + } + + return vt.basicType.name + optsSuffix } func (vt *Vartype) IsShell() bool { diff --git a/pkgtools/pkglint/files/vartype_test.go b/pkgtools/pkglint/files/vartype_test.go index 2200f0c8ad5..cf7ff24a00d 100644 --- a/pkgtools/pkglint/files/vartype_test.go +++ b/pkgtools/pkglint/files/vartype_test.go @@ -28,7 +28,7 @@ func (s *Suite) Test_Vartype_AlternativeFiles(c *check.C) { // test generates the files description for the "set" permission. test := func(rules []string, alternatives string) { aclEntries := (*VarTypeRegistry).parseACLEntries(nil, "", rules...) - vartype := Vartype{lkNone, BtYesNo, aclEntries, false} + vartype := Vartype{BtYesNo, NoVartypeOptions, aclEntries} alternativeFiles := vartype.AlternativeFiles(aclpSet) @@ -116,6 +116,15 @@ func (s *Suite) Test_Vartype_AlternativeFiles(c *check.C) { "builtin.mk, but not buildlink3.mk, Makefile or *.mk") } +func (s *Suite) Test_Vartype_String(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + vartype := G.Pkgsrc.VariableType(nil, "PKG_DEBUG_LEVEL") + t.Check(vartype.String(), equals, "Integer (command-line-provided)") +} + func (s *Suite) Test_BasicType_HasEnum(c *check.C) { vc := enum("start middle end") @@ -151,13 +160,13 @@ func (s *Suite) Test_ACLPermissions_HumanString(c *check.C) { equals, "set, given a default value, appended to, used at load time, or used") } -func (s *Suite) Test_Vartype_IsConsideredList(c *check.C) { +func (s *Suite) Test_Vartype_MayBeAppendedTo(c *check.C) { t := s.Init(c) t.SetUpVartypes() - c.Check(G.Pkgsrc.VariableType(nil, "COMMENT").IsConsideredList(), equals, false) - c.Check(G.Pkgsrc.VariableType(nil, "DEPENDS").IsConsideredList(), equals, true) - c.Check(G.Pkgsrc.VariableType(nil, "PKG_FAIL_REASON").IsConsideredList(), equals, true) - c.Check(G.Pkgsrc.VariableType(nil, "CONF_FILES").IsConsideredList(), equals, true) + c.Check(G.Pkgsrc.VariableType(nil, "COMMENT").MayBeAppendedTo(), equals, true) + c.Check(G.Pkgsrc.VariableType(nil, "DEPENDS").MayBeAppendedTo(), equals, true) + c.Check(G.Pkgsrc.VariableType(nil, "PKG_FAIL_REASON").MayBeAppendedTo(), equals, true) + c.Check(G.Pkgsrc.VariableType(nil, "CONF_FILES").MayBeAppendedTo(), equals, true) } diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go index 9322056eec5..92f471983b2 100644 --- a/pkgtools/pkglint/files/vartypecheck.go +++ b/pkgtools/pkglint/files/vartypecheck.go @@ -238,7 +238,7 @@ func (cv *VartypeCheck) Comment() { pkgbase := G.Pkg.EffectivePkgbase if hasPrefix(strings.ToLower(value), strings.ToLower(pkgbase+" ")) { cv.Warnf("COMMENT should not start with the package name.") - G.Explain( + cv.Explain( "The COMMENT is usually displayed together with the package name.", "Therefore it does not need to repeat the package name but should", "provide additional information instead.") @@ -251,7 +251,7 @@ func (cv *VartypeCheck) Comment() { if m, isA := match1(value, `\b(is an?)\b`); m { cv.Warnf("COMMENT should not contain %q.", isA) - G.Explain( + cv.Explain( "The words \"package is a\" are redundant.", "Since every package comment could start with them,", "it is better to remove this redundancy in all cases.") @@ -286,7 +286,7 @@ func (cv *VartypeCheck) ConfFiles() { if i%2 == 1 && !hasPrefix(word, "${") { cv.Warnf("The destination file %q should start with a variable reference.", word) - G.Explain( + cv.Explain( "Since pkgsrc can be installed in different locations, the", "configuration files will also end up in different locations.", "Typical variables that are used for configuration files are", @@ -304,7 +304,7 @@ func (cv *VartypeCheck) Dependency() { if deppat != nil && deppat.Wildcard == "" && (rest == "{,nb*}" || rest == "{,nb[0-9]*}") { cv.Warnf("Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.") - G.Explain( + cv.Explain( "The \"{,nb*}\" extension is only necessary for dependencies of the", "form \"pkgbase-1.2\", since the pattern \"pkgbase-1.2\" doesn't match", "the version \"pkgbase-1.2nb5\".", @@ -318,7 +318,7 @@ func (cv *VartypeCheck) Dependency() { } else if deppat == nil || rest != "" { cv.Warnf("Invalid dependency pattern %q.", value) - G.Explain( + cv.Explain( "Typical dependencies have the following forms:", "", "\tpackage>=2.5", @@ -332,7 +332,7 @@ func (cv *VartypeCheck) Dependency() { if m, inside := match1(wildcard, `^\[(.*)\]\*$`); m { if inside != "0-9" { cv.Warnf("Only [0-9]* is allowed in the numeric part of a dependency.") - G.Explain( + cv.Explain( "The pattern -[0-9] means any version.", "All other version patterns should be expressed using", "the comparison operators like < or >= or even >=2<3.", @@ -345,21 +345,21 @@ func (cv *VartypeCheck) Dependency() { } else if m, ver, suffix := match2(wildcard, `^(\d\w*(?:\.\w+)*)(\.\*|\{,nb\*\}|\{,nb\[0-9\]\*\}|\*|)$`); m { if suffix == "" { cv.Warnf("Please use %q instead of %q as the version pattern.", ver+"{,nb*}", ver) - G.Explain( + cv.Explain( "Without the \"{,nb*}\" suffix, this version pattern only matches", "package versions that don't have a PKGREVISION (which is the part", "after the \"nb\").") } if suffix == "*" { cv.Warnf("Please use %q instead of %q as the version pattern.", ver+".*", ver+"*") - G.Explain( + cv.Explain( "For example, the version \"1*\" also matches \"10.0.0\", which is", "probably not intended.") } } else if wildcard == "*" { cv.Warnf("Please use \"%[1]s-[0-9]*\" instead of \"%[1]s-*\".", deppat.Pkgbase) - G.Explain( + cv.Explain( "If you use a * alone, the package specification may match other", "packages that have the same prefix but a longer name.", "For example, foo-* matches foo-1.2 but also", @@ -369,7 +369,7 @@ func (cv *VartypeCheck) Dependency() { withoutCharClasses := replaceAll(wildcard, `\[[\d-]+\]`, "") if contains(withoutCharClasses, "-") { cv.Warnf("The version pattern %q should not contain a hyphen.", wildcard) - G.Explain( + cv.Explain( "Pkgsrc interprets package names with version numbers like this:", "", "\t\"foo-2.0-2.1.x\" => pkgbase \"foo\", version \"2.0-2.1.x\"", @@ -418,7 +418,7 @@ func (cv *VartypeCheck) DependencyWithPath() { } cv.Warnf("Invalid dependency pattern with path %q.", value) - G.Explain( + cv.Explain( "Examples for valid dependency patterns with path are:", " package-[0-9]*:../../category/package", " package>=3.41:../../category/package", @@ -451,7 +451,7 @@ func (cv *VartypeCheck) EmulPlatform() { enumEmulArch.checker(archCv) } else { cv.Warnf("%q is not a valid emulation platform.", cv.Value) - G.Explain( + cv.Explain( "An emulation platform has the form <OPSYS>-<MACHINE_ARCH>.", "OPSYS is the lower-case name of the operating system, and", "MACHINE_ARCH is the hardware architecture.", @@ -706,7 +706,7 @@ func (cv *VartypeCheck) MachineGnuPlatform() { } else { cv.Warnf("%q is not a valid platform pattern.", cv.Value) - G.Explain( + cv.Explain( "A platform pattern has the form <OPSYS>-<OS_VERSION>-<MACHINE_ARCH>.", "Each of these components may use wildcards.", "", @@ -745,7 +745,7 @@ func (cv *VartypeCheck) Message() { if matches(value, `^[\"'].*[\"']$`) { cv.Warnf("%s should not be quoted.", varname) - G.Explain( + cv.Explain( "The quoting is only needed for variables which are interpreted as", "multiple words (or, generally speaking, a list of something).", "A single text message does not belong to this class,", @@ -769,7 +769,7 @@ func (cv *VartypeCheck) Option() { // There's a difference between empty and absent here. if _, found := G.Pkgsrc.PkgOptions[optname]; !found { cv.Warnf("Unknown option %q.", optname) - G.Explain( + cv.Explain( "This option is not documented in the mk/defaults/options.description file.", "Please think of a brief but precise description and either", "update that file yourself or suggest a description for this option", @@ -806,7 +806,7 @@ func (cv *VartypeCheck) Pathlist() { if !hasPrefix(dir, "/") && !hasPrefix(dir, "${") { cv.Errorf("The component %q of %s must be an absolute path.", dir, cv.Varname) - G.Explain( + cv.Explain( "Relative paths in the PATH variable are a security risk.", "They also make the execution unreliable since they are", "evaluated relative to the current directory of the process,", @@ -864,7 +864,7 @@ func (cv *VartypeCheck) Pkgname() { if cv.Op != opUseMatch && value == cv.ValueNoVar && !matches(value, rePkgname) { cv.Warnf("%q is not a valid package name.", value) - G.Explain( + cv.Explain( "A valid package name has the form packagename-version, where version", "consists only of digits, letters and dots.") } @@ -876,7 +876,7 @@ func (cv *VartypeCheck) PkgOptionsVar() { // TODO: Replace regex with proper VarUse. if matches(cv.Value, `\$\{PKGBASE[:\}]`) { cv.Errorf("PKGBASE must not be used in PKG_OPTIONS_VAR.") - G.Explain( + cv.Explain( "PKGBASE is defined in bsd.pkg.mk, which is included as the", "very last file, but PKG_OPTIONS_VAR is evaluated earlier.", "Use ${PKGNAME:C/-[0-9].*//} instead.") @@ -902,7 +902,7 @@ func (cv *VartypeCheck) PkgRevision() { } if cv.MkLine.Basename != "Makefile" { cv.Errorf("%s only makes sense directly in the package Makefile.", cv.Varname) - G.Explain( + cv.Explain( "Usually, different packages using the same Makefile.common have", "different dependencies and will be bumped at different times (e.g.", "for shlib major bumps) and thus the PKGREVISIONs must be in the", @@ -941,7 +941,7 @@ func (cv *VartypeCheck) MachinePlatformPattern() { } else { cv.Warnf("%q is not a valid platform pattern.", cv.Value) - G.Explain( + cv.Explain( "A platform pattern has the form <OPSYS>-<OS_VERSION>-<MACHINE_ARCH>.", "Each of these components may be a shell globbing expression.", "", @@ -968,7 +968,7 @@ func (cv *VartypeCheck) PythonDependency() { cv.Warnf("Python dependencies should not contain variables.") } else if !matches(cv.ValueNoVar, `^[+\-.0-9A-Z_a-z]+(?:|:link|:build)$`) { cv.Warnf("Invalid Python dependency %q.", cv.Value) - G.Explain( + cv.Explain( "Python dependencies must be an identifier for a package, as", "specified in lang/python/versioned_dependencies.mk.", "This identifier may be followed by :build for a build-time only", @@ -992,7 +992,7 @@ func (cv *VartypeCheck) RelativePkgPath() { func (cv *VartypeCheck) Restricted() { if cv.Value != "${RESTRICTED}" { cv.Warnf("The only valid value for %s is ${RESTRICTED}.", cv.Varname) - G.Explain( + cv.Explain( "These variables are used to control which files may be mirrored on", "FTP servers or CD-ROM collections.", "They are not intended to mark", @@ -1005,7 +1005,7 @@ func (cv *VartypeCheck) SedCommands() { if rest != "" { if contains(cv.MkLine.Text, "#") { cv.Errorf("Invalid shell words %q in sed commands.", rest) - G.Explain( + cv.Explain( "When sed commands have embedded \"#\" characters, they need to be", "escaped with a backslash, otherwise make(1) will interpret them as a", "comment, even if they occur in single or double quotes or whatever.") @@ -1027,7 +1027,7 @@ func (cv *VartypeCheck) SedCommands() { ncommands++ if ncommands > 1 { cv.Notef("Each sed command should appear in an assignment of its own.") - G.Explain( + cv.Explain( "For example, instead of", " SUBST_SED.foo+= -e s,command1,, -e s,command2,,", "use", @@ -1059,16 +1059,22 @@ func (cv *VartypeCheck) ShellCommand() { return } setE := true - NewShellLineChecker(cv.MkLines, cv.MkLine).CheckShellCommand(cv.Value, &setE, RunTime) + ck := NewShellLineChecker(cv.MkLines, cv.MkLine) + ck.checkVarUse = false + ck.CheckShellCommand(cv.Value, &setE, RunTime) } // ShellCommands checks for zero or more shell commands, each terminated with a semicolon. func (cv *VartypeCheck) ShellCommands() { - NewShellLineChecker(cv.MkLines, cv.MkLine).CheckShellCommands(cv.Value, RunTime) + ck := NewShellLineChecker(cv.MkLines, cv.MkLine) + ck.checkVarUse = false + ck.CheckShellCommands(cv.Value, RunTime) } func (cv *VartypeCheck) ShellWord() { - NewShellLineChecker(cv.MkLines, cv.MkLine).CheckWord(cv.Value, true, RunTime) + ck := NewShellLineChecker(cv.MkLines, cv.MkLine) + ck.checkVarUse = false + ck.CheckWord(cv.Value, true, RunTime) } func (cv *VartypeCheck) Stage() { @@ -1097,7 +1103,7 @@ func (cv *VartypeCheck) Tool() { } else if cv.Op != opUseMatch && cv.Value == cv.ValueNoVar { cv.Errorf("Invalid tool dependency %q.", cv.Value) - G.Explain( + cv.Explain( "A tool dependency typically looks like \"sed\" or \"sed:run\".") } } @@ -1156,7 +1162,7 @@ func (cv *VartypeCheck) UserGroupName() { func (cv *VartypeCheck) VariableName() { if cv.Value == cv.ValueNoVar && !matches(cv.Value, `^[A-Z_][0-9A-Z_]*(?:[.].*)?$`) { cv.Warnf("%q is not a valid variable name.", cv.Value) - G.Explain( + cv.Explain( "Variable names are restricted to only uppercase letters and the", "underscore in the basename, and arbitrary characters in the", "parameterized part, following the dot.", @@ -1181,7 +1187,7 @@ func (cv *VartypeCheck) Version() { if m, ver, suffix := match2(value, `^(`+digit+alnum+`*(?:\.`+alnum+`+)*)(\.\*|\*|)$`); m { if suffix == "*" && ver != "[0-9]" { cv.Warnf("Please use %q instead of %q as the version pattern.", ver+".*", ver+"*") - G.Explain( + cv.Explain( "For example, the version \"1*\" also matches \"10.0.0\", which is", "probably not intended.") } @@ -1259,7 +1265,7 @@ func (cv *VartypeCheck) Yes() { switch cv.Op { case opUseMatch: cv.Warnf("%s should only be used in a \".if defined(...)\" condition.", cv.Varname) - G.Explain( + cv.Explain( "This variable can have only two values: defined or undefined.", "When it is defined, it means \"yes\", even when its value is", "\"no\" or the empty string.", @@ -1270,7 +1276,7 @@ func (cv *VartypeCheck) Yes() { default: if !matches(cv.Value, `^(?:YES|yes)(?:[\t ]+#.*)?$`) { cv.Warnf("%s should be set to YES or yes.", cv.Varname) - G.Explain( + cv.Explain( "This variable means \"yes\" if it is defined, and \"no\" if it is undefined.", "Even when it has the value \"no\", this means \"yes\".", "Therefore when it is defined, its value should correspond to its meaning.") @@ -1301,7 +1307,7 @@ func (cv *VartypeCheck) YesNo() { } } else if cv.Op == opUseCompare { cv.Warnf("%s should be matched against %q or %q, not compared with %q.", cv.Varname, yes1, no1, cv.Value) - G.Explain( + cv.Explain( "The yes/no value can be written in either upper or lower case, and", "both forms are actually used.", "As long as this is the case, when checking the variable value,", diff --git a/pkgtools/pkglint/files/vartypecheck_test.go b/pkgtools/pkglint/files/vartypecheck_test.go index 4da93c0dd14..25fa0045a0e 100644 --- a/pkgtools/pkglint/files/vartypecheck_test.go +++ b/pkgtools/pkglint/files/vartypecheck_test.go @@ -308,11 +308,10 @@ func (s *Suite) Test_VartypeCheck_DependencyWithPath(c *check.C) { vt.Output( "WARN: ~/category/package/filename.mk:1: Invalid dependency pattern with path \"Perl\".", "WARN: ~/category/package/filename.mk:2: Dependency paths should have the form \"../../category/package\".", - "ERROR: ~/category/package/filename.mk:2: Relative path \"../perl5\" does not exist.", + "ERROR: ~/category/package/filename.mk:2: Relative path \"../perl5/Makefile\" does not exist.", "WARN: ~/category/package/filename.mk:2: \"../perl5\" is not a valid relative package directory.", "WARN: ~/category/package/filename.mk:2: Please use USE_TOOLS+=perl:run instead of this dependency.", - "ERROR: ~/category/package/filename.mk:3: Relative path \"../../lang/perl5\" does not exist.", - "ERROR: ~/category/package/filename.mk:3: There is no package in \"lang/perl5\".", + "ERROR: ~/category/package/filename.mk:3: Relative path \"../../lang/perl5/Makefile\" does not exist.", "WARN: ~/category/package/filename.mk:3: Please use USE_TOOLS+=perl:run instead of this dependency.", "WARN: ~/category/package/filename.mk:5: Please use USE_TOOLS+=msgfmt instead of this dependency.", "WARN: ~/category/package/filename.mk:6: Please use USE_TOOLS+=gmake instead of this dependency.") @@ -704,9 +703,11 @@ func (s *Suite) Test_VartypeCheck_LdFlag(c *check.C) { func (s *Suite) Test_VartypeCheck_License(c *check.C) { t := s.Init(c) - t.SetUpPkgsrc() // Adds the gnu-gpl-v2 and 2-clause-bsd licenses + t.SetUpPkgsrc() // Adds the gnu-gpl-v2 and 2-clause-bsd licenses t.SetUpPackage("category/package") + t.FinishSetUp() + G.Pkg = NewPackage(t.File("category/package")) mklines := t.NewMkLines("perl5.mk", @@ -914,7 +915,6 @@ func (s *Suite) Test_VartypeCheck_Pathname(c *check.C) { vt.Values( "anything") - // FIXME: Warn about the absolute pathname in line 4. vt.Output( "WARN: filename.mk:1: \"${PREFIX}/*\" is not a valid pathname.") } @@ -1004,9 +1004,9 @@ func (s *Suite) Test_VartypeCheck_PkgPath(c *check.C) { "../../invalid/relative") vt.Output( - "ERROR: filename.mk:3: Relative path \"../../invalid\" does not exist.", + "ERROR: filename.mk:3: Relative path \"../../invalid/Makefile\" does not exist.", "WARN: filename.mk:3: \"../../invalid\" is not a valid relative package directory.", - "ERROR: filename.mk:4: Relative path \"../../../../invalid/relative\" does not exist.", + "ERROR: filename.mk:4: Relative path \"../../../../invalid/relative/Makefile\" does not exist.", "WARN: filename.mk:4: \"../../../../invalid/relative\" is not a valid relative package directory.") } @@ -1112,6 +1112,8 @@ func (s *Suite) Test_VartypeCheck_SedCommands(c *check.C) { "ERROR: filename.mk:9: The -e option to sed requires an argument.", "WARN: filename.mk:10: Unknown sed command \"-i\".", "NOTE: filename.mk:10: Please always use \"-e\" in sed commands, even if there is only one substitution.", + // TODO: duplicate warning + "WARN: filename.mk:11: Unclosed shell variable starting at \"$${unclosedShellVar\".", "WARN: filename.mk:11: Unclosed shell variable starting at \"$${unclosedShellVar\".") } @@ -1209,9 +1211,9 @@ func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { vt.Op(opUseMatch) vt.Values( "tool1", - "tool1:build", - "tool1:*", - "${t}:build") + "tool1\\:build", + "tool1\\:*", + "${t}\\:build") vt.OutputEmpty() } @@ -1485,7 +1487,9 @@ func (vt *VartypeCheckTester) Op(op MkOperator) { // Values feeds each of the values to the actual check. // Each value is interpreted as if it were written verbatim into a Makefile line. -// That is, # starts a comment, and for the opUseMatch operator, all closing braces must be escaped. +// That is, # starts a comment. +// +// For the opUseMatch operator, all colons and closing braces must be escaped. func (vt *VartypeCheckTester) Values(values ...string) { toText := func(value string) string { @@ -1522,7 +1526,7 @@ func (vt *VartypeCheckTester) Values(values ...string) { // See MkLineChecker.checkVartype. var lineValues []string - if vartype == nil || vartype.kindOfList == lkNone { + if vartype == nil || !vartype.List() { lineValues = []string{effectiveValue} } else { lineValues = mkline.ValueFields(effectiveValue) |