diff options
| author | rillig <rillig@pkgsrc.org> | 2019-03-10 19:01:50 +0000 |
|---|---|---|
| committer | rillig <rillig@pkgsrc.org> | 2019-03-10 19:01:50 +0000 |
| commit | 1a353ea4eceb65b1a01e10e3d93ceec432295415 (patch) | |
| tree | 36ba46b3523986b5c416549295ef05841d7a470b | |
| parent | 0e81c7a00d92f6a32b2d176f0cbd0bcc295234f8 (diff) | |
| download | pkgsrc-1a353ea4eceb65b1a01e10e3d93ceec432295415.tar.gz | |
pkgtools/pkglint: update to 5.7.2
Changes since 5.7.1:
* Fixed detection of GNU_CONFIGURE=yes combined with USE_LANGUAGES
missing c. This combination tends to fail in the configure phase.
* When the distinfo doesn't contain all hashes for the downloaded
distfiles (typically SHA512 is missing) and the distfiles are actually
downloaded to ${PKGSRCDIR}/distfiles, pkglint can now add the missing
hashes. It only does this if there is at least one existing hash
and if all existing hashes are correct.
* The check for redundant variables has been improved considerably.
Before there were several situations in which pkglint didn't get the
redundant variable definitions right because its internal model only
mimicked reality. The model has been improved and so have the
diagnostics.
* Pkglint only warns about wrong permissions (for defining or using
a variable) when it knows the type of the variable and the permissions
for the current file. Before, it had also warned if the permissions
for the current file were not explicitly defined.
* CFLAGS and LDFLAGS may be appended in buildlink3.mk files. This
had been disallowed before, for no apparent reason. There are several
places in pkgsrc where especially CFLAGS.${OPSYS} is appended to.
* Cleaned up internal handling of relative paths. Previously pkglint
sometimes resolved relative paths using the wrong base directory,
which led to all kinds of wrong warnings and strange behavior.
* Fixed lots of edge cases when parsing Makefile lines. These cases
don't occur often but experience tells that the most fundamental code
must be as correct as possible (see the handling of relative paths
above).
* Lots of refactoring and housekeeping, as always.
47 files changed, 5707 insertions, 1328 deletions
diff --git a/pkgtools/pkglint/Makefile b/pkgtools/pkglint/Makefile index 9cfabc82615..23ed228fc44 100644 --- a/pkgtools/pkglint/Makefile +++ b/pkgtools/pkglint/Makefile @@ -1,7 +1,6 @@ -# $NetBSD: Makefile,v 1.569 2019/03/09 10:05:10 bsiegert Exp $ +# $NetBSD: Makefile,v 1.570 2019/03/10 19:01:50 rillig Exp $ -PKGNAME= pkglint-5.7.1 -PKGREVISION= 1 +PKGNAME= pkglint-5.7.2 CATEGORIES= pkgtools DISTNAME= tools MASTER_SITES= ${MASTER_SITE_GITHUB:=golang/} @@ -80,4 +79,5 @@ do-install-man: .PHONY BUILDLINK_DEPMETHOD.go-check= full .include "../../devel/go-check/buildlink3.mk" +.include "../../security/go-crypto/buildlink3.mk" .include "../../mk/bsd.pkg.mk" diff --git a/pkgtools/pkglint/PLIST b/pkgtools/pkglint/PLIST index 7b41ad65334..5808399a521 100644 --- a/pkgtools/pkglint/PLIST +++ b/pkgtools/pkglint/PLIST @@ -1,4 +1,4 @@ -@comment $NetBSD: PLIST,v 1.10 2019/01/26 16:31:33 rillig Exp $ +@comment $NetBSD: PLIST,v 1.11 2019/03/10 19:01:50 rillig Exp $ bin/pkglint gopkg/pkg/${GO_PLATFORM}/netbsd.org/pkglint.a gopkg/pkg/${GO_PLATFORM}/netbsd.org/pkglint/getopt.a @@ -63,6 +63,8 @@ gopkg/src/netbsd.org/pkglint/mkshtypes.go gopkg/src/netbsd.org/pkglint/mkshtypes_test.go gopkg/src/netbsd.org/pkglint/mkshwalker.go gopkg/src/netbsd.org/pkglint/mkshwalker_test.go +gopkg/src/netbsd.org/pkglint/mktokenslexer.go +gopkg/src/netbsd.org/pkglint/mktokenslexer_test.go gopkg/src/netbsd.org/pkglint/mktypes.go gopkg/src/netbsd.org/pkglint/mktypes_test.go gopkg/src/netbsd.org/pkglint/options.go @@ -81,6 +83,8 @@ gopkg/src/netbsd.org/pkglint/pkgver/vercmp.go gopkg/src/netbsd.org/pkglint/pkgver/vercmp_test.go gopkg/src/netbsd.org/pkglint/plist.go gopkg/src/netbsd.org/pkglint/plist_test.go +gopkg/src/netbsd.org/pkglint/redundantscope.go +gopkg/src/netbsd.org/pkglint/redundantscope_test.go gopkg/src/netbsd.org/pkglint/regex/regex.go gopkg/src/netbsd.org/pkglint/shell.go gopkg/src/netbsd.org/pkglint/shell.y diff --git a/pkgtools/pkglint/files/autofix.go b/pkgtools/pkglint/files/autofix.go index 8e579a12fb5..8ca3f993521 100644 --- a/pkgtools/pkglint/files/autofix.go +++ b/pkgtools/pkglint/files/autofix.go @@ -328,9 +328,9 @@ func (fix *Autofix) Realign(mkline MkLine, newWidth int) { { // Parsing the continuation marker as variable value is cheating but works well. text := strings.TrimSuffix(mkline.raw[0].orignl, "\n") - _, _, _, _, _, valueAlign, value, _, _ := MatchVarassign(text) - if value != "\\" { - oldWidth = tabWidth(valueAlign) + _, a := MatchVarassign(text) + if a.value != "\\" { + oldWidth = tabWidth(a.valueAlign) } } diff --git a/pkgtools/pkglint/files/autofix_test.go b/pkgtools/pkglint/files/autofix_test.go index 5b7febc46de..5d8bcc7de9f 100644 --- a/pkgtools/pkglint/files/autofix_test.go +++ b/pkgtools/pkglint/files/autofix_test.go @@ -161,7 +161,6 @@ func (s *Suite) Test_Autofix_ReplaceRegex__autofix(c *check.C) { fix.Apply() t.CheckOutputLines( - "", "AUTOFIX: ~/Makefile:2: Replacing \"X\" with \"Y\".", "-\tline2", "+\tYXXe2") diff --git a/pkgtools/pkglint/files/buildlink3_test.go b/pkgtools/pkglint/files/buildlink3_test.go index 8de6c282ce4..8208945b63d 100644 --- a/pkgtools/pkglint/files/buildlink3_test.go +++ b/pkgtools/pkglint/files/buildlink3_test.go @@ -474,22 +474,13 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_unknown_variable(c *ch t.CheckOutputLines( "WARN: buildlink3.mk:3: LICENSE may not be used in any file; it is a write-only variable.", "WARN: buildlink3.mk:3: The variable LICENSE should be quoted as part of a shell word.", - "WARN: buildlink3.mk:8: LICENSE should not be evaluated at load time.", - "WARN: buildlink3.mk:8: LICENSE may not be used in any file; it is a write-only variable.", "WARN: buildlink3.mk:8: LICENSE should not be evaluated indirectly at load time.", - "WARN: buildlink3.mk:8: LICENSE may not be used in any file; it is a write-only variable.", "WARN: buildlink3.mk:8: The variable LICENSE should be quoted as part of a shell word.", - "WARN: buildlink3.mk:9: LICENSE should not be evaluated at load time.", - "WARN: buildlink3.mk:9: LICENSE may not be used in any file; it is a write-only variable.", "WARN: buildlink3.mk:9: LICENSE should not be evaluated indirectly at load time.", - "WARN: buildlink3.mk:9: LICENSE may not be used in any file; it is a write-only variable.", "WARN: buildlink3.mk:9: The variable LICENSE should be quoted as part of a shell word.", - - "WARN: buildlink3.mk:13: LICENSE may not be used in any file; it is a write-only variable.", "WARN: buildlink3.mk:13: The variable LICENSE should be quoted as part of a shell word.", - "WARN: buildlink3.mk:3: Please replace \"${LICENSE}\" with a simple string (also in other variables in this file).") } @@ -640,13 +631,9 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__other_variables(c *check. G.Check(t.File("category/package")) - // FIXME: Why is appending to LDFLAGS forbidden? It sounds useful. t.CheckOutputLines( - "WARN: ~/category/package/buildlink3.mk:14: "+ - "The variable LDFLAGS.NetBSD may not be appended to in this file; "+ - "it would be ok in Makefile, Makefile.common, options.mk or *.mk.", - "WARN: ~/category/package/buildlink3.mk:16: "+ - "Only buildlink variables for \"package\", "+ + "WARN: ~/category/package/buildlink3.mk:16: " + + "Only buildlink variables for \"package\", " + "not \"other\" may be set in this file.") } diff --git a/pkgtools/pkglint/files/category_test.go b/pkgtools/pkglint/files/category_test.go index d8f22a218a3..e56b7c2af4f 100644 --- a/pkgtools/pkglint/files/category_test.go +++ b/pkgtools/pkglint/files/category_test.go @@ -290,8 +290,11 @@ func (s *Suite) Test_CheckdirCategory__comment_at_the_top(c *check.C) { CheckdirCategory(t.File("category")) - // FIXME: Wow. These are quite a few warnings and errors, just because there is - // an additional comment above the COMMENT definition. + // These are quite a few warnings and errors, just because there is + // an additional comment above the COMMENT definition. + // On the other hand, the category Makefiles are so simple and their + // structure has been fixed for at least 20 years, therefore this case + // is rather exotic anyway. t.CheckOutputLines( "ERROR: ~/category/Makefile:3: COMMENT= line expected.", "NOTE: ~/category/Makefile:2: Empty line expected after this line.", diff --git a/pkgtools/pkglint/files/check_test.go b/pkgtools/pkglint/files/check_test.go index 154ad8e2f92..362c778d9c8 100644 --- a/pkgtools/pkglint/files/check_test.go +++ b/pkgtools/pkglint/files/check_test.go @@ -92,7 +92,6 @@ func (s *Suite) TearDownTest(c *check.C) { _, _ = fmt.Fprintf(os.Stderr, "Cannot chdir back to previous dir: %s", err) } - G = Pkglint{} // unusable because of missing Logger.out and Logger.err if out := t.Output(); out != "" { var msg strings.Builder msg.WriteString("\n") @@ -106,8 +105,11 @@ func (s *Suite) TearDownTest(c *check.C) { _, _ = fmt.Fprintf(&msg, "\n") _, _ = os.Stderr.WriteString(msg.String()) } + t.tmpdir = "" t.DisableTracing() + + G = Pkglint{} // unusable because of missing Logger.out and Logger.err } var _ = check.Suite(new(Suite)) @@ -196,6 +198,36 @@ func (t *Tester) SetUpFileMkLines(relativeFileName string, lines ...string) MkLi return LoadMk(filename, MustSucceed) } +// LoadMkInclude loads the given Makefile fragment and all the files it includes, +// merging all the lines into a single MkLines object. +// +// This is useful for testing code related to Package.readMakefile. +func (t *Tester) LoadMkInclude(relativeFileName string) MkLines { + var lines []Line + + // TODO: Include files with multiple-inclusion guard only once. + // TODO: Include files without multiple-inclusion guard as often as needed. + // TODO: Set an upper limit, to prevent denial of service. + + var load func(filename string) + load = func(filename string) { + for _, mkline := range NewMkLines(Load(filename, MustSucceed)).mklines { + lines = append(lines, mkline.Line) + + if mkline.IsInclude() { + included := cleanpath(path.Dir(filename) + "/" + mkline.IncludedFile()) + load(included) + } + } + } + + load(t.File(relativeFileName)) + + // This assumes that the test files do not contain parse errors. + // Otherwise the diagnostics would appear twice. + return NewMkLines(NewLines(t.File(relativeFileName), lines)) +} + // SetUpPkgsrc sets up a minimal but complete pkgsrc installation in the // temporary folder, so that pkglint runs without any errors. // Individual files may be overwritten by calling other SetUp* methods. @@ -257,6 +289,8 @@ func (t *Tester) SetUpPkgsrc() { // used at load time by packages. t.CreateFileLines("mk/bsd.prefs.mk", MkRcsID) + t.CreateFileLines("mk/bsd.fast.prefs.mk", + MkRcsID) // Category Makefiles require this file for the common definitions. t.CreateFileLines("mk/misc/category.mk") @@ -486,6 +520,69 @@ func (t *Tester) Remove(relativeFileName string) { G.fileCache.Evict(filename) } +// SetUpHierarchy provides a function for creating hierarchies of MkLines +// that include each other. +// The hierarchy is created only in memory, nothing is written to disk. +// +// include, get := t.SetUpHierarchy() +// +// include("including.mk", +// include("other.mk", +// "VAR= other"), +// include("module.mk", +// "VAR= module", +// include("version.mk", +// "VAR= version"), +// include("env.mk", +// "VAR= env"))) +// +// mklines := get("including.mk") +// module := get("module.mk") +func (t *Tester) SetUpHierarchy() ( + include func(filename string, args ...interface{}) MkLines, + get func(string) MkLines) { + + files := map[string]MkLines{} + + // FIXME: Define where the filename is relative to: to the file, or to the current directory. + include = func(filename string, args ...interface{}) MkLines { + var lines []Line + lineno := 1 + + addLine := func(text string) { + lines = append(lines, t.NewLine(filename, lineno, text)) + lineno++ + } + + for _, arg := range args { + switch arg := arg.(type) { + case string: + addLine(arg) + case MkLines: + text := sprintf(".include %q", arg.lines.FileName) + addLine(text) + lines = append(lines, arg.lines.Lines...) + default: + panic("invalid type") + } + } + + mklines := NewMkLines(NewLines(filename, lines)) + // FIXME: This filename must be relative to the including file. + G.Assertf(files[filename] == nil, "MkLines with name %q already exist.", filename) + // FIXME: This filename must be relative to the base directory. + files[filename] = mklines + return mklines + } + + get = func(filename string) MkLines { + G.Assertf(files[filename] != nil, "MkLines with name %q doesn't exist.", filename) + return files[filename] + } + + return +} + // Check delegates a check to the check.Check function. // Thereby, there is no need to distinguish between c.Check and t.Check // in the test code. @@ -584,6 +681,11 @@ func (t *Tester) NewLine(filename string, lineno int, text string) Line { // NewMkLine creates an in-memory line in the Makefile format with the given text. func (t *Tester) NewMkLine(filename string, lineno int, text string) MkLine { + basename := path.Base(filename) + G.Assertf( + hasSuffix(basename, ".mk") || basename == "Makefile" || hasPrefix(basename, "Makefile."), + "filename %q must be realistic, otherwise the variable permissions are wrong", filename) + return NewMkLine(t.NewLine(filename, lineno, text)) } @@ -616,6 +718,11 @@ func (t *Tester) NewLinesAt(filename string, firstLine int, texts ...string) Lin // No actual file is created for the lines; // see SetUpFileMkLines for loading Makefile fragments with line continuations. func (t *Tester) NewMkLines(filename string, lines ...string) MkLines { + basename := path.Base(filename) + G.Assertf( + hasSuffix(basename, ".mk") || basename == "Makefile" || hasPrefix(basename, "Makefile."), + "filename %q must be realistic, otherwise the variable permissions are wrong", filename) + var rawText strings.Builder for _, line := range lines { rawText.WriteString(line) @@ -633,13 +740,18 @@ func (t *Tester) Output() string { t.stdout.Reset() t.stderr.Reset() G.Logger.logged = Once{} + if G.Logger.out != nil { // Necessary because Main resets the G variable. + G.Logger.out.state = 0 // Prevent an empty line at the beginning of the next output. + G.Logger.err.state = 0 + } + G.Assertf(t.tmpdir != "", "Tester must be initialized before checking the output.") output := stdout + stderr - if t.tmpdir != "" { - output = strings.Replace(output, t.tmpdir, "~", -1) - } else { - panic("asdfgsfas") - } + // TODO: The explanations are wrapped. Because of this it can happen + // that t.tmpdir is spread among multiple lines if that directory + // name contains spaces, which is common on Windows. A temporary + // workaround is to set TMP=/path/without/spaces. + output = strings.Replace(output, t.tmpdir, "~", -1) return output } diff --git a/pkgtools/pkglint/files/distinfo.go b/pkgtools/pkglint/files/distinfo.go index 034a849dc9b..9bef89ff565 100644 --- a/pkgtools/pkglint/files/distinfo.go +++ b/pkgtools/pkglint/files/distinfo.go @@ -3,9 +3,13 @@ package pkglint import ( "bytes" "crypto/sha1" + "crypto/sha512" "encoding/hex" + "golang.org/x/crypto/ripemd160" + "hash" + "io" "io/ioutil" - "path" + "os" "strings" ) @@ -26,106 +30,273 @@ func CheckLinesDistinfo(lines Lines) { distinfoIsCommitted := isCommitted(filename) ck := distinfoLinesChecker{ lines, patchdir, distinfoIsCommitted, - make(map[string]bool), "", nil, unknown, nil} - ck.checkLines(lines) + nil, make(map[string]distinfoFileInfo)} + ck.parse() + ck.check() CheckLinesTrailingEmptyLines(lines) ck.checkUnrecordedPatches() + SaveAutofixChanges(lines) } -// XXX: Maybe an approach that first groups the lines by filename -// is easier to understand. - type distinfoLinesChecker struct { - distinfoLines Lines + lines Lines patchdir string // Relative to G.Pkg distinfoIsCommitted bool - // All patches that are mentioned in the distinfo file. - patches map[string]bool // "patch-aa" => true - - currentFileName string - currentFirstLine Line // The first line of the currentFileName group - isPatch YesNoUnknown // Whether currentFileName is a patch, as opposed to a distfile - algorithms []string // The algorithms seen for currentFileName + filenames []string // For keeping the order from top to bottom + infos map[string]distinfoFileInfo } -func (ck *distinfoLinesChecker) checkLines(lines Lines) { - lines.CheckRcsID(0, ``, "") - if 1 < len(lines.Lines) && lines.Lines[1].Text != "" { - lines.Lines[1].Notef("Empty line expected.") - } +func (ck *distinfoLinesChecker) parse() { + lines := ck.lines - for i, line := range lines.Lines { - if i < 2 { - continue + llex := NewLinesLexer(lines) + if lines.CheckRcsID(0, ``, "") { + llex.Skip() + } + llex.SkipEmptyOrNote() + + prevFilename := "" + var hashes []distinfoHash + + isPatch := func() YesNoUnknown { + switch { + case !hasPrefix(prevFilename, "patch-"): + return no + case G.Pkg == nil: + return unknown + case fileExists(G.Pkg.File(ck.patchdir + "/" + prevFilename)): + return yes + default: + return no } - m, alg, filename, hash := match3(line.Text, `^(\w+) \((\w[^)]*)\) = (.*)(?: bytes)?$`) + } + + finishGroup := func() { + ck.filenames = append(ck.filenames, prevFilename) + ck.infos[prevFilename] = distinfoFileInfo{isPatch(), hashes} + hashes = nil + } + + for !llex.EOF() { + line := llex.CurrentLine() + llex.Skip() + + m, alg, filename, hash := match3(line.Text, `^(\w+) \((\w[^)]*)\) = (\S+(?: bytes)?)$`) if !m { line.Errorf("Invalid line: %s", line.Text) continue } - if filename != ck.currentFileName { - ck.onFilenameChange(line, filename) + if prevFilename != "" && filename != prevFilename { + finishGroup() } - ck.algorithms = append(ck.algorithms, alg) + prevFilename = filename - ck.checkGlobalDistfileMismatch(line, filename, alg, hash) - ck.checkUncommittedPatch(line, filename, alg, hash) + hashes = append(hashes, distinfoHash{line, filename, alg, hash}) } - ck.onFilenameChange(ck.distinfoLines.EOFLine(), "") -} -func (ck *distinfoLinesChecker) onFilenameChange(line Line, nextFname string) { - if ck.currentFileName != "" { - ck.checkAlgorithms(line) + if prevFilename != "" { + finishGroup() } +} - if !hasPrefix(nextFname, "patch-") { - ck.isPatch = no - } else if G.Pkg == nil { - ck.isPatch = unknown - } else if fileExists(G.Pkg.File(ck.patchdir + "/" + nextFname)) { - ck.isPatch = yes - } else { - ck.isPatch = no - } +func (ck *distinfoLinesChecker) check() { + for _, filename := range ck.filenames { + info := ck.infos[filename] - ck.currentFileName = nextFname - ck.currentFirstLine = line - ck.algorithms = nil + ck.checkAlgorithms(info) + for _, hash := range info.hashes { + ck.checkGlobalDistfileMismatch(hash) + if info.isPatch == yes { + ck.checkUncommittedPatch(hash) + } + } + } } -func (ck *distinfoLinesChecker) checkAlgorithms(line Line) { - filename := ck.currentFileName - algorithms := strings.Join(ck.algorithms, ", ") +func (ck *distinfoLinesChecker) checkAlgorithms(info distinfoFileInfo) { + filename := info.filename() + algorithms := info.algorithms() + line := info.line() + + isPatch := info.isPatch switch { + case algorithms == "SHA1" && isPatch != no: + return - case ck.isPatch == yes: - if algorithms != "SHA1" { - line.Errorf("Expected SHA1 hash for %s, got %s.", filename, algorithms) - } + case algorithms == "SHA1, RMD160, SHA512, Size" && isPatch != yes: + return + } - case ck.isPatch == unknown: - break + switch { + case isPatch == yes: + line.Errorf("Expected SHA1 hash for %s, got %s.", filename, algorithms) + + case isPatch == unknown: + line.Errorf("Wrong checksum algorithms %s for %s.", algorithms, filename) + line.Explain( + "Distfiles that are downloaded from external sources must have the", + "checksum algorithms SHA1, RMD160, SHA512, Size.", + "", + "Patch files from pkgsrc must have only the SHA1 hash.") - case G.Pkg != nil && G.Pkg.IgnoreMissingPatches: - break + // At this point, the file is either a missing patch file or a distfile. case hasPrefix(filename, "patch-") && algorithms == "SHA1": - pathToPatchdir := relpath(path.Dir(ck.currentFirstLine.Filename), G.Pkg.File(ck.patchdir)) - ck.currentFirstLine.Warnf("Patch file %q does not exist in directory %q.", filename, pathToPatchdir) + if G.Pkg.IgnoreMissingPatches { + break + } + + line.Warnf("Patch file %q does not exist in directory %q.", + filename, line.PathToFile(G.Pkg.File(ck.patchdir))) G.Explain( "If the patches directory looks correct, the patch may have been", "removed without updating the distinfo file.", "In such a case please update the distinfo file.", "", - "If the patches directory looks wrong, pkglint needs to be improved.") + "In rare cases, pkglint cannot determine the correct location of the patches directory.", + "In that case, see the pkglint man page for contact information.") + + default: + ck.checkAlgorithmsDistfile(info) + } +} + +// checkAlgorithmsDistfile checks whether some of the standard algorithms are +// missing. If so and the downloaded distfile exists, they are calculated and +// added to the distinfo file via an autofix. +func (ck *distinfoLinesChecker) checkAlgorithmsDistfile(info distinfoFileInfo) { + line := info.line() + line.Errorf("Expected SHA1, RMD160, SHA512, Size checksums for %q, got %s.", info.filename(), info.algorithms()) + + algorithms := [...]string{"SHA1", "RMD160", "SHA512", "Size"} + + missing := map[string]bool{} + for _, alg := range algorithms { + missing[alg] = true + } + seen := map[string]distinfoHash{} + + for _, hash := range info.hashes { + alg := hash.algorithm + if missing[alg] { + seen[alg] = hash + delete(missing, alg) + } + } + + if len(missing) == 0 || len(seen) == 0 { + return + } + + distdir := G.Pkgsrc.File("distfiles") + distSubdir := "" + if G.Pkg != nil { + distSubdir = G.Pkg.vars.LastValue("DIST_SUBDIR") + } + + // It's a rare situation that the explanation is generated + // this far from the corresponding diagnostic. + // This explanation only makes sense when there are some + // hashes missing that can be automatically added by pkglint. + line.Explain( + "To add the missing lines to the distinfo file, run", + sprintf("\t%s", bmake("distinfo")), + "for each variant of the package until all distfiles are downloaded to", + sprintf("%q.", cleanpath("${PKGSRCDIR}/distfiles/"+distSubdir)), + "", + "The variants are typically selected by setting EMUL_PLATFORM", + "or similar variables in the command line.", + "", + "After that, run", + sprintf("%q", "cvs update -C distinfo"), + "to revert the distinfo file to the previous state, since the above", + "commands have removed some of the entries.", + "", + "After downloading all possible distfiles, run", + sprintf("%q,", "pkglint --autofix"), + "which will find the downloaded distfiles and add the missing", + "hashes to the distinfo file.") + + distfile := cleanpath(distdir + "/" + distSubdir + "/" + info.filename()) + if !fileExists(distfile) { + return + } + + computeHash := func(hasher hash.Hash) string { + f, err := os.Open(distfile) + G.AssertNil(err, "Opening distfile") + + // Don't load the distfile into memory since some of them + // are hundreds of MB in size. + _, err = io.Copy(hasher, f) + G.AssertNil(err, "Computing hash of distfile") + + hexHash := hex.EncodeToString(hasher.Sum(nil)) + + err = f.Close() + G.AssertNil(err, "Closing distfile") + + return hexHash + } - case algorithms != "SHA1, RMD160, SHA512, Size": - line.Errorf("Expected SHA1, RMD160, SHA512, Size checksums for %q, got %s.", filename, algorithms) + compute := func(alg string) string { + switch alg { + case "SHA1": + return computeHash(sha1.New()) + case "RMD160": + return computeHash(ripemd160.New()) + case "SHA512": + return computeHash(sha512.New()) + default: + fileInfo, err := os.Lstat(distfile) + G.AssertNil(err, "Inaccessible distfile info") + return sprintf("%d bytes", fileInfo.Size()) + } + } + + for alg, hash := range seen { + computed := compute(alg) + + if computed != hash.hash { + // Do not try to autofix anything in this situation. + // Wrong hashes are a serious issue. + line.Errorf("The %s checksum for %q is %s in distinfo, %s in %s.", + alg, hash.filename, hash.hash, computed, line.PathToFile(distfile)) + return + } + } + + // At this point, all the existing hash algorithms are correct, + // and there is at least one hash algorithm. This is evidence enough + // that the distfile is the expected one. Now generate the missing hashes + // and insert them, in the correct order. + + var insertion Line + var remainingHashes = info.hashes + for _, alg := range algorithms { + if missing[alg] { + computed := compute(alg) + + if insertion == nil { + fix := line.Autofix() + fix.Errorf("Missing %s hash for %s.", alg, info.filename()) + fix.InsertBefore(sprintf("%s (%s) = %s", alg, info.filename(), computed)) + fix.Apply() + } else { + fix := insertion.Autofix() + fix.Errorf("Missing %s hash for %s.", alg, info.filename()) + fix.InsertAfter(sprintf("%s (%s) = %s", alg, info.filename(), computed)) + fix.Apply() + } + + } else if remainingHashes[0].algorithm == alg { + insertion = remainingHashes[0].line + remainingHashes = remainingHashes[1:] + } } } @@ -143,16 +314,26 @@ func (ck *distinfoLinesChecker) checkUnrecordedPatches() { for _, file := range patchFiles { patchName := file.Name() - if file.Mode().IsRegular() && !ck.patches[patchName] && hasPrefix(patchName, "patch-") { - ck.distinfoLines.Errorf("Patch %q is not recorded. Run %q.", - cleanpath(relpath(path.Dir(ck.distinfoLines.FileName), G.Pkg.File(ck.patchdir+"/"+patchName))), + if file.Mode().IsRegular() && ck.infos[patchName].isPatch != yes && hasPrefix(patchName, "patch-") { + line := NewLineWhole(ck.lines.FileName) + line.Errorf("Patch %q is not recorded. Run %q.", + line.PathToFile(G.Pkg.File(ck.patchdir+"/"+patchName)), bmake("makepatchsum")) } } } // Inter-package check for differing distfile checksums. -func (ck *distinfoLinesChecker) checkGlobalDistfileMismatch(line Line, filename, alg, hash string) { +func (ck *distinfoLinesChecker) checkGlobalDistfileMismatch(info distinfoHash) { + hashes := G.Hashes + if hashes == nil { + return + } + + filename := info.filename + alg := info.algorithm + hash := info.hash + line := info.line // Intentionally checking the filename instead of ck.isPatch. // Missing the few distfiles that actually start with patch-* @@ -161,11 +342,6 @@ func (ck *distinfoLinesChecker) checkGlobalDistfileMismatch(line Line, filename, return } - hashes := G.Hashes - if hashes == nil { - return - } - // The Size hash is not encoded in hex, therefore it would trigger wrong error messages below. // Since the Size hash is targeted towards humans and not really useful for detecting duplicates, // omitting the check here is ok. Any mismatches will be reliably detected because the other @@ -194,17 +370,19 @@ func (ck *distinfoLinesChecker) checkGlobalDistfileMismatch(line Line, filename, } } -func (ck *distinfoLinesChecker) checkUncommittedPatch(line Line, patchName, alg, hash string) { - if ck.isPatch == yes { - patchFileName := ck.patchdir + "/" + patchName - resolvedPatchFileName := G.Pkg.File(patchFileName) - if ck.distinfoIsCommitted && !isCommitted(resolvedPatchFileName) { - line.Warnf("%s is registered in distinfo but not added to CVS.", line.PathToFile(resolvedPatchFileName)) - } - if alg == "SHA1" { - ck.checkPatchSha1(line, patchFileName, hash) - } - ck.patches[patchName] = true +func (ck *distinfoLinesChecker) checkUncommittedPatch(info distinfoHash) { + patchName := info.filename + alg := info.algorithm + hash := info.hash + line := info.line + + patchFileName := ck.patchdir + "/" + patchName + resolvedPatchFileName := G.Pkg.File(patchFileName) + if ck.distinfoIsCommitted && !isCommitted(resolvedPatchFileName) { + line.Warnf("%s is registered in distinfo but not added to CVS.", line.PathToFile(resolvedPatchFileName)) + } + if alg == "SHA1" { + ck.checkPatchSha1(line, patchFileName, hash) } } @@ -226,6 +404,32 @@ func (ck *distinfoLinesChecker) checkPatchSha1(line Line, patchFileName, distinf } } +type distinfoFileInfo struct { + // yes = the patch file exists + // unknown = distinfo file is checked without a pkgsrc package + // no = distfile or nonexistent patch file + isPatch YesNoUnknown + hashes []distinfoHash +} + +func (info *distinfoFileInfo) filename() string { return info.hashes[0].filename } +func (info *distinfoFileInfo) line() Line { return info.hashes[0].line } + +func (info *distinfoFileInfo) algorithms() string { + var algs []string + for _, hash := range info.hashes { + algs = append(algs, hash.algorithm) + } + return strings.Join(algs, ", ") +} + +type distinfoHash struct { + line Line + filename string + algorithm string + hash string +} + // Same as in mk/checksum/distinfo.awk:/function patchsum/ func computePatchSha1Hex(patchFilename string) (string, error) { patchBytes, err := ioutil.ReadFile(patchFilename) diff --git a/pkgtools/pkglint/files/distinfo_test.go b/pkgtools/pkglint/files/distinfo_test.go index d3927af5d85..1231b8f3819 100644 --- a/pkgtools/pkglint/files/distinfo_test.go +++ b/pkgtools/pkglint/files/distinfo_test.go @@ -2,7 +2,7 @@ package pkglint import "gopkg.in/check.v1" -func (s *Suite) Test_CheckLinesDistinfo(c *check.C) { +func (s *Suite) Test_CheckLinesDistinfo__parse_errors(c *check.C) { t := s.Init(c) t.Chdir("category/package") @@ -27,14 +27,16 @@ func (s *Suite) Test_CheckLinesDistinfo(c *check.C) { t.CheckOutputLines( "ERROR: distinfo:1: Expected \"$"+"NetBSD$\".", - "NOTE: distinfo:2: Empty line expected.", - "ERROR: distinfo:5: Expected SHA1, RMD160, SHA512, Size checksums for \"distfile.tar.gz\", got MD5, SHA1.", - "ERROR: distinfo:7: Expected SHA1 hash for patch-aa, got SHA1, Size.", + "NOTE: distinfo:1: Empty line expected before this line.", + "ERROR: distinfo:1: Invalid line: should be the RCS ID", + "ERROR: distinfo:2: Invalid line: should be empty", "ERROR: distinfo:8: Invalid line: Another invalid line", + "ERROR: distinfo:3: Expected SHA1, RMD160, SHA512, Size checksums for \"distfile.tar.gz\", got MD5, SHA1.", + "ERROR: distinfo:5: Expected SHA1 hash for patch-aa, got SHA1, Size.", "WARN: distinfo:9: Patch file \"patch-nonexistent\" does not exist in directory \"patches\".") } -func (s *Suite) Test_CheckLinesDistinfo__nonexistent_distfile_called_patch(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__nonexistent_distfile_called_patch(c *check.C) { t := s.Init(c) t.Chdir("category/package") @@ -51,11 +53,11 @@ func (s *Suite) Test_CheckLinesDistinfo__nonexistent_distfile_called_patch(c *ch // a patch, it is a normal distfile because it has other hash algorithms // than exactly SHA1. t.CheckOutputLines( - "ERROR: distinfo:EOF: Expected SHA1, RMD160, SHA512, Size checksums " + + "ERROR: distinfo:3: Expected SHA1, RMD160, SHA512, Size checksums " + "for \"patch-5.3.tar.gz\", got MD5, SHA1.") } -func (s *Suite) Test_CheckLinesDistinfo__wrong_distfile_algorithms(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__wrong_distfile_algorithms(c *check.C) { t := s.Init(c) t.Chdir("category/package") @@ -68,11 +70,39 @@ func (s *Suite) Test_CheckLinesDistinfo__wrong_distfile_algorithms(c *check.C) { CheckLinesDistinfo(lines) t.CheckOutputLines( - "ERROR: distinfo:EOF: Expected SHA1, RMD160, SHA512, Size checksums " + + "ERROR: distinfo:3: Expected SHA1, RMD160, SHA512, Size checksums " + "for \"distfile.tar.gz\", got MD5, SHA1.") } -func (s *Suite) Test_CheckLinesDistinfo__wrong_patch_algorithms(c *check.C) { +// This case only happens when a distinfo file is checked on its own, +// without any reference to a pkgsrc package. Additionally the distfile +// must start with the patch- prefix and the algorithms must be wrong +// for both distfile or patch. +// +// This test only demonstrates the edge case. +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__ambiguous_distfile(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--explain") + t.Chdir("category/package") + lines := t.SetUpFileLines("distinfo", + RcsID, + "", + "MD5 (patch-4.2.tar.gz) = 12345678901234567890123456789012") + + CheckLinesDistinfo(lines) + + t.CheckOutputLines( + "ERROR: distinfo:3: Wrong checksum algorithms MD5 for patch-4.2.tar.gz.", + "", + "\tDistfiles that are downloaded from external sources must have the", + "\tchecksum algorithms SHA1, RMD160, SHA512, Size.", + "", + "\tPatch files from pkgsrc must have only the SHA1 hash.", + "") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__wrong_patch_algorithms(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package") @@ -87,10 +117,23 @@ func (s *Suite) Test_CheckLinesDistinfo__wrong_patch_algorithms(c *check.C) { G.Check(".") t.CheckOutputLines( + "ERROR: distinfo:3: Expected SHA1 hash for patch-aa, got MD5, SHA1.", "ERROR: distinfo:4: SHA1 hash of patches/patch-aa differs "+ "(distinfo has 1234567890123456789012345678901234567890, "+ - "patch file has ebbf34b0641bcb508f17d5a27f2bf2a536d810ac).", - "ERROR: distinfo:EOF: Expected SHA1 hash for patch-aa, got MD5, SHA1.") + "patch file has ebbf34b0641bcb508f17d5a27f2bf2a536d810ac).") +} + +func (s *Suite) Test_distinfoLinesChecker_parse__empty(c *check.C) { + t := s.Init(c) + + lines := t.SetUpFileLines("distinfo", + RcsID, + "") + + CheckLinesDistinfo(lines) + + t.CheckOutputLines( + "NOTE: ~/distinfo:2: Trailing empty lines.") } // When checking the complete pkgsrc tree, pkglint has all information it needs @@ -109,7 +152,8 @@ func (s *Suite) Test_distinfoLinesChecker_checkGlobalDistfileMismatch(c *check.C RcsID, "", "SHA512 (distfile-1.0.tar.gz) = 1234567811111111", - "SHA512 (distfile-1.1.tar.gz) = 1111111111111111") + "SHA512 (distfile-1.1.tar.gz) = 1111111111111111", + "SHA512 (patch-4.2.tar.gz) = 1234567812345678") t.CreateFileLines("category/package2/distinfo", RcsID, "", @@ -135,29 +179,104 @@ func (s *Suite) Test_distinfoLinesChecker_checkGlobalDistfileMismatch(c *check.C G.Main("pkglint", "-r", "-Wall", "-Call", t.File(".")) t.CheckOutputLines( - "ERROR: ~/category/package1/distinfo:4: "+ + "ERROR: ~/category/package1/distinfo:3: "+ "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.0.tar.gz\", got SHA512.", - "ERROR: ~/category/package1/distinfo:EOF: "+ + "ERROR: ~/category/package1/distinfo:4: "+ "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.1.tar.gz\", got SHA512.", - "ERROR: ~/category/package2/distinfo:3: The SHA512 hash for distfile-1.0.tar.gz is 1234567822222222, "+ - "which conflicts with 1234567811111111 in ../package1/distinfo:3.", - "ERROR: ~/category/package2/distinfo:4: "+ + "ERROR: ~/category/package1/distinfo:5: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"patch-4.2.tar.gz\", got SHA512.", + + "ERROR: ~/category/package2/distinfo:3: "+ "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.0.tar.gz\", got SHA512.", - "ERROR: ~/category/package2/distinfo:5: "+ + "ERROR: ~/category/package2/distinfo:3: "+ + "The SHA512 hash for distfile-1.0.tar.gz is 1234567822222222, "+ + "which conflicts with 1234567811111111 in ../../category/package1/distinfo:3.", + "ERROR: ~/category/package2/distinfo:4: "+ "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.1.tar.gz\", got SHA512.", "ERROR: ~/category/package2/distinfo:5: "+ - "The SHA512 hash for encoding-error.tar.gz contains a non-hex character.", - "ERROR: ~/category/package2/distinfo:EOF: "+ "Expected SHA1, RMD160, SHA512, Size checksums for \"encoding-error.tar.gz\", got SHA512.", + "ERROR: ~/category/package2/distinfo:5: "+ + "The SHA512 hash for encoding-error.tar.gz contains a non-hex character.", + "WARN: ~/licenses/gnu-gpl-v2: This license seems to be unused.", - "7 errors and 1 warning found.") + "8 errors and 1 warning found.", + "(Run \"pkglint -e\" to show explanations.)") // Ensure that hex.DecodeString does not waste memory here. t.Check(len(G.Hashes["SHA512:distfile-1.0.tar.gz"].hash), equals, 8) t.Check(cap(G.Hashes["SHA512:distfile-1.0.tar.gz"].hash), equals, 8) } -func (s *Suite) Test_CheckLinesDistinfo__uncommitted_patch(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__missing_patch_with_distfile_checksums(c *check.C) { + t := s.Init(c) + + lines := t.SetUpFileLines("distinfo", + RcsID, + "", + "SHA1 (patch-aa) = ...", + "RMD160 (patch-aa) = ...", + "SHA512 (patch-aa) = ...", + "Size (patch-aa) = ... bytes") + + CheckLinesDistinfo(lines) + + // The file name certainly looks like a pkgsrc patch, but there + // is no corresponding file in the file system, and there is no + // current package to correctly determine the PATCHDIR. Therefore + // pkglint doesn't know whether this is a distfile or a missing + // patch file and doesn't warn at all. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__existing_patch_with_distfile_checksums(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "SHA1 (patch-aa) = ...", + "RMD160 (patch-aa) = ...", + "SHA512 (patch-aa) = ...", + "Size (patch-aa) = ... bytes") + t.CreateFileDummyPatch("category/package/patches/patch-aa") + + G.Check(t.File("category/package")) + + // Even though the checksums in the distinfo file look as if they + // refer to a distfile, there is a patch file in the file system + // that matches the distinfo lines. When checking a pkgsrc package + // (as opposed to checking a distinfo file on its own), this means + // that the distinfo lines clearly refer to that patch file and not + // to a distfile. + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: "+ + "Expected SHA1 hash for patch-aa, got SHA1, RMD160, SHA512, Size.", + "ERROR: ~/category/package/distinfo:3: "+ + "SHA1 hash of patches/patch-aa differs (distinfo has ..., "+ + "patch file has ebbf34b0641bcb508f17d5a27f2bf2a536d810ac).") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__missing_patch_with_wrong_algorithms(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.SetUpFileLines("category/package/distinfo", + RcsID, + "", + "RMD160 (patch-aa) = ...") + + G.Check(t.File("category/package")) + + // Patch files usually have the SHA1 hash or none at all if they are fresh. + // In all other cases pkglint assumes that the file is a distfile, + // therefore it requires the usual distfile checksum algorithms here. + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: " + + "Expected SHA1, RMD160, SHA512, Size checksums for \"patch-aa\", got RMD160.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch__bad(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package") @@ -176,7 +295,27 @@ func (s *Suite) Test_CheckLinesDistinfo__uncommitted_patch(c *check.C) { "WARN: distinfo:3: patches/patch-aa is registered in distinfo but not added to CVS.") } -func (s *Suite) Test_CheckLinesDistinfo__unrecorded_patches(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch__good(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.CreateFileDummyPatch("patches/patch-aa") + t.CreateFileLines("CVS/Entries", + "/distinfo/...") + t.CreateFileLines("patches/CVS/Entries", + "/patch-aa/...") + t.SetUpFileLines("distinfo", + RcsID, + "", + "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") + + G.checkdirPackage(".") + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_distinfoLinesChecker_checkUnrecordedPatches(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package") @@ -202,7 +341,7 @@ func (s *Suite) Test_CheckLinesDistinfo__unrecorded_patches(c *check.C) { // The distinfo file and the patches are usually placed in the package // directory. By defining PATCHDIR or DISTINFO_FILE, a package can define // that they are somewhere else in pkgsrc. -func (s *Suite) Test_CheckLinesDistinfo__relative_path_in_distinfo(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1__relative_path_in_distinfo(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package", @@ -340,35 +479,251 @@ func (s *Suite) Test_CheckLinesDistinfo__missing_php_patches(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1(c *check.C) { t := s.Init(c) + G.Pkg = NewPackage(t.File("category/package")) + distinfoLine := t.NewLine(t.File("category/package/distinfo"), 5, "") + + checker := distinfoLinesChecker{} + checker.checkPatchSha1(distinfoLine, "patch-nonexistent", "distinfo-sha1") + + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:5: Patch patch-nonexistent does not exist.") +} + +// When there is at least one correct hash for a distfile, running +// pkglint --autofix adds the missing hashes, provided the distfile has been +// downloaded to pkgsrc/distfiles, which is the standard distfiles location. +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__add_missing_hashes(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "--explain") t.SetUpPackage("category/package") - t.Chdir("category/package") - t.CreateFileDummyPatch("patches/patch-aa") - t.CreateFileLines("CVS/Entries", - "/distinfo/...") - t.CreateFileLines("patches/CVS/Entries", - "/patch-aa/...") - t.SetUpFileLines("distinfo", + t.CreateFileLines("category/package/distinfo", RcsID, "", - "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") + "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", + "Size (package-1.0.txt) = 13 bytes", + "CRC32 (package-1.0.txt) = asdf") + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() - G.checkdirPackage(".") + // This run is only used to verify that the RMD160 hash is correct, and if + // it should ever differ, the correct hash will appear in an error message. + G.Check(t.File("category/package")) - t.CheckOutputEmpty() + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", "+ + "got RMD160, Size, CRC32.", + "", + "\tTo add the missing lines to the distinfo file, run", + "\t\t"+confMake+" distinfo", + "\tfor each variant of the package until all distfiles are downloaded", + "\tto \"${PKGSRCDIR}/distfiles\".", + "", + "\tThe variants are typically selected by setting EMUL_PLATFORM or", + "\tsimilar variables in the command line.", + "", + "\tAfter that, run \"cvs update -C distinfo\" to revert the distinfo file", + "\tto the previous state, since the above commands have removed some of", + "\tthe entries.", + "", + "\tAfter downloading all possible distfiles, run \"pkglint --autofix\",", + "\twhich will find the downloaded distfiles and add the missing hashes", + "\tto the distinfo file.", + "", + "ERROR: ~/category/package/distinfo:3: Missing SHA1 hash for package-1.0.txt.", + "ERROR: ~/category/package/distinfo:3: Missing SHA512 hash for package-1.0.txt.") + + t.SetUpCommandLine("-Wall", "--autofix", "--show-autofix", "--source") + + G.Check(t.File("category/package")) + + // Since the file exists in the distfiles directory, pkglint checks the + // hash right away. It also adds the missing hashes since this file is + // not a patch file. + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: Missing SHA1 hash for package-1.0.txt.", + "AUTOFIX: ~/category/package/distinfo:3: "+ + "Inserting a line \"SHA1 (package-1.0.txt) "+ + "= cd50d19784897085a8d0e3e413f8612b097c03f1\" "+ + "before this line.", + "+\tSHA1 (package-1.0.txt) = cd50d19784897085a8d0e3e413f8612b097c03f1", + ">\tRMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", + "", + "ERROR: ~/category/package/distinfo:3: Missing SHA512 hash for package-1.0.txt.", + "AUTOFIX: ~/category/package/distinfo:3: "+ + "Inserting a line \"SHA512 (package-1.0.txt) "+ + "= f65f341b35981fda842b09b2c8af9bcdb7602a4c2e6fa1f7d41f0974d3e3122f"+ + "268fc79d5a4af66358f5133885cd1c165c916f80ab25e5d8d95db46f803c782c\" after this line.", + "+\tSHA1 (package-1.0.txt) = cd50d19784897085a8d0e3e413f8612b097c03f1", + ">\tRMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", + "+\tSHA512 (package-1.0.txt) = f65f341b35981fda842b09b2c8af9bcdb7602a4c2e6fa1f7d41f0974d3e3122f"+ + "268fc79d5a4af66358f5133885cd1c165c916f80ab25e5d8d95db46f803c782c") + + t.SetUpCommandLine("-Wall") + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: " + + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", " + + "got SHA1, RMD160, SHA512, Size, CRC32.") } -func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__wrong_distfile_hash(c *check.C) { t := s.Init(c) - G.Pkg = NewPackage(t.File("category/package")) - distinfoLine := t.NewLine(t.File("category/package/distinfo"), 5, "") + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "RMD160 (package-1.0.txt) = 1234wrongHash1234") + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() - checker := distinfoLinesChecker{} - checker.checkPatchSha1(distinfoLine, "patch-nonexistent", "distinfo-sha1") + G.Check(t.File("category/package")) t.CheckOutputLines( - "ERROR: ~/category/package/distinfo:5: Patch patch-nonexistent does not exist.") + "ERROR: ~/category/package/distinfo:3: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", "+ + "got RMD160.", + "ERROR: ~/category/package/distinfo:3: "+ + "The RMD160 checksum for \"package-1.0.txt\" is 1234wrongHash1234 in distinfo, "+ + "1a88147a0344137404c63f3b695366eab869a98a in ../../distfiles/package-1.0.txt.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__no_usual_algorithm(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "MD5 (package-1.0.txt) = 1234wrongHash1234") + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: " + + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", " + + "got MD5.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__top_algorithms_missing(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "SHA512 (package-1.0.txt) = f65f341b35981fda842b09b2c8af9bcdb7602a4c2e6fa1f7"+ + "d41f0974d3e3122f268fc79d5a4af66358f5133885cd1c165c916f80ab25e5d8d95db46f803c782c", + "Size (package-1.0.txt) = 13 bytes") + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", "+ + "got SHA512, Size.", + "ERROR: ~/category/package/distinfo:3: Missing SHA1 hash for package-1.0.txt.", + "ERROR: ~/category/package/distinfo:3: Missing RMD160 hash for package-1.0.txt.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__bottom_algorithms_missing(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "SHA1 (package-1.0.txt) = cd50d19784897085a8d0e3e413f8612b097c03f1", + "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a") + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", "+ + "got SHA1, RMD160.", + "ERROR: ~/category/package/distinfo:4: Missing SHA512 hash for package-1.0.txt.", + "ERROR: ~/category/package/distinfo:4: Missing Size hash for package-1.0.txt.") + + t.SetUpCommandLine("-Wall", "--autofix") + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "AUTOFIX: ~/category/package/distinfo:4: "+ + "Inserting a line \"SHA512 (package-1.0.txt) = f65f341b35981fda842b"+ + "09b2c8af9bcdb7602a4c2e6fa1f7d41f0974d3e3122f268fc79d5a4af66358f513"+ + "3885cd1c165c916f80ab25e5d8d95db46f803c782c\" after this line.", + "AUTOFIX: ~/category/package/distinfo:4: "+ + "Inserting a line \"Size (package-1.0.txt) = 13 bytes\" after this line.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__algorithms_in_wrong_order(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", + "SHA1 (package-1.0.txt) = cd50d19784897085a8d0e3e413f8612b097c03f1", + "Size (package-1.0.txt) = 13 bytes", + "SHA512 (package-1.0.txt) = f65f341b35981fda842b09b2c8af9bcdb7602a4c2e6fa1f7"+ + "d41f0974d3e3122f268fc79d5a4af66358f5133885cd1c165c916f80ab25e5d8d95db46f803c782c") + + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + // This case doesn't happen in practice, therefore there's no autofix for it. + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: " + + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", " + + "got RMD160, SHA1, Size, SHA512.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__some_algorithms_in_wrong_order(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", + "Size (package-1.0.txt) = 13 bytes", + "SHA512 (package-1.0.txt) = f65f341b35981fda842b09b2c8af9bcdb7602a4c2e6fa1f7"+ + "d41f0974d3e3122f268fc79d5a4af66358f5133885cd1c165c916f80ab25e5d8d95db46f803c782c") + + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + // This case doesn't happen in practice, therefore there's no autofix for it. + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", "+ + "got RMD160, Size, SHA512.", + "ERROR: ~/category/package/distinfo:3: Missing SHA1 hash for package-1.0.txt.") } diff --git a/pkgtools/pkglint/files/line.go b/pkgtools/pkglint/files/line.go index 19e6eb19c95..376f44bc715 100644 --- a/pkgtools/pkglint/files/line.go +++ b/pkgtools/pkglint/files/line.go @@ -105,15 +105,12 @@ func NewLineWhole(filename string) Line { // RefTo returns a reference to another line, // which can be in the same file or in a different file. func (line *LineImpl) RefTo(other Line) string { - if line.Filename != other.Filename { - return cleanpath(relpath(path.Dir(line.Filename), other.Filename)) + ":" + other.Linenos() - } - return "line " + other.Linenos() + return line.RefToLocation(other.Location) } func (line *LineImpl) RefToLocation(other Location) string { if line.Filename != other.Filename { - return cleanpath(relpath(path.Dir(line.Filename), other.Filename)) + ":" + other.Linenos() + return line.PathToFile(other.Filename) + ":" + other.Linenos() } return "line " + other.Linenos() } diff --git a/pkgtools/pkglint/files/lines.go b/pkgtools/pkglint/files/lines.go index dbcb70ab32e..0d4b9914497 100644 --- a/pkgtools/pkglint/files/lines.go +++ b/pkgtools/pkglint/files/lines.go @@ -35,6 +35,7 @@ func (ls *LinesImpl) SaveAutofixChanges() bool { return SaveAutofixChanges(ls) } +// CheckRcsID returns true if the expected RCS Id was found. func (ls *LinesImpl) CheckRcsID(index int, prefixRe regex.Pattern, suggestedPrefix string) bool { if trace.Tracing { defer trace.Call(prefixRe, suggestedPrefix)() diff --git a/pkgtools/pkglint/files/logging_test.go b/pkgtools/pkglint/files/logging_test.go index 6f3fe96cb0c..6b1dcaad546 100644 --- a/pkgtools/pkglint/files/logging_test.go +++ b/pkgtools/pkglint/files/logging_test.go @@ -749,7 +749,7 @@ func (s *Suite) Test_Logger_Diag__source_duplicates(c *check.C) { t.CheckOutputLines( "ERROR: ~/category/package1/distinfo: "+ - "Patch \"../dependency/patches/patch-aa\" is not recorded. "+ + "Patch \"../../category/dependency/patches/patch-aa\" is not recorded. "+ "Run \""+confMake+" makepatchsum\".", "", ">\t--- old file", @@ -757,7 +757,7 @@ func (s *Suite) Test_Logger_Diag__source_duplicates(c *check.C) { "Each patch must be documented.", "", "ERROR: ~/category/package2/distinfo: "+ - "Patch \"../dependency/patches/patch-aa\" is not recorded. "+ + "Patch \"../../category/dependency/patches/patch-aa\" is not recorded. "+ "Run \""+confMake+" makepatchsum\".", "", "3 errors and 0 warnings found.", diff --git a/pkgtools/pkglint/files/mkline.go b/pkgtools/pkglint/files/mkline.go index a0e78802ec7..a444dea21aa 100644 --- a/pkgtools/pkglint/files/mkline.go +++ b/pkgtools/pkglint/files/mkline.go @@ -22,17 +22,19 @@ type MkLineImpl struct { } type mkLineAssign = *mkLineAssignImpl // See https://github.com/golang/go/issues/28045 type mkLineAssignImpl struct { - commented bool // Whether the whole variable assignment is commented out - varname string // e.g. "HOMEPAGE", "SUBST_SED.perl" - varcanon string // e.g. "HOMEPAGE", "SUBST_SED.*" - varparam string // e.g. "", "perl" - op MkOperator // - valueAlign string // The text up to and including the assignment operator, e.g. VARNAME+=\t - value string // The trimmed value - valueMk []*MkToken // The value, sent through splitIntoMkWords - valueMkRest string // nonempty in case of parse errors - fields []string // The value, space-separated according to shell quoting rules - comment string + commented bool // Whether the whole variable assignment is commented out + varname string // e.g. "HOMEPAGE", "SUBST_SED.perl" + varcanon string // e.g. "HOMEPAGE", "SUBST_SED.*" + varparam string // e.g. "", "perl" + spaceAfterVarname string + op MkOperator // + valueAlign string // The text up to and including the assignment operator, e.g. VARNAME+=\t + value string // The trimmed value + valueMk []*MkToken // The value, sent through splitIntoMkWords + valueMkRest string // nonempty in case of parse errors + fields []string // The value, space-separated according to shell quoting rules + spaceAfterValue string + comment string } type mkLineShell struct { command string @@ -77,24 +79,26 @@ func NewMkLine(line Line) *MkLineImpl { "Otherwise remove the leading whitespace.") } - if m, commented, varname, spaceAfterVarname, op, valueAlign, value, spaceAfterValue, comment := MatchVarassign(text); m { - if spaceAfterVarname != "" { + if m, a := MatchVarassign(text); m { + if a.spaceAfterVarname != "" { + varname := a.varname + op := a.op switch { - case hasSuffix(varname, "+") && op == "=": + case hasSuffix(varname, "+") && (op == opAssign || op == opAssignAppend): break - case matches(varname, `^[a-z]`) && op == ":=": + case matches(varname, `^[a-z]`) && op == opAssignEval: break default: // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing. fix := line.Autofix() fix.Notef("Unnecessary space after variable name %q.", varname) - fix.Replace(varname+spaceAfterVarname+op, varname+op) + fix.Replace(varname+a.spaceAfterVarname+op.String(), varname+op.String()) fix.Apply() } } // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing. - if comment != "" && value != "" && spaceAfterValue == "" { + if a.comment != "" && a.value != "" && a.spaceAfterValue == "" { line.Warnf("The # character starts a Makefile comment.") G.Explain( "In a variable assignment, an unescaped # starts a comment that", @@ -102,18 +106,7 @@ func NewMkLine(line Line) *MkLineImpl { "To escape the #, write \\#.") } - return &MkLineImpl{line, &mkLineAssignImpl{ - commented, - varname, - varnameCanon(varname), - varnameParam(varname), - NewMkOperator(op), - valueAlign, - strings.Replace(value, "\\#", "#", -1), - nil, - "", - nil, - comment}} + return &MkLineImpl{line, a} } if hasPrefix(text, "\t") { @@ -131,7 +124,13 @@ func NewMkLine(line Line) *MkLineImpl { } if m, indent, directive, args, comment := matchMkDirective(text); m { - return &MkLineImpl{line, &mkLineDirectiveImpl{indent, directive, args, comment, nil, nil, nil}} + + // In .if and .endif lines the space surrounding the comment is irrelevant. + // Especially for checking that the .endif comment matches the .if condition, + // it must be trimmed. + trimmedComment := trimHspace(comment) + + return &MkLineImpl{line, &mkLineDirectiveImpl{indent, directive, args, trimmedComment, nil, nil, nil}} } if m, indent, directive, includedFile := MatchMkInclude(text); m { @@ -161,6 +160,7 @@ func NewMkLine(line Line) *MkLineImpl { return &MkLineImpl{line, nil} } +// String returns the filename and line numbers. func (mkline *MkLineImpl) String() string { return sprintf("%s:%s", mkline.Filename, mkline.Linenos()) } @@ -423,6 +423,8 @@ func (mkline *MkLineImpl) ValueSplit(value string, separator string) []string { return split } +var notSpace = textproc.Space.Inverse() + // ValueFields splits the given value, taking care of variable references. // Example: // @@ -456,7 +458,7 @@ func (mkline *MkLineImpl) ValueFields(value string) []string { for lexer.NextBytesSet(textproc.Space) != "" { cont = false } - if word := lexer.NextBytesSet(textproc.Space.Inverse()); word != "" { + if word := lexer.NextBytesSet(notSpace); word != "" { out(word) cont = true } @@ -529,6 +531,9 @@ func (mkline *MkLineImpl) WithoutMakeVariables(value string) string { } func (mkline *MkLineImpl) ResolveVarsInRelativePath(relativePath string) string { + if !contains(relativePath, "$") { + return cleanpath(relativePath) + } var basedir string if G.Pkg != nil { @@ -551,8 +556,22 @@ func (mkline *MkLineImpl) ResolveVarsInRelativePath(relativePath string) string } tmp = strings.Replace(tmp, "${PKGSRCDIR}", pkgsrcdir, -1) } - tmp = strings.Replace(tmp, "${.CURDIR}", ".", -1) // TODO: Replace with the "typical" os.Getwd(). - tmp = strings.Replace(tmp, "${.PARSEDIR}", ".", -1) // FIXME + + // Strictly speaking, the .CURDIR should be replaced with the basedir. + // Depending on whether pkglint is executed with a relative or an absolute + // path, this would produce diagnostics that "this relative path must not + // be absolute". Since ${.CURDIR} is usually used in package Makefiles and + // followed by "../.." anyway, the exact directory doesn't matter. + tmp = strings.Replace(tmp, "${.CURDIR}", ".", -1) + + // TODO: Add test for exists(${.PARSEDIR}/file). + // TODO: Add test for evaluating ${.PARSEDIR} in an included package. + // TODO: Add test for including ${.PARSEDIR}/other.mk. + // TODO: Add test for evaluating ${.PARSEDIR} in the infrastructure. + // This is the only practically relevant use case since the category + // directories don't contain any *.mk files that could be included. + // TODO: Add test that suggests ${.PARSEDIR} in .include to be omitted. + tmp = strings.Replace(tmp, "${.PARSEDIR}", ".", -1) replaceLatest := func(varuse, category string, pattern regex.Pattern, replacement string) { if contains(tmp, varuse) { @@ -603,16 +622,151 @@ func (mkline *MkLineImpl) RefTo(other MkLine) string { } var ( - LowerDash = textproc.NewByteSet("a-z---") - AlnumDot = textproc.NewByteSet("A-Za-z0-9_.") + LowerDash = textproc.NewByteSet("a-z---") + AlnumDot = textproc.NewByteSet("A-Za-z0-9_.") + unescapeMkCommentSafeChars = textproc.NewByteSet("\\#[$").Inverse() ) -func matchMkDirective(text string) (m bool, indent, directive, args, comment string) { +// unescapeMkComment takes a Makefile line, as written in a file, and splits +// it into the main part and the comment. +// +// The comment starts at the first #. Except if it is preceded by an odd number +// of backslashes. Or by an opening bracket. +// +// The main text is returned including leading and trailing whitespace. Any +// escaped # is returned in its unescaped form, that is, \# becomes #. +// +// The comment is returned including the leading "#", if any. If the line has +// no comment, it is an empty string. +func unescapeMkComment(text string) (main, comment string) { + var sb strings.Builder + lexer := textproc.NewLexer(text) - if !lexer.SkipByte('.') { + +again: + if plain := lexer.NextBytesSet(unescapeMkCommentSafeChars); plain != "" { + sb.WriteString(plain) + goto again + } + + switch { + case lexer.SkipByte('$'): + sb.WriteByte('$') + + case lexer.SkipString("\\#"): + sb.WriteByte('#') + + case lexer.PeekByte() == '\\' && len(lexer.Rest()) >= 2: + sb.WriteString(lexer.Rest()[:2]) + lexer.Skip(2) + + case lexer.SkipByte('\\'): + sb.WriteByte('\\') + + case lexer.SkipString("[#"): + // See devel/bmake/files/parse.c:/as in modifier/ + sb.WriteString("[#") + + case lexer.SkipByte('['): + sb.WriteByte('[') + + default: + main = sb.String() + if lexer.PeekByte() == '#' { + return main, lexer.Rest() + } + + G.Assertf(lexer.EOF(), "unescapeMkComment(%q): sb = %q, rest = %q", text, main, lexer.Rest()) + return main, "" + } + + goto again +} + +// splitMkLine parses a logical line from a Makefile (that is, after joining +// the lines that end in a backslash) into two parts: the main part and the +// comment. +// +// This applies to all line types except those starting with a tab, which +// contain the shell commands to be associated with make targets. These cannot +// have comments. +func splitMkLine(text string) (main string, tokens []*MkToken, rest string, spaceBeforeComment string, hasComment bool, comment string) { + + main, comment = unescapeMkComment(text) + + p := NewMkParser(nil, main, false) + lexer := p.lexer + + rtrimHspace := func(s string) string { + end := len(s) + for end > 0 && isHspace(s[end-1]) { + end-- + } + return s[:end] + } + + parseToken := func() string { + var sb strings.Builder + + for !lexer.EOF() { + if lexer.SkipString("$$") { + sb.WriteString("$$") + continue + } + + other := lexer.NextBytesFunc(func(b byte) bool { return b != '$' }) + if other == "" { + break + } + + sb.WriteString(other) + } + + return sb.String() + } + + for !lexer.EOF() { + mark := lexer.Mark() + + if varUse := p.VarUse(); varUse != nil { + tokens = append(tokens, &MkToken{lexer.Since(mark), varUse}) + + } else if token := parseToken(); token != "" { + tokens = append(tokens, &MkToken{token, nil}) + + } else { + break + } + } + + if comment != "" { + hasComment = true + comment = comment[1:] + } + rest = lexer.Rest() + main = main[:len(main)-len(rest)] + + if rest == "" { + mainWithSpaces := main + main = rtrimHspace(main) + spaceBeforeComment = mainWithSpaces[len(main):] + } + + return +} + +func matchMkDirective(text string) (m bool, indent, directive, args, comment string) { + if !hasPrefix(text, ".") { + return + } + + main, _, rest, _, hasComment, trailingComment := splitMkLine(text) + if rest != "" { return } + lexer := textproc.NewLexer(main[1:]) + indent = lexer.NextHspace() directive = lexer.NextBytesSet(LowerDash) switch directive { @@ -629,27 +783,10 @@ func matchMkDirective(text string) (m bool, indent, directive, args, comment str lexer.SkipHspace() - argsStart := lexer.Mark() - for !lexer.EOF() && lexer.PeekByte() != '#' { - switch { - case lexer.SkipString("[#"): - // See devel/bmake/files/parse.c:/as in modifier/ + args = lexer.Rest() - case lexer.PeekByte() == '\\' && len(lexer.Rest()) > 1: - lexer.Skip(2) - - default: - lexer.Skip(1) - } - } - args = lexer.Since(argsStart) - args = strings.TrimFunc(args, func(r rune) bool { return isHspace(byte(r)) }) - args = strings.Replace(args, "\\#", "#", -1) - - if !lexer.EOF() { - lexer.Skip(1) - lexer.SkipHspace() - comment = lexer.Rest() + if hasComment { + comment = trailingComment } m = true @@ -1204,48 +1341,55 @@ func (ind *Indentation) CheckFinish(filename string) { } } -// VarnameBytes contains characters that may be used in variable names. -// The bracket is included only for the tool of the same name, e.g. "TOOLS_PATH.[". +// VarbaseBytes contains characters that may be used in the main part of variable names. +// VarparamBytes contains characters that may be used in the parameter part of variable names. +// +// For example, TOOLS_PATH.[ is a valid variable name but [ alone isn't since +// the opening bracket is only allowed in the parameter part of variable names. // // This approach differs from the one in devel/bmake/files/parse.c:/^Parse_IsVar, // but in practice it works equally well. Luckily there aren't many situations // where a complicated variable name contains unbalanced parentheses or braces, // which would confuse the devel/bmake parser. -var VarnameBytes = textproc.NewByteSet("A-Za-z_0-9*+---.[") +// +// TODO: The allowed characters differ between the basename and the parameter +// of the variable. The square bracket is only allowed in the parameter part. +var ( + VarbaseBytes = textproc.NewByteSet("A-Za-z_0-9+---") + VarparamBytes = textproc.NewByteSet("A-Za-z_0-9#*+---.[") +) -func MatchVarassign(text string) (m, commented bool, varname, spaceAfterVarname, op, valueAlign, value, spaceAfterValue, comment string) { - lexer := textproc.NewLexer(text) +func MatchVarassign(text string) (m bool, assignment mkLineAssign) { + commented := hasPrefix(text, "#") + withoutLeadingComment := text + if commented { + withoutLeadingComment = withoutLeadingComment[1:] + } + + main, tokens, rest, spaceBeforeComment, hasComment, comment := splitMkLine(withoutLeadingComment) + + lexer := NewMkTokensLexer(tokens) + mainStart := lexer.Mark() - commented = lexer.SkipByte('#') for !commented && lexer.SkipByte(' ') { } varnameStart := lexer.Mark() - for !lexer.EOF() { - switch { - - case lexer.NextBytesSet(VarnameBytes) != "": - continue - - case lexer.PeekByte() == '$': - parser := NewMkParser(nil, lexer.Rest(), false) - varuse := parser.VarUse() - if varuse == nil { - return - } - varuseLen := len(lexer.Rest()) - len(parser.Rest()) - lexer.Skip(varuseLen) - continue + // TODO: duplicated code in MkParser.Varname + for lexer.NextBytesSet(VarbaseBytes) != "" || lexer.NextVarUse() != nil { + } + if lexer.SkipByte('.') || hasPrefix(main, "SITES_") { + for lexer.NextBytesSet(VarparamBytes) != "" || lexer.NextVarUse() != nil { } - break } - varname = lexer.Since(varnameStart) + + varname := lexer.Since(varnameStart) if varname == "" { return } - spaceAfterVarname = lexer.NextHspace() + spaceAfterVarname := lexer.NextHspace() opStart := lexer.Mark() switch lexer.PeekByte() { @@ -1255,37 +1399,36 @@ func MatchVarassign(text string) (m, commented bool, varname, spaceAfterVarname, if !lexer.SkipByte('=') { return } - op = lexer.Since(opStart) + op := NewMkOperator(lexer.Since(opStart)) - if hasSuffix(varname, "+") && op == "=" && spaceAfterVarname == "" { + if hasSuffix(varname, "+") && op == opAssign && spaceAfterVarname == "" { varname = varname[:len(varname)-1] - op = "+=" + op = opAssignAppend } lexer.SkipHspace() - valueAlign = text[:len(text)-len(lexer.Rest())] - valueStart := lexer.Mark() - // FIXME: This is the same code as in matchMkDirective. - for !lexer.EOF() && lexer.PeekByte() != '#' { - switch { - case lexer.SkipString("[#"): - break - - case lexer.PeekByte() == '\\' && len(lexer.Rest()) > 1: - lexer.Skip(2) - - default: - lexer.Skip(1) - } + value := trimHspace(lexer.Rest() + rest) + if value == "" { + spaceBeforeComment = "" + } + valueAlign := ifelseStr(commented, "#", "") + lexer.Since(mainStart) + + return true, &mkLineAssignImpl{ + commented: commented, + varname: varname, + varcanon: varnameCanon(varname), + varparam: varnameParam(varname), + spaceAfterVarname: spaceAfterVarname, + op: op, + valueAlign: valueAlign, + value: value, + valueMk: nil, // filled in lazily + valueMkRest: "", // filled in lazily + fields: nil, // filled in lazily + spaceAfterValue: spaceBeforeComment, + comment: ifelseStr(hasComment, "#", "") + comment, } - rawValueWithSpace := lexer.Since(valueStart) - spaceAfterValue = rawValueWithSpace[len(strings.TrimRight(rawValueWithSpace, " \t")):] - value = trimHspace(strings.Replace(lexer.Since(valueStart), "\\#", "#", -1)) - comment = lexer.Rest() - - m = true - return } func MatchMkInclude(text string) (m bool, indentation, directive, filename string) { diff --git a/pkgtools/pkglint/files/mkline_test.go b/pkgtools/pkglint/files/mkline_test.go index 52581146361..8325f1c18a7 100644 --- a/pkgtools/pkglint/files/mkline_test.go +++ b/pkgtools/pkglint/files/mkline_test.go @@ -165,23 +165,19 @@ func (s *Suite) Test_NewMkLine__autofix_space_after_varname(c *check.C) { CheckFileMk(filename) t.CheckOutputLines( - "NOTE: ~/Makefile:2: Unnecessary space after variable name \"VARNAME\".", - // FIXME: Don't say anything here because the spaced form is clearer that the compressed form. - "NOTE: ~/Makefile:4: Unnecessary space after variable name \"VARNAME+\".") + "NOTE: ~/Makefile:2: Unnecessary space after variable name \"VARNAME\".") t.SetUpCommandLine("-Wspace", "--autofix") CheckFileMk(filename) t.CheckOutputLines( - "AUTOFIX: ~/Makefile:2: Replacing \"VARNAME +=\" with \"VARNAME+=\".", - // FIXME: Don't fix anything here because the spaced form is clearer that the compressed form. - "AUTOFIX: ~/Makefile:4: Replacing \"VARNAME+ +=\" with \"VARNAME++=\".") + "AUTOFIX: ~/Makefile:2: Replacing \"VARNAME +=\" with \"VARNAME+=\".") t.CheckFileLines("Makefile", MkRcsID+"", "VARNAME+=\t${VARNAME}", "VARNAME+ =\t${VARNAME+}", - "VARNAME++=\t${VARNAME+}", + "VARNAME+ +=\t${VARNAME+}", "pkgbase := pkglint") } @@ -193,14 +189,45 @@ func (s *Suite) Test_NewMkLine__varname_with_hash(c *check.C) { // Parse error because the # starts a comment. c.Check(mkline.IsVarassign(), equals, false) - mkline2 := t.NewMkLine("Makefile", 123, "VARNAME.\\#=\tvalue") + mkline2 := t.NewMkLine("Makefile", 124, "VARNAME.\\#=\tvalue") + + c.Check(mkline2.IsVarassign(), equals, true) + c.Check(mkline2.Varname(), equals, "VARNAME.#") + + t.CheckOutputLines( + "ERROR: Makefile:123: Unknown Makefile line format: \"VARNAME.#=\\tvalue\".") +} + +// Ensures that pkglint parses escaped # characters in the same way as bmake. +// +// To check that bmake parses them the same, set a breakpoint after the t.NewMkLines +// and look in t.tmpdir for the location of the file. Then run bmake with that file. +func (s *Suite) Test_NewMkLine__escaped_hash_in_value(c *check.C) { + t := s.Init(c) - // FIXME: Varname() should be "VARNAME.#". - c.Check(mkline2.IsVarassign(), equals, false) + mklines := t.SetUpFileMkLines("Makefile", + "VAR0=\tvalue#", + "VAR1=\tvalue\\#", + "VAR2=\tvalue\\\\#", + "VAR3=\tvalue\\\\\\#", + "VAR4=\tvalue\\\\\\\\#", + "", + "all:", + ".for var in VAR0 VAR1 VAR2 VAR3 VAR4", + "\t@printf '%s\\n' ${${var}}''", + ".endfor") + parsed := mklines.mklines + + c.Check(parsed[0].Value(), equals, "value") + c.Check(parsed[1].Value(), equals, "value#") + c.Check(parsed[2].Value(), equals, "value\\\\") + c.Check(parsed[3].Value(), equals, "value\\\\#") + c.Check(parsed[4].Value(), equals, "value\\\\\\\\") t.CheckOutputLines( - "ERROR: Makefile:123: Unknown Makefile line format: \"VARNAME.#=\\tvalue\".", - "ERROR: Makefile:123: Unknown Makefile line format: \"VARNAME.\\\\#=\\tvalue\".") + "WARN: ~/Makefile:1: The # character starts a Makefile comment.", + "WARN: ~/Makefile:3: The # character starts a Makefile comment.", + "WARN: ~/Makefile:5: The # character starts a Makefile comment.") } func (s *Suite) Test_MkLine_Varparam(c *check.C) { @@ -254,26 +281,26 @@ func (s *Suite) Test_VarUseContext_String(c *check.C) { func (s *Suite) Test_NewMkLine__number_sign(c *check.C) { t := s.Init(c) - mklineVarassignEscaped := t.NewMkLine("filename", 1, "SED_CMD=\t's,\\#,hash,g'") + mklineVarassignEscaped := t.NewMkLine("filename.mk", 1, "SED_CMD=\t's,\\#,hash,g'") c.Check(mklineVarassignEscaped.Varname(), equals, "SED_CMD") c.Check(mklineVarassignEscaped.Value(), equals, "'s,#,hash,g'") - mklineCommandEscaped := t.NewMkLine("filename", 1, "\tsed -e 's,\\#,hash,g'") + mklineCommandEscaped := t.NewMkLine("filename.mk", 1, "\tsed -e 's,\\#,hash,g'") c.Check(mklineCommandEscaped.ShellCommand(), equals, "sed -e 's,\\#,hash,g'") // From shells/zsh/Makefile.common, rev. 1.78 - mklineCommandUnescaped := t.NewMkLine("filename", 1, "\t# $ sha1 patches/patch-ac") + mklineCommandUnescaped := t.NewMkLine("filename.mk", 1, "\t# $ sha1 patches/patch-ac") c.Check(mklineCommandUnescaped.ShellCommand(), equals, "# $ sha1 patches/patch-ac") t.CheckOutputEmpty() // No warning about parsing the lonely dollar sign. - mklineVarassignUnescaped := t.NewMkLine("filename", 1, "SED_CMD=\t's,#,hash,'") + mklineVarassignUnescaped := t.NewMkLine("filename.mk", 1, "SED_CMD=\t's,#,hash,'") c.Check(mklineVarassignUnescaped.Value(), equals, "'s,") t.CheckOutputLines( - "WARN: filename:1: The # character starts a Makefile comment.") + "WARN: filename.mk:1: The # character starts a Makefile comment.") } func (s *Suite) Test_NewMkLine__varassign_leading_space(c *check.C) { @@ -331,7 +358,7 @@ func (s *Suite) Test_NewMkLine__infrastructure(c *check.C) { func (s *Suite) Test_MkLine_VariableNeedsQuoting__unknown_rhs(c *check.C) { t := s.Init(c) - mkline := t.NewMkLine("filename", 1, "PKGNAME:= ${UNKNOWN}") + mkline := t.NewMkLine("filename.mk", 1, "PKGNAME:= ${UNKNOWN}") t.SetUpVartypes() vuc := VarUseContext{G.Pkgsrc.VariableType("PKGNAME"), vucTimeParse, VucQuotUnknown, false} @@ -728,7 +755,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__shellword_part(c *check.C) { t.CheckOutputLines( "NOTE: ~/Makefile:6: The substitution command \"s:@LINKER_RPATH_FLAG@:${LINKER_RPATH_FLAG}:g\" " + - "can be replaced with \"SUBST_VARS.class+= LINKER_RPATH_FLAG\".") + "can be replaced with \"SUBST_VARS.class= LINKER_RPATH_FLAG\".") } // Tools, when used in a shell command, must not be quoted. @@ -1050,6 +1077,14 @@ func (s *Suite) Test_MkLine_ValueTokens__warnings(c *check.C) { "WARN: Makefile:2: Please use curly braces {} instead of round parentheses () for ROUND.") } +func (s *Suite) Test_MkLine_Tokenize__commented_varassign(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("filename.mk", 123, "#VAR=\tvalue ${VAR} suffix text") + + t.Check(mkline.Tokenize(mkline.Value(), false), check.HasLen, 3) +} + func (s *Suite) Test_MkLine_ResolveVarsInRelativePath(c *check.C) { t := s.Init(c) @@ -1103,25 +1138,32 @@ func (s *Suite) Test_MatchVarassign(c *check.C) { s.Init(c) test := func(text string, commented bool, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment string) { - type VarAssign struct { - commented bool - varname, spaceAfterVarname string - op, align string - value, spaceAfterValue string - comment string - } - expected := VarAssign{commented, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment} - am, acommented, avarname, aspaceAfterVarname, aop, aalign, avalue, aspaceAfterValue, acomment := MatchVarassign(text) - if !am { + m, actual := MatchVarassign(text) + if !m { c.Errorf("Text %q doesn't match variable assignment", text) return } - actual := VarAssign{acommented, avarname, aspaceAfterVarname, aop, aalign, avalue, aspaceAfterValue, acomment} - c.Check(actual, equals, expected) + + expected := mkLineAssignImpl{ + commented: commented, + varname: varname, + varcanon: varnameCanon(varname), + varparam: varnameParam(varname), + spaceAfterVarname: spaceAfterVarname, + op: NewMkOperator(op), + valueAlign: align, + value: value, + valueMk: nil, + valueMkRest: "", + fields: nil, + spaceAfterValue: spaceAfterValue, + comment: comment, + } + c.Check(*actual, deepEquals, expected) } testInvalid := func(text string) { - m, _, _, _, _, _, _, _, _ := MatchVarassign(text) + m, _ := MatchVarassign(text) if m { c.Errorf("Text %q matches variable assignment but shouldn't.", text) } @@ -1151,6 +1193,69 @@ func (s *Suite) Test_MatchVarassign(c *check.C) { // A single space is typically used for writing documentation, not for commenting out code. // Therefore this line doesn't count as commented variable assignment. testInvalid("# VAR=value") + + // Ensure that the alignment for the variable value is correct. + test("BUILD_DIRS=\tdir1 dir2", + false, + "BUILD_DIRS", + "", + "=", + "BUILD_DIRS=\t", + "dir1 dir2", + "", + "") + + // Ensure that the alignment for the variable value is correct, + // even if the whole line is commented. + test("#BUILD_DIRS=\tdir1 dir2", + true, + "BUILD_DIRS", + "", + "=", + "#BUILD_DIRS=\t", + "dir1 dir2", + "", + "") + + test("MASTER_SITES=\t#none", + false, + "MASTER_SITES", + "", + "=", + "MASTER_SITES=\t", + "", + "", + "#none") + + test("MASTER_SITES=\t# none", + false, + "MASTER_SITES", + "", + "=", + "MASTER_SITES=\t", + "", + "", + "# none") + + test("EGDIRS=\t${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d", + false, + "EGDIRS", + "", + "=", + "EGDIRS=\t", + "${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d", + "", + "") + + test("VAR:=\t${VAR:M-*:[\\#]}", + false, + "VAR", + "", + ":=", + "VAR:=\t", + "${VAR:M-*:[#]}", + "", + "") } func (s *Suite) Test_NewMkOperator(c *check.C) { @@ -1254,6 +1359,32 @@ func (s *Suite) Test_Indentation_TrackAfter__lonely_else(c *check.C) { t.CheckOutputEmpty() } +func (s *Suite) Test_Indentation_Varnames__repetition(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpPackage("category/other") + t.CreateFileDummyBuildlink3("category/other/buildlink3.mk") + t.SetUpPackage("category/package", + "DISTNAME=\tpackage-1.0", + ".include \"../../category/other/buildlink3.mk\"") + t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", + ".if ${OPSYS} == NetBSD || ${OPSYS} == FreeBSD", + ". if ${OPSYS} == NetBSD", + ". include \"../../category/other/buildlink3.mk\"", + ". endif", + ".endif") + + G.Check(t.File("category/package")) + + // TODO: It feels wrong that OPSYS is mentioned twice here. + // Why only twice and not three times? + t.CheckOutputLines( + "WARN: ~/category/package/buildlink3.mk:14: " + + "\"../../category/other/buildlink3.mk\" is included conditionally here " + + "(depending on OPSYS, OPSYS) and unconditionally in Makefile:20.") +} + func (s *Suite) Test_MkLine_DetermineUsedVariables(c *check.C) { t := s.Init(c) @@ -1330,6 +1461,362 @@ func (s *Suite) Test_MkLine_UnquoteShell(c *check.C) { test("`", "`") } +func (s *Suite) Test_unescapeMkComment(c *check.C) { + t := s.Init(c) + + test := func(text string, main, comment string) { + aMain, aComment := unescapeMkComment(text) + t.Check( + []interface{}{text, aMain, aComment}, + deepEquals, + []interface{}{text, main, comment}) + } + + test("", + "", + "") + test("text", + "text", + "") + + // The leading space from the comment is preserved to make parsing as exact + // as possible. + // + // The difference between "#defined" and "# defined" is relevant in a few + // cases, such as the API documentation of the infrastructure files. + test("# comment", + "", + "# comment") + test("#\tcomment", + "", + "#\tcomment") + test("# comment", + "", + "# comment") + + // Other than in the shell, # also starts a comment in the middle of a word. + test("COMMENT=\tThe C# compiler", + "COMMENT=\tThe C", + "# compiler") + test("COMMENT=\tThe C\\# compiler", + "COMMENT=\tThe C# compiler", + "") + + test("${TARGET}: ${SOURCES} # comment", + "${TARGET}: ${SOURCES} ", + "# comment") + + // A # starts a comment, except if it immediately follows a [. + // This is done so that the length modifier :[#] can be written without + // escaping the #. + test("VAR=\t${OTHER:[#]} # comment", + "VAR=\t${OTHER:[#]} ", + "# comment") + + // The # in the :[#] modifier may be escaped or not. Both forms are equivalent. + test("VAR:=\t${VAR:M-*:[\\#]}", + "VAR:=\t${VAR:M-*:[#]}", + "") + + // The character [ prevents the following # from starting a comment, even + // outside of variable modifiers. + test("COMMENT=\t[#] $$\\# $$# comment", + "COMMENT=\t[#] $$# $$", + "# comment") + + // A backslash always escapes the next character, be it a # for a comment + // or something else. This makes it difficult to write a literal \# in a + // Makefile, but that's an edge case anyway. + test("VAR0=\t#comment", + "VAR0=\t", + "#comment") + test("VAR1=\t\\#no-comment", + "VAR1=\t#no-comment", + "") + test("VAR2=\t\\\\#comment", + "VAR2=\t\\\\", + "#comment") + + // The backslash is only removed when it escapes a comment. + // In particular, it cannot be used to escape a dollar that starts a + // variable use. + test("VAR0=\t$T", + "VAR0=\t$T", + "") + test("VAR1=\t\\$T", + "VAR1=\t\\$T", + "") + test("VAR2=\t\\\\$T", + "VAR2=\t\\\\$T", + "") + + // To escape a dollar, write it twice. + test("$$shellvar $${shellvar} \\${MKVAR} [] \\x", + "$$shellvar $${shellvar} \\${MKVAR} [] \\x", + "") + + // Parse errors are recorded in the rest return value. + test("${UNCLOSED", + "${UNCLOSED", + "") + + // In this early phase of parsing, unfinished variable uses are not + // interpreted and do not influence the detection of the comment start. + test("text before ${UNCLOSED # comment", + "text before ${UNCLOSED ", + "# comment") + + // The dollar-space refers to a normal Make variable named " ". + // The lonely dollar at the very end refers to the variable named "", + // which is specially protected in bmake to always contain the empty string. + // It is heavily used in .for loops in the form ${:Uvalue}. + test("Lonely $ character $", + "Lonely $ character $", + "") + + // An even number of backslashes does not escape the #. + // Therefore it starts a comment here. + test("VAR2=\t\\\\#comment", + "VAR2=\t\\\\", + "#comment") +} + +func (s *Suite) Test_splitMkLine(c *check.C) { + t := s.Init(c) + + varuse := func(varname string, modifiers ...string) *MkToken { + text := "${" + varname + for _, modifier := range modifiers { + text += ":" + modifier + } + text += "}" + return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)} + } + varuseText := func(text, varname string, modifiers ...string) *MkToken { + return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)} + } + text := func(text string) *MkToken { + return &MkToken{text, nil} + } + tokens := func(tokens ...*MkToken) []*MkToken { + return tokens + } + _, _, _, _ = text, varuse, varuseText, tokens + + test := func(text string, main string, tokens []*MkToken, rest string, spaceBeforeComment string, hasComment bool, comment string) { + aMain, aTokens, aRest, aSpaceBeforeComment, aHasComment, aComment := splitMkLine(text) + t.Check( + []interface{}{text, aTokens, aMain, aRest, aSpaceBeforeComment, aHasComment, aComment}, + deepEquals, + []interface{}{text, tokens, main, rest, spaceBeforeComment, hasComment, comment}) + } + + test("", + "", + tokens(), + "", + "", + false, + "") + test("text", + "text", + tokens(text("text")), + "", + "", + false, + "") + + // The leading space from the comment is preserved to make parsing as exact + // as possible. + // + // The difference between "#defined" and "# defined" is relevant in a few + // cases, such as the API documentation of the infrastructure files. + test("# comment", + "", + tokens(), + "", + "", + true, + " comment") + test("#\tcomment", + "", + tokens(), + "", + "", + true, + "\tcomment") + test("# comment", + "", + tokens(), + "", + "", + true, + " comment") + + // Other than in the shell, # also starts a comment in the middle of a word. + test("COMMENT=\tThe C# compiler", + "COMMENT=\tThe C", + tokens(text("COMMENT=\tThe C")), + "", + "", + true, + " compiler") + test("COMMENT=\tThe C\\# compiler", + "COMMENT=\tThe C# compiler", + tokens(text("COMMENT=\tThe C# compiler")), + "", + "", + false, + "") + + test("${TARGET}: ${SOURCES} # comment", + "${TARGET}: ${SOURCES}", + tokens(varuse("TARGET"), text(": "), varuse("SOURCES"), text(" ")), + "", + " ", + true, + " comment") + + // A # starts a comment, except if it immediately follows a [. + // This is done so that the length modifier :[#] can be written without + // escaping the #. + test("VAR=\t${OTHER:[#]} # comment", + "VAR=\t${OTHER:[#]}", + tokens(text("VAR=\t"), varuse("OTHER", "[#]"), text(" ")), + "", + " ", + true, + " comment") + + // The # in the :[#] modifier may be escaped or not. Both forms are equivalent. + test("VAR:=\t${VAR:M-*:[\\#]}", + "VAR:=\t${VAR:M-*:[#]}", + tokens(text("VAR:=\t"), varuse("VAR", "M-*", "[#]")), + "", + "", + false, + "") + + // A backslash always escapes the next character, be it a # for a comment + // or something else. This makes it difficult to write a literal \# in a + // Makefile, but that's an edge case anyway. + test("VAR0=\t#comment", + "VAR0=", + tokens(text("VAR0=\t")), + "", + // Later, when converting this result into a proper variable assignment, + // this "space before comment" is reclassified as "space before the value", + // in order to align the "#comment" with the other variable values. + "\t", + true, + "comment") + test("VAR1=\t\\#no-comment", + "VAR1=\t#no-comment", + tokens(text("VAR1=\t#no-comment")), + "", + "", + false, + "") + test("VAR2=\t\\\\#comment", + "VAR2=\t\\\\", + tokens(text("VAR2=\t\\\\")), + "", + "", + true, + "comment") + + // The backslash is only removed when it escapes a comment. + // In particular, it cannot be used to escape a dollar that starts a + // variable use. + test("VAR0=\t$T", + "VAR0=\t$T", + tokens(text("VAR0=\t"), varuseText("$T", "T")), + "", + "", + false, + "") + test("VAR1=\t\\$T", + "VAR1=\t\\$T", + tokens(text("VAR1=\t\\"), varuseText("$T", "T")), + "", + "", + false, + "") + test("VAR2=\t\\\\$T", + "VAR2=\t\\\\$T", + tokens(text("VAR2=\t\\\\"), varuseText("$T", "T")), + "", + "", + false, + "") + + // To escape a dollar, write it twice. + test("$$shellvar $${shellvar} \\${MKVAR} [] \\x", + "$$shellvar $${shellvar} \\${MKVAR} [] \\x", + tokens(text("$$shellvar $${shellvar} \\"), varuse("MKVAR"), text(" [] \\x")), + "", + "", + false, + "") + + // Parse errors are recorded in the rest return value. + test("${UNCLOSED", + "", + tokens(), + "${UNCLOSED", + "", + false, + "") + + // When a parse error occurs, the comment is not parsed and the main text + // is not trimmed to the right, to keep as much original information as + // possible. + test("text before ${UNCLOSED # comment", + "text before ", + tokens(text("text before ")), + "${UNCLOSED ", // FIXME: put the space into spaceBeforeComment + "", // FIXME: the space is missing here + true, + " comment") + + // The dollar-space refers to a normal Make variable named " ". + // The lonely dollar at the very end refers to the variable named "", + // which is specially protected in bmake to always contain the empty string. + // It is heavily used in .for loops in the form ${:Uvalue}. + // + // TODO: The rest of pkglint assumes that the empty string is not a valid + // variable name, mainly because the empty variable name is not visible + // outside of the bmake debugging mode. + test("Lonely $ character $", + "Lonely $ character ", + tokens( + text("Lonely "), + varuseText("$ " /* instead of "${ }" */, " "), + text("character ")), + "$", + "", + false, + "") + + // The character [ prevents the following # from starting a comment, even + // outside of variable modifiers. + test("COMMENT=\t[#] $$\\# $$# comment", + "COMMENT=\t[#] $$# $$", + tokens(text("COMMENT=\t[#] $$# $$")), + "", + "", + true, + " comment") + + test("VAR2=\t\\\\#comment", + "VAR2=\t\\\\", + tokens(text("VAR2=\t\\\\")), + "", + "", + true, + "comment") +} + func (s *Suite) Test_matchMkDirective(c *check.C) { test := func(input, expectedIndent, expectedDirective, expectedArgs, expectedComment string) { @@ -1340,11 +1827,19 @@ func (s *Suite) Test_matchMkDirective(c *check.C) { []interface{}{true, expectedIndent, expectedDirective, expectedArgs, expectedComment}) } + testFail := func(input string) { + m, indent, directive, args, comment := matchMkDirective(input) + if m { + c.Errorf("The line %q could be parsed as directive (%q, %q, %q, %q) but shouldn't.", + indent, directive, args, comment) + } + } + test(".if ${VAR} == value", "", "if", "${VAR} == value", "") test(".\tendif # comment", - "\t", "endif", "", "comment") + "\t", "endif", "", " comment") test(".if ${VAR} == \"#\"", "", "if", "${VAR} == \"", "\"") @@ -1354,6 +1849,9 @@ func (s *Suite) Test_matchMkDirective(c *check.C) { test(".if ${VAR} == \\", "", "if", "${VAR} == \\", "") + + // Unclosed variable + testFail(".if ${VAR") } func (s *Suite) Test_MatchMkInclude(c *check.C) { diff --git a/pkgtools/pkglint/files/mklinechecker.go b/pkgtools/pkglint/files/mklinechecker.go index 111eab92c9a..f9edd9c5cae 100644 --- a/pkgtools/pkglint/files/mklinechecker.go +++ b/pkgtools/pkglint/files/mklinechecker.go @@ -423,7 +423,7 @@ func (ck MkLineChecker) CheckVaruse(varuse *MkVarUse, vuc *VarUseContext) { if G.Opts.WarnQuoting && vuc.quoting != VucQuotUnknown && needsQuoting != unknown { // FIXME: Why "Shellword" when there's no indication that this is actually a shell type? // It's for splitting the value into tokens, taking "double" and 'single' quotes into account. - ck.CheckVaruseShellword(varname, vartype, vuc, varuse.Mod(), needsQuoting) + ck.CheckVaruseShellword(varname, vartype, vuc, varuse.Mod(), needsQuoting == yes) } if G.Pkgsrc.UserDefinedVars.Defined(varname) && !G.Pkgsrc.IsBuildDef(varname) { @@ -544,17 +544,16 @@ func (ck MkLineChecker) checkVarusePermissions(varname string, vartype *Vartype, mkline := ck.MkLine effPerms := vartype.EffectivePermissions(mkline.Basename) + if effPerms == aclpUnknown { + return + } + if effPerms.Contains(aclpUseLoadtime) { + return + } // Is the variable used at load time although that is not allowed? - directly := false - indirectly := false - if !effPerms.Contains(aclpUseLoadtime) { // May not be used at load time. - if vuc.time == vucTimeParse { - directly = true - } else if vuc.vartype != nil && vuc.vartype.Union().Contains(aclpUseLoadtime) { - indirectly = true - } - } + directly := vuc.time == vucTimeParse + indirectly := !directly && vuc.vartype != nil && vuc.vartype.Union().Contains(aclpUseLoadtime) if (directly || indirectly) && !vartype.guessed { if tool := G.ToolByVarname(varname); tool != nil { @@ -566,17 +565,21 @@ func (ck MkLineChecker) checkVarusePermissions(varname string, vartype *Vartype, } } - if !effPerms.Contains(aclpUseLoadtime) && !effPerms.Contains(aclpUse) { + if !effPerms.Contains(aclpUse) { needed := aclpUse if directly || indirectly { needed = aclpUseLoadtime } alternativeFiles := vartype.AllowedFiles(needed) if alternativeFiles != "" { - mkline.Warnf("%s may not be used in this file; it would be ok in %s.", - varname, alternativeFiles) + if G.Mk == nil || G.Mk.FirstTimeSlice("don't-use", varname, mkline.Filename) { + mkline.Warnf("%s may not be used in this file; it would be ok in %s.", + varname, alternativeFiles) + } } else { - mkline.Warnf("%s may not be used in any file; it is a write-only variable.", varname) + if G.Mk == nil || G.Mk.FirstTimeSlice("write-only", varname) { + mkline.Warnf("%s may not be used in any file; it is a write-only variable.", varname) + } } ck.explainPermissions(varname, vartype) @@ -654,7 +657,7 @@ func (ck MkLineChecker) warnVaruseLoadTime(varname string, isIndirect bool) { // CheckVaruseShellword checks whether a variable use of the form ${VAR} // or ${VAR:modifiers} is allowed in a certain context. -func (ck MkLineChecker) CheckVaruseShellword(varname string, vartype *Vartype, vuc *VarUseContext, mod string, needsQuoting YesNoUnknown) { +func (ck MkLineChecker) CheckVaruseShellword(varname string, vartype *Vartype, vuc *VarUseContext, mod string, needsQuoting bool) { if trace.Tracing { defer trace.Call(varname, vartype, vuc, mod, needsQuoting)() } @@ -672,7 +675,7 @@ func (ck MkLineChecker) CheckVaruseShellword(varname string, vartype *Vartype, v if mod == ":M*:Q" && !needMstar { mkline.Notef("The :M* modifier is not needed here.") - } else if needsQuoting == yes { + } else if needsQuoting { modNoQ := strings.TrimSuffix(mod, ":Q") modNoM := strings.TrimSuffix(modNoQ, ":M*") correctMod := modNoM + ifelseStr(needMstar, ":M*:Q", ":Q") @@ -744,14 +747,12 @@ func (ck MkLineChecker) CheckVaruseShellword(varname string, vartype *Vartype, v } } - if hasSuffix(mod, ":Q") && needsQuoting != yes { + if hasSuffix(mod, ":Q") && !needsQuoting { bad := "${" + varname + mod + "}" good := "${" + varname + strings.TrimSuffix(mod, ":Q") + "}" fix := mkline.Line.Autofix() - if needsQuoting == no { - fix.Notef("The :Q operator isn't necessary for ${%s} here.", varname) - } + fix.Notef("The :Q operator isn't necessary for ${%s} here.", varname) fix.Explain( "Many variables in pkgsrc do not need the :Q operator since they", "are not expected to contain whitespace or other special characters.", @@ -780,17 +781,13 @@ func (ck MkLineChecker) checkVaruseDeprecated(varuse *MkVarUse) { } func (ck MkLineChecker) checkVarassignDecreasingVersions() { - if trace.Tracing { - defer trace.Call0()() - } - mkline := ck.MkLine strVersions := mkline.Fields() intVersions := make([]int, len(strVersions)) for i, strVersion := range strVersions { iver, err := strconv.Atoi(strVersion) if err != nil || !(iver > 0) { - mkline.Errorf("All values for %s must be positive integers.", mkline.Varname()) + mkline.Errorf("Value %q for %s must be a positive integer.", strVersion, mkline.Varname()) return } intVersions[i] = iver @@ -798,7 +795,8 @@ func (ck MkLineChecker) checkVarassignDecreasingVersions() { for i, ver := range intVersions { if i > 0 && ver >= intVersions[i-1] { - mkline.Warnf("The values for %s should be in decreasing order.", mkline.Varname()) + mkline.Warnf("The values for %s should be in decreasing order (%d before %d).", + mkline.Varname(), ver, intVersions[i-1]) G.Explain( "If they aren't, it may be possible that needless versions of", "packages are installed.") @@ -858,27 +856,37 @@ func (ck MkLineChecker) checkVarassignLeftDeprecated() { } } +// checkVarassignLeftNotUsed checks whether the left-hand side of a variable +// assignment is not used. If it is unused and also doesn't have a predefined +// data type, it may be a spelling mistake. func (ck MkLineChecker) checkVarassignLeftNotUsed() { varname := ck.MkLine.Varname() varcanon := varnameCanon(varname) - // If the variable is not used and is untyped, it may be a spelling mistake. if ck.MkLine.Op() == opAssignEval && varname == strings.ToLower(varname) { if trace.Tracing { trace.Step1("%s might be unused unless it is an argument to a procedure file.", varname) } + return + } - } else if !varIsUsedSimilar(varname) { - if vartypes := G.Pkgsrc.vartypes; vartypes[varname] != nil || vartypes[varcanon] != nil { - // Ok - } else if deprecated := G.Pkgsrc.Deprecated; deprecated[varname] != "" || deprecated[varcanon] != "" { - // Ok - } else if G.Mk != nil && !G.Mk.FirstTimeSlice("defined but not used: ", varname) { - // Skip - } else { - ck.MkLine.Warnf("%s is defined but not used.", varname) - } + if varIsUsedSimilar(varname) { + return + } + + if vartypes := G.Pkgsrc.vartypes; vartypes[varname] != nil || vartypes[varcanon] != nil { + return + } + + if deprecated := G.Pkgsrc.Deprecated; deprecated[varname] != "" || deprecated[varcanon] != "" { + return + } + + if G.Mk != nil && !G.Mk.FirstTimeSlice("defined but not used: ", varname) { + return } + + ck.MkLine.Warnf("%s is defined but not used.", varname) } // checkVarassignRightVaruse checks that in a variable assignment, @@ -1070,7 +1078,7 @@ func (ck MkLineChecker) checkVartype(varname string, op MkOperator, value, comme case value == "": break - case vartype.kindOfList == lkShell: + default: words, _ := splitIntoMkWords(mkline.Line, value) for _, word := range words { ck.CheckVartypeBasic(varname, vartype.basicType, op, word, comment, vartype.guessed) @@ -1146,26 +1154,34 @@ func (ck MkLineChecker) checkDirectiveCond() { return } - checkCompareVarStr := func(varuse *MkVarUse, op string, value string) { + checkCompareVarStr := func(varuse *MkVarUse, op string, str string) { varname := varuse.varname varmods := varuse.modifiers switch len(varmods) { case 0: - ck.checkCompareVarStr(varname, op, value) + ck.checkCompareVarStr(varname, op, str) case 1: - if m, _, _ := varmods[0].MatchMatch(); m && value != "" { - ck.checkVartype(varname, opUseMatch, value, "") + if m, _, pattern := varmods[0].MatchMatch(); m { + ck.checkVartype(varname, opUseMatch, pattern, "") + + // After applying the :M or :N modifier, every expression may end up empty, + // regardless of its data type. Therefore there's no point in type-checking that case. + if str != "" { + ck.checkVartype(varname, opUseCompare, str, "") + } } default: // This case covers ${VAR:Mfilter:O:u} or similar uses in conditions. - // To check these properly, pkglint first needs to know the most common modifiers and how they interact. - // As of November 2018, the modifiers are not modeled. + // To check these properly, pkglint first needs to know the most common + // modifiers and how they interact. + // As of March 2019, the modifiers are not modeled. // The following tracing statement makes it easy to discover these cases, // in order to decide whether checking them is worthwhile. if trace.Tracing { - trace.Stepf("checkCompareVarStr ${%s%s} %s %s", varuse.varname, varuse.Mod(), op, value) + trace.Stepf("checkCompareVarStr ${%s%s} %s %s", + varuse.varname, varuse.Mod(), op, str) } } } @@ -1285,10 +1301,12 @@ func (ck MkLineChecker) CheckRelativePath(relativePath string, mustExist bool) { return } - abs := resolvedPath - if !filepath.IsAbs(abs) { - abs = path.Dir(mkline.Filename) + "/" + abs + if filepath.IsAbs(resolvedPath) { + mkline.Errorf("The path %q must be relative.", resolvedPath) + return } + + abs := path.Dir(mkline.Filename) + "/" + resolvedPath if _, err := os.Stat(abs); err != nil { if mustExist && !(G.Mk != nil && G.Mk.indentation.IsCheckedFile(resolvedPath)) { mkline.Errorf("Relative path %q does not exist.", resolvedPath) @@ -1305,6 +1323,7 @@ func (ck MkLineChecker) CheckRelativePath(relativePath string, mustExist bool) { // From a package to another package. case hasPrefix(relativePath, "../mk/") && relpath(path.Dir(mkline.Filename), G.Pkgsrc.File(".")) == "..": // For category Makefiles. + // TODO: Or from a pkgsrc wip package to wip/mk. default: mkline.Warnf("Invalid relative path %q.", relativePath) // TODO: Explain this warning. diff --git a/pkgtools/pkglint/files/mklinechecker_test.go b/pkgtools/pkglint/files/mklinechecker_test.go index cd86056879a..57a5b34d7db 100644 --- a/pkgtools/pkglint/files/mklinechecker_test.go +++ b/pkgtools/pkglint/files/mklinechecker_test.go @@ -1,6 +1,9 @@ package pkglint -import "gopkg.in/check.v1" +import ( + "gopkg.in/check.v1" + "runtime" +) func (s *Suite) Test_MkLineChecker_checkVarassignLeft(c *check.C) { t := s.Init(c) @@ -15,6 +18,48 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeft(c *check.C) { "WARN: module.mk:123: _VARNAME is defined but not used.") } +func (s *Suite) Test_MkLineChecker_checkVarassignLeftNotUsed__procedure_call(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("mk/pkg-build-options.mk") + mklines := t.SetUpFileMkLines("category/package/filename.mk", + MkRcsID, + "", + "pkgbase := glib2", + ".include \"../../mk/pkg-build-options.mk\"", + "", + "VAR=\tvalue") + + mklines.Check() + + // There is no warning for pkgbase although it looks unused as well. + // The file pkg-build-options.mk is essentially a procedure call, + // and pkgbase is its parameter. + // + // To distinguish these parameters from ordinary variables, they are + // usually written with the := operator instead of the = operator. + // This has the added benefit that the parameter is only evaluated + // once, especially if it contains references to other variables. + t.CheckOutputLines( + "WARN: ~/category/package/filename.mk:6: VAR is defined but not used.") +} + +// Files from the pkgsrc infrastructure may define and use variables +// whose name starts with an underscore. +func (s *Suite) Test_MkLineChecker_checkVarassignLeft__infrastructure(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("mk/infra.mk", + MkRcsID, + "_VARNAME=\tvalue") + + G.Check(t.File("mk/infra.mk")) + + t.CheckOutputLines( + "WARN: ~/mk/infra.mk:2: _VARNAME is defined but not used.") +} + func (s *Suite) Test_MkLineChecker_Check__url2pkg(c *check.C) { t := s.Init(c) @@ -286,7 +331,7 @@ func (s *Suite) Test_MkLineChecker_checkVartype(c *check.C) { t := s.Init(c) t.SetUpVartypes() - mkline := t.NewMkLine("filename", 1, "DISTNAME=gcc-${GCC_VERSION}") + mkline := t.NewMkLine("filename.mk", 1, "DISTNAME=gcc-${GCC_VERSION}") MkLineChecker{mkline}.checkVartype("DISTNAME", opAssign, "gcc-${GCC_VERSION}", "") @@ -318,11 +363,14 @@ func (s *Suite) Test_MkLineChecker_checkVarassign__URL_with_shell_special_charac G.Pkg = NewPackage(t.File("graphics/gimp-fix-ca")) t.SetUpVartypes() - mkline := t.NewMkLine("filename", 10, "MASTER_SITES=http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=") + mkline := t.NewMkLine("filename.mk", 10, "MASTER_SITES=http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=") MkLineChecker{mkline}.checkVarassign() - t.CheckOutputEmpty() + t.CheckOutputLines( + "WARN: filename.mk:10: The variable MASTER_SITES may not be set " + + "(only given a default value, or appended to) in this file; " + + "it would be ok in Makefile, Makefile.common or options.mk.") } func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { @@ -331,7 +379,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { t.SetUpVartypes() test := func(cond string, output ...string) { - MkLineChecker{t.NewMkLine("filename", 1, cond)}.checkDirectiveCond() + MkLineChecker{t.NewMkLine("filename.mk", 1, cond)}.checkDirectiveCond() if len(output) > 0 { t.CheckOutputLines(output...) } else { @@ -340,50 +388,43 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { } test(".if !empty(PKGSRC_COMPILER:Mmycc)", - "WARN: filename:1: The pattern \"mycc\" cannot match any of "+ + "WARN: filename.mk:1: The pattern \"mycc\" cannot match any of "+ "{ ccache ccc clang distcc f2c gcc hp icc ido "+ "mipspro mipspro-ucode pcc sunpro xlc } for PKGSRC_COMPILER.") test(".elif ${A} != ${B}", - "WARN: filename:1: A is used but not defined.", - "WARN: filename:1: B is used but not defined.") + "WARN: filename.mk:1: A is used but not defined.", + "WARN: filename.mk:1: B is used but not defined.") test(".if ${HOMEPAGE} == \"mailto:someone@example.org\"", - "WARN: filename:1: \"mailto:someone@example.org\" is not a valid URL.", - "WARN: filename:1: HOMEPAGE should not be evaluated at load time.", - "WARN: filename:1: HOMEPAGE may not be used in any file; it is a write-only variable.") + "WARN: filename.mk:1: \"mailto:someone@example.org\" is not a valid URL.", + "WARN: filename.mk:1: HOMEPAGE should not be evaluated at load time.") test(".if !empty(PKGSRC_RUN_TEST:M[Y][eE][sS])", - "WARN: filename:1: PKGSRC_RUN_TEST should be matched "+ + "WARN: filename.mk:1: PKGSRC_RUN_TEST should be matched "+ "against \"[yY][eE][sS]\" or \"[nN][oO]\", not \"[Y][eE][sS]\".") - test(".if !empty(IS_BUILTIN.Xfixes:M[yY][eE][sS])", - "WARN: filename:1: IS_BUILTIN.Xfixes should not be evaluated at load time.", - "WARN: filename:1: IS_BUILTIN.Xfixes may not be used in this file; it would be ok in builtin.mk.") + test(".if !empty(IS_BUILTIN.Xfixes:M[yY][eE][sS])") test(".if !empty(${IS_BUILTIN.Xfixes:M[yY][eE][sS]})", - "WARN: filename:1: The empty() function takes a variable name as parameter, "+ - "not a variable expression.", - "WARN: filename:1: IS_BUILTIN.Xfixes should not be evaluated at load time.", - "WARN: filename:1: IS_BUILTIN.Xfixes may not be used in this file; it would be ok in builtin.mk.") + "WARN: filename.mk:1: The empty() function takes a variable name as parameter, "+ + "not a variable expression.") test(".if ${PKGSRC_COMPILER} == \"msvc\"", - "WARN: filename:1: \"msvc\" is not valid for PKGSRC_COMPILER. "+ + "WARN: filename.mk:1: \"msvc\" is not valid for PKGSRC_COMPILER. "+ "Use one of { ccache ccc clang distcc f2c gcc hp icc ido mipspro mipspro-ucode pcc sunpro xlc } instead.", - "WARN: filename:1: Use ${PKGSRC_COMPILER:Mmsvc} instead of the == operator.") + "WARN: filename.mk:1: Use ${PKGSRC_COMPILER:Mmsvc} instead of the == operator.") test(".if ${PKG_LIBTOOL:Mlibtool}", - "NOTE: filename:1: PKG_LIBTOOL should be compared using == instead of matching against \":Mlibtool\".", - "WARN: filename:1: PKG_LIBTOOL should not be evaluated at load time.", - "WARN: filename:1: PKG_LIBTOOL may not be used in any file; it is a write-only variable.") + "NOTE: filename.mk:1: PKG_LIBTOOL should be compared using == instead of matching against \":Mlibtool\".") test(".if ${MACHINE_PLATFORM:MUnknownOS-*-*} || ${MACHINE_ARCH:Mx86}", - "WARN: filename:1: "+ + "WARN: filename.mk:1: "+ "The pattern \"UnknownOS\" cannot match any of "+ "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ "} for the operating system part of MACHINE_PLATFORM.", - "WARN: filename:1: "+ + "WARN: filename.mk:1: "+ "The pattern \"x86\" cannot match any of "+ "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm "+ "earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb "+ @@ -391,11 +432,13 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { "m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax "+ "powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ "} for MACHINE_ARCH.", - "NOTE: filename:1: MACHINE_ARCH should be compared using == instead of matching against \":Mx86\".") + "NOTE: filename.mk:1: MACHINE_ARCH should be compared using == instead of matching against \":Mx86\".") test(".if ${MASTER_SITES:Mftp://*} == \"ftp://netbsd.org/\"", - "WARN: filename:1: MASTER_SITES should not be evaluated at load time.", - "WARN: filename:1: MASTER_SITES may not be used in any file; it is a write-only variable.") + // FIXME: Indeed, indeed, the :M modifier ends at the colon. + // Why doesn't pkglint complain loudly about the unknown "//*" modifier? + "WARN: filename.mk:1: \"ftp\" is not a valid URL.", + "WARN: filename.mk:1: MASTER_SITES should not be evaluated at load time.") // The only interesting line from the below tracing output is the one // containing "checkCompareVarStr". @@ -405,17 +448,17 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { "TRACE: 1 + (*MkParser).mkCondAtom(\"${VAR:Mpattern1:Mpattern2} == comparison\")", "TRACE: 1 - (*MkParser).mkCondAtom(\"${VAR:Mpattern1:Mpattern2} == comparison\")", "TRACE: 1 checkCompareVarStr ${VAR:Mpattern1:Mpattern2} == comparison", - "TRACE: 1 + MkLineChecker.CheckVaruse(filename:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:parse quoting:plain wordpart:false))", + "TRACE: 1 + MkLineChecker.CheckVaruse(filename.mk:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:parse quoting:plain wordpart:false))", "TRACE: 1 2 + (*Pkgsrc).VariableType(\"VAR\")", "TRACE: 1 2 3 No type definition found for \"VAR\".", "TRACE: 1 2 - (*Pkgsrc).VariableType(\"VAR\", \"=>\", (*pkglint.Vartype)(nil))", - "WARN: filename:1: VAR is used but not defined.", + "WARN: filename.mk:1: VAR is used but not defined.", "TRACE: 1 2 + MkLineChecker.checkVarusePermissions(\"VAR\", (no-type time:parse quoting:plain wordpart:false))", "TRACE: 1 2 3 No type definition found for \"VAR\".", "TRACE: 1 2 - MkLineChecker.checkVarusePermissions(\"VAR\", (no-type time:parse quoting:plain wordpart:false))", "TRACE: 1 2 + (*MkLineImpl).VariableNeedsQuoting(\"VAR\", (*pkglint.Vartype)(nil), (no-type time:parse quoting:plain wordpart:false))", "TRACE: 1 2 - (*MkLineImpl).VariableNeedsQuoting(\"VAR\", (*pkglint.Vartype)(nil), (no-type time:parse quoting:plain wordpart:false), \"=>\", unknown)", - "TRACE: 1 - MkLineChecker.CheckVaruse(filename:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:parse quoting:plain wordpart:false))", + "TRACE: 1 - MkLineChecker.CheckVaruse(filename.mk:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:parse quoting:plain wordpart:false))", "TRACE: - MkLineChecker.checkDirectiveCond(\"${VAR:Mpattern1:Mpattern2} == comparison\")") t.EnableSilentTracing() } @@ -471,6 +514,40 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__no_tracing(c * mklines.Check() } +// Setting a default license is typical for big software projects +// like GNOME or KDE that consist of many packages, or for programming +// languages like Perl or Python that suggest certain licenses. +// +// The default license is typically set in a Makefile.common or module.mk. +func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__license_default(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mkline := t.NewMkLine("filename.mk", 123, "LICENSE?=\tgnu-gpl-v2") + + MkLineChecker{mkline}.checkVarassignLeftPermissions() + + t.CheckOutputEmpty() +} + +// Setting a default license doesn't make sense in a package Makefile +// since that Makefile is only used for a single package. +// It only makes sense to set the license unconditionally there. +func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__license_default_Makefile(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mkline := t.NewMkLine("Makefile", 123, "LICENSE?=\tgnu-gpl-v2") + + MkLineChecker{mkline}.checkVarassignLeftPermissions() + + t.CheckOutputLines( + "WARN: Makefile:123: " + + "The variable LICENSE may not be given a default value " + + "(only set, or appended to) in this file; " + + "it would be ok in *.") +} + // Don't check the permissions for infrastructure files since they have their own rules. func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__infrastructure(c *check.C) { t := s.Init(c) @@ -651,12 +728,13 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__PKGREVISION(c *check. t.SetUpVartypes() mklines := t.NewMkLines("any.mk", MkRcsID, - // PKGREVISION may only be set in Makefile, not used at load time; see vardefs.go. ".if defined(PKGREVISION)", ".endif") mklines.Check() + // Since PKGREVISION may only be set in the package Makefile directly, + // there is no other file that could be mentioned as "it would be ok in". t.CheckOutputLines( "WARN: any.mk:2: PKGREVISION should not be evaluated at load time.", "WARN: any.mk:2: PKGREVISION may not be used in any file; it is a write-only variable.") @@ -677,6 +755,127 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__indirectly(c *check.C "WARN: file.mk:2: ONLY_FOR_UNPRIVILEGED should not be evaluated indirectly at load time.") } +// This test is only here for branch coverage. +// It does not demonstrate anything useful. +func (s *Suite) Test_MkLineChecker_checkVarusePermissions__indirectly_tool(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("file.mk", + MkRcsID, + "USE_TOOLS+=\t${PKGREVISION}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: file.mk:2: PKGREVISION should not be evaluated indirectly at load time.", + "WARN: file.mk:2: PKGREVISION may not be used in any file; it is a write-only variable.") +} + +func (s *Suite) Test_MkLineChecker_checkVarusePermissions__write_only_usable_in_other_file(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("buildlink3.mk", + MkRcsID, + "VAR=\t${VAR} ${AUTO_MKDIRS}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: buildlink3.mk:2: " + + "AUTO_MKDIRS may not be used in this file; " + + "it would be ok in Makefile, Makefile.* or *.mk.") +} + +func (s *Suite) Test_MkLineChecker_checkVarusePermissions__multiple_times_per_file(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("buildlink3.mk", + MkRcsID, + "VAR=\t${VAR} ${AUTO_MKDIRS} ${AUTO_MKDIRS} ${PKGREVISION} ${PKGREVISION}", + "VAR=\t${VAR} ${AUTO_MKDIRS} ${AUTO_MKDIRS} ${PKGREVISION} ${PKGREVISION}") + + mklines.Check() + + // Since these warnings are valid for the whole file, duplicates are suppressed. + t.CheckOutputLines( + "WARN: buildlink3.mk:2: "+ + "AUTO_MKDIRS may not be used in this file; "+ + "it would be ok in Makefile, Makefile.* or *.mk.", + "WARN: buildlink3.mk:2: "+ + "PKGREVISION may not be used in any file; "+ + "it is a write-only variable.") +} + +// In some pkglint tests, the method is called directly without G.Mk being set. +// In practice this doesn't happen. +func (s *Suite) Test_MkLineChecker_checkVarusePermissions__without_mklines(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mkline := t.NewMkLine("buildlink3.mk", 123, + "VAR=\t${VAR} ${AUTO_MKDIRS} ${AUTO_MKDIRS} ${PKGREVISION} ${PKGREVISION}") + + MkLineChecker{mkline}.Check() + + // Since G.Mk is not set, the duplicates are not suppressed. + // Therefore in this case there are more warnings than in realistic situations. + t.CheckOutputLines( + "WARN: buildlink3.mk:123: VAR is defined but not used.", + "WARN: buildlink3.mk:123: VAR is used but not defined.", + "WARN: buildlink3.mk:123: "+ + "AUTO_MKDIRS may not be used in this file; "+ + "it would be ok in Makefile, Makefile.* or *.mk.", + "WARN: buildlink3.mk:123: "+ + "AUTO_MKDIRS may not be used in this file; "+ + "it would be ok in Makefile, Makefile.* or *.mk.", + "WARN: buildlink3.mk:123: "+ + "PKGREVISION may not be used in any file; "+ + "it is a write-only variable.", + "WARN: buildlink3.mk:123: "+ + "PKGREVISION may not be used in any file; "+ + "it is a write-only variable.") +} + +func (s *Suite) Test_MkLineChecker_checkVarassignDecreasingVersions(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("Makefile", + MkRcsID, + "PYTHON_VERSIONS_ACCEPTED=\t36 __future__", + "PYTHON_VERSIONS_ACCEPTED=\t36 -13", + "PYTHON_VERSIONS_ACCEPTED=\t36 ${PKGVERSION_NOREV}", + "PYTHON_VERSIONS_ACCEPTED=\t36 37", + "PYTHON_VERSIONS_ACCEPTED=\t37 36 27 25") + + // TODO: All but the last of the above assignments should be flagged as + // redundant by RedundantScope; as of March 2019, that check is only + // implemented for package Makefiles, not for individual files. + + mklines.Check() + + // Half of these warnings are from VartypeCheck.Version, the + // other half are from checkVarassignDecreasingVersions. + // Strictly speaking some of them are redundant, but that would + // mean to reject only variable references in checkVarassignDecreasingVersions. + // This is probably ok. + // TODO: Fix this. + t.CheckOutputLines( + "WARN: Makefile:2: Invalid version number \"__future__\".", + "ERROR: Makefile:2: Value \"__future__\" for "+ + "PYTHON_VERSIONS_ACCEPTED must be a positive integer.", + "WARN: Makefile:3: Invalid version number \"-13\".", + "ERROR: Makefile:3: Value \"-13\" for "+ + "PYTHON_VERSIONS_ACCEPTED must be a positive integer.", + "ERROR: Makefile:4: Value \"${PKGVERSION_NOREV}\" for "+ + "PYTHON_VERSIONS_ACCEPTED must be a positive integer.", + "WARN: Makefile:5: The values for PYTHON_VERSIONS_ACCEPTED "+ + "should be in decreasing order (37 before 36).") +} + func (s *Suite) Test_MkLineChecker_warnVaruseToolLoadTime(c *check.C) { t := s.Init(c) @@ -707,6 +906,33 @@ func (s *Suite) Test_MkLineChecker_warnVaruseToolLoadTime(c *check.C) { "WARN: Makefile:6: _TOOLS_VARNAME.mk-tool is defined but not used.") } +// This somewhat unrealistic case demonstrates how there can be a tool in a +// Makefile that is not known to the global pkgsrc. +// +// This test covers the "pkgsrcTool != nil" condition. +func (s *Suite) Test_MkLineChecker_warnVaruseToolLoadTime__local_tool(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.CreateFileLines("mk/bsd.prefs.mk") + mklines := t.SetUpFileMkLines("category/package/Makefile", + MkRcsID, + ".include \"../../mk/bsd.prefs.mk\"", + "", + "TOOLS_CREATE+=\t\tmk-tool", + "_TOOLS_VARNAME.mk-tool=\tMK_TOOL", + "", + "TOOL_OUTPUT!=\t${MK_TOOL}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:5: Variable names starting with an underscore (_TOOLS_VARNAME.mk-tool) are reserved for internal pkgsrc use.", + "WARN: ~/category/package/Makefile:5: _TOOLS_VARNAME.mk-tool is defined but not used.", + "WARN: ~/category/package/Makefile:7: TOOL_OUTPUT is defined but not used.", + "WARN: ~/category/package/Makefile:7: The tool ${MK_TOOL} cannot be used at load time.") +} + func (s *Suite) Test_MkLineChecker_Check__warn_varuse_LOCALBASE(c *check.C) { t := s.Init(c) @@ -802,6 +1028,32 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparison_with_shell_com "WARN: security/openssl/Makefile:2: Use ${PKGSRC_COMPILER:Mgcc} instead of the == operator.") } +// The :N modifier filters unwanted values. After this filter, any variable value +// may be compared with the empty string, regardless of the variable type. +// Effectively, the :N modifier changes the type from T to Option(T). +func (s *Suite) Test_MkLineChecker_checkDirectiveCond__compare_pattern_with_empty(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", + MkRcsID, + ".if ${X11BASE:Npattern} == \"\"", + ".endif", + "", + ".if ${X11BASE:N<>} == \"*\"", + ".endif", + "", + ".if !(${OPSYS:M*BSD} != \"\")", + ".endif") + + mklines.Check() + + // TODO: There should be a warning about "<>" containing invalid + // characters for a path. See VartypeCheck.Pathname + t.CheckOutputLines( + "WARN: filename.mk:5: \"*\" is not a valid pathname.") +} + func (s *Suite) Test_MkLineChecker_checkDirectiveCondEmpty(c *check.C) { t := s.Init(c) @@ -840,15 +1092,17 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparing_PKGSRC_COMPILER t := s.Init(c) t.SetUpVartypes() - G.Mk = t.NewMkLines("audio/pulseaudio/Makefile", + G.Mk = t.NewMkLines("Makefile", MkRcsID, - ".if ${OPSYS} == \"Darwin\" && ${PKGSRC_COMPILER} == \"clang\"", + ".if ${PKGSRC_COMPILER} == \"clang\"", + ".elif ${PKGSRC_COMPILER} != \"gcc\"", ".endif") G.Mk.Check() t.CheckOutputLines( - "WARN: audio/pulseaudio/Makefile:2: Use ${PKGSRC_COMPILER:Mclang} instead of the == operator.") + "WARN: Makefile:2: Use ${PKGSRC_COMPILER:Mclang} instead of the == operator.", + "WARN: Makefile:3: Use ${PKGSRC_COMPILER:Ngcc} instead of the != operator.") } func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS_with_backticks(c *check.C) { @@ -1298,35 +1552,54 @@ func (s *Suite) Test_MkLineChecker_checkVarassignMisc(c *check.C) { mklines := t.SetUpFileMkLines("module.mk", MkRcsID, "EGDIR= ${PREFIX}/etc/rc.d", + "RPMIGNOREPATH+= ${PREFIX}/etc/rc.d", "_TOOLS_VARNAME.sed= SED", "DIST_SUBDIR= ${PKGNAME}", "WRKSRC= ${PKGNAME}", - "SITES_distfile.tar.gz= ${MASTER_SITE_GITHUB:=user/}", - // TODO: The first of the below assignments should be flagged as redundant by RedundantScope; - // as of January 2019, that check is only implemented for package Makefiles, not for other files. - "PYTHON_VERSIONS_ACCEPTED= -13", - "PYTHON_VERSIONS_ACCEPTED= 27 36") + "SITES_distfile.tar.gz= ${MASTER_SITE_GITHUB:=user/}") mklines.Check() // TODO: Split this test into several, one for each topic. t.CheckOutputLines( "WARN: ~/module.mk:2: Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.", - "WARN: ~/module.mk:3: Variable names starting with an underscore (_TOOLS_VARNAME.sed) are reserved for internal pkgsrc use.", - "WARN: ~/module.mk:3: _TOOLS_VARNAME.sed is defined but not used.", - "WARN: ~/module.mk:4: PKGNAME should not be used in DIST_SUBDIR as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", - "WARN: ~/module.mk:5: PKGNAME should not be used in WRKSRC as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", - "WARN: ~/module.mk:6: SITES_distfile.tar.gz is defined but not used.", - "WARN: ~/module.mk:6: SITES_* is deprecated. Please use SITES.* instead.", - "WARN: ~/module.mk:7: The variable PYTHON_VERSIONS_ACCEPTED may not be set "+ - "(only given a default value, or appended to) in this file; "+ - "it would be ok in Makefile, Makefile.common or options.mk.", - "WARN: ~/module.mk:7: Invalid version number \"-13\".", - "ERROR: ~/module.mk:7: All values for PYTHON_VERSIONS_ACCEPTED must be positive integers.", - "WARN: ~/module.mk:8: The variable PYTHON_VERSIONS_ACCEPTED may not be set "+ - "(only given a default value, or appended to) in this file; "+ - "it would be ok in Makefile, Makefile.common or options.mk.", - "WARN: ~/module.mk:8: The values for PYTHON_VERSIONS_ACCEPTED should be in decreasing order.") + "WARN: ~/module.mk:4: Variable names starting with an underscore (_TOOLS_VARNAME.sed) are reserved for internal pkgsrc use.", + "WARN: ~/module.mk:4: _TOOLS_VARNAME.sed is defined but not used.", + "WARN: ~/module.mk:5: PKGNAME should not be used in DIST_SUBDIR as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", + "WARN: ~/module.mk:6: PKGNAME should not be used in WRKSRC as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", + "WARN: ~/module.mk:7: SITES_distfile.tar.gz is defined but not used.", + "WARN: ~/module.mk:7: SITES_* is deprecated. Please use SITES.* instead.") +} + +func (s *Suite) Test_MkLineChecker_checkVarassignMisc__multiple_inclusion_guards(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.SetUpVartypes() + t.CreateFileLines("filename.mk", + MkRcsID, + ".if !defined(FILENAME_MK)", + "FILENAME_MK=\t# defined", + ".endif") + t.CreateFileLines("Makefile.common", + MkRcsID, + ".if !defined(MAKEFILE_COMMON)", + "MAKEFILE_COMMON=\t# defined", + "", + ".endif") + t.CreateFileLines("other.mk", + MkRcsID, + "COMMENT=\t# defined") + + G.Check(t.File("filename.mk")) + G.Check(t.File("Makefile.common")) + G.Check(t.File("other.mk")) + + // For multiple-inclusion guards, the meaning of the variable value + // is clear, therefore they are exempted from the warnings. + t.CheckOutputLines( + "NOTE: ~/other.mk:2: Please use \"# empty\", \"# none\" or \"# yes\" " + + "instead of \"# defined\".") } func (s *Suite) Test_MkLineChecker_checkText(c *check.C) { @@ -1409,3 +1682,55 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePath(c *check.C) { // TODO: This warning is unspecific, there is also a pkglint warning "should be ../../category/package". "WARN: ~/category/package/module.mk:11: Invalid relative path \"../package/module.mk\".") } + +func (s *Suite) Test_MkLineChecker_CheckRelativePath__absolute_path(c *check.C) { + t := s.Init(c) + + absDir := ifelseStr(runtime.GOOS == "windows", "C:/", "/") + // Just a random UUID, to really guarantee that the file does not exist. + absPath := absDir + "0f5c2d56-8a7a-4c9d-9caa-859b52bbc8c7" + + t.SetUpPkgsrc() + G.Pkgsrc.LoadInfrastructure() + mklines := t.SetUpFileMkLines("category/package/module.mk", + MkRcsID, + "DISTINFO_FILE=\t"+absPath) + + mklines.Check() + + t.CheckOutputLines( + "ERROR: ~/category/package/module.mk:2: The path \"" + absPath + "\" must be relative.") +} + +func (s *Suite) Test_MkLineChecker_CheckRelativePath__include_if_exists(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("filename.mk", + MkRcsID, + ".include \"included.mk\"", + ".sinclude \"included.mk\"") + + mklines.Check() + + // There is no warning for line 3 because of the "s" in "sinclude". + t.CheckOutputLines( + "ERROR: ~/filename.mk:2: Relative path \"included.mk\" does not exist.") +} + +func (s *Suite) Test_MkLineChecker_CheckRelativePath__wip_mk(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("wip/mk/git-package.mk", + MkRcsID) + t.SetUpPackage("wip/package", + ".include \"../mk/git-package.mk\"") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("wip/package")) + + t.CheckOutputLines( + "WARN: ~/wip/package/Makefile:20: "+ + "References to the pkgsrc-wip infrastructure should look like \"../../wip/mk\", "+ + "not \"../mk\".", + "WARN: ~/wip/package/Makefile:20: Invalid relative path \"../mk/git-package.mk\".") +} diff --git a/pkgtools/pkglint/files/mklines.go b/pkgtools/pkglint/files/mklines.go index 1f875d68698..987b9cb9cd6 100644 --- a/pkgtools/pkglint/files/mklines.go +++ b/pkgtools/pkglint/files/mklines.go @@ -115,7 +115,6 @@ func (mklines *MkLinesImpl) checkAll() { "pre-install": true, "do-install": true, "post-install": true, "pre-package": true, "do-package": true, "post-package": true, "pre-clean": true, "do-clean": true, "post-clean": true} - G.Assertf(len(allowedTargets) == 33, "Error in allowedTargets initialization") mklines.lines.CheckRcsID(0, `#[\t ]+`, "# ") @@ -378,7 +377,13 @@ func (mklines *MkLinesImpl) collectDocumentedVariables() { parser := NewMkParser(nil, words[1], false) varname := parser.Varname() - if hasSuffix(varname, ".") && parser.lexer.SkipRegexp(G.res.Compile(`^<\w+>`)) { + if len(varname) < 3 { + break + } + if hasSuffix(varname, ".") { + if !parser.lexer.SkipRegexp(G.res.Compile(`^<\w+>`)) { + break + } varname += "*" } parser.lexer.SkipByte(':') @@ -389,7 +394,7 @@ func (mklines *MkLinesImpl) collectDocumentedVariables() { scope.Use(varcanon, mkline) } - if 1 < len(words) && words[1] == "Copyright" { + if words[1] == "Copyright" { relevant = false } @@ -401,37 +406,6 @@ func (mklines *MkLinesImpl) collectDocumentedVariables() { finish() } -func (mklines *MkLinesImpl) CheckRedundantAssignments() { - scope := NewRedundantScope() - - isRelevant := func(old, new MkLine) bool { - if new.Op() == opAssignEval { - return false - } - return true - } - - scope.OnRedundant = func(old, new MkLine) { - if isRelevant(old, new) && old.Value() == new.Value() { - new.Notef("Definition of %s is redundant because of %s.", old.Varname(), new.RefTo(old)) - } - } - - scope.OnOverwrite = func(old, new MkLine) { - if isRelevant(old, new) { - old.Warnf("Variable %s is overwritten in %s.", new.Varname(), old.RefTo(new)) - G.Explain( - "The variable definition in this line does not have an effect since", - "it is overwritten elsewhere.", - "This typically happens because of a typo (writing = instead of +=)", - "or because the line that overwrites", - "is in another file that is used by several packages.") - } - } - - mklines.ForEach(scope.Handle) -} - // CheckForUsedComment checks that this file (a Makefile.common) has the given // relativeName in one of the "# used by" comments at the beginning of the file. func (mklines *MkLinesImpl) CheckForUsedComment(relativeName string) { @@ -550,8 +524,8 @@ func (va *VaralignBlock) processVarassign(mkline MkLine) { if mkline.IsMultiline() { // Parsing the continuation marker as variable value is cheating but works well. text := strings.TrimSuffix(mkline.raw[0].orignl, "\n") - m, _, _, _, _, _, value, _, _ := MatchVarassign(text) - continuation = m && value == "\\" + m, a := MatchVarassign(text) + continuation = m && a.value == "\\" } valueAlign := mkline.ValueAlign() diff --git a/pkgtools/pkglint/files/mklines_test.go b/pkgtools/pkglint/files/mklines_test.go index 9bf8fb574f8..5fe9a1c8046 100644 --- a/pkgtools/pkglint/files/mklines_test.go +++ b/pkgtools/pkglint/files/mklines_test.go @@ -335,12 +335,7 @@ func (s *Suite) Test_MkLines_collectDefinedVariables(c *check.C) { // The tools autoreconf and autoheader213 are known at this point because of the USE_TOOLS line. // The SUV variable is used implicitly by the SUBST framework, therefore no warning. // The OSV.NetBSD variable is used implicitly via the OSV variable, therefore no warning. - t.CheckOutputLines( - // FIXME: the below warning is wrong; it's ok to have SUBST blocks in all files, - // maybe except buildlink3.mk. - "WARN: determine-defined-variables.mk:12: The variable SUBST_VARS.subst may not be set " + - "(only given a default value, or appended to) in this file; " + - "it would be ok in Makefile, Makefile.common or options.mk.") + t.CheckOutputEmpty() } func (s *Suite) Test_MkLines_collectDefinedVariables__BUILTIN_FIND_FILES_VAR(c *check.C) { @@ -371,7 +366,7 @@ func (s *Suite) Test_MkLines_collectDefinedVariables__BUILTIN_FIND_FILES_VAR(c * func (s *Suite) Test_MkLines_collectUsedVariables__simple(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("filename", + mklines := t.NewMkLines("filename.mk", "\t${VAR}") mkline := mklines.mklines[0] G.Mk = mklines @@ -410,7 +405,7 @@ func (s *Suite) Test_MkLines__private_tool_undefined(c *check.C) { t := s.Init(c) t.SetUpVartypes() - mklines := t.NewMkLines("filename", + mklines := t.NewMkLines("filename.mk", MkRcsID, "", "\tmd5sum filename") @@ -418,14 +413,14 @@ func (s *Suite) Test_MkLines__private_tool_undefined(c *check.C) { mklines.Check() t.CheckOutputLines( - "WARN: filename:3: Unknown shell command \"md5sum\".") + "WARN: filename.mk:3: Unknown shell command \"md5sum\".") } func (s *Suite) Test_MkLines__private_tool_defined(c *check.C) { t := s.Init(c) t.SetUpVartypes() - mklines := t.NewMkLines("filename", + mklines := t.NewMkLines("filename.mk", MkRcsID, "TOOLS_CREATE+=\tmd5sum", "", @@ -435,7 +430,7 @@ func (s *Suite) Test_MkLines__private_tool_defined(c *check.C) { // TODO: Is it necessary to add the tool to USE_TOOLS? If not, why not? t.CheckOutputLines( - "WARN: filename:4: The \"md5sum\" tool is used but not added to USE_TOOLS.") + "WARN: filename.mk:4: The \"md5sum\" tool is used but not added to USE_TOOLS.") } func (s *Suite) Test_MkLines_Check__indentation(c *check.C) { @@ -644,6 +639,10 @@ func (s *Suite) Test_MkLines_collectDocumentedVariables(c *check.C) { "# VARIABLE", "#\tA paragraph of a single line is not enough to be recognized as \"relevant\".", "", + "# PARAGRAPH", + "#\tA paragraph may end in a", + "#\tPARA_END_VARNAME.", + "", "# VARBASE1.<param1>", "# VARBASE2.*", "# VARBASE3.${id}") @@ -659,11 +658,12 @@ func (s *Suite) Test_MkLines_collectDocumentedVariables(c *check.C) { sort.Strings(varnames) expected := []string{ + "PARAGRAPH (line 23)", "PKG_DEBUG_LEVEL (line 11)", "PKG_VERBOSE (line 16)", - "VARBASE1.* (line 23)", - "VARBASE2.* (line 24)", - "VARBASE3.* (line 25)"} + "VARBASE1.* (line 27)", + "VARBASE2.* (line 28)", + "VARBASE3.* (line 29)"} c.Check(varnames, deepEquals, expected) } @@ -704,237 +704,6 @@ func (s *Suite) Test_MkLines__unknown_options(c *check.C) { "WARN: options.mk:4: Unknown option \"unknown\".") } -func (s *Suite) Test_MkLines_CheckRedundantAssignments__override_in_mk(c *check.C) { - t := s.Init(c) - included := t.NewMkLines("included.mk", - "OVERRIDE=\tprevious value", - "REDUNDANT=\tredundant") - including := t.NewMkLines("including.mk", - ".include \"included.mk\"", - "OVERRIDE=\toverridden value", - "REDUNDANT=\tredundant") - - var allLines []Line - allLines = append(allLines, including.lines.Lines[:1]...) - allLines = append(allLines, included.lines.Lines...) - allLines = append(allLines, including.lines.Lines[1:]...) - mklines := NewMkLines(NewLines(included.lines.FileName, allLines)) - - // XXX: The warnings from here are not in the same order as the other warnings. - // XXX: There may be some warnings for the same file separated by warnings for other files. - mklines.CheckRedundantAssignments() - - t.CheckOutputLines( - "NOTE: including.mk:3: Definition of REDUNDANT is redundant because of included.mk:2.") -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__override_in_Makefile(c *check.C) { - t := s.Init(c) - included := t.NewMkLines("module.mk", - "VAR=\tvalue ${OTHER}", - "VAR?=\tvalue ${OTHER}", - "VAR=\tnew value") - including := t.NewMkLines("Makefile", - ".include \"module.mk\"", - "VAR=\tthe package may overwrite variables from other files") - - var allLines []Line - allLines = append(allLines, including.lines.Lines[:1]...) - allLines = append(allLines, included.lines.Lines...) - allLines = append(allLines, including.lines.Lines[1:]...) - mklines := NewMkLines(NewLines(including.lines.FileName, allLines)) - - // XXX: The warnings from here are not in the same order as the other warnings. - // XXX: There may be some warnings for the same file separated by warnings for other files. - mklines.CheckRedundantAssignments() - - // No warning for VAR=... in Makefile since it makes sense to have common files - // with default values for variables, overriding some of them in each package. - t.CheckOutputLines( - "NOTE: module.mk:2: Definition of VAR is redundant because of line 1.", - "WARN: module.mk:1: Variable VAR is overwritten in line 3.") -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__default_value_definitely_unused(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR=\tvalue ${OTHER}", - "VAR?=\tdifferent value") - - mklines.CheckRedundantAssignments() - - // FIXME: A default assignment after an unconditional assignment is redundant. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__default_value_overridden(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR?=\tdefault value", - "VAR=\toverridden value") - - mklines.CheckRedundantAssignments() - - t.CheckOutputLines( - "WARN: module.mk:1: Variable VAR is overwritten in line 2.") -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__overwrite_same_value(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR=\tvalue ${OTHER}", - "VAR=\tvalue ${OTHER}") - - mklines.CheckRedundantAssignments() - - t.CheckOutputLines( - "NOTE: module.mk:2: Definition of VAR is redundant because of line 1.") -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__conditional_overwrite(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR=\tdefault", - ".if ${OPSYS} == NetBSD", - "VAR=\topsys", - ".endif") - - mklines.CheckRedundantAssignments() - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__conditional_default(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR=\tdefault", - ".if ${OPSYS} == NetBSD", - "VAR?=\topsys", - ".endif") - - mklines.CheckRedundantAssignments() - - // TODO: WARN: module.mk:3: The value \"opsys\" will never be assigned to VAR because it is defined unconditionally in line 1. - t.CheckOutputEmpty() -} - -// These warnings are precise and accurate since the value of VAR is not used between line 2 and 4. -func (s *Suite) Test_MkLines_CheckRedundantAssignments__overwrite_same_variable_different_value(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "OTHER=\tvalue before", - "VAR=\tvalue ${OTHER}", - "OTHER=\tvalue after", - "VAR=\tvalue ${OTHER}") - - mklines.CheckRedundantAssignments() - - t.CheckOutputLines( - "WARN: module.mk:1: Variable OTHER is overwritten in line 3.", - "NOTE: module.mk:4: Definition of VAR is redundant because of line 2.") -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__overwrite_different_value_used_between(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "OTHER=\tvalue before", - "VAR=\tvalue ${OTHER}", - - // VAR is used here at load time, therefore it must be defined at this point. - // At this point, VAR uses the \"before\" value of OTHER. - "RESULT1:=\t${VAR}", - - "OTHER=\tvalue after", - - // VAR is used here again at load time, this time using the \"after\" value of OTHER. - "RESULT2:=\t${VAR}", - - // Still this definition is redundant. - "VAR=\tvalue ${OTHER}") - - mklines.CheckRedundantAssignments() - - t.CheckOutputLines( - "WARN: module.mk:1: Variable OTHER is overwritten in line 4.", - "NOTE: module.mk:6: Definition of VAR is redundant because of line 2.") -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__procedure_call(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("mk/pthread.buildlink3.mk", - "CHECK_BUILTIN.pthread:=\tyes", - ".include \"../../mk/pthread.builtin.mk\"", - "CHECK_BUILTIN.pthread:=\tno") - - mklines.CheckRedundantAssignments() - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__shell_and_eval(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR:=\tvalue ${OTHER}", - "VAR!=\tvalue ${OTHER}") - - mklines.CheckRedundantAssignments() - - // As of November 2018, pkglint doesn't check redundancies that involve the := or != operators. - // - // What happens here is: - // - // Line 1 evaluates OTHER at load time. - // Line 1 assigns its value to VAR. - // Line 2 evaluates OTHER at load time. - // Line 2 passes its value through the shell and assigns the result to VAR. - // - // Since VAR is defined in line 1, not used afterwards and overwritten in line 2, it is redundant. - // Well, not quite, because evaluating ${OTHER} might have side-effects from :sh or ::= modifiers, - // but these are so rare that they are frowned upon and are not considered by pkglint. - // - // Expected result: - // WARN: module.mk:2: Previous definition of VAR in line 1 is unused. - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__shell_and_eval_literal(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR:=\tvalue", - "VAR!=\tvalue") - - mklines.CheckRedundantAssignments() - - // Even when := is used with a literal value (which is usually - // only done for procedure calls), the shell evaluation can have - // so many different side effects that pkglint cannot reliably - // help in this situation. - // - // TODO: Why not? The evaluation in line 1 is trivial to analyze. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__included_OPSYS_variable(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package", - ".include \"../../category/dependency/buildlink3.mk\"", - "CONFIGURE_ARGS+=\tone", - "CONFIGURE_ARGS=\ttwo", - "CONFIGURE_ARGS+=\tthree") - t.SetUpPackage("category/dependency") - t.CreateFileDummyBuildlink3("category/dependency/buildlink3.mk") - t.CreateFileLines("category/dependency/builtin.mk", - MkRcsID, - "CONFIGURE_ARGS.Darwin+=\tdarwin") - - G.Check(t.File("category/package")) - - t.CheckOutputLines( - "WARN: ~/category/package/Makefile:21: Variable CONFIGURE_ARGS is overwritten in line 22.") -} - func (s *Suite) Test_MkLines_Check__PLIST_VARS(c *check.C) { t := s.Init(c) @@ -1101,7 +870,8 @@ func (s *Suite) Test_MkLines_Check__hacks_mk(c *check.C) { mklines.Check() // No warning about including bsd.prefs.mk before using the ?= operator. - // FIXME: Why not? + // This is because the hacks.mk files are included implicitly by the + // pkgsrc infrastructure right after bsd.prefs.mk. t.CheckOutputEmpty() } @@ -1167,8 +937,8 @@ func (s *Suite) Test_MkLines_Check__extra_warnings(c *check.C) { G.Pkg = NewPackage(t.File("category/pkgbase")) G.Mk = t.NewMkLines("options.mk", MkRcsID, + "", ".for word in ${PKG_FAIL_REASON}", - "PYTHON_VERSIONS_ACCEPTED=\t27 35 30", "CONFIGURE_ARGS+=\t--sharedir=${PREFIX}/share/kde", "COMMENT=\t# defined", ".endfor", @@ -1181,7 +951,6 @@ func (s *Suite) Test_MkLines_Check__extra_warnings(c *check.C) { G.Mk.Check() t.CheckOutputLines( - "WARN: options.mk:3: The values for PYTHON_VERSIONS_ACCEPTED should be in decreasing order.", "NOTE: options.mk:5: Please use \"# empty\", \"# none\" or \"# yes\" instead of \"# defined\".", "WARN: options.mk:7: Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".", "WARN: options.mk:11: Building the package should take place entirely inside ${WRKSRC}, not \"${WRKSRC}/..\".", diff --git a/pkgtools/pkglint/files/mkparser.go b/pkgtools/pkglint/files/mkparser.go index b10888013ea..4cbcb8877f3 100644 --- a/pkgtools/pkglint/files/mkparser.go +++ b/pkgtools/pkglint/files/mkparser.go @@ -27,6 +27,7 @@ func (p *MkParser) Rest() string { // // The text argument is assumed to be after unescaping the # character, // which means the # is a normal character and does not introduce a Makefile comment. +// For VarUse, this distinction is irrelevant. func NewMkParser(line Line, text string, emitWarnings bool) *MkParser { G.Assertf((line != nil) == emitWarnings, "line must be given iff emitWarnings is set") return &MkParser{line, textproc.NewLexer(text), emitWarnings} @@ -120,13 +121,7 @@ func (p *MkParser) VarUse() *MkVarUse { } lexer.Reset(mark) - } - - if lexer.SkipByte('@') { - return &MkVarUse{"@", nil} - } - if lexer.SkipByte('<') { - return &MkVarUse{"<", nil} + return nil } varname := lexer.NextByteSet(textproc.AlnumU) @@ -152,6 +147,26 @@ func (p *MkParser) VarUse() *MkVarUse { return &MkVarUse{sprintf("%c", varname), nil} } + if !lexer.EOF() { + symbol := lexer.Rest()[:1] + switch symbol { + case "$": + // This is an escaped dollar character and not a variable use. + + case "@", "<", " ": + // These variable names are known to exist. + // + // Many others are also possible but not used in practice. + // In particular, when parsing the :C or :S modifier, + // the $ must not be interpreted as a variable name, + // even when it looks like $/ could refer to the "/" variable. + // + // TODO: Find out whether $" is a variable use when it appears in the :M modifier. + lexer.Skip(1) + return &MkVarUse{symbol, nil} + } + } + lexer.Reset(mark) return nil } @@ -466,6 +481,29 @@ func (p *MkParser) mkCondAtom() MkCond { } } lexer.Reset(mark) + + lexer.Skip(1) + var rhsText strings.Builder + loop: + for { + m := lexer.Mark() + switch { + case p.VarUse() != nil, + lexer.NextBytesSet(textproc.Alnum) != "", + lexer.NextBytesFunc(func(b byte) bool { return b != '"' && b != '\\' }) != "": + rhsText.WriteString(lexer.Since(m)) + + case lexer.SkipString("\\\""), + lexer.SkipString("\\\\"): + rhsText.WriteByte(lexer.Since(m)[1]) + + case lexer.SkipByte('"'): + return &mkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, rhsText.String()}} + default: + break loop + } + } + lexer.Reset(mark) } } @@ -524,9 +562,14 @@ func (p *MkParser) mkCondFunc() *mkCond { func (p *MkParser) Varname() string { lexer := p.lexer + // TODO: duplicated code in MatchVarassign mark := lexer.Mark() lexer.SkipByte('.') - for p.VarUse() != nil || lexer.NextBytesSet(VarnameBytes) != "" { + for lexer.NextBytesSet(VarbaseBytes) != "" || p.VarUse() != nil { + } + if lexer.SkipByte('.') || hasPrefix(lexer.Since(mark), "SITES_") { + for lexer.NextBytesSet(VarparamBytes) != "" || p.VarUse() != nil { + } } return lexer.Since(mark) } @@ -799,6 +842,7 @@ func (w *MkCondWalker) Walk(cond MkCond, callback *MkCondCallback) { if callback.VarUse != nil { callback.VarUse(cond.CompareVarStr.Var) } + w.walkStr(cond.CompareVarStr.Str, callback) case cond.CompareVarNum != nil: if callback.CompareVarNum != nil { @@ -814,5 +858,17 @@ func (w *MkCondWalker) Walk(cond MkCond, callback *MkCondCallback) { call := cond.Call callback.Call(call.Name, call.Arg) } + w.walkStr(cond.Call.Arg, callback) + } +} + +func (w *MkCondWalker) walkStr(str string, callback *MkCondCallback) { + if callback.VarUse != nil { + tokens := NewMkParser(nil, str, false).MkTokens() + for _, token := range tokens { + if token.Varuse != nil { + callback.VarUse(token.Varuse) + } + } } } diff --git a/pkgtools/pkglint/files/mkparser_test.go b/pkgtools/pkglint/files/mkparser_test.go index ae3b03a73d1..e3a4b8aafae 100644 --- a/pkgtools/pkglint/files/mkparser_test.go +++ b/pkgtools/pkglint/files/mkparser_test.go @@ -192,6 +192,11 @@ func (s *Suite) Test_MkParser_VarUse(c *check.C) { test("${PKGNAME:C/-[0-9].*$/-[0-9]*/}", varuse("PKGNAME", "C/-[0-9].*$/-[0-9]*/")) + // TODO: Does the $@ refer to ${.TARGET}, or is it just an unmatchable + // regular expression? + test("${PKGNAME:C/$@/target?/}", + varuse("PKGNAME", "C/$@/target?/")) + test("${PKGNAME:S/py${_PYTHON_VERSION}/py${i}/:C/-[0-9].*$/-[0-9]*/}", varuse("PKGNAME", "S/py${_PYTHON_VERSION}/py${i}/", "C/-[0-9].*$/-[0-9]*/")) @@ -331,6 +336,12 @@ func (s *Suite) Test_MkParser_VarUse(c *check.C) { t.CheckOutputLines( "WARN: Test_MkParser_VarUse.mk:1: Modifier ${PLIST_SUBST_VARS:@var@...@} is missing the final \"@\".") + + // Unfinished variable use + testRest("${", nil, "${") + + // Unfinished nested variable use + testRest("${${", nil, "${${") } func (s *Suite) Test_MkParser_VarUse__ambiguous(c *check.C) { @@ -412,6 +423,13 @@ func (s *Suite) Test_MkParser_MkCond(c *check.C) { test("\"${pkg}\" == \"${name}\"", &mkCond{CompareVarVar: &MkCondCompareVarVar{varuse("pkg"), "==", varuse("name")}}) + // The right-hand side is not analyzed further to keep the data types simple. + test("${ABC} == \"${A}B${C}\"", + &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("ABC"), "==", "${A}B${C}"}}) + + test("${ABC} == \"${A}\\\"${B}\\\\${C}$${shellvar}${D}\"", + &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("ABC"), "==", "${A}\"${B}\\${C}$${shellvar}${D}"}}) + test("exists(/etc/hosts)", &mkCond{Call: &MkCondCall{"exists", "/etc/hosts"}}) @@ -776,9 +794,11 @@ func (s *Suite) Test_MkCondWalker_Walk(c *check.C) { mkline := t.NewMkLine("Makefile", 4, ""+ ".if ${VAR:Mmatch} == ${OTHER} || "+ "${STR} == Str || "+ + "${VAR} == \"${PRE}text${POST}\" || "+ "${NUM} == 3 && "+ "defined(VAR) && "+ "!exists(file.mk) && "+ + "exists(${FILE}) && "+ "(((${NONEMPTY})))") var events []string @@ -830,11 +850,17 @@ func (s *Suite) Test_MkCondWalker_Walk(c *check.C) { " varUse OTHER", " compareVarStr STR, Str", " varUse STR", + " compareVarStr VAR, ${PRE}text${POST}", + " varUse VAR", + " varUse PRE", + " varUse POST", " compareVarNum NUM, 3", " varUse NUM", " defined VAR", " varUse VAR", " call exists, file.mk", + " call exists, ${FILE}", + " varUse FILE", " var NONEMPTY", " varUse NONEMPTY"}) } diff --git a/pkgtools/pkglint/files/mktokenslexer.go b/pkgtools/pkglint/files/mktokenslexer.go new file mode 100644 index 00000000000..0330d52480a --- /dev/null +++ b/pkgtools/pkglint/files/mktokenslexer.go @@ -0,0 +1,89 @@ +package pkglint + +import ( + "netbsd.org/pkglint/textproc" + "strings" +) + +// MkTokensLexer parses a sequence of variable uses (like ${VAR:Mpattern}) +// interleaved with other text that is uninterpreted by bmake. +type MkTokensLexer struct { + // The lexer for the current text-only token. + // If the current token is a variable use, the lexer will always return + // EOF internally. That is not visible from the outside though, as EOF is + // overridden in this type. + *textproc.Lexer + + // The remaining tokens. + tokens []*MkToken +} + +func NewMkTokensLexer(tokens []*MkToken) *MkTokensLexer { + lexer := &MkTokensLexer{nil, tokens} + lexer.next() + return lexer +} + +func (m *MkTokensLexer) next() { + if len(m.tokens) > 0 && m.tokens[0].Varuse == nil { + m.Lexer = textproc.NewLexer(m.tokens[0].Text) + m.tokens = m.tokens[1:] + } else { + m.Lexer = textproc.NewLexer("") + } +} + +// EOF returns whether the whole input has been consumed. +func (m *MkTokensLexer) EOF() bool { return m.Lexer.EOF() && len(m.tokens) == 0 } + +// Rest returns the string concatenation of the tokens that have not yet been consumed. +func (m *MkTokensLexer) Rest() string { + var sb strings.Builder + sb.WriteString(m.Lexer.Rest()) + for _, token := range m.tokens { + sb.WriteString(token.Text) + } + return sb.String() +} + +// NextVarUse returns the next varuse token, unless there is some plain text +// before it. In that case or at EOF, it returns nil. +func (m *MkTokensLexer) NextVarUse() *MkVarUse { + if m.Lexer.EOF() && len(m.tokens) > 0 && m.tokens[0].Varuse != nil { + token := m.tokens[0] + m.tokens = m.tokens[1:] + m.next() + return token.Varuse + } + return nil +} + +// Mark remembers the current position of the lexer. +// The lexer can later be reset to that position by calling Reset. +func (m *MkTokensLexer) Mark() MkTokensLexerMark { + return MkTokensLexerMark{m.Lexer.Rest(), m.tokens} +} + +// Since returns the text between the given mark and the current position +// of the lexer. +func (m *MkTokensLexer) Since(mark MkTokensLexerMark) string { + early := (&MkTokensLexer{textproc.NewLexer(mark.rest), mark.tokens}).Rest() + late := m.Rest() + + return strings.TrimSuffix(early, late) +} + +// Reset sets the lexer back to the given position. +// The lexer may be reset to the same mark multiple times, +// that is, the mark is not destroyed. +func (m *MkTokensLexer) Reset(mark MkTokensLexerMark) { + m.Lexer = textproc.NewLexer(mark.rest) + m.tokens = mark.tokens +} + +// MkTokensLexerMark remembers a position of a token lexer, +// to be restored later. +type MkTokensLexerMark struct { + rest string + tokens []*MkToken +} diff --git a/pkgtools/pkglint/files/mktokenslexer_test.go b/pkgtools/pkglint/files/mktokenslexer_test.go new file mode 100644 index 00000000000..00926c1e369 --- /dev/null +++ b/pkgtools/pkglint/files/mktokenslexer_test.go @@ -0,0 +1,291 @@ +package pkglint + +import ( + "gopkg.in/check.v1" + "netbsd.org/pkglint/textproc" +) + +func (s *Suite) Test_MkTokensLexer__empty_slice_returns_EOF(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer(nil) + + t.Check(lexer.EOF(), equals, true) +} + +// A slice of a single token behaves like textproc.Lexer. +func (s *Suite) Test_MkTokensLexer__single_plain_text_token(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{{"\\# $$ [#] $V", nil}}) + + t.Check(lexer.SkipByte('\\'), equals, true) + t.Check(lexer.Rest(), equals, "# $$ [#] $V") + t.Check(lexer.SkipByte('#'), equals, true) + t.Check(lexer.NextHspace(), equals, " ") + t.Check(lexer.NextBytesSet(textproc.Space.Inverse()), equals, "$$") + t.Check(lexer.Skip(len(lexer.Rest())), equals, true) + t.Check(lexer.EOF(), equals, true) +} + +// If the first element of the slice is a variable use, none of the plain +// text patterns matches. +// +// The code that uses the MkTokensLexer needs to distinguish these cases +// anyway, therefore it doesn't make sense to treat variable uses as plain +// text. +func (s *Suite) Test_MkTokensLexer__single_varuse_token(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{{"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}}) + + t.Check(lexer.EOF(), equals, false) + t.Check(lexer.PeekByte(), equals, -1) + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR", "Mpattern")) +} + +func (s *Suite) Test_MkTokensLexer__plain_then_varuse(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"plain text", nil}, + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}}) + + t.Check(lexer.NextBytesSet(textproc.Digit.Inverse()), equals, "plain text") + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR", "Mpattern")) + t.Check(lexer.EOF(), equals, true) +} + +func (s *Suite) Test_MkTokensLexer__varuse_varuse_varuse(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"${dirs:O:u}", NewMkVarUse("dirs", "O", "u")}, + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}, + {"${.TARGET}", NewMkVarUse(".TARGET")}}) + + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("dirs", "O", "u")) + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR", "Mpattern")) + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse(".TARGET")) + t.Check(lexer.NextVarUse(), check.IsNil) +} + +func (s *Suite) Test_MkTokensLexer__mark_reset_since_in_initial_state(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"${dirs:O:u}", NewMkVarUse("dirs", "O", "u")}, + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}, + {"${.TARGET}", NewMkVarUse(".TARGET")}}) + + start := lexer.Mark() + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("dirs", "O", "u")) + middle := lexer.Mark() + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}${.TARGET}") + lexer.Reset(start) + t.Check(lexer.Rest(), equals, "${dirs:O:u}${VAR:Mpattern}${.TARGET}") + lexer.Reset(middle) + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}${.TARGET}") +} + +func (s *Suite) Test_MkTokensLexer__mark_reset_since_inside_plain_text(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"plain text", nil}, + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}, + {"rest", nil}}) + + start := lexer.Mark() + t.Check(lexer.NextBytesSet(textproc.Alpha), equals, "plain") + middle := lexer.Mark() + t.Check(lexer.Rest(), equals, " text${VAR:Mpattern}rest") + lexer.Reset(start) + t.Check(lexer.Rest(), equals, "plain text${VAR:Mpattern}rest") + lexer.Reset(middle) + t.Check(lexer.Rest(), equals, " text${VAR:Mpattern}rest") +} + +func (s *Suite) Test_MkTokensLexer__mark_reset_since_after_plain_text(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"plain text", nil}, + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}, + {"rest", nil}}) + + start := lexer.Mark() + t.Check(lexer.SkipString("plain text"), equals, true) + end := lexer.Mark() + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}rest") + lexer.Reset(start) + t.Check(lexer.Rest(), equals, "plain text${VAR:Mpattern}rest") + lexer.Reset(end) + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}rest") +} + +func (s *Suite) Test_MkTokensLexer__mark_reset_since_after_varuse(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}, + {"rest", nil}}) + + start := lexer.Mark() + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR", "Mpattern")) + end := lexer.Mark() + t.Check(lexer.Rest(), equals, "rest") + lexer.Reset(start) + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}rest") + lexer.Reset(end) + t.Check(lexer.Rest(), equals, "rest") +} + +func (s *Suite) Test_MkTokensLexer__multiple_marks_in_same_plain_text(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"plain text", nil}, + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}, + {"rest", nil}}) + + start := lexer.Mark() + t.Check(lexer.NextString("plain "), equals, "plain ") + middle := lexer.Mark() + t.Check(lexer.NextString("text"), equals, "text") + end := lexer.Mark() + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}rest") + lexer.Reset(start) + t.Check(lexer.Rest(), equals, "plain text${VAR:Mpattern}rest") + lexer.Reset(middle) + t.Check(lexer.Rest(), equals, "text${VAR:Mpattern}rest") + lexer.Reset(end) + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}rest") +} + +func (s *Suite) Test_MkTokensLexer__multiple_marks_in_varuse(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"${VAR1}", NewMkVarUse("VAR1")}, + {"${VAR2}", NewMkVarUse("VAR2")}, + {"${VAR3}", NewMkVarUse("VAR3")}}) + + start := lexer.Mark() + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR1")) + middle := lexer.Mark() + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR2")) + further := lexer.Mark() + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR3")) + end := lexer.Mark() + t.Check(lexer.Rest(), equals, "") + lexer.Reset(middle) + t.Check(lexer.Rest(), equals, "${VAR2}${VAR3}") + lexer.Reset(further) + t.Check(lexer.Rest(), equals, "${VAR3}") + lexer.Reset(start) + t.Check(lexer.Rest(), equals, "${VAR1}${VAR2}${VAR3}") + lexer.Reset(end) + t.Check(lexer.Rest(), equals, "") +} + +func (s *Suite) Test_MkTokensLexer__EOF_before_plain_text(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{{"rest", nil}}) + + t.Check(lexer.EOF(), equals, false) +} + +func (s *Suite) Test_MkTokensLexer__EOF_before_varuse(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{{"${VAR}", NewMkVarUse("VAR")}}) + + t.Check(lexer.EOF(), equals, false) +} + +// When the MkTokensLexer is constructed, it gets a copy of the tokens array. +// In theory it would be possible to change the tokens after starting lexing, +// but there is no practical case where that would be useful. +// +// Since each slice is a separate view on the underlying array, modifying the +// size of the outside slice does not affect parsing. This is also only a +// theoretical case. +// +// Because all these cases are only theoretical, the MkTokensLexer doesn't +// bother to make this unnecessary copy and works on the shared slice. +func (s *Suite) Test_MkTokensLexer__constructor_uses_shared_array(c *check.C) { + t := s.Init(c) + + tokens := []*MkToken{{"${VAR}", NewMkVarUse("VAR")}} + lexer := NewMkTokensLexer(tokens) + + t.Check(lexer.Rest(), equals, "${VAR}") + + tokens[0].Text = "modified text" + tokens[0].Varuse = NewMkVarUse("MODIFIED", "Mpattern") + tokens = tokens[0:0] + + t.Check(lexer.Rest(), equals, "modified text") +} + +func (s *Suite) Test_MkTokensLexer__peek_after_varuse(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"${VAR}", NewMkVarUse("VAR")}, + {"${VAR}", NewMkVarUse("VAR")}, + {"text", nil}}) + + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR")) + t.Check(lexer.PeekByte(), equals, -1) + + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR")) + t.Check(lexer.PeekByte(), equals, int('t')) +} + +func (s *Suite) Test_MkTokensLexer__varuse_when_plain_text(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{{"text", nil}}) + + t.Check(lexer.NextVarUse(), check.IsNil) + t.Check(lexer.NextString("te"), equals, "te") + t.Check(lexer.NextVarUse(), check.IsNil) + t.Check(lexer.NextString("xt"), equals, "xt") + t.Check(lexer.NextVarUse(), check.IsNil) +} + +// The code that creates the tokens for the lexer never puts two +// plain text MkTokens besides each other. There's no point in doing +// that since they could have been combined into a single token from +// the beginning. +func (s *Suite) Test_MkTokensLexer__adjacent_plain_text(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"text1", nil}, + {"text2", nil}}) + + // Returns false since the string is distributed over two separate tokens. + t.Check(lexer.SkipString("text1text2"), equals, false) + + t.Check(lexer.SkipString("text1"), equals, true) + + // This returns false since the internal lexer is not advanced to the + // next text token. To do that, all methods from the internal lexer + // would have to be redefined by MkTokensLexer in order to advance the + // internal lexer to the next token. + // + // Since this situation doesn't occur in practice, there's no point in + // implementing it. + t.Check(lexer.SkipString("text2"), equals, false) + + // Just for covering the "Varuse != nil" branch in MkTokensLexer.NextVarUse. + t.Check(lexer.NextVarUse(), check.IsNil) + + // The string is still not found since the next token is only consumed + // by the NextVarUse above if it is indeed a VarUse. + t.Check(lexer.SkipString("text2"), equals, false) +} diff --git a/pkgtools/pkglint/files/package.go b/pkgtools/pkglint/files/package.go index 7e7a3a07199..7f7ec79cd56 100644 --- a/pkgtools/pkglint/files/package.go +++ b/pkgtools/pkglint/files/package.go @@ -28,10 +28,21 @@ type Package struct { EffectivePkgnameLine MkLine // The origin of the three Effective* values Plist PlistContent // Files and directories mentioned in the PLIST files - vars Scope - bl3 map[string]MkLine // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. - included map[string]MkLine // filename => line - seenMakefileCommon bool // Does the package have any .includes? + vars Scope + bl3 map[string]MkLine // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. + + // Remembers the Makefile fragments that have already been included. + // The key to the map is the filename relative to the package directory. + // Typical keys are "../../category/package/buildlink3.mk". + // + // TODO: Include files with multiple-inclusion guard only once. + // + // TODO: Include files without multiple-inclusion guard as often as needed. + // + // TODO: Set an upper limit, to prevent denial of service. + included Once + + seenMakefileCommon bool // Does the package have any .includes? // Files from .include lines that are nested inside .if. // They often depend on OPSYS or on the existence of files in the build environment. @@ -61,7 +72,7 @@ func NewPackage(dir string) *Package { Plist: NewPlistContent(), vars: NewScope(), bl3: make(map[string]MkLine), - included: make(map[string]MkLine), + included: Once{}, conditionalIncludes: make(map[string]MkLine), unconditionalIncludes: make(map[string]MkLine), } @@ -152,7 +163,7 @@ func (pkg *Package) checkLinesBuildlink3Inclusion(mklines MkLines) { } } -func (pkg *Package) loadPackageMakefile() MkLines { +func (pkg *Package) loadPackageMakefile() (MkLines, MkLines) { filename := pkg.File("Makefile") if trace.Tracing { defer trace.Call1(filename)() @@ -162,7 +173,7 @@ func (pkg *Package) loadPackageMakefile() MkLines { allLines := NewMkLines(NewLines("", nil)) if _, result := pkg.readMakefile(filename, mainLines, allLines, ""); !result { LoadMk(filename, NotEmpty|LogErrors) // Just for the LogErrors. - return nil + return nil, nil } // TODO: Is this still necessary? This code is 20 years old and was introduced @@ -183,7 +194,6 @@ func (pkg *Package) loadPackageMakefile() MkLines { } allLines.collectUsedVariables() - allLines.CheckRedundantAssignments() pkg.Pkgdir = pkg.vars.LastValue("PKGDIR") pkg.DistinfoFile = pkg.vars.LastValue("DISTINFO_FILE") @@ -212,7 +222,7 @@ func (pkg *Package) loadPackageMakefile() MkLines { trace.Step1("PKGDIR=%s", pkg.Pkgdir) } - return mainLines + return mainLines, allLines } // TODO: What is allLines used for, is it still necessary? Would it be better as a field in Package? @@ -225,71 +235,92 @@ func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines Mk if fileMklines == nil { return false, false } + exists = true + result = true isMainMakefile := len(mainLines.mklines) == 0 - result = true - lineAction := func(mkline MkLine) bool { - if isMainMakefile { - mainLines.mklines = append(mainLines.mklines, mkline) - mainLines.lines.Lines = append(mainLines.lines.Lines, mkline.Line) + handleIncludeLine := func(mkline MkLine) YesNoUnknown { + includedFile, incDir, incBase := pkg.findIncludedFile(mkline, filename) + + if includedFile == "" { + return unknown } - allLines.mklines = append(allLines.mklines, mkline) - allLines.lines.Lines = append(allLines.lines.Lines, mkline.Line) - includedFile, incDir, incBase := pkg.findIncludedFile(mkline, filename) + dirname, _ := path.Split(filename) + dirname = cleanpath(dirname) + fullIncluded := dirname + "/" + includedFile + relIncludedFile := relpath(pkg.dir, fullIncluded) - if includedFile != "" && pkg.included[includedFile] == nil { - pkg.included[includedFile] = mkline + if !pkg.diveInto(filename, includedFile) { + return unknown + } + + if !pkg.included.FirstTime(relIncludedFile) { + return unknown + } - // TODO: "../../../.." also matches but shouldn't. - if matches(includedFile, `^\.\./[^./][^/]*/[^/]+`) { + if matches(includedFile, `^\.\./[^./][^/]*/[^/]+`) { + if G.Wip && contains(includedFile, "/mk/") { + mkline.Warnf("References to the pkgsrc-wip infrastructure should look like \"../../wip/mk\", not \"../mk\".") + } else { mkline.Warnf("References to other packages should look like \"../../category/package\", not \"../package\".") - mkline.ExplainRelativeDirs() } + mkline.ExplainRelativeDirs() + } - pkg.collectUsedBy(mkline, incDir, incBase, includedFile) + pkg.collectUsedBy(mkline, incDir, incBase, includedFile) - skip := contains(filename, "/mk/") || hasSuffix(includedFile, "/bsd.pkg.mk") || IsPrefs(includedFile) - if !skip { - dirname, _ := path.Split(filename) - dirname = cleanpath(dirname) + if trace.Tracing { + trace.Step1("Including %q.", fullIncluded) + } + fullIncluding := ifelseStr(incBase == "Makefile.common" && incDir != "", filename, "") + innerExists, innerResult := pkg.readMakefile(fullIncluded, mainLines, allLines, fullIncluding) - fullIncluded := dirname + "/" + includedFile - if trace.Tracing { - trace.Step1("Including %q.", fullIncluded) - } - fullIncluding := ifelseStr(incBase == "Makefile.common" && incDir != "", filename, "") - innerExists, innerResult := pkg.readMakefile(fullIncluded, mainLines, allLines, fullIncluding) + if !innerExists { + if fileMklines.indentation.IsCheckedFile(includedFile) { + return yes // See https://github.com/rillig/pkglint/issues/1 + } - if !innerExists { - if fileMklines.indentation.IsCheckedFile(includedFile) { - return true // See https://github.com/rillig/pkglint/issues/1 - } + // Only look in the directory relative to the + // current file and in the package directory. + // Make(1) has a list of include directories, but pkgsrc + // doesn't make use of that, so pkglint also doesn't + // need this extra complexity. + pkgBasedir := pkg.File(".") + if dirname != pkgBasedir { // Prevent unnecessary syscalls + dirname = pkgBasedir + + fullIncludedFallback := dirname + "/" + includedFile + innerExists, innerResult = pkg.readMakefile(fullIncludedFallback, mainLines, allLines, fullIncluding) + } - // Only look in the directory relative to the - // current file and in the package directory. - // Make(1) has a list of include directories, but pkgsrc - // doesn't make use of that, so pkglint also doesn't - // need this extra complexity. - pkgBasedir := pkg.File(".") - if dirname != pkgBasedir { // Prevent unnecessary syscalls - dirname = pkgBasedir - - fullIncludedFallback := dirname + "/" + includedFile - innerExists, innerResult = pkg.readMakefile(fullIncludedFallback, mainLines, allLines, fullIncluding) - } + if !innerExists { + mkline.Errorf("Cannot read %q.", includedFile) + } + } - if !innerExists { - mkline.Errorf("Cannot read %q.", includedFile) - } - } + if !innerResult { + result = false + return no + } - if !innerResult { - result = false - return false - } + return unknown + } + + lineAction := func(mkline MkLine) bool { + if isMainMakefile { + mainLines.mklines = append(mainLines.mklines, mkline) + mainLines.lines.Lines = append(mainLines.lines.Lines, mkline.Line) + } + allLines.mklines = append(allLines.mklines, mkline) + allLines.lines.Lines = append(allLines.lines.Lines, mkline.Line) + + if mkline.IsInclude() { + includeResult := handleIncludeLine(mkline) + if includeResult != unknown { + return includeResult == yes } } @@ -305,6 +336,7 @@ func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines Mk } return true } + atEnd := func(mkline MkLine) {} fileMklines.ForEachEnd(lineAction, atEnd) @@ -315,8 +347,9 @@ func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines Mk // For every included buildlink3.mk, include the corresponding builtin.mk // automatically since the pkgsrc infrastructure does the same. if path.Base(filename) == "buildlink3.mk" { - builtin := path.Join(path.Dir(filename), "builtin.mk") - if fileExists(builtin) { + builtin := cleanpath(path.Dir(filename) + "/builtin.mk") + builtinRel := relpath(pkg.dir, builtin) + if pkg.included.FirstTime(builtinRel) && fileExists(builtin) { pkg.readMakefile(builtin, mainLines, allLines, "") } } @@ -324,6 +357,17 @@ func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines Mk return } +func (pkg *Package) diveInto(includingFile string, includedFile string) bool { + skip := hasSuffix(includedFile, "/bsd.pkg.mk") || IsPrefs(includedFile) + if !skip && contains(includingFile, "/mk/") { + skip = true + if contains(includingFile, "buildlink3.mk") && contains(includedFile, "builtin.mk") { + skip = false + } + } + return !skip +} + func (pkg *Package) collectUsedBy(mkline MkLine, incDir string, incBase string, includedFile string) { switch { case @@ -343,19 +387,17 @@ func (pkg *Package) collectUsedBy(mkline MkLine, incDir string, incBase string, func (pkg *Package) findIncludedFile(mkline MkLine, includingFilename string) (includedFile, incDir, incBase string) { - if mkline.IsInclude() { - // TODO: resolveVariableRefs uses G.Pkg implicitly. It should be made explicit. - // TODO: Try to combine resolveVariableRefs and ResolveVarsInRelativePath. - includedFile = resolveVariableRefs(mkline.ResolveVarsInRelativePath(mkline.IncludedFile())) - if containsVarRef(includedFile) { - if trace.Tracing && !contains(includingFilename, "/mk/") { - trace.Stepf("%s:%s: Skipping include file %q. This may result in false warnings.", - mkline.Filename, mkline.Linenos(), includedFile) - } - includedFile = "" + // TODO: resolveVariableRefs uses G.Pkg implicitly. It should be made explicit. + // TODO: Try to combine resolveVariableRefs and ResolveVarsInRelativePath. + includedFile = resolveVariableRefs(mkline.ResolveVarsInRelativePath(mkline.IncludedFile())) + if containsVarRef(includedFile) { + if trace.Tracing && !contains(includingFilename, "/mk/") { + trace.Stepf("%s:%s: Skipping include file %q. This may result in false warnings.", + mkline.Filename, mkline.Linenos(), includedFile) } - incDir, incBase = path.Split(includedFile) + includedFile = "" } + incDir, incBase = path.Split(includedFile) if includedFile != "" { if mkline.Basename != "buildlink3.mk" { @@ -371,7 +413,7 @@ func (pkg *Package) findIncludedFile(mkline MkLine, includingFilename string) (i return } -func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines) { +func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines, allLines MkLines) { if trace.Tracing { defer trace.Call1(filename)() } @@ -408,11 +450,18 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines) { } if !vars.Defined("LICENSE") && !vars.Defined("META_PACKAGE") && pkg.once.FirstTime("LICENSE") { - NewLineWhole(filename).Errorf("Each package must define its LICENSE.") + line := NewLineWhole(filename) + line.Errorf("Each package must define its LICENSE.") // TODO: Explain why the LICENSE is necessary. + line.Explain( + "To take a good guess on the license of a package,", + sprintf("run %q.", bmake("guess-license"))) } - pkg.checkGnuConfigureUseLanguages() + scope := NewRedundantScope() + scope.Check(allLines) // Updates the variables in the scope + pkg.checkGnuConfigureUseLanguages(scope) + pkg.determineEffectivePkgVars() pkg.checkPossibleDowngrade() @@ -434,28 +483,45 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines) { SaveAutofixChanges(mklines.lines) } -func (pkg *Package) checkGnuConfigureUseLanguages() { - vars := pkg.vars +func (pkg *Package) checkGnuConfigureUseLanguages(s *RedundantScope) { - if gnuLine := vars.FirstDefinition("GNU_CONFIGURE"); gnuLine != nil { + gnuConfigure := s.vars["GNU_CONFIGURE"] + if gnuConfigure == nil || !gnuConfigure.vari.Constant() { + return + } - // FIXME: Instead of using the first definition here, a better approach - // is probably to use all the definitions except those from mk/compiler.mk. - // In real pkgsrc, the last definition is typically from mk/compiler.mk - // and only contains c++. - if useLine := vars.FirstDefinition("USE_LANGUAGES"); useLine != nil { + useLanguages := s.vars["USE_LANGUAGES"] + if useLanguages == nil || !useLanguages.vari.Constant() { + return + } - if matches(useLine.VarassignComment(), `(?-i)\b(?:c|empty|none)\b`) { - // Don't emit a warning since the comment probably contains a - // statement that C is really not needed. + var wrongLines []MkLine + for _, mkline := range useLanguages.vari.WriteLocations() { - } else if !matches(useLine.Value(), `(?:^|[\t ]+)(?:c|c99|objc)(?:[\t ]+|$)`) { - gnuLine.Warnf( - "GNU_CONFIGURE almost always needs a C compiler, "+ - "but \"c\" is not added to USE_LANGUAGES in %s.", - gnuLine.RefTo(useLine)) - } + if G.Pkgsrc.IsInfra(mkline.Line.Filename) { + continue } + + if matches(mkline.VarassignComment(), `(?-i)\b(?:c|empty|none)\b`) { + // Don't emit a warning since the comment probably contains a + // statement that C is really not needed. + return + } + + languages := mkline.Value() + if matches(languages, `(?:^|[\t ]+)(?:c|c99|objc)(?:[\t ]+|$)`) { + return + } + + wrongLines = append(wrongLines, mkline) + } + + gnuLine := gnuConfigure.vari.WriteLocations()[0] + for _, useLine := range wrongLines { + gnuLine.Warnf( + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in %s.", + gnuLine.RefTo(useLine)) } } diff --git a/pkgtools/pkglint/files/package_test.go b/pkgtools/pkglint/files/package_test.go index 6162c4c4c80..3dae5b5bd82 100644 --- a/pkgtools/pkglint/files/package_test.go +++ b/pkgtools/pkglint/files/package_test.go @@ -28,7 +28,8 @@ func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__package_but_not_file t.CreateFileLines("category/dependency/buildlink3.mk") G.Pkg = NewPackage(t.File("category/package")) - G.Pkg.bl3["../../category/dependency/buildlink3.mk"] = t.NewMkLine("filename", 1, "") + G.Pkg.bl3["../../category/dependency/buildlink3.mk"] = + t.NewMkLine("../../category/dependency/buildlink3.mk", 1, "") mklines := t.NewMkLines("category/package/buildlink3.mk", MkRcsID) @@ -212,7 +213,6 @@ func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { t.CreateFileLines("mk/bsd.pkg.mk", "# dummy") t.CreateFileLines("x11/Makefile", MkRcsID) t.CreateFileLines("x11/9term/PLIST", PlistRcsID, "bin/9term") - t.CreateFileLines("x11/9term/distinfo", RcsID) t.CreateFileLines("x11/9term/Makefile", MkRcsID, "", @@ -221,6 +221,8 @@ func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { "", "COMMENT=\tTerminal", "", + "NO_CHECKSUM=\tyes", + "", ".include \"../../mk/bsd.pkg.mk\"") t.SetUpVartypes() @@ -228,6 +230,7 @@ func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { G.Check(t.File("x11/9term")) // Since the error is grave enough, the warning about the correct position is suppressed. + // TODO: Knowing the correct position helps, though. t.CheckOutputLines( "ERROR: ~/x11/9term/Makefile: Each package must define its LICENSE.") } @@ -524,13 +527,39 @@ func (s *Suite) Test_Package_loadPackageMakefile(c *check.C) { // Including a package Makefile directly is an error (in the last line), // but that is checked later. - // A file including itself does not lead to an endless loop while parsing - // but may still produce unexpected warnings, such as redundant definitions. + // This test demonstrates that a file including itself does not lead to an + // endless loop while parsing. It might trigger an error in the future. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package__relative_included_filenames_in_same_directory(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "PKGNAME=\tpkgname-1.67", + "DISTNAME=\tdistfile_1_67", + ".include \"../../category/package/other.mk\"") + t.CreateFileLines("category/package/other.mk", + MkRcsID, + "PKGNAME=\tpkgname-1.67", + "DISTNAME=\tdistfile_1_67", + ".include \"../../category/package/other.mk\"") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + // TODO: Since other.mk is referenced via "../../category/package", + // it would be nice if this relative path would be reflected in the output + // instead of referring just to "other.mk". + // This needs to be fixed somewhere near relpath. + // + // The notes are in reverse order because they are produced when checking + // other.mk, and there the relative order is correct (line 2 before line 3). t.CheckOutputLines( - "NOTE: ~/category/package/Makefile:3: "+ - "Definition of PKGNAME is redundant because of ../../category/package/Makefile:3.", "NOTE: ~/category/package/Makefile:4: "+ - "Definition of DISTNAME is redundant because of ../../category/package/Makefile:4.") + "Definition of PKGNAME is redundant because of other.mk:2.", + "NOTE: ~/category/package/Makefile:3: "+ + "Definition of DISTNAME is redundant because of other.mk:3.") } func (s *Suite) Test_Package_loadPackageMakefile__PECL_VERSION(c *check.C) { @@ -668,14 +697,16 @@ func (s *Suite) Test_Package__redundant_master_sites(c *check.C) { G.checkdirPackage(t.File("math/R-date")) // The definition in Makefile:6 is redundant because the same definition - // occurs later in Makefile.extension:4. Usually the later definition gets - // the note. In this case though, it would be wrong to mark the - // definition in Makefile.extension as redundant because that definition is - // probably used by other packages as well. Therefore in this case the roles - // of the two lines are swapped; see RedundantScope.Handle, the ".includes" line. + // occurs later in Makefile.extension:4. + // + // When a file includes another file, it's always the including file that + // is marked as redundant since the included file typically provides the + // generally useful value for several packages; + // see RedundantScope.handleVarassign, keyword includePath. t.CheckOutputLines( "NOTE: ~/math/R-date/Makefile:6: " + - "Definition of MASTER_SITES is redundant because of ../../math/R/Makefile.extension:4.") + "Definition of MASTER_SITES is redundant " + + "because of ../../math/R/Makefile.extension:4.") } func (s *Suite) Test_Package_checkUpdate(c *check.C) { @@ -828,6 +859,131 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__USE_IMAKE_and_USE_X11(c * "NOTE: ~/category/package/Makefile:21: USE_IMAKE makes USE_X11 in line 20 redundant.") } +func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__no_C(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "USE_LANGUAGES=\tfortran77", + "USE_LANGUAGES+=\tc++14", + "USE_LANGUAGES+=\tada", + "GNU_CONFIGURE=\tyes") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:23: "+ + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in line 20.", + "WARN: ~/category/package/Makefile:23: "+ + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in line 21.", + "WARN: ~/category/package/Makefile:23: "+ + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in line 22.") +} + +func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__C_in_the_middle(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "USE_LANGUAGES=\tfortran77", + "USE_LANGUAGES+=\tc99", + "USE_LANGUAGES+=\tada", + "GNU_CONFIGURE=\tyes") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + // Until March 2019 pkglint wrongly warned that USE_LANGUAGES would not + // include c or c99, although c99 was added. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__realistic_compiler_mk(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "USE_LANGUAGES=\tfortran77", + "USE_LANGUAGES+=\tc++", + "USE_LANGUAGES+=\tada", + "GNU_CONFIGURE=\tyes", + "", + ".include \"../../mk/compiler.mk\"") + t.CreateFileLines("mk/compiler.mk", + MkRcsID, + ".include \"bsd.prefs.mk\"", + "", + "USE_LANGUAGES?=\tc", + "USE_LANGUAGES+=\tc", + "USE_LANGUAGES+=\tc++") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + // The package defines several languages it needs, but C is not one of them. + // When the package is loaded, the included files are read in recursively, even + // when they come from the pkgsrc infrastructure. + // + // Up to March 2019, the USE_LANGUAGES definitions from mk/compiler.mk were + // loaded as if they were defined by the package, without taking the conditionals + // into account. Thereby "c" was added unconditionally to USE_LANGUAGES. + // + // Since March 2019 the infrastructure files are ignored when determining the value + // of USE_LANGUAGES. + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:23: "+ + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in line 20.", + "WARN: ~/category/package/Makefile:23: "+ + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in line 21.", + "WARN: ~/category/package/Makefile:23: "+ + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in line 22.") +} + +func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__only_GNU_CONFIGURE(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "GNU_CONFIGURE=\tyes") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__ok(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "GNU_CONFIGURE=\tyes", + "USE_LANGUAGES=\tc++ objc") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package__USE_LANGUAGES_too_late(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../mk/compiler.mk\"", + "USE_LANGUAGES=\tc c99 fortran ada c++14") + t.CreateFileLines("mk/compiler.mk", + MkRcsID) + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + // FIXME: There must be a warning "USE_LANGUAGES must be added before including compiler.mk." + t.CheckOutputEmpty() +} + func (s *Suite) Test_Package_readMakefile__skipping(c *check.C) { t := s.Init(c) @@ -914,6 +1070,43 @@ func (s *Suite) Test_Package_readMakefile__builtin_mk(c *check.C) { "WARN: ~/category/package/Makefile:23: OTHER_VAR is used but not defined.") } +// Ensures that the paths in Package.included are indeed relative to the +// package directory. This hadn't been the case until March 2019. +func (s *Suite) Test_Package_readMakefile__included(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../devel/library/buildlink3.mk\"", + ".include \"../../lang/language/module.mk\"") + t.SetUpPackage("devel/library") + t.CreateFileDummyBuildlink3("devel/library/buildlink3.mk") + t.CreateFileLines("devel/library/builtin.mk", + MkRcsID) + t.CreateFileLines("lang/language/module.mk", + MkRcsID, + ".include \"version.mk\"") + t.CreateFileLines("lang/language/version.mk", + MkRcsID) + pkg := NewPackage(t.File("category/package")) + + pkg.loadPackageMakefile() + + expected := []string{ + "../../devel/library/buildlink3.mk", + "../../devel/library/builtin.mk", + "../../lang/language/module.mk", + "../../lang/language/version.mk", + "suppress-varorder.mk"} + + seen := pkg.included + for _, filename := range expected { + if !seen.Seen(filename) { + c.Errorf("File %q is not seen.", filename) + } + } + t.Check(seen.seen, check.HasLen, 5) +} + func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { t := s.Init(c) @@ -1040,3 +1233,61 @@ func (s *Suite) Test_Package__redundant_variable_in_unrelated_files(c *check.C) // PY_PATCHPLIST is not redundant in these files. t.CheckOutputEmpty() } + +// Pkglint loads some files from the pkgsrc infrastructure and skips others. +// +// When a buildlink3.mk file from the infrastructure is included, it should +// be allowed to include its corresponding builtin.mk file in turn. +// +// This is necessary to load the correct variable assignments for the +// redundancy check, in particular variable assignments that serve as +// arguments to "procedure calls", such as mk/find-files.mk. +func (s *Suite) Test_Package_readMakefile__include_infrastructure(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--dumpmakefile") + t.SetUpPackage("category/package", + ".include \"../../mk/dlopen.buildlink3.mk\"", + ".include \"../../mk/pthread.buildlink3.mk\"") + t.CreateFileLines("mk/dlopen.buildlink3.mk", + ".include \"dlopen.builtin.mk\"") + t.CreateFileLines("mk/dlopen.builtin.mk", + ".include \"pthread.builtin.mk\"") + t.CreateFileLines("mk/pthread.buildlink3.mk", + ".include \"pthread.builtin.mk\"") + t.CreateFileLines("mk/pthread.builtin.mk", + "# This should be included by pthread.buildlink3.mk") + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "Whole Makefile (with all included files) follows:", + "~/category/package/Makefile:1: "+MkRcsID, + "~/category/package/Makefile:2: ", + "~/category/package/Makefile:3: DISTNAME=\tdistname-1.0", + "~/category/package/Makefile:4: #PKGNAME=\tpackage-1.0", + "~/category/package/Makefile:5: CATEGORIES=\tcategory", + "~/category/package/Makefile:6: MASTER_SITES=\t# none", + "~/category/package/Makefile:7: ", + "~/category/package/Makefile:8: MAINTAINER=\tpkgsrc-users@NetBSD.org", + "~/category/package/Makefile:9: HOMEPAGE=\t# none", + "~/category/package/Makefile:10: COMMENT=\tDummy package", + "~/category/package/Makefile:11: LICENSE=\t2-clause-bsd", + "~/category/package/Makefile:12: ", + "~/category/package/Makefile:13: .include \"suppress-varorder.mk\"", + "~/category/package/suppress-varorder.mk:1: "+MkRcsID, + "~/category/package/Makefile:14: # empty", + "~/category/package/Makefile:15: # empty", + "~/category/package/Makefile:16: # empty", + "~/category/package/Makefile:17: # empty", + "~/category/package/Makefile:18: # empty", + "~/category/package/Makefile:19: # empty", + "~/category/package/Makefile:20: .include \"../../mk/dlopen.buildlink3.mk\"", + "~/category/package/../../mk/dlopen.buildlink3.mk:1: .include \"dlopen.builtin.mk\"", + "~/mk/dlopen.builtin.mk:1: .include \"pthread.builtin.mk\"", + "~/category/package/Makefile:21: .include \"../../mk/pthread.buildlink3.mk\"", + "~/category/package/../../mk/pthread.buildlink3.mk:1: .include \"pthread.builtin.mk\"", + "~/mk/pthread.builtin.mk:1: # This should be included by pthread.buildlink3.mk", + "~/category/package/Makefile:22: ", + "~/category/package/Makefile:23: .include \"../../mk/bsd.pkg.mk\"") +} diff --git a/pkgtools/pkglint/files/pkglint.1 b/pkgtools/pkglint/files/pkglint.1 index 566d72a1236..85e02e03a0b 100644 --- a/pkgtools/pkglint/files/pkglint.1 +++ b/pkgtools/pkglint/files/pkglint.1 @@ -1,4 +1,4 @@ -.\" $NetBSD: pkglint.1,v 1.54 2019/02/21 23:44:55 rillig Exp $ +.\" $NetBSD: pkglint.1,v 1.55 2019/03/10 19:01:50 rillig Exp $ .\" From FreeBSD: portlint.1,v 1.8 1997/11/25 14:53:14 itojun Exp .\" .\" Copyright (c) 1997 by Jun-ichiro Itoh <itojun@itojun.org>. diff --git a/pkgtools/pkglint/files/pkglint.go b/pkgtools/pkglint/files/pkglint.go index e5096bc488c..71fe509c840 100644 --- a/pkgtools/pkglint/files/pkglint.go +++ b/pkgtools/pkglint/files/pkglint.go @@ -380,7 +380,7 @@ func (pkglint *Pkglint) checkdirPackage(dir string) { // Load the package Makefile and all included files, // to collect all used and defined variables and similar data. - mklines := pkg.loadPackageMakefile() + mklines, allLines := pkg.loadPackageMakefile() if mklines == nil { return } @@ -437,7 +437,7 @@ func (pkglint *Pkglint) checkdirPackage(dir string) { case path.Base(filename) == "Makefile": pkglint.checkExecutable(filename, st.Mode()) - pkg.checkfilePackageMakefile(filename, mklines) + pkg.checkfilePackageMakefile(filename, mklines, allLines) default: pkglint.checkDirent(filename, st.Mode()) diff --git a/pkgtools/pkglint/files/pkglint_test.go b/pkgtools/pkglint/files/pkglint_test.go index ea11ec91aea..567f772fec3 100644 --- a/pkgtools/pkglint/files/pkglint_test.go +++ b/pkgtools/pkglint/files/pkglint_test.go @@ -419,7 +419,7 @@ func (s *Suite) Test_Pkglint_Check(c *check.C) { func (s *Suite) Test_resolveVariableRefs__circular_reference(c *check.C) { t := s.Init(c) - mkline := t.NewMkLine("filename", 1, "GCC_VERSION=${GCC_VERSION}") + mkline := t.NewMkLine("filename.mk", 1, "GCC_VERSION=${GCC_VERSION}") G.Pkg = NewPackage(t.File("category/pkgbase")) G.Pkg.vars.Define("GCC_VERSION", mkline) @@ -433,9 +433,9 @@ func (s *Suite) Test_resolveVariableRefs__circular_reference(c *check.C) { func (s *Suite) Test_resolveVariableRefs__multilevel(c *check.C) { t := s.Init(c) - mkline1 := t.NewMkLine("filename", 10, "FIRST=\t${SECOND}") - mkline2 := t.NewMkLine("filename", 11, "SECOND=\t${THIRD}") - mkline3 := t.NewMkLine("filename", 12, "THIRD=\tgot it") + mkline1 := t.NewMkLine("filename.mk", 10, "FIRST=\t${SECOND}") + mkline2 := t.NewMkLine("filename.mk", 11, "SECOND=\t${THIRD}") + mkline3 := t.NewMkLine("filename.mk", 12, "THIRD=\tgot it") G.Pkg = NewPackage(t.File("category/pkgbase")) defineVar(mkline1, "FIRST") defineVar(mkline2, "SECOND") @@ -455,7 +455,7 @@ func (s *Suite) Test_resolveVariableRefs__multilevel(c *check.C) { func (s *Suite) Test_resolveVariableRefs__special_chars(c *check.C) { t := s.Init(c) - mkline := t.NewMkLine("filename", 10, "_=x11") + mkline := t.NewMkLine("filename.mk", 10, "_=x11") G.Pkg = NewPackage(t.File("category/pkg")) G.Pkg.vars.Define("GST_PLUGINS0.10_TYPE", mkline) @@ -551,6 +551,15 @@ func (s *Suite) Test_CheckLinesMessage__autofix(c *check.C) { "===========================================================================") } +func (s *Suite) Test_CheckLinesMessage__common(c *check.C) { + t := s.Init(c) + + // FIXME: If there is a MESSAGE.common, it is combined with MESSAGE. + // See meta-pkgs/ruby-redmine-plugins for an example. + + t.CheckOutputEmpty() +} + // Demonstrates that an ALTERNATIVES file can be tested individually, // without any dependencies on a whole package or a PLIST file. func (s *Suite) Test_Pkglint_checkReg__alternatives(c *check.C) { diff --git a/pkgtools/pkglint/files/pkgsrc.go b/pkgtools/pkglint/files/pkgsrc.go index 2af34590b56..f8f58cbf642 100644 --- a/pkgtools/pkglint/files/pkgsrc.go +++ b/pkgtools/pkglint/files/pkgsrc.go @@ -814,6 +814,13 @@ func (src *Pkgsrc) ToRel(filename string) string { return relpath(src.topdir, filename) } +// IsInfra returns whether the given filename (relative to the pkglint +// working directory) is part of the pkgsrc infrastructure. +func (src *Pkgsrc) IsInfra(filename string) bool { + rel := src.ToRel(filename) + return hasPrefix(rel, "mk/") || hasPrefix(rel, "wip/mk/") +} + func (src *Pkgsrc) addBuildDefs(varnames ...string) { for _, varname := range varnames { src.buildDefs[varname] = true @@ -953,6 +960,8 @@ func (src *Pkgsrc) guessVariableType(varname string) (vartype *Vartype) { case hasSuffix(varbase, "_MK"): // TODO: Add BtGuard for inclusion guards, since these variables may only be checked using defined(). gtype = &Vartype{lkNone, BtUnknown, allowAll, true} + case hasSuffix(varbase, "_SKIP"): + gtype = &Vartype{lkShell, BtPathmask, allowRuntime, true} } if gtype == nil { diff --git a/pkgtools/pkglint/files/pkgsrc_test.go b/pkgtools/pkglint/files/pkgsrc_test.go index 3a5201bc3dd..ca54cf82981 100644 --- a/pkgtools/pkglint/files/pkgsrc_test.go +++ b/pkgtools/pkglint/files/pkgsrc_test.go @@ -659,3 +659,26 @@ func (s *Suite) Test_Pkgsrc_VariableType__from_mk(c *check.C) { "WARN: ~/category/package/Makefile:21: ABCPATH is used but not defined.", "0 errors and 2 warnings found.") } + +func (s *Suite) Test_Pkgsrc_guessVariableType__SKIP(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("filename.mk", + MkRcsID, + "MY_CHECK_SKIP=\t*.c \"bad*pathname\"", + "MY_CHECK_SKIP+=\t*.cpp", + ".if ${MY_CHECK_SKIP}", + ".endif") + + mklines.Check() + + // FIXME: The permissions in guessVariableType say allRuntime, which excludes + // aclpUseLoadtime. Therefore there should be a warning about the VarUse in + // the .if line. + // The check in MkLineChecker.checkVarusePermissions is disabled for guessed types. + // + // There is no warning for the += operator in line 3 since the variable type + // (although guessed) is a list of things, and lists may be appended to. + t.CheckOutputLines( + "WARN: filename.mk:2: \"\\\"bad*pathname\\\"\" is not a valid pathname mask.") +} diff --git a/pkgtools/pkglint/files/plist.go b/pkgtools/pkglint/files/plist.go index 0f5f602dd3c..09630961103 100644 --- a/pkgtools/pkglint/files/plist.go +++ b/pkgtools/pkglint/files/plist.go @@ -344,7 +344,7 @@ func (ck *PlistChecker) checkPathShare(pline *PlistLine) { case hasPrefix(text, "share/icons/") && G.Pkg != nil: if hasPrefix(text, "share/icons/hicolor/") && G.Pkg.Pkgpath != "graphics/hicolor-icon-theme" { f := "../../graphics/hicolor-icon-theme/buildlink3.mk" - if G.Pkg.included[f] == nil && ck.once.FirstTime("hicolor-icon-theme") { + if !G.Pkg.included.Seen(f) && ck.once.FirstTime("hicolor-icon-theme") { pline.Errorf("Packages that install hicolor icons must include %q in the Makefile.", f) } } @@ -359,7 +359,7 @@ func (ck *PlistChecker) checkPathShare(pline *PlistLine) { if hasPrefix(text, "share/icons/gnome") && G.Pkg.Pkgpath != "graphics/gnome-icon-theme" { f := "../../graphics/gnome-icon-theme/buildlink3.mk" - if G.Pkg.included[f] == nil { + if !G.Pkg.included.Seen(f) { pline.Errorf("The package Makefile must include %q.", f) G.Explain( "Packages that install GNOME icons must maintain the icon theme", diff --git a/pkgtools/pkglint/files/plist_test.go b/pkgtools/pkglint/files/plist_test.go index 469b1aef6c6..68dfc06f18f 100644 --- a/pkgtools/pkglint/files/plist_test.go +++ b/pkgtools/pkglint/files/plist_test.go @@ -566,6 +566,34 @@ func (s *Suite) Test_PlistChecker_checkPathShare(c *check.C) { "WARN: ~/PLIST:6: Man pages should be installed into man/, not share/man/.") } +func (s *Suite) Test_PlistChecker_checkPathShare__gnome_icon_theme(c *check.C) { + t := s.Init(c) + + t.CreateFileDummyBuildlink3("graphics/gnome-icon-theme/buildlink3.mk") + t.SetUpPackage("graphics/gnome-icon-theme-extras", + "ICON_THEMES=\tyes", + ".include \"../../graphics/gnome-icon-theme/buildlink3.mk\"") + t.CreateFileLines("graphics/gnome-icon-theme-extras/PLIST", + PlistRcsID, + "share/icons/gnome/16x16/devices/media-optical-cd-audio.png", + "share/icons/gnome/16x16/devices/media-optical-dvd.png") + G.Pkgsrc.LoadInfrastructure() + t.Chdir(".") + + // This variant is typically run interactively. + G.Check("graphics/gnome-icon-theme-extras") + + t.CheckOutputEmpty() + + // Note the leading "./". + // This variant is typical for recursive runs of pkglint. + G.Check("./graphics/gnome-icon-theme-extras") + + // Up to March 2019, a bug in relpath produced different behavior + // depending on the leading dot. + t.CheckOutputEmpty() +} + func (s *Suite) Test_PlistLine_CheckTrailingWhitespace(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/redundantscope.go b/pkgtools/pkglint/files/redundantscope.go new file mode 100644 index 00000000000..ae0dbdea83f --- /dev/null +++ b/pkgtools/pkglint/files/redundantscope.go @@ -0,0 +1,279 @@ +package pkglint + +// RedundantScope checks for redundant variable definitions and for variables +// that are accidentally overwritten. It tries to be as correct as possible +// by not flagging anything that is defined conditionally. +// +// There may be some edge cases though like defining PKGNAME, then evaluating +// it using :=, then defining it again. This pattern is so error-prone that +// it should not appear in pkgsrc at all, thus pkglint doesn't even expect it. +// (Well, except for the PKGNAME case, but that's deep in the infrastructure +// and only affects the "nb13" extension.) +// +// TODO: This scope is not only used for detecting redundancies. It also +// provides information about whether the variables are constant or depend on +// other variables. Therefore the name may change soon. +type RedundantScope struct { + vars map[string]*redundantScopeVarinfo + includePath includePath +} +type redundantScopeVarinfo struct { + vari *Var + includePaths []includePath + lastAction uint8 // 0 = none, 1 = read, 2 = write +} + +func NewRedundantScope() *RedundantScope { + return &RedundantScope{vars: make(map[string]*redundantScopeVarinfo)} +} + +func (s *RedundantScope) Check(mklines MkLines) { + mklines.ForEach(func(mkline MkLine) { + s.Handle(mkline, mklines.indentation) + }) +} + +func (s *RedundantScope) Handle(mkline MkLine, ind *Indentation) { + s.updateIncludePath(mkline) + + switch { + case mkline.IsVarassign(): + s.handleVarassign(mkline, ind) + } + + s.handleVarUse(mkline) +} + +func (s *RedundantScope) updateIncludePath(mkline MkLine) { + if mkline.firstLine == 1 { + s.includePath.push(mkline.Location.Filename) + } else { + s.includePath.popUntil(mkline.Location.Filename) + } +} + +func (s *RedundantScope) handleVarassign(mkline MkLine, ind *Indentation) { + varname := mkline.Varname() + info := s.get(varname) + + defer func() { + info.vari.Write(mkline, ind.Depth("") > 0, ind.Varnames()...) + info.lastAction = 2 + s.access(varname) + }() + + // In the very first assignment, no redundancy can occur. + prevWrites := info.vari.WriteLocations() + if len(prevWrites) == 0 { + return + } + + // TODO: Just being conditional is only half the truth. + // To be precise, the "conditional path" must differ between + // this variable assignment and the/any? previous one. + // See Test_RedundantScope__overwrite_inside_conditional. + // Anyway, too few warnings are better than wrong warnings. + if info.vari.Conditional() || ind.Depth("") > 0 { + return + } + + // When the variable has been read after the previous write, + // it is not redundant. + if info.lastAction == 1 { + return + } + + effOp := mkline.Op() + value := mkline.Value() + + // FIXME: Skip the whole redundancy check if the value is not known to be constant. + if effOp == opAssign && info.vari.Value() == value { + effOp = opAssignDefault + } + + if effOp == opAssignEval && value == mkline.WithoutMakeVariables(value) { + // Maybe add support for VAR:= ${OTHER} later. This involves evaluating + // the OTHER variable though using the appropriate scope. Oh, wait, + // there _is_ a scope here. So if OTHER doesn't refer to further + // variables it's all possible. + // + // TODO: The above idea seems possible and useful. + effOp = opAssign + } + + switch effOp { + + case opAssign: // with a different value than before + if s.includePath.includedByOrEqualsAll(info.includePaths) { + + // The situation is: + // + // including.mk: VAR= initial value + // included.mk: VAR= overwriting <-- you are here + // + // Because the included files is never wrong (by definition), + // the including file gets the warning in this case. + s.onOverwrite(prevWrites[len(prevWrites)-1], mkline) + } + + case opAssignDefault: // or opAssign with the same value as before + switch { + + case s.includePath.includesOrEqualsAll(info.includePaths): + + // The situation is: + // + // included.mk: VAR= value + // including.mk: VAR= value <-- you are here + // including.mk: VAR?= value <-- or here + // + // After including one or more files, the variable is either + // overwritten or defaulted with the same value as its + // guaranteed current value. All previous accesses to the + // variable were either in this file or in an included file. + s.onRedundant(mkline, prevWrites[len(prevWrites)-1]) + + case s.includePath.includedByOrEqualsAll(info.includePaths): + + // The situation is: + // + // including.mk: VAR= value + // included.mk: VAR?= value <-- you are here + // included.mk: VAR= value <-- or here + // + // A variable has been defined in an including file. + // The current line either has a default assignment or an + // unconditional assignment. This is common and fine. + // + // Except when this line has the same value as the guaranteed + // current value of the variable. Then it is redundant. + if info.vari.Constant() && info.vari.ConstantValue() == mkline.Value() { + s.onRedundant(prevWrites[len(prevWrites)-1], mkline) + } + } + } +} + +func (s *RedundantScope) handleVarUse(mkline MkLine) { + switch { + case mkline.IsVarassign(), mkline.IsCommentedVarassign(): + for _, varname := range mkline.DetermineUsedVariables() { + info := s.get(varname) + info.vari.Read(mkline) + info.lastAction = 1 + s.access(varname) + } + + case mkline.IsDirective(): + // TODO: Handle varuse for conditions and loops. + break + + case mkline.IsInclude(), mkline.IsSysinclude(): + // TODO: Handle VarUse for includes, which may reference variables. + break + + case mkline.IsDependency(): + // TODO: Handle VarUse for this case. + } +} + +// access returns the info for the given variable, creating it if necessary. +func (s *RedundantScope) get(varname string) *redundantScopeVarinfo { + info := s.vars[varname] + if info == nil { + v := NewVar(varname) + info = &redundantScopeVarinfo{v, nil, 0} + s.vars[varname] = info + } + return info +} + +// access records the current file location, to be used in later inclusion checks. +func (s *RedundantScope) access(varname string) { + info := s.vars[varname] + info.includePaths = append(info.includePaths, s.includePath.copy()) +} + +func (s *RedundantScope) onRedundant(redundant MkLine, because MkLine) { + if redundant.Op() == opAssignDefault { + redundant.Notef("Default assignment of %s has no effect because of %s.", + because.Varname(), redundant.RefTo(because)) + } else { + redundant.Notef("Definition of %s is redundant because of %s.", + because.Varname(), redundant.RefTo(because)) + } +} + +func (s *RedundantScope) onOverwrite(overwritten MkLine, by MkLine) { + overwritten.Warnf("Variable %s is overwritten in %s.", + overwritten.Varname(), overwritten.RefTo(by)) + G.Explain( + "The variable definition in this line does not have an effect since", + "it is overwritten elsewhere.", + "This typically happens because of a typo (writing = instead of +=)", + "or because the line that overwrites", + "is in another file that is used by several packages.") +} + +// includePath remembers the whole sequence of included files, +// such as Makefile includes ../../a/b/buildlink3.mk includes ../../c/d/buildlink3.mk. +// +// This information is used by the RedundantScope to decide whether +// one of two variable assignments is redundant. Two assignments can +// only be redundant if one location includes the other. +type includePath struct { + files []string +} + +func (p *includePath) push(filename string) { + p.files = append(p.files, filename) +} + +func (p *includePath) popUntil(filename string) { + for p.files[len(p.files)-1] != filename { + p.files = p.files[:len(p.files)-1] + } +} + +func (p *includePath) includes(other includePath) bool { + for i, filename := range p.files { + if i >= len(other.files) || other.files[i] != filename { + return false + } + } + return len(p.files) < len(other.files) +} + +func (p *includePath) includesOrEqualsAll(others []includePath) bool { + for _, other := range others { + if !(p.includes(other) || p.equals(other)) { + return false + } + } + return true +} + +func (p *includePath) includedByOrEqualsAll(others []includePath) bool { + for _, other := range others { + if !(other.includes(*p) || p.equals(other)) { + return false + } + } + return true +} + +func (p *includePath) equals(other includePath) bool { + if len(p.files) != len(other.files) { + return false + } + for i, filename := range p.files { + if other.files[i] != filename { + return false + } + } + return true +} + +func (p *includePath) copy() includePath { + return includePath{append([]string(nil), p.files...)} +} diff --git a/pkgtools/pkglint/files/redundantscope_test.go b/pkgtools/pkglint/files/redundantscope_test.go new file mode 100644 index 00000000000..ebe18bacfae --- /dev/null +++ b/pkgtools/pkglint/files/redundantscope_test.go @@ -0,0 +1,1284 @@ +package pkglint + +import "gopkg.in/check.v1" + +// In a single file, five variables get a default value and are later overridden +// with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_default(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT?=\tvalue", + "ASSIGN?=\tvalue", + "APPEND?=\tvalue", + "EVAL?=\tvalue", + "SHELL?=\tvalue", + "", + "DEFAULT?=\tvalue", + "ASSIGN=\tvalue", + "APPEND+=\tvalue", + "EVAL:=\tvalue", + "SHELL!=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "NOTE: file.mk:8: Definition of ASSIGN is redundant because of line 2.", + "WARN: file.mk:4: Variable EVAL is overwritten in line 10.") + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get assigned are value and are later overridden +// with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_assign(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT=\tvalue", + "ASSIGN=\tvalue", + "APPEND=\tvalue", + "EVAL=\tvalue", + "SHELL=\tvalue", + "", + "DEFAULT?=\tvalue", + "ASSIGN=\tvalue", + "APPEND+=\tvalue", + "EVAL:=\tvalue", + "SHELL!=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "NOTE: file.mk:8: Definition of ASSIGN is redundant because of line 2.", + "WARN: file.mk:4: Variable EVAL is overwritten in line 10.") + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get appended a value and are later overridden +// with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_append(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT+=\tvalue", + "ASSIGN+=\tvalue", + "APPEND+=\tvalue", + "EVAL+=\tvalue", + "SHELL+=\tvalue", + "", + "DEFAULT?=\tvalue", + "ASSIGN=\tvalue", + "APPEND+=\tvalue", + "EVAL:=\tvalue", + "SHELL!=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "WARN: file.mk:2: Variable ASSIGN is overwritten in line 8.", + "WARN: file.mk:4: Variable EVAL is overwritten in line 10.") + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get assigned a value using the := operator, +// which in this simple case is equivalent to the = operator. The variables are +// later overridden with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_eval(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT:=\tvalue", + "ASSIGN:=\tvalue", + "APPEND:=\tvalue", + "EVAL:=\tvalue", + "SHELL:=\tvalue", + "", + "DEFAULT?=\tvalue", + "ASSIGN=\tvalue", + "APPEND+=\tvalue", + "EVAL:=\tvalue", + "SHELL!=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "NOTE: file.mk:8: Definition of ASSIGN is redundant because of line 2.", + "WARN: file.mk:4: Variable EVAL is overwritten in line 10.") + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get assigned a value using the != operator, +// which runs a shell command. As of March 2019 pkglint doesn't try to evaluate +// the shell commands, therefore the variable values are unknown. The variables +// are later overridden using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_shell(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT!=\tvalue", + "ASSIGN!=\tvalue", + "APPEND!=\tvalue", + "EVAL!=\tvalue", + "SHELL!=\tvalue", + "", + "DEFAULT?=\tvalue", + "ASSIGN=\tvalue", + "APPEND+=\tvalue", + "EVAL:=\tvalue", + "SHELL!=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "WARN: file.mk:2: Variable ASSIGN is overwritten in line 8.", + "WARN: file.mk:4: Variable EVAL is overwritten in line 10.") + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get a default value and are later overridden +// with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_default_ref(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT?=\t${OTHER}", + "ASSIGN?=\t${OTHER}", + "APPEND?=\t${OTHER}", + "EVAL?=\t${OTHER}", + "SHELL?=\t${OTHER}", + "", + "DEFAULT?=\t${OTHER}", + "ASSIGN=\t${OTHER}", + "APPEND+=\t${OTHER}", + "EVAL:=\t${OTHER}", + "SHELL!=\t${OTHER}") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "NOTE: file.mk:8: Definition of ASSIGN is redundant because of line 2.") + // TODO: "4: is overwritten later", + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get assigned are value and are later overridden +// with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_assign_ref(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT=\t${OTHER}", + "ASSIGN=\t${OTHER}", + "APPEND=\t${OTHER}", + "EVAL=\t${OTHER}", + "SHELL=\t${OTHER}", + "", + "DEFAULT?=\t${OTHER}", + "ASSIGN=\t${OTHER}", + "APPEND+=\t${OTHER}", + "EVAL:=\t${OTHER}", + "SHELL!=\t${OTHER}") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "NOTE: file.mk:8: Definition of ASSIGN is redundant because of line 2.") + // TODO: "4: is overwritten later", + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get appended a value and are later overridden +// with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_append_ref(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT+=\t${OTHER}", + "ASSIGN+=\t${OTHER}", + "APPEND+=\t${OTHER}", + "EVAL+=\t${OTHER}", + "SHELL+=\t${OTHER}", + "", + "DEFAULT?=\t${OTHER}", + "ASSIGN=\t${OTHER}", + "APPEND+=\t${OTHER}", + "EVAL:=\t${OTHER}", + "SHELL!=\t${OTHER}") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "WARN: file.mk:2: Variable ASSIGN is overwritten in line 8.") + // TODO: "4: is overwritten later", + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get assigned a value using the := operator, +// which in this simple case is equivalent to the = operator. The variables are +// later overridden with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_eval_ref(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT:=\t${OTHER}", + "ASSIGN:=\t${OTHER}", + "APPEND:=\t${OTHER}", + "EVAL:=\t${OTHER}", + "SHELL:=\t${OTHER}", + "", + "DEFAULT?=\t${OTHER}", + "ASSIGN=\t${OTHER}", + "APPEND+=\t${OTHER}", + "EVAL:=\t${OTHER}", + "SHELL!=\t${OTHER}") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "NOTE: file.mk:8: Definition of ASSIGN is redundant because of line 2.") + // TODO: "4: is overwritten later", + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get assigned a value using the != operator, +// which runs a shell command. As of March 2019 pkglint doesn't try to evaluate +// the shell commands, therefore the variable values are unknown. The variables +// are later overridden using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_shell_ref(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT!=\t${OTHER}", + "ASSIGN!=\t${OTHER}", + "APPEND!=\t${OTHER}", + "EVAL!=\t${OTHER}", + "SHELL!=\t${OTHER}", + "", + "DEFAULT?=\t${OTHER}", + "ASSIGN=\t${OTHER}", + "APPEND+=\t${OTHER}", + "EVAL:=\t${OTHER}", + "SHELL!=\t${OTHER}") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "WARN: file.mk:2: Variable ASSIGN is overwritten in line 8.") + // TODO: "4: is overwritten later", + // TODO: "5: is overwritten later" +} + +func (s *Suite) Test_RedundantScope__after_including_same_value(c *check.C) { + t := s.Init(c) + + // Only test the ?=, = and += operators since the others are ignored, + // as of March 2019. + include, get := t.SetUpHierarchy() + include("including.mk", + include("included.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg?= ${OTHER}", + "VAR.def.app?= ${OTHER}", + "VAR.asg.def= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app= ${OTHER}", + "VAR.app.def+= ${OTHER}", + "VAR.app.asg+= ${OTHER}", + "VAR.app.app+= ${OTHER}"), + "VAR.def.def?= ${OTHER}", + "VAR.def.asg= ${OTHER}", + "VAR.def.app+= ${OTHER}", + "VAR.asg.def?= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app+= ${OTHER}", + "VAR.app.def?= ${OTHER}", + "VAR.app.asg= ${OTHER}", + "VAR.app.app+= ${OTHER}") + mklines := get("including.mk") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: including.mk:2: Default assignment of VAR.def.def has no effect because of included.mk:1.", + "NOTE: including.mk:3: Definition of VAR.def.asg is redundant because of included.mk:2.", + // VAR.def.app defines a default value and then appends to it. This is a common pattern. + // Appending the same value feels redundant but probably doesn't happen in practice. + // If it does, there should be a note for it. + "NOTE: including.mk:5: Default assignment of VAR.asg.def has no effect because of included.mk:4.", + "NOTE: including.mk:6: Definition of VAR.asg.asg is redundant because of included.mk:5.", + // VAR.asg.app defines a variable and later appends to it. This is a common pattern. + // Appending the same value feels redundant but probably doesn't happen in practice. + // If it does, there should be a note for it. + "NOTE: including.mk:8: Default assignment of VAR.app.def has no effect because of included.mk:7.", + // VAR.app.asg first appends and then overwrites. This might be a mistake. + // TODO: Find out whether this case happens in actual pkgsrc and if it's accidental. + // VAR.app.app first appends and then appends one more. This is a common pattern. + ) +} + +func (s *Suite) Test_RedundantScope__after_including_different_value(c *check.C) { + t := s.Init(c) + + // Only test the ?=, = and += operators since the others are ignored, + // as of March 2019. + include, get := t.SetUpHierarchy() + include("including.mk", + include("included.mk", + "VAR.def.def?= ${VALUE}", + "VAR.def.asg?= ${VALUE}", + "VAR.def.app?= ${VALUE}", + "VAR.asg.def= ${VALUE}", + "VAR.asg.asg= ${VALUE}", + "VAR.asg.app= ${VALUE}", + "VAR.app.def+= ${VALUE}", + "VAR.app.asg+= ${VALUE}", + "VAR.app.app+= ${VALUE}"), + "VAR.def.def?= ${OTHER}", + "VAR.def.asg= ${OTHER}", + "VAR.def.app+= ${OTHER}", + "VAR.asg.def?= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app+= ${OTHER}", + "VAR.app.def?= ${OTHER}", + "VAR.app.asg= ${OTHER}", + "VAR.app.app+= ${OTHER}") + mklines := get("including.mk") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: including.mk:2: Default assignment of VAR.def.def has no effect because of included.mk:1.", + "NOTE: including.mk:5: Default assignment of VAR.asg.def has no effect because of included.mk:4.", + "NOTE: including.mk:8: Default assignment of VAR.app.def has no effect because of included.mk:7.") +} + +func (s *Suite) Test_RedundantScope__before_including_same_value(c *check.C) { + t := s.Init(c) + + // Only test the ?=, = and += operators since the others are ignored, + // as of March 2019. + include, get := t.SetUpHierarchy() + include("including.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg?= ${OTHER}", + "VAR.def.app?= ${OTHER}", + "VAR.asg.def= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app= ${OTHER}", + "VAR.app.def+= ${OTHER}", + "VAR.app.asg+= ${OTHER}", + "VAR.app.app+= ${OTHER}", + include("included.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg= ${OTHER}", + "VAR.def.app+= ${OTHER}", + "VAR.asg.def?= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app+= ${OTHER}", + "VAR.app.def?= ${OTHER}", + "VAR.app.asg= ${OTHER}", + "VAR.app.app+= ${OTHER}")) + mklines := get("including.mk") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: including.mk:1: Default assignment of VAR.def.def has no effect because of included.mk:1.", + "NOTE: including.mk:2: Default assignment of VAR.def.asg has no effect because of included.mk:2.", + "NOTE: including.mk:4: Definition of VAR.asg.def is redundant because of included.mk:4.", + "NOTE: including.mk:5: Definition of VAR.asg.asg is redundant because of included.mk:5.", + "WARN: including.mk:8: Variable VAR.app.asg is overwritten in included.mk:8.") +} + +func (s *Suite) Test_RedundantScope__before_including_different_value(c *check.C) { + t := s.Init(c) + + // Only test the ?=, = and += operators since the others are ignored, + // as of March 2019. + include, get := t.SetUpHierarchy() + include("including.mk", + "VAR.def.def?= ${VALUE}", + "VAR.def.asg?= ${VALUE}", + "VAR.def.app?= ${VALUE}", + "VAR.asg.def= ${VALUE}", + "VAR.asg.asg= ${VALUE}", + "VAR.asg.app= ${VALUE}", + "VAR.app.def+= ${VALUE}", + "VAR.app.asg+= ${VALUE}", + "VAR.app.app+= ${VALUE}", + include("included.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg= ${OTHER}", + "VAR.def.app+= ${OTHER}", + "VAR.asg.def?= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app+= ${OTHER}", + "VAR.app.def?= ${OTHER}", + "VAR.app.asg= ${OTHER}", + "VAR.app.app+= ${OTHER}")) + mklines := get("including.mk") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "WARN: including.mk:2: Variable VAR.def.asg is overwritten in included.mk:2.", + "WARN: including.mk:5: Variable VAR.asg.asg is overwritten in included.mk:5.", + "WARN: including.mk:8: Variable VAR.app.asg is overwritten in included.mk:8.") +} + +func (s *Suite) Test_RedundantScope__independent_same_value(c *check.C) { + t := s.Init(c) + + // Only test the ?=, = and += operators since the others are ignored, + // as of March 2019. + include, get := t.SetUpHierarchy() + include("including.mk", + include("included1.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg?= ${OTHER}", + "VAR.def.app?= ${OTHER}", + "VAR.asg.def= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app= ${OTHER}", + "VAR.app.def+= ${OTHER}", + "VAR.app.asg+= ${OTHER}", + "VAR.app.app+= ${OTHER}"), + include("included2.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg= ${OTHER}", + "VAR.def.app+= ${OTHER}", + "VAR.asg.def?= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app+= ${OTHER}", + "VAR.app.def?= ${OTHER}", + "VAR.app.asg= ${OTHER}", + "VAR.app.app+= ${OTHER}")) + mklines := get("including.mk") + + NewRedundantScope().Check(mklines) + + // Since the two included files are independent, there cannot be any + // redundancies between them. These redundancies can only be discovered + // when one of them includes the other. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__independent_different_value(c *check.C) { + t := s.Init(c) + + // Only test the ?=, = and += operators since the others are ignored, + // as of March 2019. + include, get := t.SetUpHierarchy() + include("including.mk", + include("included1.mk", + "VAR.def.def?= ${VALUE}", + "VAR.def.asg?= ${VALUE}", + "VAR.def.app?= ${VALUE}", + "VAR.asg.def= ${VALUE}", + "VAR.asg.asg= ${VALUE}", + "VAR.asg.app= ${VALUE}", + "VAR.app.def+= ${VALUE}", + "VAR.app.asg+= ${VALUE}", + "VAR.app.app+= ${VALUE}"), + include("included2.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg= ${OTHER}", + "VAR.def.app+= ${OTHER}", + "VAR.asg.def?= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app+= ${OTHER}", + "VAR.app.def?= ${OTHER}", + "VAR.app.asg= ${OTHER}", + "VAR.app.app+= ${OTHER}")) + mklines := get("including.mk") + + NewRedundantScope().Check(mklines) + + // Since the two included files are independent, there cannot be any + // redundancies between them. Redundancies can only be discovered + // when one of them includes the other. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__file_hierarchy(c *check.C) { + t := s.Init(c) + + include, get := t.SetUpHierarchy() + + include("including.mk", + include("other.mk", + "VAR= other"), + include("module.mk", + "VAR= module", + include("version.mk", + "VAR= version"), + include("env.mk", + "VAR= env"))) + + NewRedundantScope().Check(get("including.mk")) + + // No output since the included files are independent. + t.CheckOutputEmpty() + + NewRedundantScope().Check(get("other.mk")) + + // No output since the file by itself in neither redundant nor + // does it include any other file. + t.CheckOutputEmpty() + + NewRedundantScope().Check(get("module.mk")) + + // No warning about env.mk because it is independent from version.mk. + // Pkglint only produces warnings when it is very sure that the variable + // definition is really redundant in all cases. + // + // One reason to not warn is that at the point where env.mk is evaluated, + // version.mk had last written to the variable. Since version.mk is + // independent from env.mk, there is nothing redundant here. + // Pkglint doesn't do this, but it could. + // + // Another reason not to warn is that all locations where the variable has + // ever been accessed are saved. And if the current location neither includes + // all of the others nor is included by all of the others, there is at least + // one access that is in an unrelated file. This is what pkglint does. + t.CheckOutputLines( + "WARN: module.mk:1: Variable VAR is overwritten in version.mk:1.") +} + +// FIXME: Continue the systematic redundancy tests. +// +// A test where the operators = and += define a variable that afterwards +// is assigned the same value using the ?= operator. +// +// Tests where the variables refer to other variables. These variables may +// be read and written between the relevant assignments. +// +// Tests where the variables are defined conditionally using .if, .else, .endif. +// +// Tests where the variables are defined in a .for loop that might not be +// evaluated at all. +// +// Tests where files are included conditionally and additionally have conditional +// sections, arbitrarily nested. +// +// Tests that show how to suppress the notes about redundant assignments +// and overwritten variables. The explanation must be helpful. +// +// Tests for dynamic variable assignments. For example BUILD_DIRS.NetBSD may +// be modified by any assignment of the form BUILD_DIRS.${var} or even ${var}. +// Without further analysis, pkglint cannot report redundancy warnings for any +// package that uses such variable assignments. + +func (s *Suite) Test_RedundantScope__override_after_including(c *check.C) { + t := s.Init(c) + t.CreateFileLines("included.mk", + "OVERRIDE=\tprevious value", + "REDUNDANT=\tredundant") + t.CreateFileLines("including.mk", + ".include \"included.mk\"", + "OVERRIDE=\toverridden value", + "REDUNDANT=\tredundant") + t.Chdir(".") + mklines := t.LoadMkInclude("including.mk") + + // XXX: The warnings from here are not in the same order as the other warnings. + // XXX: There may be some warnings for the same file separated by warnings for other files. + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: including.mk:3: Definition of REDUNDANT is redundant because of included.mk:2.") +} + +func (s *Suite) Test_RedundantScope__redundant_assign_after_including(c *check.C) { + t := s.Init(c) + t.CreateFileLines("included.mk", + "REDUNDANT=\tredundant") + t.CreateFileLines("including.mk", + ".include \"included.mk\"", + "REDUNDANT=\tredundant") + t.Chdir(".") + mklines := t.LoadMkInclude("including.mk") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: including.mk:2: Definition of REDUNDANT is redundant because of included.mk:1.") +} + +func (s *Suite) Test_RedundantScope__override_in_Makefile_after_including(c *check.C) { + t := s.Init(c) + t.CreateFileLines("module.mk", + "VAR=\tvalue ${OTHER}", + "VAR?=\tvalue ${OTHER}", + "VAR=\tnew value") + t.CreateFileLines("Makefile", + ".include \"module.mk\"", + "VAR=\tthe package may overwrite variables from other files") + t.Chdir(".") + + mklines := t.LoadMkInclude("Makefile") + + // XXX: The warnings from here are not in the same order as the other warnings. + // XXX: There may be some warnings for the same file separated by warnings for other files. + NewRedundantScope().Check(mklines) + + // No warning for VAR=... in Makefile since it makes sense to have common files + // with default values for variables, overriding some of them in each package. + t.CheckOutputLines( + "NOTE: module.mk:2: Default assignment of VAR has no effect because of line 1.", + "WARN: module.mk:2: Variable VAR is overwritten in line 3.") +} + +func (s *Suite) Test_RedundantScope__default_value_definitely_unused(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR=\tvalue ${OTHER}", + "VAR?=\tdifferent value") + + NewRedundantScope().Check(mklines) + + // A default assignment after an unconditional assignment is redundant. + // Even more so when the variable is not used between the two assignments. + t.CheckOutputLines( + "NOTE: module.mk:2: Default assignment of VAR has no effect because of line 1.") +} + +func (s *Suite) Test_RedundantScope__default_value_overridden(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR?=\tdefault value", + "VAR=\toverridden value") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "WARN: module.mk:1: Variable VAR is overwritten in line 2.") +} + +func (s *Suite) Test_RedundantScope__overwrite_same_value(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR=\tvalue ${OTHER}", + "VAR=\tvalue ${OTHER}") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: module.mk:2: Definition of VAR is redundant because of line 1.") +} + +func (s *Suite) Test_RedundantScope__conditional_overwrite(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR=\tdefault", + ".if ${OPSYS} == NetBSD", + "VAR=\topsys", + ".endif") + + NewRedundantScope().Check(mklines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__overwrite_inside_conditional(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR=\tgeneric", + ".if ${OPSYS} == NetBSD", + "VAR=\tignored", + "VAR=\toverwritten", + ".endif") + + NewRedundantScope().Check(mklines) + + // TODO: expected a warning "WARN: module.mk:4: line 3 is ignored" + // Since line 3 and line 4 are in the same basic block, line 3 is definitely ignored. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__conditionally_include(c *check.C) { + t := s.Init(c) + t.CreateFileLines("module.mk", + "VAR=\tgeneric", + ".if ${OPSYS} == NetBSD", + ". include \"included.mk\"", + ".endif") + t.CreateFileLines("included.mk", + "VAR=\tignored", + "VAR=\toverwritten") + mklines := t.LoadMkInclude("module.mk") + + NewRedundantScope().Check(mklines) + + // TODO: expected a warning "WARN: module.mk:4: line 3 is ignored" + // Since line 3 and line 4 are in the same basic block, line 3 is definitely ignored. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__conditional_default(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR=\tdefault", + ".if ${OPSYS} == NetBSD", + "VAR?=\topsys", + ".endif") + + NewRedundantScope().Check(mklines) + + // TODO: WARN: module.mk:3: The value \"opsys\" will never be assigned to VAR because it is defined unconditionally in line 1. + t.CheckOutputEmpty() +} + +// These warnings are precise and accurate since the value of VAR is not used between line 2 and 4. +func (s *Suite) Test_RedundantScope__overwrite_same_variable_different_value(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "OTHER=\tvalue before", + "VAR=\tvalue ${OTHER}", + "OTHER=\tvalue after", + "VAR=\tvalue ${OTHER}") + + NewRedundantScope().Check(mklines) + + // Strictly speaking, line 1 is redundant because OTHER is not evaluated + // at load time and then immediately overwritten in line 3. If the operator + // in line 2 were a := instead of a =, the situation would be clear. + // Pkglint doesn't warn about the redundancy in line 1 because it prefers + // to omit warnings instead of giving wrong advice. + t.CheckOutputLines( + "NOTE: module.mk:4: Definition of VAR is redundant because of line 2.") +} + +func (s *Suite) Test_RedundantScope__overwrite_different_value_used_between(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "OTHER=\tvalue before", + "VAR=\tvalue ${OTHER}", + + // VAR is used here at load time, therefore it must be defined at this point. + // At this point, VAR uses the \"before\" value of OTHER. + "RESULT1:=\t${VAR}", + + "OTHER=\tvalue after", + + // VAR is used here again at load time, this time using the \"after\" value of OTHER. + "RESULT2:=\t${VAR}", + + // Still this definition is redundant. + "VAR=\tvalue ${OTHER}") + + NewRedundantScope().Check(mklines) + + // There is nothing redundant here. Each write is followed by a + // corresponding read, except for the last one. That is ok though + // because in pkgsrc the last action of a package is to include + // bsd.pkg.mk, which reads almost all variables. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__procedure_call_to_noop(c *check.C) { + t := s.Init(c) + + include, get := t.SetUpHierarchy() + include("mk/pthread.buildlink3.mk", + "CHECK_BUILTIN.pthread:= yes", + include("pthread.builtin.mk", + "# Nothing happens here."), + "CHECK_BUILTIN.pthread:= no") + + NewRedundantScope().Check(get("mk/pthread.buildlink3.mk")) + + t.CheckOutputLines( + "WARN: mk/pthread.buildlink3.mk:1: Variable CHECK_BUILTIN.pthread is overwritten in line 3.") +} + +func (s *Suite) Test_RedundantScope__procedure_call_implemented(c *check.C) { + t := s.Init(c) + + include, get := t.SetUpHierarchy() + include("mk/pthread.buildlink3.mk", + "CHECK_BUILTIN.pthread:= yes", + include("pthread.builtin.mk", + "CHECK_BUILTIN.pthread?= no", + ".if !empty(CHECK_BUILTIN.pthread:M[Nn][Oo])", + ".endif"), + "CHECK_BUILTIN.pthread:= no") + + NewRedundantScope().Check(get("mk/pthread.buildlink3.mk")) + + // This test is a bit unrealistic. It wrongly assumes that all files from + // an .include directive are actually included by pkglint. + // + // See Package.readMakefile/handleIncludeLine/skip. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__procedure_call_implemented_package(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.SetUpPackage("devel/gettext-lib") + t.SetUpPackage("x11/Xaos", + ".include \"../../devel/gettext-lib/buildlink3.mk\"") + t.CreateFileLines("devel/gettext-lib/builtin.mk", + MkRcsID, + "", + ".include \"../../mk/bsd.fast.prefs.mk\"", + "", + "CHECK_BUILTIN.gettext?=\tno", + ".if !empty(CHECK_BUILTIN.gettext:M[nN][oO])", + ".endif") + t.CreateFileLines("devel/gettext-lib/buildlink3.mk", + MkRcsID, + "CHECK_BUILTIN.gettext:=\tyes", + ".include \"builtin.mk\"", + "CHECK_BUILTIN.gettext:=\tno") + G.Pkgsrc.LoadInfrastructure() + + // Checking x11/Xaos instead of devel/gettext-lib avoids warnings + // about the minimal buildlink3.mk file. + G.Check(t.File("x11/Xaos")) + + // There is nothing redundant here. + // Up to March 2019, pkglint didn't pass the correct pathnames to Package.included, + // which triggered a wrong note here. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__procedure_call_infrastructure(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("x11/alacarte", + ".include \"../../mk/pthread.buildlink3.mk\"") + t.CreateFileLines("mk/pthread.buildlink3.mk", + MkRcsID, + "CHECK_BUILTIN.gettext:=\tyes", + ".include \"pthread.builtin.mk\"", + "CHECK_BUILTIN.gettext:=\tno") + t.CreateFileLines("mk/pthread.builtin.mk", + MkRcsID, + "CHECK_BUILTIN.gettext?=\tno", + ".if !empty(CHECK_BUILTIN.gettext:M[nN][oO])", + ".endif") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("x11/alacarte")) + + // There is nothing redundant here. + // + // 1. pthread.buildlink3.mk sets the variable + // 2. pthread.builtin.mk assigns it a default value + // (which is common practice) + // 3. pthread.builtin.mk then reads it + // (which marks the next write as non-redundant) + // 4. pthread.buildlink3.mk sets the variable again + // (this is considered neither overwriting nor redundant) + // + // Up to March 2019, pkglint complained: + // + // WARN: ~/mk/pthread.buildlink3.mk:2: + // Variable CHECK_BUILTIN.gettext is overwritten in line 4. + // + // The cause for the warning is that when including files from the + // infrastructure, pkglint only includes the outermost level of files. + // If an infrastructure file includes another infrastructure file, + // pkglint skips that, for performance reasons. + // + // This optimization effectively made the .include for pthread.builtin.mk + // a no-op, therefore it was correct to issue a warning here. + // + // Since this warning is wrong, in March 2019 another special rule has + // been added to Package.readMakefile.handleIncludeLine.skip saying that + // including a buildlink3.mk file also includes the corresponding + // builtin.mk file. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__shell_and_eval(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR:=\tvalue ${OTHER}", + "VAR!=\tvalue ${OTHER}") + + NewRedundantScope().Check(mklines) + + // As of November 2018, pkglint doesn't check redundancies that involve the := or != operators. + // + // What happens here is: + // + // Line 1 evaluates OTHER at load time. + // Line 1 assigns its value to VAR. + // Line 2 evaluates OTHER at load time. + // Line 2 passes its value through the shell and assigns the result to VAR. + // + // Since VAR is defined in line 1, not used afterwards and overwritten in line 2, it is redundant. + // Well, not quite, because evaluating ${OTHER} might have side-effects from :sh or ::= modifiers, + // but these are so rare that they are frowned upon and are not considered by pkglint. + // + // Expected result: + // WARN: module.mk:2: Previous definition of VAR in line 1 is unused. + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__shell_and_eval_literal(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR:=\tvalue", + "VAR!=\tvalue") + + NewRedundantScope().Check(mklines) + + // Even when := is used with a literal value (which is usually + // only done for procedure calls), the shell evaluation can have + // so many different side effects that pkglint cannot reliably + // help in this situation. + // + // TODO: Why not? The evaluation in line 1 is trivial to analyze. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__included_OPSYS_variable(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../category/dependency/buildlink3.mk\"", + "CONFIGURE_ARGS+=\tone", + "CONFIGURE_ARGS=\ttwo", + "CONFIGURE_ARGS+=\tthree") + t.SetUpPackage("category/dependency") + t.CreateFileDummyBuildlink3("category/dependency/buildlink3.mk") + t.CreateFileLines("category/dependency/builtin.mk", + MkRcsID, + "CONFIGURE_ARGS.Darwin+=\tdarwin") + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:21: Variable CONFIGURE_ARGS is overwritten in line 22.") +} + +func (s *Suite) Test_RedundantScope__if_then_else(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("if-then-else.mk", + MkRcsID, + ".if exists(${FILE})", + "OS=\tNetBSD", + ".else", + "OS=\tOTHER", + ".endif") + + NewRedundantScope().Check(mklines) + + // These two definitions are of course not redundant since they happen in + // different branches of the same .if statement. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__if_then_else_without_variable(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("if-then-else.mk", + MkRcsID, + ".if exists(/nonexistent)", + "IT=\texists", + ".else", + "IT=\tdoesn't exist", + ".endif") + + NewRedundantScope().Check(mklines) + + // These two definitions are of course not redundant since they happen in + // different branches of the same .if statement. + // Even though the .if condition does not refer to any variables, + // this still means that the variable assignments are conditional. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__append_then_default(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("append-then-default.mk", + MkRcsID, + "VAR+=\tvalue", + "VAR?=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: ~/append-then-default.mk:3: Default assignment of VAR has no effect because of line 2.") +} + +func (s *Suite) Test_RedundantScope__assign_then_default_in_same_file(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("assign-then-default.mk", + MkRcsID, + "VAR=\tvalue", + "VAR?=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: ~/assign-then-default.mk:3: " + + "Default assignment of VAR has no effect because of line 2.") +} + +func (s *Suite) Test_RedundantScope__eval_then_eval(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("filename.mk", + MkRcsID, + "VAR:=\tvalue", + "VAR:=\tvalue", + "VAR:=\tother") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "WARN: ~/filename.mk:2: Variable VAR is overwritten in line 3.", + "WARN: ~/filename.mk:3: Variable VAR is overwritten in line 4.") +} + +func (s *Suite) Test_RedundantScope__shell_then_assign(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("filename.mk", + MkRcsID, + "VAR!=\techo echo", + "VAR=\techo echo") + + NewRedundantScope().Check(mklines) + + // Although the two variable assignments look very similar, they do + // something entirely different. The first executes the echo command, + // and the second just assigns a string. Therefore the actual variable + // values are different, and the second assignment is not redundant. + // It assigns a different value. Nevertheless, the shell command is + // redundant and can be removed since its result is never used. + t.CheckOutputLines( + "WARN: ~/filename.mk:2: Variable VAR is overwritten in line 3.") +} + +func (s *Suite) Test_RedundantScope__shell_then_read_then_assign(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("filename.mk", + MkRcsID, + "VAR!=\techo echo", + "OUTPUT:=${VAR}", + "VAR=\techo echo") + + NewRedundantScope().Check(mklines) + + // No warning since the value is used in-between. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__assign_then_default_in_included_file(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("assign-then-default.mk", + MkRcsID, + "VAR=\tvalue", + ".include \"included.mk\"") + t.CreateFileLines("included.mk", + MkRcsID, + "VAR?=\tvalue") + mklines := t.LoadMkInclude("assign-then-default.mk") + + NewRedundantScope().Check(mklines) + + // If assign-then-default.mk:2 is deleted, VAR still has the same value. + t.CheckOutputLines( + "NOTE: ~/assign-then-default.mk:2: Definition of VAR is redundant because of included.mk:2.") +} + +func (s *Suite) Test_RedundantScope__conditionally_included_file(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("including.mk", + MkRcsID, + "VAR=\tvalue", + ".if ${COND}", + ". include \"included.mk\"", + ".endif") + t.CreateFileLines("included.mk", + MkRcsID, + "VAR?=\tvalue") + mklines := t.LoadMkInclude("including.mk") + + NewRedundantScope().Check(mklines) + + // The assignment in including.mk:2 is only redundant if included.mk is actually included. + // Therefore both included.mk:2 nor including.mk:2 are relevant. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__procedure_parameters(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("mk/pkg-build-options.mk", + MkRcsID, + "USED:=\t${pkgbase}") + t.CreateFileLines("including.mk", + MkRcsID, + "pkgbase=\tpackage1", + ".include \"mk/pkg-build-options.mk\"", + "", + "pkgbase=\tpackage2", + ".include \"mk/pkg-build-options.mk\"", + "", + "pkgbase=\tpackage3", + ".include \"mk/pkg-build-options.mk\"") + mklines := t.LoadMkInclude("including.mk") + + NewRedundantScope().Check(mklines) + + // This variable is not overwritten since it is used in-between + // by the included file. + t.CheckOutputEmpty() +} + +// Branch coverage for info.vari.Constant(). The other tests typically +// make a variable non-constant by adding conditional assignments between +// .if/.endif. But there are other ways. The output of shell commands is +// unpredictable for pkglint (as of March 2019), therefore it treats these +// variables as non-constant. +func (s *Suite) Test_RedundantScope_handleVarassign__shell_followed_by_default(c *check.C) { + t := s.Init(c) + + include, get := t.SetUpHierarchy() + include("including.mk", + "VAR!= echo 'hello, world'", + include("included.mk", + "VAR?= hello world")) + + NewRedundantScope().Check(get("including.mk")) + + // If pkglint should ever learn to interpret simple shell commands, there + // should be a warning for including.mk:2 that the shell command generates + // the default value. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__overwrite_definition_from_included_file(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("included.mk", + MkRcsID, + "WRKSRC=\t${WRKDIR}/${PKGBASE}") + t.CreateFileLines("including.mk", + MkRcsID, + "SUBDIR=\t${WRKSRC}", + ".include \"included.mk\"", + "WRKSRC=\t${WRKDIR}/overwritten") + mklines := t.LoadMkInclude("including.mk") + + NewRedundantScope().Check(mklines) + + // Before pkglint 5.7.2 (2019-03-10), the above setup generated a warning: + // + // WARN: ~/included.mk:2: Variable WRKSRC is overwritten in including.mk:4. + // + // This warning is obviously wrong since the included file must never + // receive a warning. Of course this default definition may be overridden + // by the including file. + // + // The warning was generated because in including.mk:2 the variable WRKSRC + // was used for the first time. Back then, each variable had only a single + // include path. That include path marks where the variable is used and + // defined. + // + // The variable definition at included.mk didn't modify this include path. + // Therefore pkglint wrongly assumed that this variable was only ever + // accessed in including.mk and issued a warning. + // + // To fix this, the RedundantScope now remembers every access to the + // variable, and the redundancy warnings are only issued in cases where + // either all variable accesses are in files including the current file, + // or all variable accesses are in files included by the current file. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope_handleVarassign__conditional(c *check.C) { + t := s.Init(c) + + scope := NewRedundantScope() + mklines := t.NewMkLines("filename.mk", + MkRcsID, + "VAR=\tvalue", + ".if 1", + "VAR=\tconditional", + ".endif") + + mklines.ForEach(func(mkline MkLine) { + scope.Handle(mkline, mklines.indentation) + }) + + t.Check( + scope.get("VAR").vari.WriteLocations(), + deepEquals, + []MkLine{mklines.mklines[1], mklines.mklines[3]}) +} + +func (s *Suite) Test_includePath_includes(c *check.C) { + t := s.Init(c) + + path := func(locations ...string) includePath { + return includePath{locations} + } + + var ( + m = path("Makefile") + mc = path("Makefile", "Makefile.common") + mco = path("Makefile", "Makefile.common", "other.mk") + mo = path("Makefile", "other.mk") + ) + + t.Check(m.includes(m), equals, false) + + t.Check(m.includes(mc), equals, true) + t.Check(m.includes(mco), equals, true) + t.Check(mc.includes(mco), equals, true) + + t.Check(mc.includes(m), equals, false) + t.Check(mc.includes(mo), equals, false) + t.Check(mo.includes(mc), equals, false) +} + +func (s *Suite) Test_includePath_equals(c *check.C) { + t := s.Init(c) + + path := func(locations ...string) includePath { + return includePath{locations} + } + + var ( + m = path("Makefile") + mc = path("Makefile", "Makefile.common") + mco = path("Makefile", "Makefile.common", "other.mk") + mo = path("Makefile", "other.mk") + ) + + t.Check(m.equals(m), equals, true) + + t.Check(m.equals(mc), equals, false) + t.Check(m.equals(mco), equals, false) + t.Check(mc.equals(mco), equals, false) + + t.Check(mc.equals(m), equals, false) + t.Check(mc.equals(mo), equals, false) + t.Check(mo.equals(mc), equals, false) +} diff --git a/pkgtools/pkglint/files/shell.go b/pkgtools/pkglint/files/shell.go index 5315865f42b..b9892dc043d 100644 --- a/pkgtools/pkglint/files/shell.go +++ b/pkgtools/pkglint/files/shell.go @@ -530,14 +530,6 @@ func (scc *SimpleCommandChecker) handleCommandVariable() bool { if varuse := parser.VarUse(); varuse != nil && parser.EOF() { varname := varuse.varname - if tool := G.ToolByVarname(varname); tool != nil { - if tool.Validity == Nowhere { - scc.shline.mkline.Warnf("The %q tool is used but not added to USE_TOOLS.", tool.Name) - } - scc.shline.checkInstallCommand(shellword) - return true - } - if vartype := G.Pkgsrc.VariableType(varname); vartype != nil && vartype.basicType.name == "ShellCommand" { scc.shline.checkInstallCommand(shellword) return true diff --git a/pkgtools/pkglint/files/shell_test.go b/pkgtools/pkglint/files/shell_test.go index bc9177d5b29..dd2add2ef2c 100644 --- a/pkgtools/pkglint/files/shell_test.go +++ b/pkgtools/pkglint/files/shell_test.go @@ -154,7 +154,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { t.SetUpTool("unzip", "UNZIP_CMD", AtRunTime) test := func(shellCommand string) { - G.Mk = t.NewMkLines("filename", + G.Mk = t.NewMkLines("filename.mk", "\t"+shellCommand) shline := NewShellLine(G.Mk.mklines[0]) @@ -170,8 +170,8 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { test("uname=`uname`; echo $$uname; echo; ${PREFIX}/bin/command") t.CheckOutputLines( - "WARN: filename:1: Unknown shell command \"uname\".", - "WARN: filename:1: Please switch to \"set -e\" mode "+ + "WARN: filename.mk:1: Unknown shell command \"uname\".", + "WARN: filename.mk:1: Please switch to \"set -e\" mode "+ "before using a semicolon (after \"uname=`uname`\") to separate commands.") t.SetUpTool("echo", "", AtRunTime) @@ -180,35 +180,30 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { test("echo ${PKGNAME:Q}") // VucQuotPlain t.CheckOutputLines( - "WARN: filename:1: PKGNAME may not be used in this file; "+ - "it would be ok in Makefile, Makefile.* or *.mk.", - "NOTE: filename:1: The :Q operator isn't necessary for ${PKGNAME} here.") + "NOTE: filename.mk:1: The :Q operator isn't necessary for ${PKGNAME} here.") test("echo \"${CFLAGS:Q}\"") // VucQuotDquot t.CheckOutputLines( - "WARN: filename:1: The :Q modifier should not be used inside double quotes.", - "WARN: filename:1: CFLAGS may not be used in this file; "+ - "it would be ok in Makefile, Makefile.common, options.mk or *.mk.", - "WARN: filename:1: Please use ${CFLAGS:M*:Q} instead of ${CFLAGS:Q} "+ + "WARN: filename.mk:1: The :Q modifier should not be used inside double quotes.", + "WARN: filename.mk:1: Please use ${CFLAGS:M*:Q} instead of ${CFLAGS:Q} "+ "and make sure the variable appears outside of any quoting characters.") test("echo '${COMMENT:Q}'") // VucQuotSquot t.CheckOutputLines( - "WARN: filename:1: COMMENT may not be used in any file; it is a write-only variable.", - "WARN: filename:1: Please move ${COMMENT:Q} outside of any quoting characters.") + "WARN: filename.mk:1: Please move ${COMMENT:Q} outside of any quoting characters.") test("echo target=$@ exitcode=$$? '$$' \"\\$$\"") t.CheckOutputLines( - "WARN: filename:1: Please use \"${.TARGET}\" instead of \"$@\".", - "WARN: filename:1: The $? shell variable is often not available in \"set -e\" mode.") + "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".", + "WARN: filename.mk:1: The $? shell variable is often not available in \"set -e\" mode.") test("echo $$@") t.CheckOutputLines( - "WARN: filename:1: The $@ shell variable should only be used in double quotes.") + "WARN: filename.mk:1: The $@ shell variable should only be used in double quotes.") test("echo \"$$\"") // As seen by make(1); the shell sees: echo "$" @@ -233,8 +228,8 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { test("${RUN} subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\"") t.CheckOutputLines( - "WARN: filename:1: Double quotes inside backticks inside double quotes are error prone.", - "WARN: filename:1: The exitcode of \"unzip\" at the left of the | operator is ignored.") + "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.", + "WARN: filename.mk:1: The exitcode of \"unzip\" at the left of the | operator is ignored.") // From mail/thunderbird/Makefile, rev. 1.159 test("" + @@ -247,9 +242,9 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { "done") t.CheckOutputLines( - "WARN: filename:1: XPI_FILES is used but not defined.", - "WARN: filename:1: Double quotes inside backticks inside double quotes are error prone.", - "WARN: filename:1: The exitcode of \"${UNZIP_CMD}\" at the left of the | operator is ignored.") + "WARN: filename.mk:1: XPI_FILES is used but not defined.", + "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.", + "WARN: filename.mk:1: The exitcode of \"${UNZIP_CMD}\" at the left of the | operator is ignored.") // From x11/wxGTK28/Makefile test("" + @@ -262,25 +257,23 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { // TODO: Why is TOOLS_PATH.msgfmt not recognized? // At least, the warning should be more specific, mentioning USE_TOOLS. t.CheckOutputLines( - "WARN: filename:1: WRKSRC may not be used in this file; "+ - "it would be ok in Makefile, Makefile.* or *.mk.", - "WARN: filename:1: Unknown shell command \"[\".", - "WARN: filename:1: Unknown shell command \"${TOOLS_PATH.msgfmt}\".") + "WARN: filename.mk:1: Unknown shell command \"[\".", + "WARN: filename.mk:1: Unknown shell command \"${TOOLS_PATH.msgfmt}\".") test("@cp from to") t.CheckOutputLines( - "WARN: filename:1: The shell command \"cp\" should not be hidden.") + "WARN: filename.mk:1: The shell command \"cp\" should not be hidden.") test("-cp from to") t.CheckOutputLines( - "WARN: filename:1: Using a leading \"-\" to suppress errors is deprecated.") + "WARN: filename.mk:1: Using a leading \"-\" to suppress errors is deprecated.") test("-${MKDIR} deeply/nested/subdir") t.CheckOutputLines( - "WARN: filename:1: Using a leading \"-\" to suppress errors is deprecated.") + "WARN: filename.mk:1: Using a leading \"-\" to suppress errors is deprecated.") G.Pkg = NewPackage(t.File("category/pkgbase")) G.Pkg.Plist.Dirs["share/pkgbase"] = true @@ -292,15 +285,15 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { // the note should not appear then. t.CheckOutputLines( - "NOTE: filename:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" "+ + "NOTE: filename.mk:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" "+ "instead of \"${INSTALL_DATA_DIR}\".", - "WARN: filename:1: The INSTALL_*_DIR commands can only handle one directory at a time.") + "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") // A directory that is not found in the PLIST. test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/share/other") t.CheckOutputLines( - "NOTE: filename:1: You can use \"INSTALLATION_DIRS+= share/other\" instead of \"${INSTALL_DATA_DIR}\".") + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= share/other\" instead of \"${INSTALL_DATA_DIR}\".") G.Pkg = nil @@ -315,7 +308,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__strip(c *check.C) { t := s.Init(c) test := func(shellCommand string) { - G.Mk = t.NewMkLines("filename", + G.Mk = t.NewMkLines("filename.mk", "\t"+shellCommand) G.Mk.ForEach(func(mkline MkLine) { @@ -327,8 +320,8 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__strip(c *check.C) { test("${STRIP} executable") t.CheckOutputLines( - "WARN: filename:1: Unknown shell command \"${STRIP}\".", - "WARN: filename:1: STRIP is used but not defined.") + "WARN: filename.mk:1: Unknown shell command \"${STRIP}\".", + "WARN: filename.mk:1: STRIP is used but not defined.") t.SetUpVartypes() @@ -430,7 +423,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__implementation(c *check.C) t := s.Init(c) t.SetUpVartypes() - G.Mk = t.NewMkLines("filename", + G.Mk = t.NewMkLines("filename.mk", "# dummy") shline := NewShellLine(G.Mk.mklines[0]) @@ -445,13 +438,13 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__implementation(c *check.C) G.Mk.ForEach(func(mkline MkLine) { shline.CheckWord(text, false, RunTime) }) t.CheckOutputLines( - "WARN: filename:1: Unknown shell command \"echo\".") + "WARN: filename.mk:1: Unknown shell command \"echo\".") G.Mk.ForEach(func(mkline MkLine) { shline.CheckShellCommandLine(text) }) // No parse errors t.CheckOutputLines( - "WARN: filename:1: Unknown shell command \"echo\".") + "WARN: filename.mk:1: Unknown shell command \"echo\".") } func (s *Suite) Test_ShellLine_CheckShellCommandLine__dollar_without_variable(c *check.C) { @@ -459,7 +452,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__dollar_without_variable(c t.SetUpVartypes() t.SetUpTool("pax", "", AtRunTime) - G.Mk = t.NewMkLines("filename", + G.Mk = t.NewMkLines("filename.mk", "# dummy") shline := NewShellLine(G.Mk.mklines[0]) @@ -516,8 +509,7 @@ func (s *Suite) Test_ShellLine_CheckWord(c *check.C) { test("${COMMENT:Q}", true) - t.CheckOutputLines( - "WARN: dummy.mk:1: COMMENT may not be used in any file; it is a write-only variable.") + t.CheckOutputEmpty() test("\"${DISTINFO_FILE:Q}\"", true) @@ -541,7 +533,7 @@ func (s *Suite) Test_ShellLine_CheckWord(c *check.C) { func (s *Suite) Test_ShellLine_CheckWord__dollar_without_variable(c *check.C) { t := s.Init(c) - shline := t.NewShellLine("filename", 1, "# dummy") + shline := t.NewShellLine("filename.mk", 1, "# dummy") shline.CheckWord("/.*~$$//g", false, RunTime) // Typical argument to pax(1). @@ -552,7 +544,7 @@ func (s *Suite) Test_ShellLine_CheckWord__backslash_plus(c *check.C) { t := s.Init(c) t.SetUpTool("find", "FIND", AtRunTime) - shline := t.NewShellLine("filename", 1, "\tfind . -exec rm -rf {} \\+") + shline := t.NewShellLine("filename.mk", 1, "\tfind . -exec rm -rf {} \\+") shline.CheckShellCommandLine(shline.mkline.ShellCommand()) @@ -563,21 +555,21 @@ func (s *Suite) Test_ShellLine_CheckWord__backslash_plus(c *check.C) { func (s *Suite) Test_ShellLine_CheckWord__squot_dollar(c *check.C) { t := s.Init(c) - shline := t.NewShellLine("filename", 1, "\t'$") + shline := t.NewShellLine("filename.mk", 1, "\t'$") shline.CheckWord(shline.mkline.ShellCommand(), false, RunTime) // FIXME: Should be parsed correctly. Make passes the dollar through (probably), // and the shell parser should complain about the unfinished string literal. t.CheckOutputLines( - "WARN: filename:1: Internal pkglint error in ShTokenizer.ShAtom at \"$\" (quoting=s).", - "WARN: filename:1: Internal pkglint error in ShellLine.CheckWord at \"'$\" (quoting=s), rest: $") + "WARN: filename.mk:1: Internal pkglint error in ShTokenizer.ShAtom at \"$\" (quoting=s).", + "WARN: filename.mk:1: Internal pkglint error in ShellLine.CheckWord at \"'$\" (quoting=s), rest: $") } func (s *Suite) Test_ShellLine_CheckWord__dquot_dollar(c *check.C) { t := s.Init(c) - shline := t.NewShellLine("filename", 1, "\t\"$") + shline := t.NewShellLine("filename.mk", 1, "\t\"$") shline.CheckWord(shline.mkline.ShellCommand(), false, RunTime) @@ -589,12 +581,12 @@ func (s *Suite) Test_ShellLine_CheckWord__dquot_dollar(c *check.C) { func (s *Suite) Test_ShellLine_CheckWord__dollar_subshell(c *check.C) { t := s.Init(c) - shline := t.NewShellLine("filename", 1, "\t$$(echo output)") + shline := t.NewShellLine("filename.mk", 1, "\t$$(echo output)") shline.CheckWord(shline.mkline.ShellCommand(), false, RunTime) t.CheckOutputLines( - "WARN: filename:1: Invoking subshells via $(...) is not portable enough.") + "WARN: filename.mk:1: Invoking subshells via $(...) is not portable enough.") } func (s *Suite) Test_ShellLine_CheckWord__PKGMANDIR(c *check.C) { @@ -709,9 +701,9 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__echo(c *check.C) { echo := t.SetUpTool("echo", "ECHO", AtRunTime) echo.MustUseVarForm = true - G.Mk = t.NewMkLines("filename", + G.Mk = t.NewMkLines("filename.mk", "# dummy") - mkline := t.NewMkLine("filename", 3, "# dummy") + mkline := t.NewMkLine("filename.mk", 3, "# dummy") MkLineChecker{mkline}.checkText("echo \"hello, world\"") @@ -720,7 +712,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__echo(c *check.C) { NewShellLine(mkline).CheckShellCommandLine("echo \"hello, world\"") t.CheckOutputLines( - "WARN: filename:3: Please use \"${ECHO}\" instead of \"echo\".") + "WARN: filename.mk:3: Please use \"${ECHO}\" instead of \"echo\".") } func (s *Suite) Test_ShellLine_CheckShellCommandLine__shell_variables(c *check.C) { @@ -762,21 +754,21 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__shell_variables(c *check.C func (s *Suite) Test_ShellLine_checkInstallCommand(c *check.C) { t := s.Init(c) - G.Mk = t.NewMkLines("filename", + G.Mk = t.NewMkLines("filename.mk", "# dummy") G.Mk.target = "do-install" - shline := t.NewShellLine("filename", 1, "\tdummy") + shline := t.NewShellLine("filename.mk", 1, "\tdummy") shline.checkInstallCommand("sed") t.CheckOutputLines( - "WARN: filename:1: The shell command \"sed\" should not be used in the install phase.") + "WARN: filename.mk:1: The shell command \"sed\" should not be used in the install phase.") shline.checkInstallCommand("cp") t.CheckOutputLines( - "WARN: filename:1: ${CP} should not be used to install files.") + "WARN: filename.mk:1: ${CP} should not be used to install files.") } func (s *Suite) Test_splitIntoMkWords(c *check.C) { diff --git a/pkgtools/pkglint/files/substcontext.go b/pkgtools/pkglint/files/substcontext.go index 45fa1ac3327..5f30908c4e6 100644 --- a/pkgtools/pkglint/files/substcontext.go +++ b/pkgtools/pkglint/files/substcontext.go @@ -283,19 +283,27 @@ func (ctx *SubstContext) suggestSubstVars(mkline MkLine) { continue } + varop := sprintf("SUBST_VARS.%s%s%s", + ctx.id, + ifelseStr(hasSuffix(ctx.id, "+"), " ", ""), + ifelseStr(ctx.curr.seenVars, "+=", "=")) + fix := mkline.Autofix() - fix.Notef("The substitution command %q can be replaced with \"SUBST_VARS.%s+= %s\".", token, ctx.id, varname) + fix.Notef("The substitution command %q can be replaced with \"%s %s\".", + token, varop, varname) fix.Explain( "Replacing @VAR@ with ${VAR} is such a typical pattern that pkgsrc has built-in support for it,", "requiring only the variable name instead of the full sed command.") if mkline.VarassignComment() == "" && len(tokens) == 2 && tokens[0] == "-e" { // TODO: Extract the alignment computation somewhere else, so that it is generally available. alignBefore := tabWidth(mkline.ValueAlign()) - alignAfter := tabWidth(sprintf("SUBST_VARS.%s+=\t", ctx.id)) + alignAfter := tabWidth(varop + "\t") tabs := strings.Repeat("\t", imax((alignAfter-alignBefore)/8, 0)) - fix.Replace(mkline.Text, sprintf("SUBST_VARS.%s+=\t%s%s", ctx.id, tabs, varname)) + fix.Replace(mkline.Text, varop+"\t"+tabs+varname) } fix.Anyway() fix.Apply() + + ctx.curr.seenVars = true } } diff --git a/pkgtools/pkglint/files/substcontext_test.go b/pkgtools/pkglint/files/substcontext_test.go index f0eca45ee8c..3fb53eb3ea1 100644 --- a/pkgtools/pkglint/files/substcontext_test.go +++ b/pkgtools/pkglint/files/substcontext_test.go @@ -31,7 +31,7 @@ func (s *Suite) Test_SubstContext__incomplete(c *check.C) { t.CheckOutputLines( "NOTE: Makefile:13: The substitution command \"s,@PREFIX@,${PREFIX},g\" "+ - "can be replaced with \"SUBST_VARS.interp+= PREFIX\".", + "can be replaced with \"SUBST_VARS.interp= PREFIX\".", "WARN: Makefile:14: Incomplete SUBST block: SUBST_STAGE.interp missing.") } @@ -56,7 +56,7 @@ func (s *Suite) Test_SubstContext__complete(c *check.C) { t.CheckOutputLines( "NOTE: Makefile:13: The substitution command \"s,@PREFIX@,${PREFIX},g\" " + - "can be replaced with \"SUBST_VARS.p+= PREFIX\".") + "can be replaced with \"SUBST_VARS.p= PREFIX\".") } func (s *Suite) Test_SubstContext__OPSYSVARS(c *check.C) { @@ -78,7 +78,7 @@ func (s *Suite) Test_SubstContext__OPSYSVARS(c *check.C) { t.CheckOutputLines( "NOTE: Makefile:14: The substitution command \"s,@PREFIX@,${PREFIX},g\" " + - "can be replaced with \"SUBST_VARS.prefix+= PREFIX\".") + "can be replaced with \"SUBST_VARS.prefix= PREFIX\".") } func (s *Suite) Test_SubstContext__no_class(c *check.C) { @@ -390,7 +390,7 @@ func (s *Suite) Test_SubstContext_suggestSubstVars(c *check.C) { t.CheckOutputLines( "WARN: subst.mk:6: Please use ${SH:Q} instead of ${SH}.", "NOTE: subst.mk:6: The substitution command \"s,@SH@,${SH},g\" "+ - "can be replaced with \"SUBST_VARS.test+= SH\".", + "can be replaced with \"SUBST_VARS.test= SH\".", "NOTE: subst.mk:7: The substitution command \"s,@SH@,${SH:Q},g\" "+ "can be replaced with \"SUBST_VARS.test+= SH\".", "WARN: subst.mk:8: Please use ${SH:T:Q} instead of ${SH:T}.", @@ -416,9 +416,9 @@ func (s *Suite) Test_SubstContext_suggestSubstVars(c *check.C) { t.CheckOutputLines( "NOTE: subst.mk:6: The substitution command \"s,@SH@,${SH},g\" "+ - "can be replaced with \"SUBST_VARS.test+= SH\".", + "can be replaced with \"SUBST_VARS.test= SH\".", "AUTOFIX: subst.mk:6: Replacing \"SUBST_SED.test+=\\t-e s,@SH@,${SH},g\" "+ - "with \"SUBST_VARS.test+=\\tSH\".", + "with \"SUBST_VARS.test=\\tSH\".", "NOTE: subst.mk:7: The substitution command \"s,@SH@,${SH:Q},g\" "+ "can be replaced with \"SUBST_VARS.test+= SH\".", "AUTOFIX: subst.mk:7: Replacing \"SUBST_SED.test+=\\t-e s,@SH@,${SH:Q},g\" "+ @@ -441,6 +441,46 @@ func (s *Suite) Test_SubstContext_suggestSubstVars(c *check.C) { "with \"SUBST_VARS.test+=\\tSH\".") } +// If the SUBST_CLASS identifier ends with a plus, the generated code must +// use the correct assignment operator and be nicely formatted. +func (s *Suite) Test_SubstContext_suggestSubstVars__plus(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpTool("sh", "SH", AtRunTime) + + mklines := t.NewMkLines("subst.mk", + MkRcsID, + "", + "SUBST_CLASSES+=\t\tgtk+", + "SUBST_STAGE.gtk+ =\tpre-configure", + "SUBST_FILES.gtk+ =\tfilename", + "SUBST_SED.gtk+ +=\t-e s,@SH@,${SH:Q},g", + "SUBST_SED.gtk+ +=\t-e s,@SH@,${SH:Q},g") + + mklines.Check() + + t.CheckOutputLines( + "NOTE: subst.mk:6: The substitution command \"s,@SH@,${SH:Q},g\" "+ + "can be replaced with \"SUBST_VARS.gtk+ = SH\".", + "NOTE: subst.mk:7: The substitution command \"s,@SH@,${SH:Q},g\" "+ + "can be replaced with \"SUBST_VARS.gtk+ += SH\".") + + t.SetUpCommandLine("--show-autofix") + + mklines.Check() + + t.CheckOutputLines( + "NOTE: subst.mk:6: The substitution command \"s,@SH@,${SH:Q},g\" "+ + "can be replaced with \"SUBST_VARS.gtk+ = SH\".", + "AUTOFIX: subst.mk:6: Replacing \"SUBST_SED.gtk+ +=\\t-e s,@SH@,${SH:Q},g\" "+ + "with \"SUBST_VARS.gtk+ =\\tSH\".", + "NOTE: subst.mk:7: The substitution command \"s,@SH@,${SH:Q},g\" "+ + "can be replaced with \"SUBST_VARS.gtk+ += SH\".", + "AUTOFIX: subst.mk:7: Replacing \"SUBST_SED.gtk+ +=\\t-e s,@SH@,${SH:Q},g\" "+ + "with \"SUBST_VARS.gtk+ +=\\tSH\".") +} + // simulateSubstLines only tests some of the inner workings of SubstContext. // It is not realistic for all cases. If in doubt, use MkLines.Check. func simulateSubstLines(t *Tester, texts ...string) { diff --git a/pkgtools/pkglint/files/textproc/lexer.go b/pkgtools/pkglint/files/textproc/lexer.go index 3d22338be95..9687e101bb2 100644 --- a/pkgtools/pkglint/files/textproc/lexer.go +++ b/pkgtools/pkglint/files/textproc/lexer.go @@ -226,6 +226,9 @@ func (l *Lexer) Copy() *Lexer { return &Lexer{l.rest} } func (l *Lexer) Commit(other *Lexer) bool { l.rest = other.rest; return true } // NewByteSet creates a bit mask out of a string like "0-9A-Za-z_". +// To add an actual hyphen to the bit mask, write it as "---" +// (a range from hyphen to hyphen). +// // The bit mask can be used with Lexer.NextBytesSet. func NewByteSet(chars string) *ByteSet { var set ByteSet diff --git a/pkgtools/pkglint/files/trace/tracing.go b/pkgtools/pkglint/files/trace/tracing.go index 0f0556041d0..28b678d6459 100644 --- a/pkgtools/pkglint/files/trace/tracing.go +++ b/pkgtools/pkglint/files/trace/tracing.go @@ -131,6 +131,8 @@ func (t *Tracer) traceCall(args ...interface{}) func() { } // Result marks an argument as a result and is only logged when the function returns. +// +// Usage: defer trace.Call(arg1, arg2, tracing.Result(&result1), tracing.Result(&result2))() func (t *Tracer) Result(rv interface{}) Result { if reflect.ValueOf(rv).Kind() != reflect.Ptr { panic(fmt.Sprintf("Result must be called with a pointer to the result, not %#v.", rv)) diff --git a/pkgtools/pkglint/files/util.go b/pkgtools/pkglint/files/util.go index 296843f73df..049bde05263 100644 --- a/pkgtools/pkglint/files/util.go +++ b/pkgtools/pkglint/files/util.go @@ -349,32 +349,68 @@ func mkopSubst(s string, left bool, from string, right bool, to string, flags st // relpath returns the relative path from the directory "from" // to the filesystem entry "to". -func relpath(from, to string) string { +// +// The relative path is built by going from the "from" directory via the +// pkgsrc root to the "to" filename. This produces the form +// "../../category/package" that is found in DEPENDS and .include lines. +// +// Both from and to are interpreted relative to the current working directory, +// unless they are absolute paths. +// +// This function should only be used if the relative path from one file to +// another cannot be computed in another way. The preferred way is to take +// the relative filenames directly from the .include or exists() where they +// appear. +// +// TODO: Invent data types for all kinds of relative paths that occur in pkgsrc +// and pkglint. Make sure that these paths cannot be accidentally mixed. +func relpath(from, to string) (result string) { + + if trace.Tracing { + defer trace.Call(from, to, trace.Result(&result))() + } + + cfrom := cleanpath(from) + cto := cleanpath(to) - // From "dir" to "dir/subdir/...". - if hasPrefix(to, from) && len(to) > len(from)+1 && to[len(from)] == '/' { - return path.Clean(to[len(from)+1:]) + if cfrom == cto { + return "." } - // Take a shortcut for the most common variant in a complete pkgsrc scan, - // which is to resolve the relative path from a package to the pkgsrc root. - // This avoids unnecessary calls to the filesystem API. - if to == "." { - fromParts := strings.FieldsFunc(from, func(r rune) bool { return r == '/' }) - if len(fromParts) == 3 && !hasPrefix(fromParts[0], ".") && !hasPrefix(fromParts[1], ".") && fromParts[2] == "." { + // Take a shortcut for the common case from "dir" to "dir/subdir/...". + if hasPrefix(cto, cfrom) && len(cto) > len(cfrom)+1 && cto[len(cfrom)] == '/' { + return cleanpath(cto[len(cfrom)+1:]) + } + + // Take a shortcut for the common case from "category/package" to ".". + // This is the most common variant in a complete pkgsrc scan. + if cto == "." { + fromParts := strings.FieldsFunc(cfrom, func(r rune) bool { return r == '/' }) + if len(fromParts) == 2 && !hasPrefix(fromParts[0], ".") && !hasPrefix(fromParts[1], ".") { return "../.." } } - absFrom := abspath(from) - absTo := abspath(to) - rel, err := filepath.Rel(absFrom, absTo) - G.AssertNil(err, "relpath %q %q", from, to) - result := filepath.ToSlash(rel) + if cfrom == "." && !filepath.IsAbs(cto) { + return path.Clean(cto) + } + + absFrom := abspath(cfrom) + absTopdir := abspath(G.Pkgsrc.topdir) + absTo := abspath(cto) + + toTop, err := filepath.Rel(absFrom, absTopdir) + G.AssertNil(err, "relpath from %q to topdir %q", absFrom, absTopdir) + + fromTop, err := filepath.Rel(absTopdir, absTo) + G.AssertNil(err, "relpath from topdir %q to %q", absTopdir, absTo) + + result = cleanpath(filepath.ToSlash(toTop) + "/" + filepath.ToSlash(fromTop)) + if trace.Tracing { - trace.Stepf("relpath from %q to %q = %q", from, to, result) + trace.Stepf("relpath from %q to %q = %q", cfrom, cto, result) } - return result + return } func abspath(filename string) string { @@ -401,6 +437,10 @@ func cleanpath(filename string) string { } } + for len(parts) > 1 && parts[len(parts)-1] == "." { + parts = parts[:len(parts)-1] + } + for i := 2; i+3 < len(parts); /* nothing */ { if parts[i] != ".." && parts[i+1] != ".." && parts[i+2] == ".." && parts[i+3] == ".." { if i+4 == len(parts) || parts[i+4] != ".." { @@ -441,6 +481,11 @@ func (o *Once) FirstTimeSlice(whats ...string) bool { return o.check(crc.Sum64()) } +func (o *Once) Seen(what string) bool { + _, seen := o.seen[crc64.Checksum([]byte(what), crc64.MakeTable(crc64.ECMA))] + return seen +} + func (o *Once) check(key uint64) bool { if _, ok := o.seen[key]; ok { return false @@ -454,6 +499,13 @@ func (o *Once) check(key uint64) bool { // Scope remembers which variables are defined and which are used // in a certain scope, such as a package or a file. +// +// TODO: Decide whether the scope should consider variable assignments +// from the pkgsrc infrastructure. For Package.checkGnuConfigureUseLanguages +// it would be better to ignore them completely. +// +// TODO: Merge this code with Var, which defines essentially the +// same features. type Scope struct { firstDef map[string]MkLine // TODO: Can this be removed? lastDef map[string]MkLine @@ -579,8 +631,9 @@ func (s *Scope) FirstDefinition(varname string) MkLine { mkline := s.firstDef[varname] if mkline != nil && mkline.IsVarassign() { lastLine := s.LastDefinition(varname) - if lastLine != mkline { - //mkline.Notef("FirstDefinition differs from LastDefinition in %s.", mkline.RefTo(lastLine)) + if trace.Tracing && lastLine != mkline { + trace.Stepf("%s: FirstDefinition differs from LastDefinition in %s.", + mkline.String(), mkline.RefTo(lastLine)) } return mkline } @@ -722,148 +775,6 @@ func naturalLess(str1, str2 string) bool { return len1 < len2 } -// RedundantScope checks for redundant variable definitions and for variables -// that are accidentally overwritten. It tries to be as correct as possible -// by not flagging anything that is defined conditionally. -// -// There may be some edge cases though like defining PKGNAME, then evaluating -// it using :=, then defining it again. This pattern is so error-prone that -// it should not appear in pkgsrc at all, thus pkglint doesn't even expect it. -// (Well, except for the PKGNAME case, but that's deep in the infrastructure -// and only affects the "nb13" extension.) -type RedundantScope struct { - vars map[string]*redundantScopeVarinfo - dirLevel int // The number of enclosing directives (.if, .for). - includePath includePath - OnRedundant func(old, new MkLine) - OnOverwrite func(old, new MkLine) -} -type redundantScopeVarinfo struct { - mkline MkLine - includePath includePath - value string -} - -func NewRedundantScope() *RedundantScope { - return &RedundantScope{vars: make(map[string]*redundantScopeVarinfo)} -} - -func (s *RedundantScope) Handle(mkline MkLine) { - if mkline.firstLine == 1 { - s.includePath.push(mkline.Location.Filename) - } else { - s.includePath.popUntil(mkline.Location.Filename) - } - - switch { - case mkline.IsVarassign(): - varname := mkline.Varname() - if s.dirLevel != 0 { - // Since the variable is defined or assigned conditionally, - // it becomes too complicated for pkglint to check all possible - // code paths. Therefore ignore the variable from now on. - s.vars[varname] = nil - break - } - - op := mkline.Op() - value := mkline.Value() - valueNovar := mkline.WithoutMakeVariables(value) - if op == opAssignEval && value == valueNovar { - op = /* effectively */ opAssign - } - - existing, found := s.vars[varname] - if !found { - if op == opAssignShell || op == opAssignEval { - s.vars[varname] = nil // Won't be checked further. - } else { - if op == opAssignAppend { - value = " " + value - } - s.vars[varname] = &redundantScopeVarinfo{mkline, s.includePath.copy(), value} - } - - } else if existing != nil { - if op == opAssign && existing.value == value { - op = /* effectively */ opAssignDefault - } - - switch op { - case opAssign: - if s.includePath.includes(existing.includePath) { - // This is the usual pattern of including a file and - // then overwriting some of them. Although technically - // this overwrites the previous definition, it is not - // worth a warning since this is used a lot and - // intentionally. - } else { - s.OnOverwrite(existing.mkline, mkline) - } - existing.value = value - case opAssignAppend: - existing.value += " " + value - case opAssignDefault: - if existing.includePath.includes(s.includePath) { - s.OnRedundant(mkline, existing.mkline) - } else if s.includePath.includes(existing.includePath) || s.includePath.equals(existing.includePath) { - s.OnRedundant(existing.mkline, mkline) - } - case opAssignShell, opAssignEval: - s.vars[varname] = nil // Won't be checked further. - } - } - - case mkline.IsDirective(): - switch mkline.Directive() { - case "for", "if", "ifdef", "ifndef": - s.dirLevel++ - case "endfor", "endif": - s.dirLevel-- - } - } -} - -type includePath struct { - files []string -} - -func (p *includePath) push(filename string) { - p.files = append(p.files, filename) -} - -func (p *includePath) popUntil(filename string) { - for p.files[len(p.files)-1] != filename { - p.files = p.files[:len(p.files)-1] - } -} - -func (p *includePath) includes(other includePath) bool { - for i, filename := range p.files { - if i < len(other.files) && other.files[i] == filename { - continue - } - return false - } - return len(p.files) < len(other.files) -} - -func (p *includePath) equals(other includePath) bool { - if len(p.files) != len(other.files) { - return false - } - for i, filename := range p.files { - if other.files[i] != filename { - return false - } - } - return true -} - -func (p *includePath) copy() includePath { - return includePath{append([]string(nil), p.files...)} -} - // IsPrefs returns whether the given file, when included, loads the user // preferences. func IsPrefs(filename string) bool { @@ -1004,7 +915,6 @@ func seeGuide(sectionName, sectionID string) string { func wrap(max int, lines ...string) []string { var wrapped []string var sb strings.Builder - nonSpace := textproc.Space.Inverse() for _, line := range lines { @@ -1024,7 +934,7 @@ func wrap(max int, lines ...string) []string { for !lexer.EOF() { bol := len(lexer.Rest()) == len(line) space := lexer.NextBytesSet(textproc.Space) - word := lexer.NextBytesSet(nonSpace) + word := lexer.NextBytesSet(notSpace) if bol && sb.Len() > 0 { space = " " @@ -1169,3 +1079,26 @@ func (si *StringInterner) Intern(str string) string { si.strs[key] = key return key } + +// StringSets stores unique strings in insertion order. +type StringSet struct { + Elements []string + seen map[string]struct{} +} + +func NewStringSet() StringSet { + return StringSet{nil, make(map[string]struct{})} +} + +func (s *StringSet) Add(element string) { + if _, found := s.seen[element]; !found { + s.seen[element] = struct{}{} + s.Elements = append(s.Elements, element) + } +} + +func (s *StringSet) AddAll(elements []string) { + for _, element := range elements { + s.Add(element) + } +} diff --git a/pkgtools/pkglint/files/util_test.go b/pkgtools/pkglint/files/util_test.go index 743a66c95ca..9bc99956c96 100644 --- a/pkgtools/pkglint/files/util_test.go +++ b/pkgtools/pkglint/files/util_test.go @@ -73,39 +73,84 @@ func (s *Suite) Test_tabWidth(c *check.C) { } func (s *Suite) Test_cleanpath(c *check.C) { - c.Check(cleanpath("simple/path"), equals, "simple/path") - c.Check(cleanpath("/absolute/path"), equals, "/absolute/path") + test := func(from, to string) { + c.Check(cleanpath(from), equals, to) + } + + test("simple/path", "simple/path") + test("/absolute/path", "/absolute/path") // Single dot components are removed, unless it's the only component of the path. - c.Check(cleanpath("./././."), equals, ".") - c.Check(cleanpath("./././"), equals, ".") - c.Check(cleanpath("dir/multi/././/file"), equals, "dir/multi/file") - c.Check(cleanpath("dir/"), equals, "dir") + test("./././.", ".") + test("./././", ".") + test("dir/multi/././/file", "dir/multi/file") + test("dir/", "dir") + + test("dir/", "dir") // Components like aa/bb/../.. are removed, but not in the initial part of the path, // and only if they are not followed by another "..". - c.Check(cleanpath("dir/../dir/../dir/../dir/subdir/../../Makefile"), equals, "dir/../dir/../dir/../Makefile") - c.Check(cleanpath("111/222/../../333/444/../../555/666/../../777/888/9"), equals, "111/222/../../777/888/9") - c.Check(cleanpath("1/2/3/../../4/5/6/../../7/8/9/../../../../10"), equals, "1/2/3/../../4/7/8/9/../../../../10") - c.Check(cleanpath("cat/pkg.v1/../../cat/pkg.v2/Makefile"), equals, "cat/pkg.v1/../../cat/pkg.v2/Makefile") - c.Check(cleanpath("aa/../../../../../a/b/c/d"), equals, "aa/../../../../../a/b/c/d") - c.Check(cleanpath("aa/bb/../../../../a/b/c/d"), equals, "aa/bb/../../../../a/b/c/d") - c.Check(cleanpath("aa/bb/cc/../../../a/b/c/d"), equals, "aa/bb/cc/../../../a/b/c/d") - c.Check(cleanpath("aa/bb/cc/dd/../../a/b/c/d"), equals, "aa/bb/a/b/c/d") - c.Check(cleanpath("aa/bb/cc/dd/ee/../a/b/c/d"), equals, "aa/bb/cc/dd/ee/../a/b/c/d") - c.Check(cleanpath("../../../../../a/b/c/d"), equals, "../../../../../a/b/c/d") - c.Check(cleanpath("aa/../../../../a/b/c/d"), equals, "aa/../../../../a/b/c/d") - c.Check(cleanpath("aa/bb/../../../a/b/c/d"), equals, "aa/bb/../../../a/b/c/d") - c.Check(cleanpath("aa/bb/cc/../../a/b/c/d"), equals, "aa/bb/cc/../../a/b/c/d") - c.Check(cleanpath("aa/bb/cc/dd/../a/b/c/d"), equals, "aa/bb/cc/dd/../a/b/c/d") - c.Check(cleanpath("aa/../cc/../../a/b/c/d"), equals, "aa/../cc/../../a/b/c/d") + test("dir/../dir/../dir/../dir/subdir/../../Makefile", "dir/../dir/../dir/../Makefile") + test("111/222/../../333/444/../../555/666/../../777/888/9", "111/222/../../777/888/9") + test("1/2/3/../../4/5/6/../../7/8/9/../../../../10", "1/2/3/../../4/7/8/9/../../../../10") + test("cat/pkg.v1/../../cat/pkg.v2/Makefile", "cat/pkg.v1/../../cat/pkg.v2/Makefile") + test("aa/../../../../../a/b/c/d", "aa/../../../../../a/b/c/d") + test("aa/bb/../../../../a/b/c/d", "aa/bb/../../../../a/b/c/d") + test("aa/bb/cc/../../../a/b/c/d", "aa/bb/cc/../../../a/b/c/d") + test("aa/bb/cc/dd/../../a/b/c/d", "aa/bb/a/b/c/d") + test("aa/bb/cc/dd/ee/../a/b/c/d", "aa/bb/cc/dd/ee/../a/b/c/d") + test("../../../../../a/b/c/d", "../../../../../a/b/c/d") + test("aa/../../../../a/b/c/d", "aa/../../../../a/b/c/d") + test("aa/bb/../../../a/b/c/d", "aa/bb/../../../a/b/c/d") + test("aa/bb/cc/../../a/b/c/d", "aa/bb/cc/../../a/b/c/d") + test("aa/bb/cc/dd/../a/b/c/d", "aa/bb/cc/dd/../a/b/c/d") + test("aa/../cc/../../a/b/c/d", "aa/../cc/../../a/b/c/d") // The initial 2 components of the path are typically category/package, when // pkglint is called from the pkgsrc top-level directory. // This path serves as the context and therefore is always kept. - c.Check(cleanpath("aa/bb/../../cc/dd/../../ee/ff"), equals, "aa/bb/../../ee/ff") - c.Check(cleanpath("aa/bb/../../cc/dd/../.."), equals, "aa/bb/../..") - c.Check(cleanpath("aa/bb/cc/dd/../.."), equals, "aa/bb") + test("aa/bb/../../cc/dd/../../ee/ff", "aa/bb/../../ee/ff") + test("aa/bb/../../cc/dd/../..", "aa/bb/../..") + test("aa/bb/cc/dd/../..", "aa/bb") + test("aa/bb/../../cc/dd/../../ee/ff/buildlink3.mk", "aa/bb/../../ee/ff/buildlink3.mk") + test("./aa/bb/../../cc/dd/../../ee/ff/buildlink3.mk", "aa/bb/../../ee/ff/buildlink3.mk") + + test("../.", "..") + test("../././././././.", "..") + test(".././././././././", "..") +} + +func (s *Suite) Test_relpath(c *check.C) { + t := s.Init(c) + + t.Chdir(".") + t.Check(G.Pkgsrc.topdir, equals, t.tmpdir) + + test := func(from, to, result string) { + c.Check(relpath(from, to), equals, result) + } + + test("some/dir", "some/directory", "../../some/directory") + + test("category/package/.", ".", "../..") + + // This case is handled by one of the shortcuts that avoid file system access. + test( + "./.", + "x11/frameworkintegration/../../meta-pkgs/kde/kf5.mk", + "meta-pkgs/kde/kf5.mk") + + // This happens when "pkglint -r x11" is run. + G.Pkgsrc.topdir = "x11/.." + + test( + "./.", + "x11/frameworkintegration/../../meta-pkgs/kde/kf5.mk", + "meta-pkgs/kde/kf5.mk") + test( + "x11/..", + "x11/frameworkintegration/../../meta-pkgs/kde/kf5.mk", + "meta-pkgs/kde/kf5.mk") } // Relpath is called so often that handling the most common calls @@ -119,9 +164,9 @@ func (s *Suite) Test_relpath__quick(c *check.C) { test("some/dir", "some/dir/../..", "../..") test("some/dir", "some/dir/./././../..", "../..") test("some/dir", "some/dir/", ".") - test("some/dir", "some/directory", "../directory") - test("category/package/.", ".", "../..") + test("some/dir", ".", "../..") + test("some/dir/.", ".", "../..") } // This is not really an internal error but won't happen in practice anyway. @@ -129,10 +174,14 @@ func (s *Suite) Test_relpath__quick(c *check.C) { func (s *Suite) Test_relpath__failure_on_Windows(c *check.C) { t := s.Init(c) - if runtime.GOOS == "windows" { + if runtime.GOOS == "windows" && hasPrefix(t.tmpdir, "C:/") { t.ExpectPanic( func() { relpath("c:/", "d:/") }, - "Pkglint internal error: relpath \"c:/\" \"d:/\": Rel: can't make d:/ relative to c:/") + sprintf( + "Pkglint internal error: "+ + "relpath from topdir %q to %q: "+ + "Rel: can't make %s relative to %s", + t.tmpdir, "D:/", "D:/", t.tmpdir)) } } @@ -710,28 +759,3 @@ func (s *Suite) Test_StringInterner(c *check.C) { t.Check(si.Intern("Hello, world"), equals, "Hello, world") t.Check(si.Intern("Hello, world"[0:5]), equals, "Hello") } - -func (s *Suite) Test_includePath_includes(c *check.C) { - t := s.Init(c) - - path := func(locations ...string) includePath { - return includePath{locations} - } - - var ( - m = path("Makefile") - mc = path("Makefile", "Makefile.common") - mco = path("Makefile", "Makefile.common", "other.mk") - mo = path("Makefile", "other.mk") - ) - - t.Check(m.includes(m), equals, false) - - t.Check(m.includes(mc), equals, true) - t.Check(m.includes(mco), equals, true) - t.Check(mc.includes(mco), equals, true) - - t.Check(mc.includes(m), equals, false) - t.Check(mc.includes(mo), equals, false) - t.Check(mo.includes(mc), equals, false) -} diff --git a/pkgtools/pkglint/files/var.go b/pkgtools/pkglint/files/var.go index 9050bb62597..553555c2997 100644 --- a/pkgtools/pkglint/files/var.go +++ b/pkgtools/pkglint/files/var.go @@ -2,23 +2,275 @@ package pkglint // Var describes a variable in a Makefile snippet. // -// TODO: Remove this type in June 2019 if it is still a stub. +// It keeps track of all places where the variable is accessed or modified (see +// ReadLocations, WriteLocations) and provides information for further static +// analysis, such as: +// +// * Whether the variable value is constant, and if so, what the constant value +// is (see Constant, ConstantValue). +// +// * What its (approximated) value is, either including values from the pkgsrc +// infrastructure (see ValueInfra) or excluding them (Value). +// +// * On which other variables this variable depends (see Conditional, +// ConditionalVars). +// +// TODO: Decide how to handle OPSYS-specific variables, such as LDFLAGS.SunOS. +// +// TODO: Decide how to handle parameterized variables, such as SUBST_MESSAGE.doc. type Var struct { Name string - Type *Vartype + + // 0 = neither written nor read + // 1 = constant + // 2 = constant and read; further writes will make it non-constant + // 3 = not constant anymore + constantState uint8 + constantValue string + + value string + valueInfra string + + readLocations []MkLine + writeLocations []MkLine + + conditional bool + conditionalVars StringSet + + refs StringSet } -func NewVar(name string) *Var { return &Var{name, nil} } +func NewVar(name string) *Var { + return &Var{name, 0, "", "", "", nil, nil, false, NewStringSet(), NewStringSet()} +} -// Constant returns whether the variable is only ever assigned a single value, -// without being dependent on any other variable. +// Conditional returns whether the variable value depends on other variables. +func (v *Var) Conditional() bool { + return v.conditional +} + +// ConditionalVars returns all variables in conditions on which the value of +// this variable depends. // -// Multiple assignments (such as VAR=1, VAR+=2, VAR+=3) are considered constant -// as well, as long as the variable is not used in-between these assignments. -// That is, no .include or .if may appear there, and none of the ::= modifiers may -// be involved. +// The returned slice must not be modified. +func (v *Var) ConditionalVars() []string { + return v.conditionalVars.Elements +} + +// Refs returns all variables on which this variable depends. These are: +// +// Variables that are referenced in the value, such as in VAR=${OTHER}. // -// Simple .for loops that append to the variable are ok though. -func (v *Var) Constant() bool { return false } +// Variables that are used in conditions that enclose one of the assignments +// to this variable, such as .if ${OPSYS} == NetBSD. +// +// Variables that are used in .for loops in which this variable is assigned +// a value, such as DIRS in: +// .for dir in ${DIRS} +// VAR+=${dir} +// .endfor +func (v *Var) Refs() []string { + return v.refs.Elements +} -func (v *Var) ConstantValue() string { return "" } +// AddRef marks this variable as being dependent on the given variable name. +// This can be used for the .for loops mentioned in Refs. +func (v *Var) AddRef(varname string) { + v.refs.Add(varname) +} + +// Constant returns whether the variable's value is a constant. +// It may reference other variables since these references are evaluated +// lazily, when the variable value is actually needed. +// +// Multiple assignments (such as VAR=1, VAR+=2, VAR+=3) are considered to +// form a single constant as well, as long as the variable is not read before +// or in-between these assignments. The definition of "read" is very strict +// here since every mention of the variable counts. This may prevent some +// essentially constant values from being detected as such, but these special +// cases may be implemented later. +// +// TODO: Simple .for loops that append to the variable are ok as well. +// (This needs to be worded more precisely since that part potentially +// adds a lot of complexity to the whole data structure.) +// +// Variable assignments in the pkgsrc infrastructure are taken into account +// for determining whether a variable is constant. +func (v *Var) Constant() bool { + return v.constantState == 1 || v.constantState == 2 +} + +// ConstantValue returns the constant value of the variable. +// It is only allowed when Constant() returns true. +// +// Variable assignments in the pkgsrc infrastructure are taken into account +// for determining the constant value. +func (v *Var) ConstantValue() string { + G.Assertf(v.Constant(), "Variable must be constant.") + return v.constantValue +} + +// Value returns the (approximated) value of the variable, taking into account +// all variable assignments that happen outside the pkgsrc infrastructure. +// +// For variables that are conditionally assigned (as in .if/.else), the +// returned value is not reliable. It may be the value from either branch, or +// even the combined value of both branches. +// +// See Constant and ConstantValue for more reliable information. +func (v *Var) Value() string { + return v.value +} + +// ValueInfra returns the (approximated) value of the variable, taking into +// account all variable assignments from the package, the user and the pkgsrc +// infrastructure. +// +// For variables that are conditionally assigned (as in .if/.else), the +// returned value is not reliable. It may be the value from either branch, or +// even the combined value of both branches. +// +// See Constant and ConstantValue for more reliable information, but these +// ignore assignments from the infrastructure. +func (v *Var) ValueInfra() string { + return v.valueInfra +} + +// ReadLocations returns the locations where the variable is read, such as +// in ${VAR} or defined(VAR) or empty(VAR). +// +// Uses inside conditionals are included, no matter whether they are actually +// reachable in practice. +// +// Indirect uses through other variables (such as VAR2=${VAR}, VAR3=${VAR2}) +// are not listed. +// +// Variable uses in the pkgsrc infrastructure are taken into account. +func (v *Var) ReadLocations() []MkLine { + return v.readLocations +} + +// WriteLocations returns the locations where the variable is modified. +// +// Assignments inside conditionals are included, no matter whether they are actually +// reachable in practice. +// +// Variable assignments in the pkgsrc infrastructure are taken into account. +func (v *Var) WriteLocations() []MkLine { + return v.writeLocations +} + +func (v *Var) Read(mkline MkLine) { + v.readLocations = append(v.readLocations, mkline) + v.constantState = [...]uint8{3, 2, 2, 3}[v.constantState] +} + +// Write marks the variable as being assigned in the given line. +// Only standard assignments (VAR=value) are handled. +// Side-effect assignments (${VAR::=value}) are not handled here since +// they don't occur in practice. +func (v *Var) Write(mkline MkLine, conditional bool, conditionVarnames ...string) { + G.Assertf(mkline.Varname() == v.Name, "wrong variable name") + + v.writeLocations = append(v.writeLocations, mkline) + + if conditional || len(conditionVarnames) > 0 { + v.conditional = true + } + v.conditionalVars.AddAll(conditionVarnames) + + v.refs.AddAll(mkline.DetermineUsedVariables()) + v.refs.AddAll(conditionVarnames) + + v.update(mkline, &v.valueInfra) + if !G.Pkgsrc.IsInfra(mkline.Line.Filename) { + v.update(mkline, &v.value) + } + + v.updateConstantValue(mkline) +} + +func (v *Var) update(mkline MkLine, update *string) { + firstWrite := len(v.writeLocations) == 1 + if v.Conditional() && !firstWrite { + return + } + + value := mkline.Value() + switch mkline.Op() { + case opAssign, opAssignEval: + *update = value + + case opAssignDefault: + if firstWrite { + *update = value + } + + case opAssignAppend: + *update += " " + value + + case opAssignShell: + // Ignore these for now. + // Later it might be useful to parse the shell commands to + // evaluate simple commands like "test && echo yes || echo no". + } +} + +func (v *Var) updateConstantValue(mkline MkLine) { + if v.constantState == 3 { + return + } + + // Even if the variable references other variables, this does not + // influence whether the variable is considered constant. (Except + // for the := operator.) + // + // Strictly speaking, the referenced variables must be still + // be constant at the end of loading the complete package. + // (And even after that, because of the ::= modifier. But luckily + // almost no one knows that modifier.) + + if v.Conditional() { + v.constantState = 3 + v.constantValue = "" + return + } + + value := mkline.Value() + switch mkline.Op() { + case opAssign: + v.constantValue = value + + case opAssignEval: + if value != mkline.WithoutMakeVariables(value) { + // To leave the variable in the constant state, the current value + // of the referenced variables would need to be resolved. + // + // This in turn requires the proper scope for resolving variable + // references. Furthermore, the referenced variables must be + // constant at this point. Later changes to these variables + // can be ignored though. + // + // Because this sounds complicated to implement, the variable + // is marked as non-constant for now. + v.constantState = 3 + v.constantValue = "" + } else { + v.constantValue = value + } + + case opAssignDefault: + if v.constantState == 0 { + v.constantValue = value + } + + case opAssignAppend: + v.constantValue += " " + value + + case opAssignShell: + v.constantState = 3 + v.constantValue = "" + } + + v.constantState = [...]uint8{1, 1, 3, 3}[v.constantState] +} diff --git a/pkgtools/pkglint/files/var_test.go b/pkgtools/pkglint/files/var_test.go index eb37ff58616..a11fa4aa145 100644 --- a/pkgtools/pkglint/files/var_test.go +++ b/pkgtools/pkglint/files/var_test.go @@ -2,18 +2,336 @@ package pkglint import "gopkg.in/check.v1" -func (s *Suite) Test_Var_Constant(c *check.C) { +func (s *Suite) Test_Var_ConstantValue__assign(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME=\toverwritten"), false) + + t.Check(v.ConstantValue(), equals, "overwritten") +} + +// Variables that reference other variable are considered constants. +// Even if these referenced variables change their value in-between, +// this does not affect the constant-ness of this variable, since the +// references are resolved lazily. +func (s *Suite) Test_Var_ConstantValue__assign_reference(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME=\t${OTHER}"), false) + + t.Check(v.Constant(), equals, true) +} + +func (s *Suite) Test_Var_ConstantValue__assign_eval_reference(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME:=\t${OTHER}"), false) + + // To analyze this case correctly, pkglint would have to know + // the current value of ${OTHER} in line 124. For that it would + // need the complete scope including all other variables. + // + // As of March 2019 this is not implemented, therefore pkglint + // doesn't treat the variable as constant, to prevent wrong warnings. + t.Check(v.Constant(), equals, false) +} + +func (s *Suite) Test_Var_ConstantValue__assign_conditional(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + t.Check(v.ConditionalVars(), check.IsNil) + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tconditional"), true, "OPSYS") + + t.Check(v.Constant(), equals, false) +} + +func (s *Suite) Test_Var_ConstantValue__default(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME?=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME?=\tignored"), false) + + t.Check(v.ConstantValue(), equals, "value") +} + +func (s *Suite) Test_Var_ConstantValue__eval_then_default(c *check.C) { + t := s.Init(c) + v := NewVar("VARNAME") - // FIXME: Replace this test with an actual use case. + v.Write(t.NewMkLine("buildlink3.mk", 123, "VARNAME:=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("builtin.mk", 124, "VARNAME?=\tignored"), false) - c.Check(v.Constant(), equals, false) + t.Check(v.ConstantValue(), equals, "value") } -func (s *Suite) Test_Var_ConstantValue(c *check.C) { +func (s *Suite) Test_Var_ConstantValue__append(c *check.C) { + t := s.Init(c) + v := NewVar("VARNAME") - // FIXME: Replace this test with an actual use case. + v.Write(t.NewMkLine("write.mk", 123, "VARNAME+=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, " value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME+=\tappended"), false) + + t.Check(v.ConstantValue(), equals, " value appended") +} + +func (s *Suite) Test_Var_ConstantValue__eval(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME:=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME:=\toverwritten"), false) + + t.Check(v.ConstantValue(), equals, "overwritten") +} + +// Variables that are based on running shell commands are never constant. +func (s *Suite) Test_Var_ConstantValue__shell(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME!=\techo hello"), false) + + t.Check(v.Constant(), equals, false) +} + +func (s *Suite) Test_Var_ConstantValue__referenced_before(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + // Since the value of VARNAME escapes here, the value is not + // guaranteed to be the same in all evaluations of ${VARNAME}. + // For example, OTHER may be used at load time in an .if + // condition. + v.Read(t.NewMkLine("readwrite.mk", 123, "OTHER=\t${VARNAME}")) + + t.Check(v.Constant(), equals, false) + + v.Write(t.NewMkLine("readwrite.mk", 124, "VARNAME=\tvalue"), false) + + t.Check(v.Constant(), equals, false) +} + +func (s *Suite) Test_Var_ConstantValue__referenced_in_between(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("readwrite.mk", 123, "VARNAME=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + // Since the value of VARNAME escapes here, the value is not + // guaranteed to be the same in all evaluations of ${VARNAME}. + // For example, OTHER may be used at load time in an .if + // condition. + v.Read(t.NewMkLine("readwrite.mk", 124, "OTHER=\t${VARNAME}")) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 125, "VARNAME=\toverwritten"), false) + + t.Check(v.Constant(), equals, false) +} + +func (s *Suite) Test_Var_ConditionalVars(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + t.Check(v.Conditional(), equals, false) + t.Check(v.ConditionalVars(), check.IsNil) + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tconditional"), true, "OPSYS") + + t.Check(v.Constant(), equals, false) + t.Check(v.Conditional(), equals, true) + t.Check(v.ConditionalVars(), deepEquals, []string{"OPSYS"}) + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME=\tconditional"), true, "OPSYS") + + t.Check(v.Conditional(), equals, true) + t.Check(v.ConditionalVars(), deepEquals, []string{"OPSYS"}) +} + +func (s *Suite) Test_Var_Value__initial_conditional_write(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME:=\toverwritten conditionally"), true, "OPSYS") + + // Since there is no previous value, the simplest choice is to just + // take the first seen value, no matter if that value is conditional + // or not. + t.Check(v.Conditional(), equals, true) + t.Check(v.Constant(), equals, false) + t.Check(v.Value(), equals, "overwritten conditionally") +} + +func (s *Suite) Test_Var_Value__conditional_write_after_unconditional(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tvalue"), false) + + t.Check(v.Value(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME+=\tappended"), false) + + t.Check(v.Value(), equals, "value appended") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME:=\toverwritten conditionally"), true, "OPSYS") + + // When there is a previous value, it's probably best to keep + // that value since this way the following code results in the + // most generic value: + // VAR= generic + // .if ${OPSYS} == NetBSD + // VAR= specific + // .endif + // The value stays the same, still it is marked as conditional and therefore + // not constant anymore. + t.Check(v.Conditional(), equals, true) + t.Check(v.Constant(), equals, false) + t.Check(v.Value(), equals, "value appended") +} + +func (s *Suite) Test_Var_Value__infrastructure(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine(t.File("write.mk"), 123, "VARNAME=\tvalue"), false) + + t.Check(v.Value(), equals, "value") + + v.Write(t.NewMkLine(t.File("mk/write.mk"), 123, "VARNAME=\tinfra"), false) + + t.Check(v.Value(), equals, "value") + + v.Write(t.NewMkLine(t.File("wip/mk/write.mk"), 123, "VARNAME=\twip infra"), false) + + t.Check(v.Value(), equals, "value") +} + +func (s *Suite) Test_Var_ValueInfra(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine(t.File("write.mk"), 123, "VARNAME=\tvalue"), false) + + t.Check(v.ValueInfra(), equals, "value") + + v.Write(t.NewMkLine(t.File("mk/write.mk"), 123, "VARNAME=\tinfra"), false) + + t.Check(v.ValueInfra(), equals, "infra") + + v.Write(t.NewMkLine(t.File("wip/mk/write.mk"), 123, "VARNAME=\twip infra"), false) + + t.Check(v.ValueInfra(), equals, "wip infra") +} + +func (s *Suite) Test_Var_ReadLocations(c *check.C) { + t := s.Init(c) + + v := NewVar("VAR") + + t.Check(v.ReadLocations(), check.IsNil) + + mkline123 := t.NewMkLine("read.mk", 123, "OTHER=\t${VAR}") + v.Read(mkline123) + + t.Check(v.ReadLocations(), deepEquals, []MkLine{mkline123}) + + mkline124 := t.NewMkLine("read.mk", 124, "OTHER=\t${VAR} ${VAR}") + v.Read(mkline124) + v.Read(mkline124) + + // For now, count every read of the variable. I'm not yet sure + // whether that's the best way or whether to make the lines unique. + t.Check(v.ReadLocations(), deepEquals, []MkLine{mkline123, mkline124, mkline124}) +} + +func (s *Suite) Test_Var_WriteLocations(c *check.C) { + t := s.Init(c) + + v := NewVar("VAR") + + t.Check(v.WriteLocations(), check.IsNil) + + mkline123 := t.NewMkLine("write.mk", 123, "VAR=\tvalue") + v.Write(mkline123, false) + + t.Check(v.WriteLocations(), deepEquals, []MkLine{mkline123}) + + // Multiple writes from the same line may happen because of a .for loop. + mkline125 := t.NewMkLine("write.mk", 125, "VAR+=\t${var}") + v.Write(mkline125, false) + v.Write(mkline125, false) + + // For now, count every write of the variable. I'm not yet sure + // whether that's the best way or whether to make the lines unique. + t.Check(v.WriteLocations(), deepEquals, []MkLine{mkline123, mkline125, mkline125}) +} + +func (s *Suite) Test_Var_Refs(c *check.C) { + t := s.Init(c) + + v := NewVar("VAR") + + t.Check(v.Refs(), check.IsNil) + + // The referenced variables are taken from the mkline. + // They don't need to be passed separately. + v.Write(t.NewMkLine("write.mk", 123, "VAR=${OTHER} ${${OPSYS} == NetBSD :? ${THEN} : ${ELSE}}"), true, "COND") + + v.AddRef("FOR") - c.Check(v.ConstantValue(), equals, "") + t.Check(v.Refs(), deepEquals, []string{"OTHER", "OPSYS", "THEN", "ELSE", "COND", "FOR"}) } diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go index 9a9d8c8f585..eb9e53f0584 100644 --- a/pkgtools/pkglint/files/vardefs.go +++ b/pkgtools/pkglint/files/vardefs.go @@ -37,7 +37,7 @@ func (src *Pkgsrc) InitVartypes() { pkg := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, ""+ "Makefile: set, use; "+ - "buildlink3.mk, builtin.mk:; "+ + "buildlink3.mk, builtin.mk: none; "+ "Makefile.*, *.mk: default, set, use") } @@ -45,7 +45,7 @@ func (src *Pkgsrc) InitVartypes() { pkgload := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, ""+ "Makefile: set, use, use-loadtime; "+ - "buildlink3.mk, builtin.mk:; "+ + "buildlink3.mk, builtin.mk: none; "+ "Makefile.*, *.mk: default, set, use, use-loadtime") } @@ -54,21 +54,29 @@ func (src *Pkgsrc) InitVartypes() { pkglist := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, ""+ "Makefile, Makefile.common, options.mk: append, default, set, use; "+ - "buildlink3.mk, builtin.mk:; "+ + "buildlink3.mk, builtin.mk: none; "+ "*.mk: append, default, use") } + // Some package-defined lists may also be appended in buildlink3.mk files, + // for example platform-specific CFLAGS and LDFLAGS. + pkglistbl3 := func(varname string, kindOfList KindOfList, checker *BasicType) { + acl(varname, kindOfList, checker, ""+ + "Makefile, Makefile.common, options.mk: append, default, set, use; "+ + "buildlink3.mk, builtin.mk, *.mk: append, default, use") + } + // sys declares a user-defined or system-defined variable that must not be modified by packages. // // It also must not be used in buildlink3.mk and builtin.mk files or at load-time, // since the system/user preferences may not have been loaded when these files are included. sys := func(varname string, kindOfList KindOfList, checker *BasicType) { - acl(varname, kindOfList, checker, "buildlink3.mk:; *: use") + acl(varname, kindOfList, checker, "buildlink3.mk: none; *: use") } // usr declares a user-defined variable that must not be modified by packages. usr := func(varname string, kindOfList KindOfList, checker *BasicType) { - acl(varname, kindOfList, checker, "buildlink3.mk:; *: use-loadtime, use") + acl(varname, kindOfList, checker, "buildlink3.mk: none; *: use-loadtime, use") } // sysload declares a system-provided variable that may already be used at load time. @@ -81,7 +89,7 @@ func (src *Pkgsrc) InitVartypes() { } cmdline := func(varname string, kindOfList KindOfList, checker *BasicType) { - acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk:; *: use-loadtime, use") + acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk: none; *: use-loadtime, use") } compilerLanguages := enum( @@ -100,7 +108,10 @@ func (src *Pkgsrc) InitVartypes() { } } } - for _, language := range [...]string{"ada", "c", "c99", "c++", "c++11", "fortran", "fortran77", "java", "objc", "obj-c++"} { + alwaysAvailable := [...]string{ + "ada", "c", "c99", "c++", "c++11", "c++14", + "fortran", "fortran77", "java", "objc", "obj-c++"} + for _, language := range alwaysAvailable { languages[language] = true } @@ -153,8 +164,8 @@ func (src *Pkgsrc) InitVartypes() { return enum(defval) } - // enumFromDirs reads the directories from category, takes all - // that have a single number in them and ranks them from earliest + // enumFromDirs reads the directories from category, takes all that have + // a single number in them (such as php72) and ranks them from earliest // to latest. // // If the directories cannot be found, the allowed values are taken @@ -232,7 +243,7 @@ func (src *Pkgsrc) InitVartypes() { usr("CROSSBASE", lkNone, BtPathname) usr("VARBASE", lkNone, BtPathname) acl("X11_TYPE", lkNone, enum("modular native"), "*: use-loadtime, use") - usr("X11BASE", lkNone, BtPathname) + acl("X11BASE", lkNone, BtPathname, "*: use-loadtime, use") usr("MOTIFBASE", lkNone, BtPathname) usr("PKGINFODIR", lkNone, BtPathname) usr("PKGMANDIR", lkNone, BtPathname) @@ -294,7 +305,7 @@ func (src *Pkgsrc) InitVartypes() { usrpkg := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, ""+ "Makefile: default, set, use, use-loadtime; "+ - "buildlink3.mk, builtin.mk:; "+ + "buildlink3.mk, builtin.mk: none; "+ "Makefile.*, *.mk: default, set, use, use-loadtime; "+ "*: use-loadtime, use") } @@ -488,10 +499,10 @@ func (src *Pkgsrc) InitVartypes() { // some other variables, sorted alphabetically - acl(".CURDIR", lkNone, BtPathname, "buildlink3.mk:; *: use, use-loadtime") - acl(".IMPSRC", lkShell, BtPathname, "buildlink3.mk:; *: use, use-loadtime") - acl(".TARGET", lkNone, BtPathname, "buildlink3.mk:; *: use, use-loadtime") - acl("@", lkNone, BtPathname, "buildlink3.mk:; *: use, use-loadtime") + acl(".CURDIR", lkNone, BtPathname, "buildlink3.mk: none; *: use, use-loadtime") + acl(".IMPSRC", lkShell, BtPathname, "buildlink3.mk: none; *: use, use-loadtime") + acl(".TARGET", lkNone, BtPathname, "buildlink3.mk: none; *: use, use-loadtime") + acl("@", lkNone, BtPathname, "buildlink3.mk: none; *: use, use-loadtime") acl("ALL_ENV", lkShell, BtShellWord, "") acl("ALTERNATIVES_FILE", lkNone, BtFileName, "") acl("ALTERNATIVES_SRC", lkShell, BtPathname, "") @@ -578,8 +589,8 @@ func (src *Pkgsrc) InitVartypes() { acl("CATEGORIES", lkShell, BtCategory, "Makefile: set, append; Makefile.common: set, default, append") sysload("CC_VERSION", lkNone, BtMessage) sysload("CC", lkNone, BtShellCommand) - pkglist("CFLAGS", lkShell, BtCFlag) // may also be changed by the user - pkglist("CFLAGS.*", lkShell, BtCFlag) // may also be changed by the user + pkglistbl3("CFLAGS", lkShell, BtCFlag) // may also be changed by the user + pkglistbl3("CFLAGS.*", lkShell, BtCFlag) // may also be changed by the user acl("CHECK_BUILTIN", lkNone, BtYesNo, "builtin.mk: default; Makefile: set") acl("CHECK_BUILTIN.*", lkNone, BtYesNo, "Makefile, options.mk, buildlink3.mk: set; builtin.mk: default, use-loadtime; *: use-loadtime") acl("CHECK_FILES_SKIP", lkShell, BtBasicRegularExpression, "Makefile, Makefile.common: append") @@ -751,7 +762,7 @@ func (src *Pkgsrc) InitVartypes() { pkg("GITHUB_TYPE", lkNone, enum("tag release")) pkg("GMAKE_REQD", lkNone, BtVersion) acl("GNU_ARCH", lkNone, enum("mips"), "") - acl("GNU_ARCH.*", lkNone, BtIdentifier, "buildlink3.mk:; *: set, use") + acl("GNU_ARCH.*", lkNone, BtIdentifier, "buildlink3.mk: none; *: set, use") acl("GNU_CONFIGURE", lkNone, BtYes, "Makefile, Makefile.common: set") acl("GNU_CONFIGURE_INFODIR", lkNone, BtPathname, "Makefile, Makefile.common: set") acl("GNU_CONFIGURE_LIBDIR", lkNone, BtPathname, "Makefile, Makefile.common: set") @@ -807,8 +818,8 @@ func (src *Pkgsrc) InitVartypes() { usr("KRB5_DEFAULT", lkNone, enum("heimdal mit-krb5")) sys("KRB5_TYPE", lkNone, BtIdentifier) sys("LD", lkNone, BtShellCommand) - pkglist("LDFLAGS", lkShell, BtLdFlag) - pkglist("LDFLAGS.*", lkShell, BtLdFlag) + pkglistbl3("LDFLAGS", lkShell, BtLdFlag) // May also be changed by the user. + pkglistbl3("LDFLAGS.*", lkShell, BtLdFlag) // May also be changed by the user. sysload("LIBABISUFFIX", lkNone, BtIdentifier) // Can also be empty. sys("LIBGRP", lkNone, BtUserGroupName) sys("LIBMODE", lkNone, BtFileMode) @@ -819,8 +830,8 @@ func (src *Pkgsrc) InitVartypes() { sys("LIBTOOL", lkNone, BtShellCommand) acl("LIBTOOL_OVERRIDE", lkShell, BtPathmask, "Makefile: set, append") pkglist("LIBTOOL_REQD", lkShell, BtVersion) - acl("LICENCE", lkNone, BtLicense, "Makefile, Makefile.common, options.mk: set, append") - acl("LICENSE", lkNone, BtLicense, "Makefile, Makefile.common, options.mk: set, append") + acl("LICENCE", lkNone, BtLicense, "buildlink3.mk, builtin.mk: none; Makefile: set, append; *: default, set, append") + acl("LICENSE", lkNone, BtLicense, "buildlink3.mk, builtin.mk: none; Makefile: set, append; *: default, set, append") pkg("LICENSE_FILE", lkNone, BtPathname) sys("LINK.*", lkNone, BtShellCommand) sys("LINKER_RPATH_FLAG", lkNone, BtShellWord) @@ -937,7 +948,7 @@ func (src *Pkgsrc) InitVartypes() { acl("PATCH_ARGS", lkShell, BtShellWord, "") acl("PATCH_DIST_ARGS", lkShell, BtShellWord, "Makefile: set, append") acl("PATCH_DIST_CAT", lkNone, BtShellCommand, "") - acl("PATCH_DIST_STRIP*", lkNone, BtShellWord, "buildlink3.mk, builtin.mk:; Makefile, Makefile.common, *.mk: set") + acl("PATCH_DIST_STRIP*", lkNone, BtShellWord, "buildlink3.mk, builtin.mk: none; Makefile, Makefile.common, *.mk: set") acl("PATCH_SITES", lkShell, BtFetchURL, "Makefile, Makefile.common, options.mk: set") acl("PATCH_STRIP", lkNone, BtShellWord, "") sys("PATH", lkNone, BtPathlist) // From the PATH environment variable. @@ -986,7 +997,7 @@ func (src *Pkgsrc) InitVartypes() { sys("PKGNAME_NOREV", lkNone, BtPkgName) sysload("PKGPATH", lkNone, BtPathname) acl("PKGREPOSITORY", lkNone, BtUnknown, "") - acl("PKGREVISION", lkNone, BtPkgRevision, "Makefile: set") + acl("PKGREVISION", lkNone, BtPkgRevision, "Makefile: set; *: none") sys("PKGSRCDIR", lkNone, BtPathname) acl("PKGSRCTOP", lkNone, BtYes, "Makefile: set") sys("PKGSRC_SETENV", lkNone, BtShellCommand) @@ -1109,6 +1120,7 @@ func (src *Pkgsrc) InitVartypes() { pkg("RESTRICTED", lkNone, BtMessage) usr("ROOT_USER", lkNone, BtUserGroupName) usr("ROOT_GROUP", lkNone, BtUserGroupName) + pkglist("RPMIGNOREPATH", lkShell, BtPathmask) acl("RUBY_BASE", lkNone, enumFromDirs("lang", `^ruby(\d+)$`, "ruby$1", "ruby22 ruby23 ruby24 ruby25"), ""+ "special:rubyversion.mk: set; "+ "*: use-loadtime, use") @@ -1142,15 +1154,15 @@ func (src *Pkgsrc) InitVartypes() { pkglist("SPECIAL_PERMS", lkShell, BtPerms) sys("STEP_MSG", lkNone, BtShellCommand) sys("STRIP", lkNone, BtShellCommand) // see mk/tools/strip.mk - acl("SUBDIR", lkShell, BtFileName, "Makefile: append; *:") + acl("SUBDIR", lkShell, BtFileName, "Makefile: append; *: none") acl("SUBST_CLASSES", lkShell, BtIdentifier, "Makefile: set, append; *: append") - acl("SUBST_CLASSES.*", lkShell, BtIdentifier, "Makefile: set, append; *: append") + acl("SUBST_CLASSES.*", lkShell, BtIdentifier, "Makefile: set, append; *: append") // OPSYS-specific acl("SUBST_FILES.*", lkShell, BtPathmask, "Makefile, Makefile.*, *.mk: set, append") acl("SUBST_FILTER_CMD.*", lkNone, BtShellCommand, "Makefile, Makefile.*, *.mk: set") acl("SUBST_MESSAGE.*", lkNone, BtMessage, "Makefile, Makefile.*, *.mk: set") acl("SUBST_SED.*", lkNone, BtSedCommands, "Makefile, Makefile.*, *.mk: set, append") pkg("SUBST_STAGE.*", lkNone, BtStage) - pkglist("SUBST_VARS.*", lkShell, BtVariableName) + acl("SUBST_VARS.*", lkShell, BtVariableName, "Makefile, Makefile.*, *.mk: set, append") pkglist("SUPERSEDES", lkShell, BtDependency) acl("TEST_DEPENDS", lkShell, BtDependencyWithPath, "Makefile, Makefile.common, *.mk: append") pkglist("TEST_DIRS", lkShell, BtWrksrcSubdirectory) @@ -1162,7 +1174,7 @@ func (src *Pkgsrc) InitVartypes() { sys("TOOLS_BROKEN", lkShell, BtTool) sys("TOOLS_CMD.*", lkNone, BtPathname) acl("TOOLS_CREATE", lkShell, BtTool, "Makefile, Makefile.common, options.mk: append") - acl("TOOLS_DEPENDS.*", lkShell, BtDependencyWithPath, "buildlink3.mk:; Makefile, Makefile.*: set, default; *: use") + acl("TOOLS_DEPENDS.*", lkShell, BtDependencyWithPath, "buildlink3.mk: none; Makefile, Makefile.*: set, default; *: use") sys("TOOLS_GNU_MISSING", lkShell, BtTool) sys("TOOLS_NOOP", lkShell, BtTool) sys("TOOLS_PATH.*", lkNone, BtPathname) @@ -1181,7 +1193,7 @@ func (src *Pkgsrc) InitVartypes() { pkg("USE_CMAKE", lkNone, BtYes) usr("USE_DESTDIR", lkNone, BtYes) pkglist("USE_FEATURES", lkShell, BtIdentifier) - acl("USE_GAMESGROUP", lkNone, BtYesNo, "buildlink3.mk, builtin.mk:; *: set, default, use") + acl("USE_GAMESGROUP", lkNone, BtYesNo, "buildlink3.mk, builtin.mk: none; *: set, default, use") pkg("USE_GCC_RUNTIME", lkNone, BtYesNo) pkg("USE_GNU_CONFIGURE_HOST", lkNone, BtYesNo) acl("USE_GNU_ICONV", lkNone, BtYes, "Makefile, Makefile.common, options.mk: set") @@ -1245,12 +1257,10 @@ func parseACLEntries(varname string, aclEntries string) []ACLEntry { var result []ACLEntry prevperms := "(first)" for _, arg := range strings.Split(aclEntries, "; ") { - var globs, perms string - if fields := strings.SplitN(arg, ": ", 2); len(fields) == 2 { - globs, perms = fields[0], fields[1] - } else { - globs = strings.TrimSuffix(arg, ":") - } + fields := strings.SplitN(arg, ": ", 2) + G.Assertf(len(fields) == 2, "Invalid ACL entry %q", arg) + globs, perms := fields[0], ifelseStr(fields[1] == "none", "", fields[1]) + G.Assertf(perms != prevperms, "Repeated permissions %q for %q.", perms, varname) prevperms = perms diff --git a/pkgtools/pkglint/files/vartype.go b/pkgtools/pkglint/files/vartype.go index b53fe35d6aa..417a84bf567 100644 --- a/pkgtools/pkglint/files/vartype.go +++ b/pkgtools/pkglint/files/vartype.go @@ -38,6 +38,10 @@ const ( aclpAppend // VAR += value aclpUseLoadtime // OTHER := ${VAR}, OTHER != ${VAR} aclpUse // OTHER = ${VAR} + + // TODO: Try what happens if this constant is removed. + // All variables should have proper permission definitions for all files. + // Missing permission definitions could also count as "none". aclpUnknown aclpAllWrite = aclpSet | aclpSetDefault | aclpAppend aclpAllRead = aclpUseLoadtime | aclpUse diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go index f5bd5e885ce..28cd5eb1a3d 100644 --- a/pkgtools/pkglint/files/vartypecheck.go +++ b/pkgtools/pkglint/files/vartypecheck.go @@ -878,7 +878,7 @@ func (cv *VartypeCheck) PkgOptionsVar() { // // Despite its name, it is more similar to RelativePkgDir than to RelativePkgPath. func (cv *VartypeCheck) PkgPath() { - pkgsrcdir := relpath(path.Dir(cv.MkLine.Filename), G.Pkgsrc.File(".")) + pkgsrcdir := cv.MkLine.PathToFile(G.Pkgsrc.File(".")) MkLineChecker{cv.MkLine}.CheckRelativePkgdir(pkgsrcdir + "/" + cv.Value) } diff --git a/pkgtools/pkglint/files/vartypecheck_test.go b/pkgtools/pkglint/files/vartypecheck_test.go index 2c9913dc4e7..a7be8db0029 100644 --- a/pkgtools/pkglint/files/vartypecheck_test.go +++ b/pkgtools/pkglint/files/vartypecheck_test.go @@ -22,9 +22,9 @@ func (s *Suite) Test_VartypeCheck_AwkCommand(c *check.C) { // The warning should be adjusted to reflect this. vt.Output( - "WARN: filename:1: $0 is ambiguous. "+ + "WARN: filename.mk:1: $0 is ambiguous. "+ "Use ${0} if you mean a Make variable or $$0 if you mean a shell variable.", - "WARN: filename:3: $0 is ambiguous. "+ + "WARN: filename.mk:3: $0 is ambiguous. "+ "Use ${0} if you mean a Make variable or $$0 if you mean a shell variable.") } @@ -42,8 +42,8 @@ func (s *Suite) Test_VartypeCheck_BasicRegularExpression(c *check.C) { ".*\\.pl$$") vt.Output( - "WARN: filename:1: Internal pkglint error in MkLine.Tokenize at \"$\".", - "WARN: filename:3: Internal pkglint error in MkLine.Tokenize at \"$\".") + "WARN: filename.mk:1: Internal pkglint error in MkLine.Tokenize at \"$\".", + "WARN: filename.mk:3: Internal pkglint error in MkLine.Tokenize at \"$\".") } @@ -58,7 +58,7 @@ func (s *Suite) Test_VartypeCheck_BuildlinkDepmethod(c *check.C) { "${BUILDLINK_DEPMETHOD.kernel}") vt.Output( - "WARN: filename:2: Invalid dependency method \"unknown\". Valid methods are \"build\" or \"full\".") + "WARN: filename.mk:2: Invalid dependency method \"unknown\". Valid methods are \"build\" or \"full\".") } func (s *Suite) Test_VartypeCheck_Category(c *check.C) { @@ -78,8 +78,8 @@ func (s *Suite) Test_VartypeCheck_Category(c *check.C) { "wip") vt.Output( - "ERROR: filename:2: Invalid category \"arabic\".", - "ERROR: filename:4: Invalid category \"wip\".") + "ERROR: filename.mk:2: Invalid category \"arabic\".", + "ERROR: filename.mk:4: Invalid category \"wip\".") } func (s *Suite) Test_VartypeCheck_CFlag(c *check.C) { @@ -103,10 +103,10 @@ func (s *Suite) Test_VartypeCheck_CFlag(c *check.C) { "`pkg-config`_plus") vt.Output( - "WARN: filename:2: Compiler flag \"/W3\" should start with a hyphen.", - "WARN: filename:3: Compiler flag \"target:sparc64\" should start with a hyphen.", - "WARN: filename:5: Unknown compiler flag \"-XX:+PrintClassHistogramAfterFullGC\".", - "WARN: filename:11: Compiler flag \"`pkg-config`_plus\" should start with a hyphen.") + "WARN: filename.mk:2: Compiler flag \"/W3\" should start with a hyphen.", + "WARN: filename.mk:3: Compiler flag \"target:sparc64\" should start with a hyphen.", + "WARN: filename.mk:5: Unknown compiler flag \"-XX:+PrintClassHistogramAfterFullGC\".", + "WARN: filename.mk:11: Compiler flag \"`pkg-config`_plus\" should start with a hyphen.") vt.Op(opUseMatch) vt.Values( @@ -139,19 +139,19 @@ func (s *Suite) Test_VartypeCheck_Comment(c *check.C) { "'SQL injection fuzzer") vt.Output( - "ERROR: filename:2: COMMENT must be set.", - "WARN: filename:3: COMMENT should not begin with \"A\".", - "WARN: filename:3: COMMENT should not end with a period.", - "WARN: filename:4: COMMENT should start with a capital letter.", - "WARN: filename:4: COMMENT should not be longer than 70 characters.", - "WARN: filename:5: COMMENT should not be enclosed in quotes.", - "WARN: filename:6: COMMENT should not be enclosed in quotes.", - "WARN: filename:7: COMMENT should not contain \"is a\".", - "WARN: filename:8: COMMENT should not contain \"is an\".", - "WARN: filename:9: COMMENT should not contain \"is a\".", - "WARN: filename:10: COMMENT should not start with the package name.", - "WARN: filename:11: COMMENT should not start with the package name.", - "WARN: filename:11: COMMENT should not contain \"is a\".") + "ERROR: filename.mk:2: COMMENT must be set.", + "WARN: filename.mk:3: COMMENT should not begin with \"A\".", + "WARN: filename.mk:3: COMMENT should not end with a period.", + "WARN: filename.mk:4: COMMENT should start with a capital letter.", + "WARN: filename.mk:4: COMMENT should not be longer than 70 characters.", + "WARN: filename.mk:5: COMMENT should not be enclosed in quotes.", + "WARN: filename.mk:6: COMMENT should not be enclosed in quotes.", + "WARN: filename.mk:7: COMMENT should not contain \"is a\".", + "WARN: filename.mk:8: COMMENT should not contain \"is an\".", + "WARN: filename.mk:9: COMMENT should not contain \"is a\".", + "WARN: filename.mk:10: COMMENT should not start with the package name.", + "WARN: filename.mk:11: COMMENT should not start with the package name.", + "WARN: filename.mk:11: COMMENT should not contain \"is a\".") } func (s *Suite) Test_VartypeCheck_ConfFiles(c *check.C) { @@ -167,9 +167,9 @@ func (s *Suite) Test_VartypeCheck_ConfFiles(c *check.C) { "share/etc/bootrc /etc/bootrc") vt.Output( - "WARN: filename:1: Values for CONF_FILES should always be pairs of paths.", - "WARN: filename:3: Values for CONF_FILES should always be pairs of paths.", - "WARN: filename:5: The destination file \"/etc/bootrc\" should start with a variable reference.") + "WARN: filename.mk:1: Values for CONF_FILES should always be pairs of paths.", + "WARN: filename.mk:3: Values for CONF_FILES should always be pairs of paths.", + "WARN: filename.mk:5: The destination file \"/etc/bootrc\" should start with a variable reference.") } func (s *Suite) Test_VartypeCheck_Dependency(c *check.C) { @@ -206,21 +206,21 @@ func (s *Suite) Test_VartypeCheck_Dependency(c *check.C) { "package-1.0>=1.0.3") vt.Output( - "WARN: filename:1: Invalid dependency pattern \"Perl\".", - "WARN: filename:3: Please use \"perl5-[0-9]*\" instead of \"perl5-*\".", - "WARN: filename:5: Only [0-9]* is allowed in the numeric part of a dependency.", - "WARN: filename:5: The version pattern \"[5.10-5.22]*\" should not contain a hyphen.", - "WARN: filename:6: Invalid dependency pattern \"py-docs\".", - "WARN: filename:10: Please use \"5.22{,nb*}\" instead of \"5.22\" as the version pattern.", - "WARN: filename:11: Please use \"5.*\" instead of \"5*\" as the version pattern.", - "WARN: filename:12: The version pattern \"2.0-[0-9]*\" should not contain a hyphen.", - "WARN: filename:21: Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.", - "WARN: filename:22: Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.", - "WARN: filename:23: Invalid dependency pattern \"package-1.0|garbage\".", + "WARN: filename.mk:1: Invalid dependency pattern \"Perl\".", + "WARN: filename.mk:3: Please use \"perl5-[0-9]*\" instead of \"perl5-*\".", + "WARN: filename.mk:5: Only [0-9]* is allowed in the numeric part of a dependency.", + "WARN: filename.mk:5: The version pattern \"[5.10-5.22]*\" should not contain a hyphen.", + "WARN: filename.mk:6: Invalid dependency pattern \"py-docs\".", + "WARN: filename.mk:10: Please use \"5.22{,nb*}\" instead of \"5.22\" as the version pattern.", + "WARN: filename.mk:11: Please use \"5.*\" instead of \"5*\" as the version pattern.", + "WARN: filename.mk:12: The version pattern \"2.0-[0-9]*\" should not contain a hyphen.", + "WARN: filename.mk:21: Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.", + "WARN: filename.mk:22: Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.", + "WARN: filename.mk:23: Invalid dependency pattern \"package-1.0|garbage\".", // TODO: Mention that the path should be removed. - "WARN: filename:25: Invalid dependency pattern \"package>=1.0:../../category/package\".", + "WARN: filename.mk:25: Invalid dependency pattern \"package>=1.0:../../category/package\".", // TODO: Mention that version numbers in a pkgbase must be appended directly, without hyphen. - "WARN: filename:26: Invalid dependency pattern \"package-1.0>=1.0.3\".") + "WARN: filename.mk:26: Invalid dependency pattern \"package-1.0>=1.0.3\".") } func (s *Suite) Test_VartypeCheck_DependencyWithPath(c *check.C) { @@ -282,7 +282,7 @@ func (s *Suite) Test_VartypeCheck_DistSuffix(c *check.C) { ".tar.gz # overrides a definition from a Makefile.common") vt.Output( - "NOTE: filename:1: EXTRACT_SUFX is \".tar.gz\" by default, so this definition may be redundant.") + "NOTE: filename.mk:1: EXTRACT_SUFX is \".tar.gz\" by default, so this definition may be redundant.") } func (s *Suite) Test_VartypeCheck_EmulPlatform(c *check.C) { @@ -295,12 +295,12 @@ func (s *Suite) Test_VartypeCheck_EmulPlatform(c *check.C) { "${LINUX}") vt.Output( - "WARN: filename:2: \"nextbsd\" is not valid for the operating system part of EMUL_PLATFORM. "+ + "WARN: filename.mk:2: \"nextbsd\" is not valid for the operating system part of EMUL_PLATFORM. "+ "Use one of "+ "{ bitrig bsdos cygwin darwin dragonfly freebsd haiku hpux "+ "interix irix linux mirbsd netbsd openbsd osf1 solaris sunos "+ "} instead.", - "WARN: filename:2: \"8087\" is not valid for the hardware architecture part of EMUL_PLATFORM. "+ + "WARN: filename.mk:2: \"8087\" is not valid for the hardware architecture part of EMUL_PLATFORM. "+ "Use one of { "+ "aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 "+ "cobalt coldfire convex dreamcast "+ @@ -313,7 +313,7 @@ func (s *Suite) Test_VartypeCheck_EmulPlatform(c *check.C) { "mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 "+ "s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ "} instead.", - "WARN: filename:3: \"${LINUX}\" is not a valid emulation platform.") + "WARN: filename.mk:3: \"${LINUX}\" is not a valid emulation platform.") } func (s *Suite) Test_VartypeCheck_Enum(c *check.C) { @@ -329,8 +329,8 @@ func (s *Suite) Test_VartypeCheck_Enum(c *check.C) { "[") vt.Output( - "WARN: filename:3: The pattern \"sun-jdk*\" cannot match any of { jdk1 jdk2 jdk4 } for JDK.", - "WARN: filename:5: Invalid match pattern \"[\".") + "WARN: filename.mk:3: The pattern \"sun-jdk*\" cannot match any of { jdk1 jdk2 jdk4 } for JDK.", + "WARN: filename.mk:5: Invalid match pattern \"[\".") } func (s *Suite) Test_VartypeCheck_Enum__use_match(c *check.C) { @@ -385,13 +385,13 @@ func (s *Suite) Test_VartypeCheck_FetchURL(c *check.C) { "${MASTER_SITE_INVALID:=subdir/}") vt.Output( - "WARN: filename:1: Please use ${MASTER_SITE_GITHUB:=example/} "+ + "WARN: filename.mk:1: Please use ${MASTER_SITE_GITHUB:=example/} "+ "instead of \"https://github.com/example/project/\" "+ "and run \""+confMake+" help topic=github\" for further tips.", - "WARN: filename:2: Please use ${MASTER_SITE_GNU:=bison} "+ + "WARN: filename.mk:2: Please use ${MASTER_SITE_GNU:=bison} "+ "instead of \"http://ftp.gnu.org/pub/gnu/bison\".", - "ERROR: filename:3: The subdirectory in MASTER_SITE_GNU must end with a slash.", - "ERROR: filename:4: The site MASTER_SITE_INVALID does not exist.") + "ERROR: filename.mk:3: The subdirectory in MASTER_SITE_GNU must end with a slash.", + "ERROR: filename.mk:4: The site MASTER_SITE_INVALID does not exist.") // PR 46570, keyword gimp-fix-ca vt.Values( @@ -405,7 +405,7 @@ func (s *Suite) Test_VartypeCheck_FetchURL(c *check.C) { "http://example.org/download?filename=<distfile>;version=<version>") vt.Output( - "WARN: filename:8: \"http://example.org/download?filename=<distfile>;version=<version>\" is not a valid URL.") + "WARN: filename.mk:8: \"http://example.org/download?filename=<distfile>;version=<version>\" is not a valid URL.") } func (s *Suite) Test_VartypeCheck_Filename(c *check.C) { @@ -417,8 +417,8 @@ func (s *Suite) Test_VartypeCheck_Filename(c *check.C) { "OS/2-manual.txt") vt.Output( - "WARN: filename:1: \"Filename with spaces.docx\" is not a valid filename.", - "WARN: filename:2: A filename should not contain a slash.") + "WARN: filename.mk:1: \"Filename with spaces.docx\" is not a valid filename.", + "WARN: filename.mk:2: A filename should not contain a slash.") vt.Op(opUseMatch) vt.Values( @@ -438,8 +438,8 @@ func (s *Suite) Test_VartypeCheck_FileMask(c *check.C) { "OS/2-manual.txt") vt.Output( - "WARN: filename:1: \"FileMask with spaces.docx\" is not a valid filename mask.", - "WARN: filename:2: A filename mask should not contain a slash.") + "WARN: filename.mk:1: \"FileMask with spaces.docx\" is not a valid filename mask.", + "WARN: filename.mk:2: A filename mask should not contain a slash.") vt.Op(opUseMatch) vt.Values( @@ -463,9 +463,9 @@ func (s *Suite) Test_VartypeCheck_FileMode(c *check.C) { "") vt.Output( - "WARN: filename:1: Invalid file mode \"u+rwx\".", - "WARN: filename:4: Invalid file mode \"12345\".", - "WARN: filename:6: Invalid file mode \"\".") + "WARN: filename.mk:1: Invalid file mode \"u+rwx\".", + "WARN: filename.mk:4: Invalid file mode \"12345\".", + "WARN: filename.mk:6: Invalid file mode \"\".") vt.Op(opUseMatch) vt.Values( @@ -474,7 +474,7 @@ func (s *Suite) Test_VartypeCheck_FileMode(c *check.C) { // There's no guarantee that a filename only contains [A-Za-z0-9.]. // Therefore there are no useful checks in this situation. vt.Output( - "WARN: filename:11: Invalid file mode \"u+rwx\".") + "WARN: filename.mk:11: Invalid file mode \"u+rwx\".") } func (s *Suite) Test_VartypeCheck_GccReqd(c *check.C) { @@ -491,8 +491,8 @@ func (s *Suite) Test_VartypeCheck_GccReqd(c *check.C) { "6", "7.3") vt.Output( - "WARN: filename:5: GCC version numbers should only contain the major version (5).", - "WARN: filename:7: GCC version numbers should only contain the major version (7).") + "WARN: filename.mk:5: GCC version numbers should only contain the major version (5).", + "WARN: filename.mk:7: GCC version numbers should only contain the major version (7).") } func (s *Suite) Test_VartypeCheck_Homepage(c *check.C) { @@ -506,7 +506,7 @@ func (s *Suite) Test_VartypeCheck_Homepage(c *check.C) { "${MASTER_SITES}") vt.Output( - "WARN: filename:3: HOMEPAGE should not be defined in terms of MASTER_SITEs.") + "WARN: filename.mk:3: HOMEPAGE should not be defined in terms of MASTER_SITEs.") G.Pkg = NewPackage(t.File("category/package")) @@ -517,7 +517,7 @@ func (s *Suite) Test_VartypeCheck_Homepage(c *check.C) { // doesn't define MASTER_SITES, that variable cannot be expanded, which means // the warning cannot refer to its value. vt.Output( - "WARN: filename:4: HOMEPAGE should not be defined in terms of MASTER_SITEs.") + "WARN: filename.mk:4: HOMEPAGE should not be defined in terms of MASTER_SITEs.") delete(G.Pkg.vars.firstDef, "MASTER_SITES") delete(G.Pkg.vars.lastDef, "MASTER_SITES") @@ -528,7 +528,7 @@ func (s *Suite) Test_VartypeCheck_Homepage(c *check.C) { "${MASTER_SITES}") vt.Output( - "WARN: filename:5: HOMEPAGE should not be defined in terms of MASTER_SITEs. " + + "WARN: filename.mk:5: HOMEPAGE should not be defined in terms of MASTER_SITEs. " + "Use https://cdn.NetBSD.org/pub/pkgsrc/distfiles/ directly.") delete(G.Pkg.vars.firstDef, "MASTER_SITES") @@ -542,7 +542,7 @@ func (s *Suite) Test_VartypeCheck_Homepage(c *check.C) { // When MASTER_SITES itself makes use of another variable, pkglint doesn't // resolve that variable and just outputs the simple variant of this warning. vt.Output( - "WARN: filename:6: HOMEPAGE should not be defined in terms of MASTER_SITEs.") + "WARN: filename.mk:6: HOMEPAGE should not be defined in terms of MASTER_SITEs.") } @@ -559,9 +559,9 @@ func (s *Suite) Test_VartypeCheck_Identifier(c *check.C) { "") vt.Output( - "WARN: filename:2: Invalid identifier \"identifiers cannot contain spaces\".", - "WARN: filename:3: Invalid identifier \"id/cannot/contain/slashes\".", - "WARN: filename:5: Invalid identifier \"\".") + "WARN: filename.mk:2: Invalid identifier \"identifiers cannot contain spaces\".", + "WARN: filename.mk:3: Invalid identifier \"id/cannot/contain/slashes\".", + "WARN: filename.mk:5: Invalid identifier \"\".") vt.Op(opUseMatch) vt.Values( @@ -571,7 +571,7 @@ func (s *Suite) Test_VartypeCheck_Identifier(c *check.C) { "A*B") vt.Output( - "WARN: filename:12: Invalid identifier pattern \"[A-Z.]\" for MYSQL_CHARSET.") + "WARN: filename.mk:12: Invalid identifier pattern \"[A-Z.]\" for MYSQL_CHARSET.") } func (s *Suite) Test_VartypeCheck_Integer(c *check.C) { @@ -586,8 +586,8 @@ func (s *Suite) Test_VartypeCheck_Integer(c *check.C) { "11111111111111111111111111111111111111111111111") vt.Output( - "WARN: filename:1: Invalid integer \"${OTHER_VAR}\".", - "WARN: filename:3: Invalid integer \"-13\".") + "WARN: filename.mk:1: Invalid integer \"${OTHER_VAR}\".", + "WARN: filename.mk:3: Invalid integer \"-13\".") } func (s *Suite) Test_VartypeCheck_LdFlag(c *check.C) { @@ -615,10 +615,10 @@ func (s *Suite) Test_VartypeCheck_LdFlag(c *check.C) { "anything") vt.Output( - "WARN: filename:4: Unknown linker flag \"-unknown\".", - "WARN: filename:5: Linker flag \"no-hyphen\" should start with a hyphen.", - "WARN: filename:6: Please use \"${COMPILER_RPATH_FLAG}\" instead of \"-Wl,--rpath\".", - "WARN: filename:12: Linker flag \"`pkg-config`_plus\" should start with a hyphen.") + "WARN: filename.mk:4: Unknown linker flag \"-unknown\".", + "WARN: filename.mk:5: Linker flag \"no-hyphen\" should start with a hyphen.", + "WARN: filename.mk:6: Please use \"${COMPILER_RPATH_FLAG}\" instead of \"-Wl,--rpath\".", + "WARN: filename.mk:12: Linker flag \"`pkg-config`_plus\" should start with a hyphen.") } func (s *Suite) Test_VartypeCheck_License(c *check.C) { @@ -640,9 +640,9 @@ func (s *Suite) Test_VartypeCheck_License(c *check.C) { "${UNKNOWN_LICENSE}") vt.Output( - "ERROR: filename:2: Parse error for license condition \"AND mit\".", - "WARN: filename:3: License file ~/licenses/artistic does not exist.", - "ERROR: filename:4: Parse error for license condition \"${UNKNOWN_LICENSE}\".") + "ERROR: filename.mk:2: Parse error for license condition \"AND mit\".", + "WARN: filename.mk:3: License file ~/licenses/artistic does not exist.", + "ERROR: filename.mk:4: Parse error for license condition \"${UNKNOWN_LICENSE}\".") vt.Op(opAssignAppend) vt.Values( @@ -650,8 +650,8 @@ func (s *Suite) Test_VartypeCheck_License(c *check.C) { "AND mit") vt.Output( - "ERROR: filename:11: Parse error for appended license condition \"gnu-gpl-v2\".", - "WARN: filename:12: License file ~/licenses/mit does not exist.") + "ERROR: filename.mk:11: Parse error for appended license condition \"gnu-gpl-v2\".", + "WARN: filename.mk:12: License file ~/licenses/mit does not exist.") } func (s *Suite) Test_VartypeCheck_MachineGnuPlatform(c *check.C) { @@ -668,18 +668,18 @@ func (s *Suite) Test_VartypeCheck_MachineGnuPlatform(c *check.C) { "x86_64-pc") // Just for code coverage. vt.Output( - "WARN: filename:2: The pattern \"Cygwin\" cannot match any of "+ + "WARN: filename.mk:2: The pattern \"Cygwin\" cannot match any of "+ "{ aarch64 aarch64_be alpha amd64 arc arm armeb armv4 armv4eb armv6 armv6eb armv7 armv7eb "+ "cobalt convex dreamcast hpcmips hpcsh hppa hppa64 i386 i486 ia64 m5407 m68010 m68k m88k "+ "mips mips64 mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 "+ "rs6000 s390 sh shle sparc sparc64 vax x86_64 "+ "} for the hardware architecture part of MACHINE_GNU_PLATFORM.", - "WARN: filename:2: The pattern \"amd64\" cannot match any of "+ + "WARN: filename.mk:2: The pattern \"amd64\" cannot match any of "+ "{ bitrig bsdos cygwin darwin dragonfly freebsd haiku hpux interix irix linux mirbsd "+ "netbsd openbsd osf1 solaris sunos } "+ "for the operating system part of MACHINE_GNU_PLATFORM.", - "WARN: filename:4: \"*-*-*-*\" is not a valid platform pattern.", - "WARN: filename:6: \"x86_64-pc\" is not a valid platform pattern.") + "WARN: filename.mk:4: \"*-*-*-*\" is not a valid platform pattern.", + "WARN: filename.mk:6: \"x86_64-pc\" is not a valid platform pattern.") } func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { @@ -698,12 +698,12 @@ func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { "NetBSD-[0-1]*-*") vt.Output( - "WARN: filename:1: \"linux-i386\" is not a valid platform pattern.", - "WARN: filename:2: The pattern \"nextbsd\" cannot match any of "+ + "WARN: filename.mk:1: \"linux-i386\" is not a valid platform pattern.", + "WARN: filename.mk:2: The pattern \"nextbsd\" cannot match any of "+ "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ "} for the operating system part of ONLY_FOR_PLATFORM.", - "WARN: filename:2: The pattern \"8087\" cannot match any of "+ + "WARN: filename.mk:2: The pattern \"8087\" cannot match any of "+ "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 "+ "cobalt coldfire convex dreamcast "+ "earm earmeb earmhf earmhfeb earmv4 earmv4eb "+ @@ -714,11 +714,11 @@ func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { "mlrisc ns32k pc532 pmax powerpc powerpc64 "+ "rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ "} for the hardware architecture part of ONLY_FOR_PLATFORM.", - "WARN: filename:3: The pattern \"netbsd\" cannot match any of "+ + "WARN: filename.mk:3: The pattern \"netbsd\" cannot match any of "+ "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ "} for the operating system part of ONLY_FOR_PLATFORM.", - "WARN: filename:3: The pattern \"l*\" cannot match any of "+ + "WARN: filename.mk:3: The pattern \"l*\" cannot match any of "+ "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 "+ "cobalt coldfire convex dreamcast "+ "earm earmeb earmhf earmhfeb earmv4 earmv4eb "+ @@ -729,8 +729,8 @@ func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { "mlrisc ns32k pc532 pmax powerpc powerpc64 "+ "rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ "} for the hardware architecture part of ONLY_FOR_PLATFORM.", - "WARN: filename:5: \"FreeBSD*\" is not a valid platform pattern.", - "WARN: filename:8: Please use \"[0-1].*\" instead of \"[0-1]*\" as the version pattern.") + "WARN: filename.mk:5: \"FreeBSD*\" is not a valid platform pattern.", + "WARN: filename.mk:8: Please use \"[0-1].*\" instead of \"[0-1]*\" as the version pattern.") } func (s *Suite) Test_VartypeCheck_MailAddress(c *check.C) { @@ -744,10 +744,10 @@ func (s *Suite) Test_VartypeCheck_MailAddress(c *check.C) { "user1@example.org,user2@example.org") vt.Output( - "WARN: filename:1: Please write \"NetBSD.org\" instead of \"netbsd.org\".", - "ERROR: filename:2: This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.", - "ERROR: filename:3: This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.", - "WARN: filename:4: \"user1@example.org,user2@example.org\" is not a valid mail address.") + "WARN: filename.mk:1: Please write \"NetBSD.org\" instead of \"netbsd.org\".", + "ERROR: filename.mk:2: This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.", + "ERROR: filename.mk:3: This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.", + "WARN: filename.mk:4: \"user1@example.org,user2@example.org\" is not a valid mail address.") } func (s *Suite) Test_VartypeCheck_Message(c *check.C) { @@ -759,7 +759,7 @@ func (s *Suite) Test_VartypeCheck_Message(c *check.C) { "Correct paths") vt.Output( - "WARN: filename:1: SUBST_MESSAGE.id should not be quoted.") + "WARN: filename.mk:1: SUBST_MESSAGE.id should not be quoted.") } func (s *Suite) Test_VartypeCheck_Option(c *check.C) { @@ -777,9 +777,9 @@ func (s *Suite) Test_VartypeCheck_Option(c *check.C) { "UPPER") vt.Output( - "WARN: filename:3: Unknown option \"unknown\".", - "WARN: filename:4: Use of the underscore character in option names is deprecated.", - "ERROR: filename:5: Invalid option name \"UPPER\". "+ + "WARN: filename.mk:3: Unknown option \"unknown\".", + "WARN: filename.mk:4: Use of the underscore character in option names is deprecated.", + "ERROR: filename.mk:5: Invalid option name \"UPPER\". "+ "Option names must start with a lowercase letter and be all-lowercase.") } @@ -792,10 +792,10 @@ func (s *Suite) Test_VartypeCheck_Pathlist(c *check.C) { "/directory with spaces") vt.Output( - "ERROR: filename:1: The component \".\" of PATH must be an absolute path.", - "ERROR: filename:1: The component \"\" of PATH must be an absolute path.", - "WARN: filename:1: \"${PREFIX}/!!!\" is not a valid pathname.", - "WARN: filename:2: \"/directory with spaces\" is not a valid pathname.") + "ERROR: filename.mk:1: The component \".\" of PATH must be an absolute path.", + "ERROR: filename.mk:1: The component \"\" of PATH must be an absolute path.", + "WARN: filename.mk:1: \"${PREFIX}/!!!\" is not a valid pathname.", + "WARN: filename.mk:2: \"/directory with spaces\" is not a valid pathname.") } func (s *Suite) Test_VartypeCheck_PathMask(c *check.C) { @@ -808,7 +808,7 @@ func (s *Suite) Test_VartypeCheck_PathMask(c *check.C) { "src/*/*") vt.Output( - "WARN: filename:2: \"src/*&*\" is not a valid pathname mask.") + "WARN: filename.mk:2: \"src/*&*\" is not a valid pathname mask.") vt.Op(opUseMatch) vt.Values("any") @@ -831,7 +831,7 @@ func (s *Suite) Test_VartypeCheck_Pathname(c *check.C) { // FIXME: Warn about the absolute pathname in line 4. vt.Output( - "WARN: filename:1: \"${PREFIX}/*\" is not a valid pathname.") + "WARN: filename.mk:1: \"${PREFIX}/*\" is not a valid pathname.") } func (s *Suite) Test_VartypeCheck_Perl5Packlist(c *check.C) { @@ -843,7 +843,7 @@ func (s *Suite) Test_VartypeCheck_Perl5Packlist(c *check.C) { "anything else") vt.Output( - "WARN: filename:1: PERL5_PACKLIST should not depend on other variables.") + "WARN: filename.mk:1: PERL5_PACKLIST should not depend on other variables.") } func (s *Suite) Test_VartypeCheck_Perms(c *check.C) { @@ -860,8 +860,8 @@ func (s *Suite) Test_VartypeCheck_Perms(c *check.C) { "${REAL_ROOT_GROUP}") vt.Output( - "ERROR: filename:2: ROOT_USER must not be used in permission definitions. Use REAL_ROOT_USER instead.", - "ERROR: filename:5: ROOT_GROUP must not be used in permission definitions. Use REAL_ROOT_GROUP instead.") + "ERROR: filename.mk:2: ROOT_USER must not be used in permission definitions. Use REAL_ROOT_USER instead.", + "ERROR: filename.mk:5: ROOT_GROUP must not be used in permission definitions. Use REAL_ROOT_GROUP instead.") } func (s *Suite) Test_VartypeCheck_Pkgname(c *check.C) { @@ -880,7 +880,7 @@ func (s *Suite) Test_VartypeCheck_Pkgname(c *check.C) { "pkgbase-3.1.4.1.5.9.2.6.5.3.5.8.9.7.9") vt.Output( - "WARN: filename:8: \"pkgbase-z1\" is not a valid package name.") + "WARN: filename.mk:8: \"pkgbase-z1\" is not a valid package name.") vt.Op(opUseMatch) vt.Values( @@ -899,8 +899,8 @@ func (s *Suite) Test_VartypeCheck_PkgOptionsVar(c *check.C) { "PKG_OPTS.mc") vt.Output( - "ERROR: filename:1: PKGBASE must not be used in PKG_OPTIONS_VAR.", - "ERROR: filename:3: PKG_OPTIONS_VAR must be "+ + "ERROR: filename.mk:1: PKGBASE must not be used in PKG_OPTIONS_VAR.", + "ERROR: filename.mk:3: PKG_OPTIONS_VAR must be "+ "of the form \"PKG_OPTIONS.*\", not \"PKG_OPTS.mc\".") } @@ -919,10 +919,10 @@ func (s *Suite) Test_VartypeCheck_PkgPath(c *check.C) { "../../invalid/relative") vt.Output( - "ERROR: filename:3: Relative path \"../../invalid\" does not exist.", - "WARN: filename:3: \"../../invalid\" is not a valid relative package directory.", - "ERROR: filename:4: Relative path \"../../../../invalid/relative\" does not exist.", - "WARN: filename:4: \"../../../../invalid/relative\" is not a valid relative package directory.") + "ERROR: filename.mk:3: Relative path \"../../invalid\" does not exist.", + "WARN: filename.mk:3: \"../../invalid\" is not a valid relative package directory.", + "ERROR: filename.mk:4: Relative path \"../../../../invalid/relative\" does not exist.", + "WARN: filename.mk:4: \"../../../../invalid/relative\" is not a valid relative package directory.") } func (s *Suite) Test_VartypeCheck_PkgRevision(c *check.C) { @@ -933,8 +933,8 @@ func (s *Suite) Test_VartypeCheck_PkgRevision(c *check.C) { "3a") vt.Output( - "WARN: filename:1: PKGREVISION must be a positive integer number.", - "ERROR: filename:1: PKGREVISION only makes sense directly in the package Makefile.") + "WARN: filename.mk:1: PKGREVISION must be a positive integer number.", + "ERROR: filename.mk:1: PKGREVISION only makes sense directly in the package Makefile.") vt.File("Makefile") vt.Values( @@ -953,8 +953,8 @@ func (s *Suite) Test_VartypeCheck_PythonDependency(c *check.C) { "cairo,X") vt.Output( - "WARN: filename:2: Python dependencies should not contain variables.", - "WARN: filename:3: Invalid Python dependency \"cairo,X\".") + "WARN: filename.mk:2: Python dependencies should not contain variables.", + "WARN: filename.mk:3: Invalid Python dependency \"cairo,X\".") } func (s *Suite) Test_VartypeCheck_PrefixPathname(c *check.C) { @@ -966,7 +966,7 @@ func (s *Suite) Test_VartypeCheck_PrefixPathname(c *check.C) { "share/locale") vt.Output( - "WARN: filename:1: Please use \"${PKGMANDIR}/man1\" instead of \"man/man1\".") + "WARN: filename.mk:1: Please use \"${PKGMANDIR}/man1\" instead of \"man/man1\".") } func (s *Suite) Test_VartypeCheck_RelativePkgPath(c *check.C) { @@ -985,9 +985,9 @@ func (s *Suite) Test_VartypeCheck_RelativePkgPath(c *check.C) { "../../invalid/relative") vt.Output( - "ERROR: filename:1: Relative path \"category/other-package\" does not exist.", - "ERROR: filename:4: Relative path \"invalid\" does not exist.", - "ERROR: filename:5: Relative path \"../../invalid/relative\" does not exist.") + "ERROR: filename.mk:1: Relative path \"category/other-package\" does not exist.", + "ERROR: filename.mk:4: Relative path \"invalid\" does not exist.", + "ERROR: filename.mk:5: Relative path \"../../invalid/relative\" does not exist.") } func (s *Suite) Test_VartypeCheck_Restricted(c *check.C) { @@ -998,7 +998,7 @@ func (s *Suite) Test_VartypeCheck_Restricted(c *check.C) { "May only be distributed free of charge") vt.Output( - "WARN: filename:1: The only valid value for NO_BIN_ON_CDROM is ${RESTRICTED}.") + "WARN: filename.mk:1: The only valid value for NO_BIN_ON_CDROM is ${RESTRICTED}.") } func (s *Suite) Test_VartypeCheck_SedCommands(c *check.C) { @@ -1019,15 +1019,15 @@ func (s *Suite) Test_VartypeCheck_SedCommands(c *check.C) { "-e s,$${unclosedShellVar") // Just for code coverage. vt.Output( - "NOTE: filename:1: Please always use \"-e\" in sed commands, even if there is only one substitution.", - "NOTE: filename:2: Each sed command should appear in an assignment of its own.", - "WARN: filename:3: The # character starts a Makefile comment.", - "ERROR: filename:3: Invalid shell words \"\\\"s,\" in sed commands.", - "WARN: filename:8: Unknown sed command \"1d\".", - "ERROR: filename:9: The -e option to sed requires an argument.", - "WARN: filename:10: Unknown sed command \"-i\".", - "NOTE: filename:10: Please always use \"-e\" in sed commands, even if there is only one substitution.", - "WARN: filename:11: Unclosed shell variable starting at \"$${unclosedShellVar\".") + "NOTE: filename.mk:1: Please always use \"-e\" in sed commands, even if there is only one substitution.", + "NOTE: filename.mk:2: Each sed command should appear in an assignment of its own.", + "WARN: filename.mk:3: The # character starts a Makefile comment.", + "ERROR: filename.mk:3: Invalid shell words \"\\\"s,\" in sed commands.", + "WARN: filename.mk:8: Unknown sed command \"1d\".", + "ERROR: filename.mk:9: The -e option to sed requires an argument.", + "WARN: filename.mk:10: Unknown sed command \"-i\".", + "NOTE: filename.mk:10: Please always use \"-e\" in sed commands, even if there is only one substitution.", + "WARN: filename.mk:11: Unclosed shell variable starting at \"$${unclosedShellVar\".") } func (s *Suite) Test_VartypeCheck_ShellCommand(c *check.C) { @@ -1057,7 +1057,7 @@ func (s *Suite) Test_VartypeCheck_ShellCommands(c *check.C) { "echo bin/program;") vt.Output( - "WARN: filename:1: This shell command list should end with a semicolon.") + "WARN: filename.mk:1: This shell command list should end with a semicolon.") } func (s *Suite) Test_VartypeCheck_Stage(c *check.C) { @@ -1070,7 +1070,7 @@ func (s *Suite) Test_VartypeCheck_Stage(c *check.C) { "pre-test") vt.Output( - "WARN: filename:2: Invalid stage name \"post-modern\". " + + "WARN: filename.mk:2: Invalid stage name \"post-modern\". " + "Use one of {pre,do,post}-{extract,patch,configure,build,test,install}.") } @@ -1092,10 +1092,10 @@ func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { "unknown") vt.Output( - "ERROR: filename:2: Invalid tool dependency \"unknown\". "+ + "ERROR: filename.mk:2: Invalid tool dependency \"unknown\". "+ "Use one of \"bootstrap\", \"build\", \"pkgsrc\", \"run\" or \"test\".", - "ERROR: filename:4: Invalid tool dependency \"mal:formed:tool\".", - "ERROR: filename:5: Unknown tool \"unknown\".") + "ERROR: filename.mk:4: Invalid tool dependency \"mal:formed:tool\".", + "ERROR: filename.mk:5: Unknown tool \"unknown\".") vt.Varname("USE_TOOLS.NetBSD") vt.Op(opAssignAppend) @@ -1104,7 +1104,7 @@ func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { "tool2:unknown") vt.Output( - "ERROR: filename:12: Invalid tool dependency \"unknown\". " + + "ERROR: filename.mk:12: Invalid tool dependency \"unknown\". " + "Use one of \"bootstrap\", \"build\", \"pkgsrc\", \"run\" or \"test\".") vt.Varname("TOOLS_NOOP") @@ -1118,7 +1118,7 @@ func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { "gmake:run") vt.Output( - "ERROR: filename:31: Unknown tool \"gmake\".") + "ERROR: filename.mk:31: Unknown tool \"gmake\".") vt.Varname("USE_TOOLS") vt.Op(opUseMatch) @@ -1156,17 +1156,17 @@ func (s *Suite) Test_VartypeCheck_URL(c *check.C) { "string with spaces") vt.Output( - "WARN: filename:4: Please write NetBSD.org instead of www.netbsd.org.", - "NOTE: filename:5: For consistency, please add a trailing slash to \"https://www.example.org\".", - "WARN: filename:8: \"\" is not a valid URL.", - "WARN: filename:9: \"ftp://example.org/<\" is not a valid URL.", - "WARN: filename:10: \"gopher://example.org/<\" is not a valid URL.", - "WARN: filename:11: \"http://example.org/<\" is not a valid URL.", - "WARN: filename:12: \"https://example.org/<\" is not a valid URL.", - "WARN: filename:13: \"https://www.example.org/path with spaces\" is not a valid URL.", - "WARN: filename:14: \"httpxs://www.example.org\" is not a valid URL. Only ftp, gopher, http, and https URLs are allowed here.", - "WARN: filename:15: \"mailto:someone@example.org\" is not a valid URL.", - "WARN: filename:16: \"string with spaces\" is not a valid URL.") + "WARN: filename.mk:4: Please write NetBSD.org instead of www.netbsd.org.", + "NOTE: filename.mk:5: For consistency, please add a trailing slash to \"https://www.example.org\".", + "WARN: filename.mk:8: \"\" is not a valid URL.", + "WARN: filename.mk:9: \"ftp://example.org/<\" is not a valid URL.", + "WARN: filename.mk:10: \"gopher://example.org/<\" is not a valid URL.", + "WARN: filename.mk:11: \"http://example.org/<\" is not a valid URL.", + "WARN: filename.mk:12: \"https://example.org/<\" is not a valid URL.", + "WARN: filename.mk:13: \"https://www.example.org/path with spaces\" is not a valid URL.", + "WARN: filename.mk:14: \"httpxs://www.example.org\" is not a valid URL. Only ftp, gopher, http, and https URLs are allowed here.", + "WARN: filename.mk:15: \"mailto:someone@example.org\" is not a valid URL.", + "WARN: filename.mk:16: \"string with spaces\" is not a valid URL.") // Yes, even in 2019, some pkgsrc-wip packages really use a gopher HOMEPAGE. vt.Values( @@ -1187,8 +1187,8 @@ func (s *Suite) Test_VartypeCheck_UserGroupName(c *check.C) { "${OTHER_VAR}") vt.Output( - "WARN: filename:1: Invalid user or group name \"user with spaces\".", - "WARN: filename:4: Invalid user or group name \"domain\\\\user\".") + "WARN: filename.mk:1: Invalid user or group name \"user with spaces\".", + "WARN: filename.mk:4: Invalid user or group name \"domain\\\\user\".") } func (s *Suite) Test_VartypeCheck_VariableName(c *check.C) { @@ -1202,7 +1202,7 @@ func (s *Suite) Test_VartypeCheck_VariableName(c *check.C) { "${INDIRECT}") vt.Output( - "WARN: filename:2: \"VarBase\" is not a valid variable name.") + "WARN: filename.mk:2: \"VarBase\" is not a valid variable name.") } func (s *Suite) Test_VartypeCheck_Version(c *check.C) { @@ -1218,7 +1218,7 @@ func (s *Suite) Test_VartypeCheck_Version(c *check.C) { "4pre7", "${VER}") vt.Output( - "WARN: filename:4: Invalid version number \"4.1-SNAPSHOT\".") + "WARN: filename.mk:4: Invalid version number \"4.1-SNAPSHOT\".") vt.Op(opUseMatch) vt.Values( @@ -1230,10 +1230,10 @@ func (s *Suite) Test_VartypeCheck_Version(c *check.C) { "1.[2-7].*", "[0-9]*") vt.Output( - "WARN: filename:11: Invalid version number pattern \"a*\".", - "WARN: filename:12: Invalid version number pattern \"1.2/456\".", - "WARN: filename:13: Please use \"4.*\" instead of \"4*\" as the version pattern.", - "WARN: filename:15: Please use \"1.[234].*\" instead of \"1.[234]*\" as the version pattern.") + "WARN: filename.mk:11: Invalid version number pattern \"a*\".", + "WARN: filename.mk:12: Invalid version number pattern \"1.2/456\".", + "WARN: filename.mk:13: Please use \"4.*\" instead of \"4*\" as the version pattern.", + "WARN: filename.mk:15: Please use \"1.[234].*\" instead of \"1.[234]*\" as the version pattern.") } func (s *Suite) Test_VartypeCheck_WrapperReorder(c *check.C) { @@ -1246,8 +1246,8 @@ func (s *Suite) Test_VartypeCheck_WrapperReorder(c *check.C) { "reorder:l:first", "omit:first") vt.Output( - "WARN: filename:2: Unknown wrapper reorder command \"reorder:l:first\".", - "WARN: filename:3: Unknown wrapper reorder command \"omit:first\".") + "WARN: filename.mk:2: Unknown wrapper reorder command \"reorder:l:first\".", + "WARN: filename.mk:3: Unknown wrapper reorder command \"omit:first\".") } func (s *Suite) Test_VartypeCheck_WrapperTransform(c *check.C) { @@ -1266,8 +1266,8 @@ func (s *Suite) Test_VartypeCheck_WrapperTransform(c *check.C) { "unknown", "-e 's,-Wall,-Wall -Wextra,'") vt.Output( - "WARN: filename:7: Unknown wrapper transform command \"rpath:/usr/lib\".", - "WARN: filename:8: Unknown wrapper transform command \"unknown\".") + "WARN: filename.mk:7: Unknown wrapper transform command \"rpath:/usr/lib\".", + "WARN: filename.mk:8: Unknown wrapper transform command \"unknown\".") } func (s *Suite) Test_VartypeCheck_WrksrcSubdirectory(c *check.C) { @@ -1287,14 +1287,14 @@ func (s *Suite) Test_VartypeCheck_WrksrcSubdirectory(c *check.C) { "${WRKDIR}/sub", "${SRCDIR}/sub") vt.Output( - "NOTE: filename:1: You can use \".\" instead of \"${WRKSRC}\".", - "NOTE: filename:2: You can use \".\" instead of \"${WRKSRC}/\".", - "NOTE: filename:3: You can use \".\" instead of \"${WRKSRC}/.\".", - "NOTE: filename:4: You can use \"subdir\" instead of \"${WRKSRC}/subdir\".", - "NOTE: filename:6: You can use \"directory\" instead of \"${WRKSRC}/directory\".", - "WARN: filename:8: \"../other\" is not a valid subdirectory of ${WRKSRC}.", - "WARN: filename:9: \"${WRKDIR}/sub\" is not a valid subdirectory of ${WRKSRC}.", - "WARN: filename:10: \"${SRCDIR}/sub\" is not a valid subdirectory of ${WRKSRC}.") + "NOTE: filename.mk:1: You can use \".\" instead of \"${WRKSRC}\".", + "NOTE: filename.mk:2: You can use \".\" instead of \"${WRKSRC}/\".", + "NOTE: filename.mk:3: You can use \".\" instead of \"${WRKSRC}/.\".", + "NOTE: filename.mk:4: You can use \"subdir\" instead of \"${WRKSRC}/subdir\".", + "NOTE: filename.mk:6: You can use \"directory\" instead of \"${WRKSRC}/directory\".", + "WARN: filename.mk:8: \"../other\" is not a valid subdirectory of ${WRKSRC}.", + "WARN: filename.mk:9: \"${WRKDIR}/sub\" is not a valid subdirectory of ${WRKSRC}.", + "WARN: filename.mk:10: \"${SRCDIR}/sub\" is not a valid subdirectory of ${WRKSRC}.") } func (s *Suite) Test_VartypeCheck_Yes(c *check.C) { @@ -1307,8 +1307,8 @@ func (s *Suite) Test_VartypeCheck_Yes(c *check.C) { "${YESVAR}") vt.Output( - "WARN: filename:2: APACHE_MODULE should be set to YES or yes.", - "WARN: filename:3: APACHE_MODULE should be set to YES or yes.") + "WARN: filename.mk:2: APACHE_MODULE should be set to YES or yes.", + "WARN: filename.mk:3: APACHE_MODULE should be set to YES or yes.") vt.Varname("PKG_DEVELOPER") vt.Op(opUseMatch) @@ -1318,9 +1318,9 @@ func (s *Suite) Test_VartypeCheck_Yes(c *check.C) { "${YESVAR}") vt.Output( - "WARN: filename:11: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.", - "WARN: filename:12: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.", - "WARN: filename:13: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.") + "WARN: filename.mk:11: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.", + "WARN: filename.mk:12: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.", + "WARN: filename.mk:13: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.") } func (s *Suite) Test_VartypeCheck_YesNo(c *check.C) { @@ -1334,8 +1334,8 @@ func (s *Suite) Test_VartypeCheck_YesNo(c *check.C) { "${YESVAR}") vt.Output( - "WARN: filename:3: GNU_CONFIGURE should be set to YES, yes, NO, or no.", - "WARN: filename:4: GNU_CONFIGURE should be set to YES, yes, NO, or no.") + "WARN: filename.mk:3: GNU_CONFIGURE should be set to YES, yes, NO, or no.", + "WARN: filename.mk:4: GNU_CONFIGURE should be set to YES, yes, NO, or no.") } func (s *Suite) Test_VartypeCheck_YesNoIndirectly(c *check.C) { @@ -1349,7 +1349,7 @@ func (s *Suite) Test_VartypeCheck_YesNoIndirectly(c *check.C) { "${YESVAR}") vt.Output( - "WARN: filename:3: GNU_CONFIGURE should be set to YES, yes, NO, or no.") + "WARN: filename.mk:3: GNU_CONFIGURE should be set to YES, yes, NO, or no.") } // VartypeCheckTester helps to test the many different checks in VartypeCheck. @@ -1375,7 +1375,7 @@ func NewVartypeCheckTester(t *Tester, checker func(cv *VartypeCheck)) *VartypeCh return &VartypeCheckTester{ t, checker, - "filename", + "filename.mk", 1, "", opAssign} |