diff options
Diffstat (limited to 'pkgtools')
47 files changed, 5707 insertions, 1328 deletions
diff --git a/pkgtools/pkglint/Makefile b/pkgtools/pkglint/Makefile index 9cfabc82615..23ed228fc44 100644 --- a/pkgtools/pkglint/Makefile +++ b/pkgtools/pkglint/Makefile @@ -1,7 +1,6 @@ -# $NetBSD: Makefile,v 1.569 2019/03/09 10:05:10 bsiegert Exp $ +# $NetBSD: Makefile,v 1.570 2019/03/10 19:01:50 rillig Exp $ -PKGNAME= pkglint-5.7.1 -PKGREVISION= 1 +PKGNAME= pkglint-5.7.2 CATEGORIES= pkgtools DISTNAME= tools MASTER_SITES= ${MASTER_SITE_GITHUB:=golang/} @@ -80,4 +79,5 @@ do-install-man: .PHONY BUILDLINK_DEPMETHOD.go-check= full .include "../../devel/go-check/buildlink3.mk" +.include "../../security/go-crypto/buildlink3.mk" .include "../../mk/bsd.pkg.mk" diff --git a/pkgtools/pkglint/PLIST b/pkgtools/pkglint/PLIST index 7b41ad65334..5808399a521 100644 --- a/pkgtools/pkglint/PLIST +++ b/pkgtools/pkglint/PLIST @@ -1,4 +1,4 @@ -@comment $NetBSD: PLIST,v 1.10 2019/01/26 16:31:33 rillig Exp $ +@comment $NetBSD: PLIST,v 1.11 2019/03/10 19:01:50 rillig Exp $ bin/pkglint gopkg/pkg/${GO_PLATFORM}/netbsd.org/pkglint.a gopkg/pkg/${GO_PLATFORM}/netbsd.org/pkglint/getopt.a @@ -63,6 +63,8 @@ gopkg/src/netbsd.org/pkglint/mkshtypes.go gopkg/src/netbsd.org/pkglint/mkshtypes_test.go gopkg/src/netbsd.org/pkglint/mkshwalker.go gopkg/src/netbsd.org/pkglint/mkshwalker_test.go +gopkg/src/netbsd.org/pkglint/mktokenslexer.go +gopkg/src/netbsd.org/pkglint/mktokenslexer_test.go gopkg/src/netbsd.org/pkglint/mktypes.go gopkg/src/netbsd.org/pkglint/mktypes_test.go gopkg/src/netbsd.org/pkglint/options.go @@ -81,6 +83,8 @@ gopkg/src/netbsd.org/pkglint/pkgver/vercmp.go gopkg/src/netbsd.org/pkglint/pkgver/vercmp_test.go gopkg/src/netbsd.org/pkglint/plist.go gopkg/src/netbsd.org/pkglint/plist_test.go +gopkg/src/netbsd.org/pkglint/redundantscope.go +gopkg/src/netbsd.org/pkglint/redundantscope_test.go gopkg/src/netbsd.org/pkglint/regex/regex.go gopkg/src/netbsd.org/pkglint/shell.go gopkg/src/netbsd.org/pkglint/shell.y diff --git a/pkgtools/pkglint/files/autofix.go b/pkgtools/pkglint/files/autofix.go index 8e579a12fb5..8ca3f993521 100644 --- a/pkgtools/pkglint/files/autofix.go +++ b/pkgtools/pkglint/files/autofix.go @@ -328,9 +328,9 @@ func (fix *Autofix) Realign(mkline MkLine, newWidth int) { { // Parsing the continuation marker as variable value is cheating but works well. text := strings.TrimSuffix(mkline.raw[0].orignl, "\n") - _, _, _, _, _, valueAlign, value, _, _ := MatchVarassign(text) - if value != "\\" { - oldWidth = tabWidth(valueAlign) + _, a := MatchVarassign(text) + if a.value != "\\" { + oldWidth = tabWidth(a.valueAlign) } } diff --git a/pkgtools/pkglint/files/autofix_test.go b/pkgtools/pkglint/files/autofix_test.go index 5b7febc46de..5d8bcc7de9f 100644 --- a/pkgtools/pkglint/files/autofix_test.go +++ b/pkgtools/pkglint/files/autofix_test.go @@ -161,7 +161,6 @@ func (s *Suite) Test_Autofix_ReplaceRegex__autofix(c *check.C) { fix.Apply() t.CheckOutputLines( - "", "AUTOFIX: ~/Makefile:2: Replacing \"X\" with \"Y\".", "-\tline2", "+\tYXXe2") diff --git a/pkgtools/pkglint/files/buildlink3_test.go b/pkgtools/pkglint/files/buildlink3_test.go index 8de6c282ce4..8208945b63d 100644 --- a/pkgtools/pkglint/files/buildlink3_test.go +++ b/pkgtools/pkglint/files/buildlink3_test.go @@ -474,22 +474,13 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_unknown_variable(c *ch t.CheckOutputLines( "WARN: buildlink3.mk:3: LICENSE may not be used in any file; it is a write-only variable.", "WARN: buildlink3.mk:3: The variable LICENSE should be quoted as part of a shell word.", - "WARN: buildlink3.mk:8: LICENSE should not be evaluated at load time.", - "WARN: buildlink3.mk:8: LICENSE may not be used in any file; it is a write-only variable.", "WARN: buildlink3.mk:8: LICENSE should not be evaluated indirectly at load time.", - "WARN: buildlink3.mk:8: LICENSE may not be used in any file; it is a write-only variable.", "WARN: buildlink3.mk:8: The variable LICENSE should be quoted as part of a shell word.", - "WARN: buildlink3.mk:9: LICENSE should not be evaluated at load time.", - "WARN: buildlink3.mk:9: LICENSE may not be used in any file; it is a write-only variable.", "WARN: buildlink3.mk:9: LICENSE should not be evaluated indirectly at load time.", - "WARN: buildlink3.mk:9: LICENSE may not be used in any file; it is a write-only variable.", "WARN: buildlink3.mk:9: The variable LICENSE should be quoted as part of a shell word.", - - "WARN: buildlink3.mk:13: LICENSE may not be used in any file; it is a write-only variable.", "WARN: buildlink3.mk:13: The variable LICENSE should be quoted as part of a shell word.", - "WARN: buildlink3.mk:3: Please replace \"${LICENSE}\" with a simple string (also in other variables in this file).") } @@ -640,13 +631,9 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__other_variables(c *check. G.Check(t.File("category/package")) - // FIXME: Why is appending to LDFLAGS forbidden? It sounds useful. t.CheckOutputLines( - "WARN: ~/category/package/buildlink3.mk:14: "+ - "The variable LDFLAGS.NetBSD may not be appended to in this file; "+ - "it would be ok in Makefile, Makefile.common, options.mk or *.mk.", - "WARN: ~/category/package/buildlink3.mk:16: "+ - "Only buildlink variables for \"package\", "+ + "WARN: ~/category/package/buildlink3.mk:16: " + + "Only buildlink variables for \"package\", " + "not \"other\" may be set in this file.") } diff --git a/pkgtools/pkglint/files/category_test.go b/pkgtools/pkglint/files/category_test.go index d8f22a218a3..e56b7c2af4f 100644 --- a/pkgtools/pkglint/files/category_test.go +++ b/pkgtools/pkglint/files/category_test.go @@ -290,8 +290,11 @@ func (s *Suite) Test_CheckdirCategory__comment_at_the_top(c *check.C) { CheckdirCategory(t.File("category")) - // FIXME: Wow. These are quite a few warnings and errors, just because there is - // an additional comment above the COMMENT definition. + // These are quite a few warnings and errors, just because there is + // an additional comment above the COMMENT definition. + // On the other hand, the category Makefiles are so simple and their + // structure has been fixed for at least 20 years, therefore this case + // is rather exotic anyway. t.CheckOutputLines( "ERROR: ~/category/Makefile:3: COMMENT= line expected.", "NOTE: ~/category/Makefile:2: Empty line expected after this line.", diff --git a/pkgtools/pkglint/files/check_test.go b/pkgtools/pkglint/files/check_test.go index 154ad8e2f92..362c778d9c8 100644 --- a/pkgtools/pkglint/files/check_test.go +++ b/pkgtools/pkglint/files/check_test.go @@ -92,7 +92,6 @@ func (s *Suite) TearDownTest(c *check.C) { _, _ = fmt.Fprintf(os.Stderr, "Cannot chdir back to previous dir: %s", err) } - G = Pkglint{} // unusable because of missing Logger.out and Logger.err if out := t.Output(); out != "" { var msg strings.Builder msg.WriteString("\n") @@ -106,8 +105,11 @@ func (s *Suite) TearDownTest(c *check.C) { _, _ = fmt.Fprintf(&msg, "\n") _, _ = os.Stderr.WriteString(msg.String()) } + t.tmpdir = "" t.DisableTracing() + + G = Pkglint{} // unusable because of missing Logger.out and Logger.err } var _ = check.Suite(new(Suite)) @@ -196,6 +198,36 @@ func (t *Tester) SetUpFileMkLines(relativeFileName string, lines ...string) MkLi return LoadMk(filename, MustSucceed) } +// LoadMkInclude loads the given Makefile fragment and all the files it includes, +// merging all the lines into a single MkLines object. +// +// This is useful for testing code related to Package.readMakefile. +func (t *Tester) LoadMkInclude(relativeFileName string) MkLines { + var lines []Line + + // TODO: Include files with multiple-inclusion guard only once. + // TODO: Include files without multiple-inclusion guard as often as needed. + // TODO: Set an upper limit, to prevent denial of service. + + var load func(filename string) + load = func(filename string) { + for _, mkline := range NewMkLines(Load(filename, MustSucceed)).mklines { + lines = append(lines, mkline.Line) + + if mkline.IsInclude() { + included := cleanpath(path.Dir(filename) + "/" + mkline.IncludedFile()) + load(included) + } + } + } + + load(t.File(relativeFileName)) + + // This assumes that the test files do not contain parse errors. + // Otherwise the diagnostics would appear twice. + return NewMkLines(NewLines(t.File(relativeFileName), lines)) +} + // SetUpPkgsrc sets up a minimal but complete pkgsrc installation in the // temporary folder, so that pkglint runs without any errors. // Individual files may be overwritten by calling other SetUp* methods. @@ -257,6 +289,8 @@ func (t *Tester) SetUpPkgsrc() { // used at load time by packages. t.CreateFileLines("mk/bsd.prefs.mk", MkRcsID) + t.CreateFileLines("mk/bsd.fast.prefs.mk", + MkRcsID) // Category Makefiles require this file for the common definitions. t.CreateFileLines("mk/misc/category.mk") @@ -486,6 +520,69 @@ func (t *Tester) Remove(relativeFileName string) { G.fileCache.Evict(filename) } +// SetUpHierarchy provides a function for creating hierarchies of MkLines +// that include each other. +// The hierarchy is created only in memory, nothing is written to disk. +// +// include, get := t.SetUpHierarchy() +// +// include("including.mk", +// include("other.mk", +// "VAR= other"), +// include("module.mk", +// "VAR= module", +// include("version.mk", +// "VAR= version"), +// include("env.mk", +// "VAR= env"))) +// +// mklines := get("including.mk") +// module := get("module.mk") +func (t *Tester) SetUpHierarchy() ( + include func(filename string, args ...interface{}) MkLines, + get func(string) MkLines) { + + files := map[string]MkLines{} + + // FIXME: Define where the filename is relative to: to the file, or to the current directory. + include = func(filename string, args ...interface{}) MkLines { + var lines []Line + lineno := 1 + + addLine := func(text string) { + lines = append(lines, t.NewLine(filename, lineno, text)) + lineno++ + } + + for _, arg := range args { + switch arg := arg.(type) { + case string: + addLine(arg) + case MkLines: + text := sprintf(".include %q", arg.lines.FileName) + addLine(text) + lines = append(lines, arg.lines.Lines...) + default: + panic("invalid type") + } + } + + mklines := NewMkLines(NewLines(filename, lines)) + // FIXME: This filename must be relative to the including file. + G.Assertf(files[filename] == nil, "MkLines with name %q already exist.", filename) + // FIXME: This filename must be relative to the base directory. + files[filename] = mklines + return mklines + } + + get = func(filename string) MkLines { + G.Assertf(files[filename] != nil, "MkLines with name %q doesn't exist.", filename) + return files[filename] + } + + return +} + // Check delegates a check to the check.Check function. // Thereby, there is no need to distinguish between c.Check and t.Check // in the test code. @@ -584,6 +681,11 @@ func (t *Tester) NewLine(filename string, lineno int, text string) Line { // NewMkLine creates an in-memory line in the Makefile format with the given text. func (t *Tester) NewMkLine(filename string, lineno int, text string) MkLine { + basename := path.Base(filename) + G.Assertf( + hasSuffix(basename, ".mk") || basename == "Makefile" || hasPrefix(basename, "Makefile."), + "filename %q must be realistic, otherwise the variable permissions are wrong", filename) + return NewMkLine(t.NewLine(filename, lineno, text)) } @@ -616,6 +718,11 @@ func (t *Tester) NewLinesAt(filename string, firstLine int, texts ...string) Lin // No actual file is created for the lines; // see SetUpFileMkLines for loading Makefile fragments with line continuations. func (t *Tester) NewMkLines(filename string, lines ...string) MkLines { + basename := path.Base(filename) + G.Assertf( + hasSuffix(basename, ".mk") || basename == "Makefile" || hasPrefix(basename, "Makefile."), + "filename %q must be realistic, otherwise the variable permissions are wrong", filename) + var rawText strings.Builder for _, line := range lines { rawText.WriteString(line) @@ -633,13 +740,18 @@ func (t *Tester) Output() string { t.stdout.Reset() t.stderr.Reset() G.Logger.logged = Once{} + if G.Logger.out != nil { // Necessary because Main resets the G variable. + G.Logger.out.state = 0 // Prevent an empty line at the beginning of the next output. + G.Logger.err.state = 0 + } + G.Assertf(t.tmpdir != "", "Tester must be initialized before checking the output.") output := stdout + stderr - if t.tmpdir != "" { - output = strings.Replace(output, t.tmpdir, "~", -1) - } else { - panic("asdfgsfas") - } + // TODO: The explanations are wrapped. Because of this it can happen + // that t.tmpdir is spread among multiple lines if that directory + // name contains spaces, which is common on Windows. A temporary + // workaround is to set TMP=/path/without/spaces. + output = strings.Replace(output, t.tmpdir, "~", -1) return output } diff --git a/pkgtools/pkglint/files/distinfo.go b/pkgtools/pkglint/files/distinfo.go index 034a849dc9b..9bef89ff565 100644 --- a/pkgtools/pkglint/files/distinfo.go +++ b/pkgtools/pkglint/files/distinfo.go @@ -3,9 +3,13 @@ package pkglint import ( "bytes" "crypto/sha1" + "crypto/sha512" "encoding/hex" + "golang.org/x/crypto/ripemd160" + "hash" + "io" "io/ioutil" - "path" + "os" "strings" ) @@ -26,106 +30,273 @@ func CheckLinesDistinfo(lines Lines) { distinfoIsCommitted := isCommitted(filename) ck := distinfoLinesChecker{ lines, patchdir, distinfoIsCommitted, - make(map[string]bool), "", nil, unknown, nil} - ck.checkLines(lines) + nil, make(map[string]distinfoFileInfo)} + ck.parse() + ck.check() CheckLinesTrailingEmptyLines(lines) ck.checkUnrecordedPatches() + SaveAutofixChanges(lines) } -// XXX: Maybe an approach that first groups the lines by filename -// is easier to understand. - type distinfoLinesChecker struct { - distinfoLines Lines + lines Lines patchdir string // Relative to G.Pkg distinfoIsCommitted bool - // All patches that are mentioned in the distinfo file. - patches map[string]bool // "patch-aa" => true - - currentFileName string - currentFirstLine Line // The first line of the currentFileName group - isPatch YesNoUnknown // Whether currentFileName is a patch, as opposed to a distfile - algorithms []string // The algorithms seen for currentFileName + filenames []string // For keeping the order from top to bottom + infos map[string]distinfoFileInfo } -func (ck *distinfoLinesChecker) checkLines(lines Lines) { - lines.CheckRcsID(0, ``, "") - if 1 < len(lines.Lines) && lines.Lines[1].Text != "" { - lines.Lines[1].Notef("Empty line expected.") - } +func (ck *distinfoLinesChecker) parse() { + lines := ck.lines - for i, line := range lines.Lines { - if i < 2 { - continue + llex := NewLinesLexer(lines) + if lines.CheckRcsID(0, ``, "") { + llex.Skip() + } + llex.SkipEmptyOrNote() + + prevFilename := "" + var hashes []distinfoHash + + isPatch := func() YesNoUnknown { + switch { + case !hasPrefix(prevFilename, "patch-"): + return no + case G.Pkg == nil: + return unknown + case fileExists(G.Pkg.File(ck.patchdir + "/" + prevFilename)): + return yes + default: + return no } - m, alg, filename, hash := match3(line.Text, `^(\w+) \((\w[^)]*)\) = (.*)(?: bytes)?$`) + } + + finishGroup := func() { + ck.filenames = append(ck.filenames, prevFilename) + ck.infos[prevFilename] = distinfoFileInfo{isPatch(), hashes} + hashes = nil + } + + for !llex.EOF() { + line := llex.CurrentLine() + llex.Skip() + + m, alg, filename, hash := match3(line.Text, `^(\w+) \((\w[^)]*)\) = (\S+(?: bytes)?)$`) if !m { line.Errorf("Invalid line: %s", line.Text) continue } - if filename != ck.currentFileName { - ck.onFilenameChange(line, filename) + if prevFilename != "" && filename != prevFilename { + finishGroup() } - ck.algorithms = append(ck.algorithms, alg) + prevFilename = filename - ck.checkGlobalDistfileMismatch(line, filename, alg, hash) - ck.checkUncommittedPatch(line, filename, alg, hash) + hashes = append(hashes, distinfoHash{line, filename, alg, hash}) } - ck.onFilenameChange(ck.distinfoLines.EOFLine(), "") -} -func (ck *distinfoLinesChecker) onFilenameChange(line Line, nextFname string) { - if ck.currentFileName != "" { - ck.checkAlgorithms(line) + if prevFilename != "" { + finishGroup() } +} - if !hasPrefix(nextFname, "patch-") { - ck.isPatch = no - } else if G.Pkg == nil { - ck.isPatch = unknown - } else if fileExists(G.Pkg.File(ck.patchdir + "/" + nextFname)) { - ck.isPatch = yes - } else { - ck.isPatch = no - } +func (ck *distinfoLinesChecker) check() { + for _, filename := range ck.filenames { + info := ck.infos[filename] - ck.currentFileName = nextFname - ck.currentFirstLine = line - ck.algorithms = nil + ck.checkAlgorithms(info) + for _, hash := range info.hashes { + ck.checkGlobalDistfileMismatch(hash) + if info.isPatch == yes { + ck.checkUncommittedPatch(hash) + } + } + } } -func (ck *distinfoLinesChecker) checkAlgorithms(line Line) { - filename := ck.currentFileName - algorithms := strings.Join(ck.algorithms, ", ") +func (ck *distinfoLinesChecker) checkAlgorithms(info distinfoFileInfo) { + filename := info.filename() + algorithms := info.algorithms() + line := info.line() + + isPatch := info.isPatch switch { + case algorithms == "SHA1" && isPatch != no: + return - case ck.isPatch == yes: - if algorithms != "SHA1" { - line.Errorf("Expected SHA1 hash for %s, got %s.", filename, algorithms) - } + case algorithms == "SHA1, RMD160, SHA512, Size" && isPatch != yes: + return + } - case ck.isPatch == unknown: - break + switch { + case isPatch == yes: + line.Errorf("Expected SHA1 hash for %s, got %s.", filename, algorithms) + + case isPatch == unknown: + line.Errorf("Wrong checksum algorithms %s for %s.", algorithms, filename) + line.Explain( + "Distfiles that are downloaded from external sources must have the", + "checksum algorithms SHA1, RMD160, SHA512, Size.", + "", + "Patch files from pkgsrc must have only the SHA1 hash.") - case G.Pkg != nil && G.Pkg.IgnoreMissingPatches: - break + // At this point, the file is either a missing patch file or a distfile. case hasPrefix(filename, "patch-") && algorithms == "SHA1": - pathToPatchdir := relpath(path.Dir(ck.currentFirstLine.Filename), G.Pkg.File(ck.patchdir)) - ck.currentFirstLine.Warnf("Patch file %q does not exist in directory %q.", filename, pathToPatchdir) + if G.Pkg.IgnoreMissingPatches { + break + } + + line.Warnf("Patch file %q does not exist in directory %q.", + filename, line.PathToFile(G.Pkg.File(ck.patchdir))) G.Explain( "If the patches directory looks correct, the patch may have been", "removed without updating the distinfo file.", "In such a case please update the distinfo file.", "", - "If the patches directory looks wrong, pkglint needs to be improved.") + "In rare cases, pkglint cannot determine the correct location of the patches directory.", + "In that case, see the pkglint man page for contact information.") + + default: + ck.checkAlgorithmsDistfile(info) + } +} + +// checkAlgorithmsDistfile checks whether some of the standard algorithms are +// missing. If so and the downloaded distfile exists, they are calculated and +// added to the distinfo file via an autofix. +func (ck *distinfoLinesChecker) checkAlgorithmsDistfile(info distinfoFileInfo) { + line := info.line() + line.Errorf("Expected SHA1, RMD160, SHA512, Size checksums for %q, got %s.", info.filename(), info.algorithms()) + + algorithms := [...]string{"SHA1", "RMD160", "SHA512", "Size"} + + missing := map[string]bool{} + for _, alg := range algorithms { + missing[alg] = true + } + seen := map[string]distinfoHash{} + + for _, hash := range info.hashes { + alg := hash.algorithm + if missing[alg] { + seen[alg] = hash + delete(missing, alg) + } + } + + if len(missing) == 0 || len(seen) == 0 { + return + } + + distdir := G.Pkgsrc.File("distfiles") + distSubdir := "" + if G.Pkg != nil { + distSubdir = G.Pkg.vars.LastValue("DIST_SUBDIR") + } + + // It's a rare situation that the explanation is generated + // this far from the corresponding diagnostic. + // This explanation only makes sense when there are some + // hashes missing that can be automatically added by pkglint. + line.Explain( + "To add the missing lines to the distinfo file, run", + sprintf("\t%s", bmake("distinfo")), + "for each variant of the package until all distfiles are downloaded to", + sprintf("%q.", cleanpath("${PKGSRCDIR}/distfiles/"+distSubdir)), + "", + "The variants are typically selected by setting EMUL_PLATFORM", + "or similar variables in the command line.", + "", + "After that, run", + sprintf("%q", "cvs update -C distinfo"), + "to revert the distinfo file to the previous state, since the above", + "commands have removed some of the entries.", + "", + "After downloading all possible distfiles, run", + sprintf("%q,", "pkglint --autofix"), + "which will find the downloaded distfiles and add the missing", + "hashes to the distinfo file.") + + distfile := cleanpath(distdir + "/" + distSubdir + "/" + info.filename()) + if !fileExists(distfile) { + return + } + + computeHash := func(hasher hash.Hash) string { + f, err := os.Open(distfile) + G.AssertNil(err, "Opening distfile") + + // Don't load the distfile into memory since some of them + // are hundreds of MB in size. + _, err = io.Copy(hasher, f) + G.AssertNil(err, "Computing hash of distfile") + + hexHash := hex.EncodeToString(hasher.Sum(nil)) + + err = f.Close() + G.AssertNil(err, "Closing distfile") + + return hexHash + } - case algorithms != "SHA1, RMD160, SHA512, Size": - line.Errorf("Expected SHA1, RMD160, SHA512, Size checksums for %q, got %s.", filename, algorithms) + compute := func(alg string) string { + switch alg { + case "SHA1": + return computeHash(sha1.New()) + case "RMD160": + return computeHash(ripemd160.New()) + case "SHA512": + return computeHash(sha512.New()) + default: + fileInfo, err := os.Lstat(distfile) + G.AssertNil(err, "Inaccessible distfile info") + return sprintf("%d bytes", fileInfo.Size()) + } + } + + for alg, hash := range seen { + computed := compute(alg) + + if computed != hash.hash { + // Do not try to autofix anything in this situation. + // Wrong hashes are a serious issue. + line.Errorf("The %s checksum for %q is %s in distinfo, %s in %s.", + alg, hash.filename, hash.hash, computed, line.PathToFile(distfile)) + return + } + } + + // At this point, all the existing hash algorithms are correct, + // and there is at least one hash algorithm. This is evidence enough + // that the distfile is the expected one. Now generate the missing hashes + // and insert them, in the correct order. + + var insertion Line + var remainingHashes = info.hashes + for _, alg := range algorithms { + if missing[alg] { + computed := compute(alg) + + if insertion == nil { + fix := line.Autofix() + fix.Errorf("Missing %s hash for %s.", alg, info.filename()) + fix.InsertBefore(sprintf("%s (%s) = %s", alg, info.filename(), computed)) + fix.Apply() + } else { + fix := insertion.Autofix() + fix.Errorf("Missing %s hash for %s.", alg, info.filename()) + fix.InsertAfter(sprintf("%s (%s) = %s", alg, info.filename(), computed)) + fix.Apply() + } + + } else if remainingHashes[0].algorithm == alg { + insertion = remainingHashes[0].line + remainingHashes = remainingHashes[1:] + } } } @@ -143,16 +314,26 @@ func (ck *distinfoLinesChecker) checkUnrecordedPatches() { for _, file := range patchFiles { patchName := file.Name() - if file.Mode().IsRegular() && !ck.patches[patchName] && hasPrefix(patchName, "patch-") { - ck.distinfoLines.Errorf("Patch %q is not recorded. Run %q.", - cleanpath(relpath(path.Dir(ck.distinfoLines.FileName), G.Pkg.File(ck.patchdir+"/"+patchName))), + if file.Mode().IsRegular() && ck.infos[patchName].isPatch != yes && hasPrefix(patchName, "patch-") { + line := NewLineWhole(ck.lines.FileName) + line.Errorf("Patch %q is not recorded. Run %q.", + line.PathToFile(G.Pkg.File(ck.patchdir+"/"+patchName)), bmake("makepatchsum")) } } } // Inter-package check for differing distfile checksums. -func (ck *distinfoLinesChecker) checkGlobalDistfileMismatch(line Line, filename, alg, hash string) { +func (ck *distinfoLinesChecker) checkGlobalDistfileMismatch(info distinfoHash) { + hashes := G.Hashes + if hashes == nil { + return + } + + filename := info.filename + alg := info.algorithm + hash := info.hash + line := info.line // Intentionally checking the filename instead of ck.isPatch. // Missing the few distfiles that actually start with patch-* @@ -161,11 +342,6 @@ func (ck *distinfoLinesChecker) checkGlobalDistfileMismatch(line Line, filename, return } - hashes := G.Hashes - if hashes == nil { - return - } - // The Size hash is not encoded in hex, therefore it would trigger wrong error messages below. // Since the Size hash is targeted towards humans and not really useful for detecting duplicates, // omitting the check here is ok. Any mismatches will be reliably detected because the other @@ -194,17 +370,19 @@ func (ck *distinfoLinesChecker) checkGlobalDistfileMismatch(line Line, filename, } } -func (ck *distinfoLinesChecker) checkUncommittedPatch(line Line, patchName, alg, hash string) { - if ck.isPatch == yes { - patchFileName := ck.patchdir + "/" + patchName - resolvedPatchFileName := G.Pkg.File(patchFileName) - if ck.distinfoIsCommitted && !isCommitted(resolvedPatchFileName) { - line.Warnf("%s is registered in distinfo but not added to CVS.", line.PathToFile(resolvedPatchFileName)) - } - if alg == "SHA1" { - ck.checkPatchSha1(line, patchFileName, hash) - } - ck.patches[patchName] = true +func (ck *distinfoLinesChecker) checkUncommittedPatch(info distinfoHash) { + patchName := info.filename + alg := info.algorithm + hash := info.hash + line := info.line + + patchFileName := ck.patchdir + "/" + patchName + resolvedPatchFileName := G.Pkg.File(patchFileName) + if ck.distinfoIsCommitted && !isCommitted(resolvedPatchFileName) { + line.Warnf("%s is registered in distinfo but not added to CVS.", line.PathToFile(resolvedPatchFileName)) + } + if alg == "SHA1" { + ck.checkPatchSha1(line, patchFileName, hash) } } @@ -226,6 +404,32 @@ func (ck *distinfoLinesChecker) checkPatchSha1(line Line, patchFileName, distinf } } +type distinfoFileInfo struct { + // yes = the patch file exists + // unknown = distinfo file is checked without a pkgsrc package + // no = distfile or nonexistent patch file + isPatch YesNoUnknown + hashes []distinfoHash +} + +func (info *distinfoFileInfo) filename() string { return info.hashes[0].filename } +func (info *distinfoFileInfo) line() Line { return info.hashes[0].line } + +func (info *distinfoFileInfo) algorithms() string { + var algs []string + for _, hash := range info.hashes { + algs = append(algs, hash.algorithm) + } + return strings.Join(algs, ", ") +} + +type distinfoHash struct { + line Line + filename string + algorithm string + hash string +} + // Same as in mk/checksum/distinfo.awk:/function patchsum/ func computePatchSha1Hex(patchFilename string) (string, error) { patchBytes, err := ioutil.ReadFile(patchFilename) diff --git a/pkgtools/pkglint/files/distinfo_test.go b/pkgtools/pkglint/files/distinfo_test.go index d3927af5d85..1231b8f3819 100644 --- a/pkgtools/pkglint/files/distinfo_test.go +++ b/pkgtools/pkglint/files/distinfo_test.go @@ -2,7 +2,7 @@ package pkglint import "gopkg.in/check.v1" -func (s *Suite) Test_CheckLinesDistinfo(c *check.C) { +func (s *Suite) Test_CheckLinesDistinfo__parse_errors(c *check.C) { t := s.Init(c) t.Chdir("category/package") @@ -27,14 +27,16 @@ func (s *Suite) Test_CheckLinesDistinfo(c *check.C) { t.CheckOutputLines( "ERROR: distinfo:1: Expected \"$"+"NetBSD$\".", - "NOTE: distinfo:2: Empty line expected.", - "ERROR: distinfo:5: Expected SHA1, RMD160, SHA512, Size checksums for \"distfile.tar.gz\", got MD5, SHA1.", - "ERROR: distinfo:7: Expected SHA1 hash for patch-aa, got SHA1, Size.", + "NOTE: distinfo:1: Empty line expected before this line.", + "ERROR: distinfo:1: Invalid line: should be the RCS ID", + "ERROR: distinfo:2: Invalid line: should be empty", "ERROR: distinfo:8: Invalid line: Another invalid line", + "ERROR: distinfo:3: Expected SHA1, RMD160, SHA512, Size checksums for \"distfile.tar.gz\", got MD5, SHA1.", + "ERROR: distinfo:5: Expected SHA1 hash for patch-aa, got SHA1, Size.", "WARN: distinfo:9: Patch file \"patch-nonexistent\" does not exist in directory \"patches\".") } -func (s *Suite) Test_CheckLinesDistinfo__nonexistent_distfile_called_patch(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__nonexistent_distfile_called_patch(c *check.C) { t := s.Init(c) t.Chdir("category/package") @@ -51,11 +53,11 @@ func (s *Suite) Test_CheckLinesDistinfo__nonexistent_distfile_called_patch(c *ch // a patch, it is a normal distfile because it has other hash algorithms // than exactly SHA1. t.CheckOutputLines( - "ERROR: distinfo:EOF: Expected SHA1, RMD160, SHA512, Size checksums " + + "ERROR: distinfo:3: Expected SHA1, RMD160, SHA512, Size checksums " + "for \"patch-5.3.tar.gz\", got MD5, SHA1.") } -func (s *Suite) Test_CheckLinesDistinfo__wrong_distfile_algorithms(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__wrong_distfile_algorithms(c *check.C) { t := s.Init(c) t.Chdir("category/package") @@ -68,11 +70,39 @@ func (s *Suite) Test_CheckLinesDistinfo__wrong_distfile_algorithms(c *check.C) { CheckLinesDistinfo(lines) t.CheckOutputLines( - "ERROR: distinfo:EOF: Expected SHA1, RMD160, SHA512, Size checksums " + + "ERROR: distinfo:3: Expected SHA1, RMD160, SHA512, Size checksums " + "for \"distfile.tar.gz\", got MD5, SHA1.") } -func (s *Suite) Test_CheckLinesDistinfo__wrong_patch_algorithms(c *check.C) { +// This case only happens when a distinfo file is checked on its own, +// without any reference to a pkgsrc package. Additionally the distfile +// must start with the patch- prefix and the algorithms must be wrong +// for both distfile or patch. +// +// This test only demonstrates the edge case. +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__ambiguous_distfile(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--explain") + t.Chdir("category/package") + lines := t.SetUpFileLines("distinfo", + RcsID, + "", + "MD5 (patch-4.2.tar.gz) = 12345678901234567890123456789012") + + CheckLinesDistinfo(lines) + + t.CheckOutputLines( + "ERROR: distinfo:3: Wrong checksum algorithms MD5 for patch-4.2.tar.gz.", + "", + "\tDistfiles that are downloaded from external sources must have the", + "\tchecksum algorithms SHA1, RMD160, SHA512, Size.", + "", + "\tPatch files from pkgsrc must have only the SHA1 hash.", + "") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__wrong_patch_algorithms(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package") @@ -87,10 +117,23 @@ func (s *Suite) Test_CheckLinesDistinfo__wrong_patch_algorithms(c *check.C) { G.Check(".") t.CheckOutputLines( + "ERROR: distinfo:3: Expected SHA1 hash for patch-aa, got MD5, SHA1.", "ERROR: distinfo:4: SHA1 hash of patches/patch-aa differs "+ "(distinfo has 1234567890123456789012345678901234567890, "+ - "patch file has ebbf34b0641bcb508f17d5a27f2bf2a536d810ac).", - "ERROR: distinfo:EOF: Expected SHA1 hash for patch-aa, got MD5, SHA1.") + "patch file has ebbf34b0641bcb508f17d5a27f2bf2a536d810ac).") +} + +func (s *Suite) Test_distinfoLinesChecker_parse__empty(c *check.C) { + t := s.Init(c) + + lines := t.SetUpFileLines("distinfo", + RcsID, + "") + + CheckLinesDistinfo(lines) + + t.CheckOutputLines( + "NOTE: ~/distinfo:2: Trailing empty lines.") } // When checking the complete pkgsrc tree, pkglint has all information it needs @@ -109,7 +152,8 @@ func (s *Suite) Test_distinfoLinesChecker_checkGlobalDistfileMismatch(c *check.C RcsID, "", "SHA512 (distfile-1.0.tar.gz) = 1234567811111111", - "SHA512 (distfile-1.1.tar.gz) = 1111111111111111") + "SHA512 (distfile-1.1.tar.gz) = 1111111111111111", + "SHA512 (patch-4.2.tar.gz) = 1234567812345678") t.CreateFileLines("category/package2/distinfo", RcsID, "", @@ -135,29 +179,104 @@ func (s *Suite) Test_distinfoLinesChecker_checkGlobalDistfileMismatch(c *check.C G.Main("pkglint", "-r", "-Wall", "-Call", t.File(".")) t.CheckOutputLines( - "ERROR: ~/category/package1/distinfo:4: "+ + "ERROR: ~/category/package1/distinfo:3: "+ "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.0.tar.gz\", got SHA512.", - "ERROR: ~/category/package1/distinfo:EOF: "+ + "ERROR: ~/category/package1/distinfo:4: "+ "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.1.tar.gz\", got SHA512.", - "ERROR: ~/category/package2/distinfo:3: The SHA512 hash for distfile-1.0.tar.gz is 1234567822222222, "+ - "which conflicts with 1234567811111111 in ../package1/distinfo:3.", - "ERROR: ~/category/package2/distinfo:4: "+ + "ERROR: ~/category/package1/distinfo:5: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"patch-4.2.tar.gz\", got SHA512.", + + "ERROR: ~/category/package2/distinfo:3: "+ "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.0.tar.gz\", got SHA512.", - "ERROR: ~/category/package2/distinfo:5: "+ + "ERROR: ~/category/package2/distinfo:3: "+ + "The SHA512 hash for distfile-1.0.tar.gz is 1234567822222222, "+ + "which conflicts with 1234567811111111 in ../../category/package1/distinfo:3.", + "ERROR: ~/category/package2/distinfo:4: "+ "Expected SHA1, RMD160, SHA512, Size checksums for \"distfile-1.1.tar.gz\", got SHA512.", "ERROR: ~/category/package2/distinfo:5: "+ - "The SHA512 hash for encoding-error.tar.gz contains a non-hex character.", - "ERROR: ~/category/package2/distinfo:EOF: "+ "Expected SHA1, RMD160, SHA512, Size checksums for \"encoding-error.tar.gz\", got SHA512.", + "ERROR: ~/category/package2/distinfo:5: "+ + "The SHA512 hash for encoding-error.tar.gz contains a non-hex character.", + "WARN: ~/licenses/gnu-gpl-v2: This license seems to be unused.", - "7 errors and 1 warning found.") + "8 errors and 1 warning found.", + "(Run \"pkglint -e\" to show explanations.)") // Ensure that hex.DecodeString does not waste memory here. t.Check(len(G.Hashes["SHA512:distfile-1.0.tar.gz"].hash), equals, 8) t.Check(cap(G.Hashes["SHA512:distfile-1.0.tar.gz"].hash), equals, 8) } -func (s *Suite) Test_CheckLinesDistinfo__uncommitted_patch(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__missing_patch_with_distfile_checksums(c *check.C) { + t := s.Init(c) + + lines := t.SetUpFileLines("distinfo", + RcsID, + "", + "SHA1 (patch-aa) = ...", + "RMD160 (patch-aa) = ...", + "SHA512 (patch-aa) = ...", + "Size (patch-aa) = ... bytes") + + CheckLinesDistinfo(lines) + + // The file name certainly looks like a pkgsrc patch, but there + // is no corresponding file in the file system, and there is no + // current package to correctly determine the PATCHDIR. Therefore + // pkglint doesn't know whether this is a distfile or a missing + // patch file and doesn't warn at all. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__existing_patch_with_distfile_checksums(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "SHA1 (patch-aa) = ...", + "RMD160 (patch-aa) = ...", + "SHA512 (patch-aa) = ...", + "Size (patch-aa) = ... bytes") + t.CreateFileDummyPatch("category/package/patches/patch-aa") + + G.Check(t.File("category/package")) + + // Even though the checksums in the distinfo file look as if they + // refer to a distfile, there is a patch file in the file system + // that matches the distinfo lines. When checking a pkgsrc package + // (as opposed to checking a distinfo file on its own), this means + // that the distinfo lines clearly refer to that patch file and not + // to a distfile. + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: "+ + "Expected SHA1 hash for patch-aa, got SHA1, RMD160, SHA512, Size.", + "ERROR: ~/category/package/distinfo:3: "+ + "SHA1 hash of patches/patch-aa differs (distinfo has ..., "+ + "patch file has ebbf34b0641bcb508f17d5a27f2bf2a536d810ac).") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithms__missing_patch_with_wrong_algorithms(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.SetUpFileLines("category/package/distinfo", + RcsID, + "", + "RMD160 (patch-aa) = ...") + + G.Check(t.File("category/package")) + + // Patch files usually have the SHA1 hash or none at all if they are fresh. + // In all other cases pkglint assumes that the file is a distfile, + // therefore it requires the usual distfile checksum algorithms here. + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: " + + "Expected SHA1, RMD160, SHA512, Size checksums for \"patch-aa\", got RMD160.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch__bad(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package") @@ -176,7 +295,27 @@ func (s *Suite) Test_CheckLinesDistinfo__uncommitted_patch(c *check.C) { "WARN: distinfo:3: patches/patch-aa is registered in distinfo but not added to CVS.") } -func (s *Suite) Test_CheckLinesDistinfo__unrecorded_patches(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch__good(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.Chdir("category/package") + t.CreateFileDummyPatch("patches/patch-aa") + t.CreateFileLines("CVS/Entries", + "/distinfo/...") + t.CreateFileLines("patches/CVS/Entries", + "/patch-aa/...") + t.SetUpFileLines("distinfo", + RcsID, + "", + "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") + + G.checkdirPackage(".") + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_distinfoLinesChecker_checkUnrecordedPatches(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package") @@ -202,7 +341,7 @@ func (s *Suite) Test_CheckLinesDistinfo__unrecorded_patches(c *check.C) { // The distinfo file and the patches are usually placed in the package // directory. By defining PATCHDIR or DISTINFO_FILE, a package can define // that they are somewhere else in pkgsrc. -func (s *Suite) Test_CheckLinesDistinfo__relative_path_in_distinfo(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1__relative_path_in_distinfo(c *check.C) { t := s.Init(c) t.SetUpPackage("category/package", @@ -340,35 +479,251 @@ func (s *Suite) Test_CheckLinesDistinfo__missing_php_patches(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_distinfoLinesChecker_checkUncommittedPatch(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1(c *check.C) { t := s.Init(c) + G.Pkg = NewPackage(t.File("category/package")) + distinfoLine := t.NewLine(t.File("category/package/distinfo"), 5, "") + + checker := distinfoLinesChecker{} + checker.checkPatchSha1(distinfoLine, "patch-nonexistent", "distinfo-sha1") + + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:5: Patch patch-nonexistent does not exist.") +} + +// When there is at least one correct hash for a distfile, running +// pkglint --autofix adds the missing hashes, provided the distfile has been +// downloaded to pkgsrc/distfiles, which is the standard distfiles location. +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__add_missing_hashes(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "--explain") t.SetUpPackage("category/package") - t.Chdir("category/package") - t.CreateFileDummyPatch("patches/patch-aa") - t.CreateFileLines("CVS/Entries", - "/distinfo/...") - t.CreateFileLines("patches/CVS/Entries", - "/patch-aa/...") - t.SetUpFileLines("distinfo", + t.CreateFileLines("category/package/distinfo", RcsID, "", - "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") + "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", + "Size (package-1.0.txt) = 13 bytes", + "CRC32 (package-1.0.txt) = asdf") + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() - G.checkdirPackage(".") + // This run is only used to verify that the RMD160 hash is correct, and if + // it should ever differ, the correct hash will appear in an error message. + G.Check(t.File("category/package")) - t.CheckOutputEmpty() + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", "+ + "got RMD160, Size, CRC32.", + "", + "\tTo add the missing lines to the distinfo file, run", + "\t\t"+confMake+" distinfo", + "\tfor each variant of the package until all distfiles are downloaded", + "\tto \"${PKGSRCDIR}/distfiles\".", + "", + "\tThe variants are typically selected by setting EMUL_PLATFORM or", + "\tsimilar variables in the command line.", + "", + "\tAfter that, run \"cvs update -C distinfo\" to revert the distinfo file", + "\tto the previous state, since the above commands have removed some of", + "\tthe entries.", + "", + "\tAfter downloading all possible distfiles, run \"pkglint --autofix\",", + "\twhich will find the downloaded distfiles and add the missing hashes", + "\tto the distinfo file.", + "", + "ERROR: ~/category/package/distinfo:3: Missing SHA1 hash for package-1.0.txt.", + "ERROR: ~/category/package/distinfo:3: Missing SHA512 hash for package-1.0.txt.") + + t.SetUpCommandLine("-Wall", "--autofix", "--show-autofix", "--source") + + G.Check(t.File("category/package")) + + // Since the file exists in the distfiles directory, pkglint checks the + // hash right away. It also adds the missing hashes since this file is + // not a patch file. + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: Missing SHA1 hash for package-1.0.txt.", + "AUTOFIX: ~/category/package/distinfo:3: "+ + "Inserting a line \"SHA1 (package-1.0.txt) "+ + "= cd50d19784897085a8d0e3e413f8612b097c03f1\" "+ + "before this line.", + "+\tSHA1 (package-1.0.txt) = cd50d19784897085a8d0e3e413f8612b097c03f1", + ">\tRMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", + "", + "ERROR: ~/category/package/distinfo:3: Missing SHA512 hash for package-1.0.txt.", + "AUTOFIX: ~/category/package/distinfo:3: "+ + "Inserting a line \"SHA512 (package-1.0.txt) "+ + "= f65f341b35981fda842b09b2c8af9bcdb7602a4c2e6fa1f7d41f0974d3e3122f"+ + "268fc79d5a4af66358f5133885cd1c165c916f80ab25e5d8d95db46f803c782c\" after this line.", + "+\tSHA1 (package-1.0.txt) = cd50d19784897085a8d0e3e413f8612b097c03f1", + ">\tRMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", + "+\tSHA512 (package-1.0.txt) = f65f341b35981fda842b09b2c8af9bcdb7602a4c2e6fa1f7d41f0974d3e3122f"+ + "268fc79d5a4af66358f5133885cd1c165c916f80ab25e5d8d95db46f803c782c") + + t.SetUpCommandLine("-Wall") + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: " + + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", " + + "got SHA1, RMD160, SHA512, Size, CRC32.") } -func (s *Suite) Test_distinfoLinesChecker_checkPatchSha1(c *check.C) { +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__wrong_distfile_hash(c *check.C) { t := s.Init(c) - G.Pkg = NewPackage(t.File("category/package")) - distinfoLine := t.NewLine(t.File("category/package/distinfo"), 5, "") + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "RMD160 (package-1.0.txt) = 1234wrongHash1234") + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() - checker := distinfoLinesChecker{} - checker.checkPatchSha1(distinfoLine, "patch-nonexistent", "distinfo-sha1") + G.Check(t.File("category/package")) t.CheckOutputLines( - "ERROR: ~/category/package/distinfo:5: Patch patch-nonexistent does not exist.") + "ERROR: ~/category/package/distinfo:3: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", "+ + "got RMD160.", + "ERROR: ~/category/package/distinfo:3: "+ + "The RMD160 checksum for \"package-1.0.txt\" is 1234wrongHash1234 in distinfo, "+ + "1a88147a0344137404c63f3b695366eab869a98a in ../../distfiles/package-1.0.txt.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__no_usual_algorithm(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "MD5 (package-1.0.txt) = 1234wrongHash1234") + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: " + + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", " + + "got MD5.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__top_algorithms_missing(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "SHA512 (package-1.0.txt) = f65f341b35981fda842b09b2c8af9bcdb7602a4c2e6fa1f7"+ + "d41f0974d3e3122f268fc79d5a4af66358f5133885cd1c165c916f80ab25e5d8d95db46f803c782c", + "Size (package-1.0.txt) = 13 bytes") + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", "+ + "got SHA512, Size.", + "ERROR: ~/category/package/distinfo:3: Missing SHA1 hash for package-1.0.txt.", + "ERROR: ~/category/package/distinfo:3: Missing RMD160 hash for package-1.0.txt.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__bottom_algorithms_missing(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "SHA1 (package-1.0.txt) = cd50d19784897085a8d0e3e413f8612b097c03f1", + "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a") + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", "+ + "got SHA1, RMD160.", + "ERROR: ~/category/package/distinfo:4: Missing SHA512 hash for package-1.0.txt.", + "ERROR: ~/category/package/distinfo:4: Missing Size hash for package-1.0.txt.") + + t.SetUpCommandLine("-Wall", "--autofix") + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "AUTOFIX: ~/category/package/distinfo:4: "+ + "Inserting a line \"SHA512 (package-1.0.txt) = f65f341b35981fda842b"+ + "09b2c8af9bcdb7602a4c2e6fa1f7d41f0974d3e3122f268fc79d5a4af66358f513"+ + "3885cd1c165c916f80ab25e5d8d95db46f803c782c\" after this line.", + "AUTOFIX: ~/category/package/distinfo:4: "+ + "Inserting a line \"Size (package-1.0.txt) = 13 bytes\" after this line.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__algorithms_in_wrong_order(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", + "SHA1 (package-1.0.txt) = cd50d19784897085a8d0e3e413f8612b097c03f1", + "Size (package-1.0.txt) = 13 bytes", + "SHA512 (package-1.0.txt) = f65f341b35981fda842b09b2c8af9bcdb7602a4c2e6fa1f7"+ + "d41f0974d3e3122f268fc79d5a4af66358f5133885cd1c165c916f80ab25e5d8d95db46f803c782c") + + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + // This case doesn't happen in practice, therefore there's no autofix for it. + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: " + + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", " + + "got RMD160, SHA1, Size, SHA512.") +} + +func (s *Suite) Test_distinfoLinesChecker_checkAlgorithmsDistfile__some_algorithms_in_wrong_order(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/distinfo", + RcsID, + "", + "RMD160 (package-1.0.txt) = 1a88147a0344137404c63f3b695366eab869a98a", + "Size (package-1.0.txt) = 13 bytes", + "SHA512 (package-1.0.txt) = f65f341b35981fda842b09b2c8af9bcdb7602a4c2e6fa1f7"+ + "d41f0974d3e3122f268fc79d5a4af66358f5133885cd1c165c916f80ab25e5d8d95db46f803c782c") + + t.CreateFileLines("distfiles/package-1.0.txt", + "hello, world") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + // This case doesn't happen in practice, therefore there's no autofix for it. + t.CheckOutputLines( + "ERROR: ~/category/package/distinfo:3: "+ + "Expected SHA1, RMD160, SHA512, Size checksums for \"package-1.0.txt\", "+ + "got RMD160, Size, SHA512.", + "ERROR: ~/category/package/distinfo:3: Missing SHA1 hash for package-1.0.txt.") } diff --git a/pkgtools/pkglint/files/line.go b/pkgtools/pkglint/files/line.go index 19e6eb19c95..376f44bc715 100644 --- a/pkgtools/pkglint/files/line.go +++ b/pkgtools/pkglint/files/line.go @@ -105,15 +105,12 @@ func NewLineWhole(filename string) Line { // RefTo returns a reference to another line, // which can be in the same file or in a different file. func (line *LineImpl) RefTo(other Line) string { - if line.Filename != other.Filename { - return cleanpath(relpath(path.Dir(line.Filename), other.Filename)) + ":" + other.Linenos() - } - return "line " + other.Linenos() + return line.RefToLocation(other.Location) } func (line *LineImpl) RefToLocation(other Location) string { if line.Filename != other.Filename { - return cleanpath(relpath(path.Dir(line.Filename), other.Filename)) + ":" + other.Linenos() + return line.PathToFile(other.Filename) + ":" + other.Linenos() } return "line " + other.Linenos() } diff --git a/pkgtools/pkglint/files/lines.go b/pkgtools/pkglint/files/lines.go index dbcb70ab32e..0d4b9914497 100644 --- a/pkgtools/pkglint/files/lines.go +++ b/pkgtools/pkglint/files/lines.go @@ -35,6 +35,7 @@ func (ls *LinesImpl) SaveAutofixChanges() bool { return SaveAutofixChanges(ls) } +// CheckRcsID returns true if the expected RCS Id was found. func (ls *LinesImpl) CheckRcsID(index int, prefixRe regex.Pattern, suggestedPrefix string) bool { if trace.Tracing { defer trace.Call(prefixRe, suggestedPrefix)() diff --git a/pkgtools/pkglint/files/logging_test.go b/pkgtools/pkglint/files/logging_test.go index 6f3fe96cb0c..6b1dcaad546 100644 --- a/pkgtools/pkglint/files/logging_test.go +++ b/pkgtools/pkglint/files/logging_test.go @@ -749,7 +749,7 @@ func (s *Suite) Test_Logger_Diag__source_duplicates(c *check.C) { t.CheckOutputLines( "ERROR: ~/category/package1/distinfo: "+ - "Patch \"../dependency/patches/patch-aa\" is not recorded. "+ + "Patch \"../../category/dependency/patches/patch-aa\" is not recorded. "+ "Run \""+confMake+" makepatchsum\".", "", ">\t--- old file", @@ -757,7 +757,7 @@ func (s *Suite) Test_Logger_Diag__source_duplicates(c *check.C) { "Each patch must be documented.", "", "ERROR: ~/category/package2/distinfo: "+ - "Patch \"../dependency/patches/patch-aa\" is not recorded. "+ + "Patch \"../../category/dependency/patches/patch-aa\" is not recorded. "+ "Run \""+confMake+" makepatchsum\".", "", "3 errors and 0 warnings found.", diff --git a/pkgtools/pkglint/files/mkline.go b/pkgtools/pkglint/files/mkline.go index a0e78802ec7..a444dea21aa 100644 --- a/pkgtools/pkglint/files/mkline.go +++ b/pkgtools/pkglint/files/mkline.go @@ -22,17 +22,19 @@ type MkLineImpl struct { } type mkLineAssign = *mkLineAssignImpl // See https://github.com/golang/go/issues/28045 type mkLineAssignImpl struct { - commented bool // Whether the whole variable assignment is commented out - varname string // e.g. "HOMEPAGE", "SUBST_SED.perl" - varcanon string // e.g. "HOMEPAGE", "SUBST_SED.*" - varparam string // e.g. "", "perl" - op MkOperator // - valueAlign string // The text up to and including the assignment operator, e.g. VARNAME+=\t - value string // The trimmed value - valueMk []*MkToken // The value, sent through splitIntoMkWords - valueMkRest string // nonempty in case of parse errors - fields []string // The value, space-separated according to shell quoting rules - comment string + commented bool // Whether the whole variable assignment is commented out + varname string // e.g. "HOMEPAGE", "SUBST_SED.perl" + varcanon string // e.g. "HOMEPAGE", "SUBST_SED.*" + varparam string // e.g. "", "perl" + spaceAfterVarname string + op MkOperator // + valueAlign string // The text up to and including the assignment operator, e.g. VARNAME+=\t + value string // The trimmed value + valueMk []*MkToken // The value, sent through splitIntoMkWords + valueMkRest string // nonempty in case of parse errors + fields []string // The value, space-separated according to shell quoting rules + spaceAfterValue string + comment string } type mkLineShell struct { command string @@ -77,24 +79,26 @@ func NewMkLine(line Line) *MkLineImpl { "Otherwise remove the leading whitespace.") } - if m, commented, varname, spaceAfterVarname, op, valueAlign, value, spaceAfterValue, comment := MatchVarassign(text); m { - if spaceAfterVarname != "" { + if m, a := MatchVarassign(text); m { + if a.spaceAfterVarname != "" { + varname := a.varname + op := a.op switch { - case hasSuffix(varname, "+") && op == "=": + case hasSuffix(varname, "+") && (op == opAssign || op == opAssignAppend): break - case matches(varname, `^[a-z]`) && op == ":=": + case matches(varname, `^[a-z]`) && op == opAssignEval: break default: // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing. fix := line.Autofix() fix.Notef("Unnecessary space after variable name %q.", varname) - fix.Replace(varname+spaceAfterVarname+op, varname+op) + fix.Replace(varname+a.spaceAfterVarname+op.String(), varname+op.String()) fix.Apply() } } // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing. - if comment != "" && value != "" && spaceAfterValue == "" { + if a.comment != "" && a.value != "" && a.spaceAfterValue == "" { line.Warnf("The # character starts a Makefile comment.") G.Explain( "In a variable assignment, an unescaped # starts a comment that", @@ -102,18 +106,7 @@ func NewMkLine(line Line) *MkLineImpl { "To escape the #, write \\#.") } - return &MkLineImpl{line, &mkLineAssignImpl{ - commented, - varname, - varnameCanon(varname), - varnameParam(varname), - NewMkOperator(op), - valueAlign, - strings.Replace(value, "\\#", "#", -1), - nil, - "", - nil, - comment}} + return &MkLineImpl{line, a} } if hasPrefix(text, "\t") { @@ -131,7 +124,13 @@ func NewMkLine(line Line) *MkLineImpl { } if m, indent, directive, args, comment := matchMkDirective(text); m { - return &MkLineImpl{line, &mkLineDirectiveImpl{indent, directive, args, comment, nil, nil, nil}} + + // In .if and .endif lines the space surrounding the comment is irrelevant. + // Especially for checking that the .endif comment matches the .if condition, + // it must be trimmed. + trimmedComment := trimHspace(comment) + + return &MkLineImpl{line, &mkLineDirectiveImpl{indent, directive, args, trimmedComment, nil, nil, nil}} } if m, indent, directive, includedFile := MatchMkInclude(text); m { @@ -161,6 +160,7 @@ func NewMkLine(line Line) *MkLineImpl { return &MkLineImpl{line, nil} } +// String returns the filename and line numbers. func (mkline *MkLineImpl) String() string { return sprintf("%s:%s", mkline.Filename, mkline.Linenos()) } @@ -423,6 +423,8 @@ func (mkline *MkLineImpl) ValueSplit(value string, separator string) []string { return split } +var notSpace = textproc.Space.Inverse() + // ValueFields splits the given value, taking care of variable references. // Example: // @@ -456,7 +458,7 @@ func (mkline *MkLineImpl) ValueFields(value string) []string { for lexer.NextBytesSet(textproc.Space) != "" { cont = false } - if word := lexer.NextBytesSet(textproc.Space.Inverse()); word != "" { + if word := lexer.NextBytesSet(notSpace); word != "" { out(word) cont = true } @@ -529,6 +531,9 @@ func (mkline *MkLineImpl) WithoutMakeVariables(value string) string { } func (mkline *MkLineImpl) ResolveVarsInRelativePath(relativePath string) string { + if !contains(relativePath, "$") { + return cleanpath(relativePath) + } var basedir string if G.Pkg != nil { @@ -551,8 +556,22 @@ func (mkline *MkLineImpl) ResolveVarsInRelativePath(relativePath string) string } tmp = strings.Replace(tmp, "${PKGSRCDIR}", pkgsrcdir, -1) } - tmp = strings.Replace(tmp, "${.CURDIR}", ".", -1) // TODO: Replace with the "typical" os.Getwd(). - tmp = strings.Replace(tmp, "${.PARSEDIR}", ".", -1) // FIXME + + // Strictly speaking, the .CURDIR should be replaced with the basedir. + // Depending on whether pkglint is executed with a relative or an absolute + // path, this would produce diagnostics that "this relative path must not + // be absolute". Since ${.CURDIR} is usually used in package Makefiles and + // followed by "../.." anyway, the exact directory doesn't matter. + tmp = strings.Replace(tmp, "${.CURDIR}", ".", -1) + + // TODO: Add test for exists(${.PARSEDIR}/file). + // TODO: Add test for evaluating ${.PARSEDIR} in an included package. + // TODO: Add test for including ${.PARSEDIR}/other.mk. + // TODO: Add test for evaluating ${.PARSEDIR} in the infrastructure. + // This is the only practically relevant use case since the category + // directories don't contain any *.mk files that could be included. + // TODO: Add test that suggests ${.PARSEDIR} in .include to be omitted. + tmp = strings.Replace(tmp, "${.PARSEDIR}", ".", -1) replaceLatest := func(varuse, category string, pattern regex.Pattern, replacement string) { if contains(tmp, varuse) { @@ -603,16 +622,151 @@ func (mkline *MkLineImpl) RefTo(other MkLine) string { } var ( - LowerDash = textproc.NewByteSet("a-z---") - AlnumDot = textproc.NewByteSet("A-Za-z0-9_.") + LowerDash = textproc.NewByteSet("a-z---") + AlnumDot = textproc.NewByteSet("A-Za-z0-9_.") + unescapeMkCommentSafeChars = textproc.NewByteSet("\\#[$").Inverse() ) -func matchMkDirective(text string) (m bool, indent, directive, args, comment string) { +// unescapeMkComment takes a Makefile line, as written in a file, and splits +// it into the main part and the comment. +// +// The comment starts at the first #. Except if it is preceded by an odd number +// of backslashes. Or by an opening bracket. +// +// The main text is returned including leading and trailing whitespace. Any +// escaped # is returned in its unescaped form, that is, \# becomes #. +// +// The comment is returned including the leading "#", if any. If the line has +// no comment, it is an empty string. +func unescapeMkComment(text string) (main, comment string) { + var sb strings.Builder + lexer := textproc.NewLexer(text) - if !lexer.SkipByte('.') { + +again: + if plain := lexer.NextBytesSet(unescapeMkCommentSafeChars); plain != "" { + sb.WriteString(plain) + goto again + } + + switch { + case lexer.SkipByte('$'): + sb.WriteByte('$') + + case lexer.SkipString("\\#"): + sb.WriteByte('#') + + case lexer.PeekByte() == '\\' && len(lexer.Rest()) >= 2: + sb.WriteString(lexer.Rest()[:2]) + lexer.Skip(2) + + case lexer.SkipByte('\\'): + sb.WriteByte('\\') + + case lexer.SkipString("[#"): + // See devel/bmake/files/parse.c:/as in modifier/ + sb.WriteString("[#") + + case lexer.SkipByte('['): + sb.WriteByte('[') + + default: + main = sb.String() + if lexer.PeekByte() == '#' { + return main, lexer.Rest() + } + + G.Assertf(lexer.EOF(), "unescapeMkComment(%q): sb = %q, rest = %q", text, main, lexer.Rest()) + return main, "" + } + + goto again +} + +// splitMkLine parses a logical line from a Makefile (that is, after joining +// the lines that end in a backslash) into two parts: the main part and the +// comment. +// +// This applies to all line types except those starting with a tab, which +// contain the shell commands to be associated with make targets. These cannot +// have comments. +func splitMkLine(text string) (main string, tokens []*MkToken, rest string, spaceBeforeComment string, hasComment bool, comment string) { + + main, comment = unescapeMkComment(text) + + p := NewMkParser(nil, main, false) + lexer := p.lexer + + rtrimHspace := func(s string) string { + end := len(s) + for end > 0 && isHspace(s[end-1]) { + end-- + } + return s[:end] + } + + parseToken := func() string { + var sb strings.Builder + + for !lexer.EOF() { + if lexer.SkipString("$$") { + sb.WriteString("$$") + continue + } + + other := lexer.NextBytesFunc(func(b byte) bool { return b != '$' }) + if other == "" { + break + } + + sb.WriteString(other) + } + + return sb.String() + } + + for !lexer.EOF() { + mark := lexer.Mark() + + if varUse := p.VarUse(); varUse != nil { + tokens = append(tokens, &MkToken{lexer.Since(mark), varUse}) + + } else if token := parseToken(); token != "" { + tokens = append(tokens, &MkToken{token, nil}) + + } else { + break + } + } + + if comment != "" { + hasComment = true + comment = comment[1:] + } + rest = lexer.Rest() + main = main[:len(main)-len(rest)] + + if rest == "" { + mainWithSpaces := main + main = rtrimHspace(main) + spaceBeforeComment = mainWithSpaces[len(main):] + } + + return +} + +func matchMkDirective(text string) (m bool, indent, directive, args, comment string) { + if !hasPrefix(text, ".") { + return + } + + main, _, rest, _, hasComment, trailingComment := splitMkLine(text) + if rest != "" { return } + lexer := textproc.NewLexer(main[1:]) + indent = lexer.NextHspace() directive = lexer.NextBytesSet(LowerDash) switch directive { @@ -629,27 +783,10 @@ func matchMkDirective(text string) (m bool, indent, directive, args, comment str lexer.SkipHspace() - argsStart := lexer.Mark() - for !lexer.EOF() && lexer.PeekByte() != '#' { - switch { - case lexer.SkipString("[#"): - // See devel/bmake/files/parse.c:/as in modifier/ + args = lexer.Rest() - case lexer.PeekByte() == '\\' && len(lexer.Rest()) > 1: - lexer.Skip(2) - - default: - lexer.Skip(1) - } - } - args = lexer.Since(argsStart) - args = strings.TrimFunc(args, func(r rune) bool { return isHspace(byte(r)) }) - args = strings.Replace(args, "\\#", "#", -1) - - if !lexer.EOF() { - lexer.Skip(1) - lexer.SkipHspace() - comment = lexer.Rest() + if hasComment { + comment = trailingComment } m = true @@ -1204,48 +1341,55 @@ func (ind *Indentation) CheckFinish(filename string) { } } -// VarnameBytes contains characters that may be used in variable names. -// The bracket is included only for the tool of the same name, e.g. "TOOLS_PATH.[". +// VarbaseBytes contains characters that may be used in the main part of variable names. +// VarparamBytes contains characters that may be used in the parameter part of variable names. +// +// For example, TOOLS_PATH.[ is a valid variable name but [ alone isn't since +// the opening bracket is only allowed in the parameter part of variable names. // // This approach differs from the one in devel/bmake/files/parse.c:/^Parse_IsVar, // but in practice it works equally well. Luckily there aren't many situations // where a complicated variable name contains unbalanced parentheses or braces, // which would confuse the devel/bmake parser. -var VarnameBytes = textproc.NewByteSet("A-Za-z_0-9*+---.[") +// +// TODO: The allowed characters differ between the basename and the parameter +// of the variable. The square bracket is only allowed in the parameter part. +var ( + VarbaseBytes = textproc.NewByteSet("A-Za-z_0-9+---") + VarparamBytes = textproc.NewByteSet("A-Za-z_0-9#*+---.[") +) -func MatchVarassign(text string) (m, commented bool, varname, spaceAfterVarname, op, valueAlign, value, spaceAfterValue, comment string) { - lexer := textproc.NewLexer(text) +func MatchVarassign(text string) (m bool, assignment mkLineAssign) { + commented := hasPrefix(text, "#") + withoutLeadingComment := text + if commented { + withoutLeadingComment = withoutLeadingComment[1:] + } + + main, tokens, rest, spaceBeforeComment, hasComment, comment := splitMkLine(withoutLeadingComment) + + lexer := NewMkTokensLexer(tokens) + mainStart := lexer.Mark() - commented = lexer.SkipByte('#') for !commented && lexer.SkipByte(' ') { } varnameStart := lexer.Mark() - for !lexer.EOF() { - switch { - - case lexer.NextBytesSet(VarnameBytes) != "": - continue - - case lexer.PeekByte() == '$': - parser := NewMkParser(nil, lexer.Rest(), false) - varuse := parser.VarUse() - if varuse == nil { - return - } - varuseLen := len(lexer.Rest()) - len(parser.Rest()) - lexer.Skip(varuseLen) - continue + // TODO: duplicated code in MkParser.Varname + for lexer.NextBytesSet(VarbaseBytes) != "" || lexer.NextVarUse() != nil { + } + if lexer.SkipByte('.') || hasPrefix(main, "SITES_") { + for lexer.NextBytesSet(VarparamBytes) != "" || lexer.NextVarUse() != nil { } - break } - varname = lexer.Since(varnameStart) + + varname := lexer.Since(varnameStart) if varname == "" { return } - spaceAfterVarname = lexer.NextHspace() + spaceAfterVarname := lexer.NextHspace() opStart := lexer.Mark() switch lexer.PeekByte() { @@ -1255,37 +1399,36 @@ func MatchVarassign(text string) (m, commented bool, varname, spaceAfterVarname, if !lexer.SkipByte('=') { return } - op = lexer.Since(opStart) + op := NewMkOperator(lexer.Since(opStart)) - if hasSuffix(varname, "+") && op == "=" && spaceAfterVarname == "" { + if hasSuffix(varname, "+") && op == opAssign && spaceAfterVarname == "" { varname = varname[:len(varname)-1] - op = "+=" + op = opAssignAppend } lexer.SkipHspace() - valueAlign = text[:len(text)-len(lexer.Rest())] - valueStart := lexer.Mark() - // FIXME: This is the same code as in matchMkDirective. - for !lexer.EOF() && lexer.PeekByte() != '#' { - switch { - case lexer.SkipString("[#"): - break - - case lexer.PeekByte() == '\\' && len(lexer.Rest()) > 1: - lexer.Skip(2) - - default: - lexer.Skip(1) - } + value := trimHspace(lexer.Rest() + rest) + if value == "" { + spaceBeforeComment = "" + } + valueAlign := ifelseStr(commented, "#", "") + lexer.Since(mainStart) + + return true, &mkLineAssignImpl{ + commented: commented, + varname: varname, + varcanon: varnameCanon(varname), + varparam: varnameParam(varname), + spaceAfterVarname: spaceAfterVarname, + op: op, + valueAlign: valueAlign, + value: value, + valueMk: nil, // filled in lazily + valueMkRest: "", // filled in lazily + fields: nil, // filled in lazily + spaceAfterValue: spaceBeforeComment, + comment: ifelseStr(hasComment, "#", "") + comment, } - rawValueWithSpace := lexer.Since(valueStart) - spaceAfterValue = rawValueWithSpace[len(strings.TrimRight(rawValueWithSpace, " \t")):] - value = trimHspace(strings.Replace(lexer.Since(valueStart), "\\#", "#", -1)) - comment = lexer.Rest() - - m = true - return } func MatchMkInclude(text string) (m bool, indentation, directive, filename string) { diff --git a/pkgtools/pkglint/files/mkline_test.go b/pkgtools/pkglint/files/mkline_test.go index 52581146361..8325f1c18a7 100644 --- a/pkgtools/pkglint/files/mkline_test.go +++ b/pkgtools/pkglint/files/mkline_test.go @@ -165,23 +165,19 @@ func (s *Suite) Test_NewMkLine__autofix_space_after_varname(c *check.C) { CheckFileMk(filename) t.CheckOutputLines( - "NOTE: ~/Makefile:2: Unnecessary space after variable name \"VARNAME\".", - // FIXME: Don't say anything here because the spaced form is clearer that the compressed form. - "NOTE: ~/Makefile:4: Unnecessary space after variable name \"VARNAME+\".") + "NOTE: ~/Makefile:2: Unnecessary space after variable name \"VARNAME\".") t.SetUpCommandLine("-Wspace", "--autofix") CheckFileMk(filename) t.CheckOutputLines( - "AUTOFIX: ~/Makefile:2: Replacing \"VARNAME +=\" with \"VARNAME+=\".", - // FIXME: Don't fix anything here because the spaced form is clearer that the compressed form. - "AUTOFIX: ~/Makefile:4: Replacing \"VARNAME+ +=\" with \"VARNAME++=\".") + "AUTOFIX: ~/Makefile:2: Replacing \"VARNAME +=\" with \"VARNAME+=\".") t.CheckFileLines("Makefile", MkRcsID+"", "VARNAME+=\t${VARNAME}", "VARNAME+ =\t${VARNAME+}", - "VARNAME++=\t${VARNAME+}", + "VARNAME+ +=\t${VARNAME+}", "pkgbase := pkglint") } @@ -193,14 +189,45 @@ func (s *Suite) Test_NewMkLine__varname_with_hash(c *check.C) { // Parse error because the # starts a comment. c.Check(mkline.IsVarassign(), equals, false) - mkline2 := t.NewMkLine("Makefile", 123, "VARNAME.\\#=\tvalue") + mkline2 := t.NewMkLine("Makefile", 124, "VARNAME.\\#=\tvalue") + + c.Check(mkline2.IsVarassign(), equals, true) + c.Check(mkline2.Varname(), equals, "VARNAME.#") + + t.CheckOutputLines( + "ERROR: Makefile:123: Unknown Makefile line format: \"VARNAME.#=\\tvalue\".") +} + +// Ensures that pkglint parses escaped # characters in the same way as bmake. +// +// To check that bmake parses them the same, set a breakpoint after the t.NewMkLines +// and look in t.tmpdir for the location of the file. Then run bmake with that file. +func (s *Suite) Test_NewMkLine__escaped_hash_in_value(c *check.C) { + t := s.Init(c) - // FIXME: Varname() should be "VARNAME.#". - c.Check(mkline2.IsVarassign(), equals, false) + mklines := t.SetUpFileMkLines("Makefile", + "VAR0=\tvalue#", + "VAR1=\tvalue\\#", + "VAR2=\tvalue\\\\#", + "VAR3=\tvalue\\\\\\#", + "VAR4=\tvalue\\\\\\\\#", + "", + "all:", + ".for var in VAR0 VAR1 VAR2 VAR3 VAR4", + "\t@printf '%s\\n' ${${var}}''", + ".endfor") + parsed := mklines.mklines + + c.Check(parsed[0].Value(), equals, "value") + c.Check(parsed[1].Value(), equals, "value#") + c.Check(parsed[2].Value(), equals, "value\\\\") + c.Check(parsed[3].Value(), equals, "value\\\\#") + c.Check(parsed[4].Value(), equals, "value\\\\\\\\") t.CheckOutputLines( - "ERROR: Makefile:123: Unknown Makefile line format: \"VARNAME.#=\\tvalue\".", - "ERROR: Makefile:123: Unknown Makefile line format: \"VARNAME.\\\\#=\\tvalue\".") + "WARN: ~/Makefile:1: The # character starts a Makefile comment.", + "WARN: ~/Makefile:3: The # character starts a Makefile comment.", + "WARN: ~/Makefile:5: The # character starts a Makefile comment.") } func (s *Suite) Test_MkLine_Varparam(c *check.C) { @@ -254,26 +281,26 @@ func (s *Suite) Test_VarUseContext_String(c *check.C) { func (s *Suite) Test_NewMkLine__number_sign(c *check.C) { t := s.Init(c) - mklineVarassignEscaped := t.NewMkLine("filename", 1, "SED_CMD=\t's,\\#,hash,g'") + mklineVarassignEscaped := t.NewMkLine("filename.mk", 1, "SED_CMD=\t's,\\#,hash,g'") c.Check(mklineVarassignEscaped.Varname(), equals, "SED_CMD") c.Check(mklineVarassignEscaped.Value(), equals, "'s,#,hash,g'") - mklineCommandEscaped := t.NewMkLine("filename", 1, "\tsed -e 's,\\#,hash,g'") + mklineCommandEscaped := t.NewMkLine("filename.mk", 1, "\tsed -e 's,\\#,hash,g'") c.Check(mklineCommandEscaped.ShellCommand(), equals, "sed -e 's,\\#,hash,g'") // From shells/zsh/Makefile.common, rev. 1.78 - mklineCommandUnescaped := t.NewMkLine("filename", 1, "\t# $ sha1 patches/patch-ac") + mklineCommandUnescaped := t.NewMkLine("filename.mk", 1, "\t# $ sha1 patches/patch-ac") c.Check(mklineCommandUnescaped.ShellCommand(), equals, "# $ sha1 patches/patch-ac") t.CheckOutputEmpty() // No warning about parsing the lonely dollar sign. - mklineVarassignUnescaped := t.NewMkLine("filename", 1, "SED_CMD=\t's,#,hash,'") + mklineVarassignUnescaped := t.NewMkLine("filename.mk", 1, "SED_CMD=\t's,#,hash,'") c.Check(mklineVarassignUnescaped.Value(), equals, "'s,") t.CheckOutputLines( - "WARN: filename:1: The # character starts a Makefile comment.") + "WARN: filename.mk:1: The # character starts a Makefile comment.") } func (s *Suite) Test_NewMkLine__varassign_leading_space(c *check.C) { @@ -331,7 +358,7 @@ func (s *Suite) Test_NewMkLine__infrastructure(c *check.C) { func (s *Suite) Test_MkLine_VariableNeedsQuoting__unknown_rhs(c *check.C) { t := s.Init(c) - mkline := t.NewMkLine("filename", 1, "PKGNAME:= ${UNKNOWN}") + mkline := t.NewMkLine("filename.mk", 1, "PKGNAME:= ${UNKNOWN}") t.SetUpVartypes() vuc := VarUseContext{G.Pkgsrc.VariableType("PKGNAME"), vucTimeParse, VucQuotUnknown, false} @@ -728,7 +755,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__shellword_part(c *check.C) { t.CheckOutputLines( "NOTE: ~/Makefile:6: The substitution command \"s:@LINKER_RPATH_FLAG@:${LINKER_RPATH_FLAG}:g\" " + - "can be replaced with \"SUBST_VARS.class+= LINKER_RPATH_FLAG\".") + "can be replaced with \"SUBST_VARS.class= LINKER_RPATH_FLAG\".") } // Tools, when used in a shell command, must not be quoted. @@ -1050,6 +1077,14 @@ func (s *Suite) Test_MkLine_ValueTokens__warnings(c *check.C) { "WARN: Makefile:2: Please use curly braces {} instead of round parentheses () for ROUND.") } +func (s *Suite) Test_MkLine_Tokenize__commented_varassign(c *check.C) { + t := s.Init(c) + + mkline := t.NewMkLine("filename.mk", 123, "#VAR=\tvalue ${VAR} suffix text") + + t.Check(mkline.Tokenize(mkline.Value(), false), check.HasLen, 3) +} + func (s *Suite) Test_MkLine_ResolveVarsInRelativePath(c *check.C) { t := s.Init(c) @@ -1103,25 +1138,32 @@ func (s *Suite) Test_MatchVarassign(c *check.C) { s.Init(c) test := func(text string, commented bool, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment string) { - type VarAssign struct { - commented bool - varname, spaceAfterVarname string - op, align string - value, spaceAfterValue string - comment string - } - expected := VarAssign{commented, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment} - am, acommented, avarname, aspaceAfterVarname, aop, aalign, avalue, aspaceAfterValue, acomment := MatchVarassign(text) - if !am { + m, actual := MatchVarassign(text) + if !m { c.Errorf("Text %q doesn't match variable assignment", text) return } - actual := VarAssign{acommented, avarname, aspaceAfterVarname, aop, aalign, avalue, aspaceAfterValue, acomment} - c.Check(actual, equals, expected) + + expected := mkLineAssignImpl{ + commented: commented, + varname: varname, + varcanon: varnameCanon(varname), + varparam: varnameParam(varname), + spaceAfterVarname: spaceAfterVarname, + op: NewMkOperator(op), + valueAlign: align, + value: value, + valueMk: nil, + valueMkRest: "", + fields: nil, + spaceAfterValue: spaceAfterValue, + comment: comment, + } + c.Check(*actual, deepEquals, expected) } testInvalid := func(text string) { - m, _, _, _, _, _, _, _, _ := MatchVarassign(text) + m, _ := MatchVarassign(text) if m { c.Errorf("Text %q matches variable assignment but shouldn't.", text) } @@ -1151,6 +1193,69 @@ func (s *Suite) Test_MatchVarassign(c *check.C) { // A single space is typically used for writing documentation, not for commenting out code. // Therefore this line doesn't count as commented variable assignment. testInvalid("# VAR=value") + + // Ensure that the alignment for the variable value is correct. + test("BUILD_DIRS=\tdir1 dir2", + false, + "BUILD_DIRS", + "", + "=", + "BUILD_DIRS=\t", + "dir1 dir2", + "", + "") + + // Ensure that the alignment for the variable value is correct, + // even if the whole line is commented. + test("#BUILD_DIRS=\tdir1 dir2", + true, + "BUILD_DIRS", + "", + "=", + "#BUILD_DIRS=\t", + "dir1 dir2", + "", + "") + + test("MASTER_SITES=\t#none", + false, + "MASTER_SITES", + "", + "=", + "MASTER_SITES=\t", + "", + "", + "#none") + + test("MASTER_SITES=\t# none", + false, + "MASTER_SITES", + "", + "=", + "MASTER_SITES=\t", + "", + "", + "# none") + + test("EGDIRS=\t${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d", + false, + "EGDIRS", + "", + "=", + "EGDIRS=\t", + "${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d", + "", + "") + + test("VAR:=\t${VAR:M-*:[\\#]}", + false, + "VAR", + "", + ":=", + "VAR:=\t", + "${VAR:M-*:[#]}", + "", + "") } func (s *Suite) Test_NewMkOperator(c *check.C) { @@ -1254,6 +1359,32 @@ func (s *Suite) Test_Indentation_TrackAfter__lonely_else(c *check.C) { t.CheckOutputEmpty() } +func (s *Suite) Test_Indentation_Varnames__repetition(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpPackage("category/other") + t.CreateFileDummyBuildlink3("category/other/buildlink3.mk") + t.SetUpPackage("category/package", + "DISTNAME=\tpackage-1.0", + ".include \"../../category/other/buildlink3.mk\"") + t.CreateFileDummyBuildlink3("category/package/buildlink3.mk", + ".if ${OPSYS} == NetBSD || ${OPSYS} == FreeBSD", + ". if ${OPSYS} == NetBSD", + ". include \"../../category/other/buildlink3.mk\"", + ". endif", + ".endif") + + G.Check(t.File("category/package")) + + // TODO: It feels wrong that OPSYS is mentioned twice here. + // Why only twice and not three times? + t.CheckOutputLines( + "WARN: ~/category/package/buildlink3.mk:14: " + + "\"../../category/other/buildlink3.mk\" is included conditionally here " + + "(depending on OPSYS, OPSYS) and unconditionally in Makefile:20.") +} + func (s *Suite) Test_MkLine_DetermineUsedVariables(c *check.C) { t := s.Init(c) @@ -1330,6 +1461,362 @@ func (s *Suite) Test_MkLine_UnquoteShell(c *check.C) { test("`", "`") } +func (s *Suite) Test_unescapeMkComment(c *check.C) { + t := s.Init(c) + + test := func(text string, main, comment string) { + aMain, aComment := unescapeMkComment(text) + t.Check( + []interface{}{text, aMain, aComment}, + deepEquals, + []interface{}{text, main, comment}) + } + + test("", + "", + "") + test("text", + "text", + "") + + // The leading space from the comment is preserved to make parsing as exact + // as possible. + // + // The difference between "#defined" and "# defined" is relevant in a few + // cases, such as the API documentation of the infrastructure files. + test("# comment", + "", + "# comment") + test("#\tcomment", + "", + "#\tcomment") + test("# comment", + "", + "# comment") + + // Other than in the shell, # also starts a comment in the middle of a word. + test("COMMENT=\tThe C# compiler", + "COMMENT=\tThe C", + "# compiler") + test("COMMENT=\tThe C\\# compiler", + "COMMENT=\tThe C# compiler", + "") + + test("${TARGET}: ${SOURCES} # comment", + "${TARGET}: ${SOURCES} ", + "# comment") + + // A # starts a comment, except if it immediately follows a [. + // This is done so that the length modifier :[#] can be written without + // escaping the #. + test("VAR=\t${OTHER:[#]} # comment", + "VAR=\t${OTHER:[#]} ", + "# comment") + + // The # in the :[#] modifier may be escaped or not. Both forms are equivalent. + test("VAR:=\t${VAR:M-*:[\\#]}", + "VAR:=\t${VAR:M-*:[#]}", + "") + + // The character [ prevents the following # from starting a comment, even + // outside of variable modifiers. + test("COMMENT=\t[#] $$\\# $$# comment", + "COMMENT=\t[#] $$# $$", + "# comment") + + // A backslash always escapes the next character, be it a # for a comment + // or something else. This makes it difficult to write a literal \# in a + // Makefile, but that's an edge case anyway. + test("VAR0=\t#comment", + "VAR0=\t", + "#comment") + test("VAR1=\t\\#no-comment", + "VAR1=\t#no-comment", + "") + test("VAR2=\t\\\\#comment", + "VAR2=\t\\\\", + "#comment") + + // The backslash is only removed when it escapes a comment. + // In particular, it cannot be used to escape a dollar that starts a + // variable use. + test("VAR0=\t$T", + "VAR0=\t$T", + "") + test("VAR1=\t\\$T", + "VAR1=\t\\$T", + "") + test("VAR2=\t\\\\$T", + "VAR2=\t\\\\$T", + "") + + // To escape a dollar, write it twice. + test("$$shellvar $${shellvar} \\${MKVAR} [] \\x", + "$$shellvar $${shellvar} \\${MKVAR} [] \\x", + "") + + // Parse errors are recorded in the rest return value. + test("${UNCLOSED", + "${UNCLOSED", + "") + + // In this early phase of parsing, unfinished variable uses are not + // interpreted and do not influence the detection of the comment start. + test("text before ${UNCLOSED # comment", + "text before ${UNCLOSED ", + "# comment") + + // The dollar-space refers to a normal Make variable named " ". + // The lonely dollar at the very end refers to the variable named "", + // which is specially protected in bmake to always contain the empty string. + // It is heavily used in .for loops in the form ${:Uvalue}. + test("Lonely $ character $", + "Lonely $ character $", + "") + + // An even number of backslashes does not escape the #. + // Therefore it starts a comment here. + test("VAR2=\t\\\\#comment", + "VAR2=\t\\\\", + "#comment") +} + +func (s *Suite) Test_splitMkLine(c *check.C) { + t := s.Init(c) + + varuse := func(varname string, modifiers ...string) *MkToken { + text := "${" + varname + for _, modifier := range modifiers { + text += ":" + modifier + } + text += "}" + return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)} + } + varuseText := func(text, varname string, modifiers ...string) *MkToken { + return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)} + } + text := func(text string) *MkToken { + return &MkToken{text, nil} + } + tokens := func(tokens ...*MkToken) []*MkToken { + return tokens + } + _, _, _, _ = text, varuse, varuseText, tokens + + test := func(text string, main string, tokens []*MkToken, rest string, spaceBeforeComment string, hasComment bool, comment string) { + aMain, aTokens, aRest, aSpaceBeforeComment, aHasComment, aComment := splitMkLine(text) + t.Check( + []interface{}{text, aTokens, aMain, aRest, aSpaceBeforeComment, aHasComment, aComment}, + deepEquals, + []interface{}{text, tokens, main, rest, spaceBeforeComment, hasComment, comment}) + } + + test("", + "", + tokens(), + "", + "", + false, + "") + test("text", + "text", + tokens(text("text")), + "", + "", + false, + "") + + // The leading space from the comment is preserved to make parsing as exact + // as possible. + // + // The difference between "#defined" and "# defined" is relevant in a few + // cases, such as the API documentation of the infrastructure files. + test("# comment", + "", + tokens(), + "", + "", + true, + " comment") + test("#\tcomment", + "", + tokens(), + "", + "", + true, + "\tcomment") + test("# comment", + "", + tokens(), + "", + "", + true, + " comment") + + // Other than in the shell, # also starts a comment in the middle of a word. + test("COMMENT=\tThe C# compiler", + "COMMENT=\tThe C", + tokens(text("COMMENT=\tThe C")), + "", + "", + true, + " compiler") + test("COMMENT=\tThe C\\# compiler", + "COMMENT=\tThe C# compiler", + tokens(text("COMMENT=\tThe C# compiler")), + "", + "", + false, + "") + + test("${TARGET}: ${SOURCES} # comment", + "${TARGET}: ${SOURCES}", + tokens(varuse("TARGET"), text(": "), varuse("SOURCES"), text(" ")), + "", + " ", + true, + " comment") + + // A # starts a comment, except if it immediately follows a [. + // This is done so that the length modifier :[#] can be written without + // escaping the #. + test("VAR=\t${OTHER:[#]} # comment", + "VAR=\t${OTHER:[#]}", + tokens(text("VAR=\t"), varuse("OTHER", "[#]"), text(" ")), + "", + " ", + true, + " comment") + + // The # in the :[#] modifier may be escaped or not. Both forms are equivalent. + test("VAR:=\t${VAR:M-*:[\\#]}", + "VAR:=\t${VAR:M-*:[#]}", + tokens(text("VAR:=\t"), varuse("VAR", "M-*", "[#]")), + "", + "", + false, + "") + + // A backslash always escapes the next character, be it a # for a comment + // or something else. This makes it difficult to write a literal \# in a + // Makefile, but that's an edge case anyway. + test("VAR0=\t#comment", + "VAR0=", + tokens(text("VAR0=\t")), + "", + // Later, when converting this result into a proper variable assignment, + // this "space before comment" is reclassified as "space before the value", + // in order to align the "#comment" with the other variable values. + "\t", + true, + "comment") + test("VAR1=\t\\#no-comment", + "VAR1=\t#no-comment", + tokens(text("VAR1=\t#no-comment")), + "", + "", + false, + "") + test("VAR2=\t\\\\#comment", + "VAR2=\t\\\\", + tokens(text("VAR2=\t\\\\")), + "", + "", + true, + "comment") + + // The backslash is only removed when it escapes a comment. + // In particular, it cannot be used to escape a dollar that starts a + // variable use. + test("VAR0=\t$T", + "VAR0=\t$T", + tokens(text("VAR0=\t"), varuseText("$T", "T")), + "", + "", + false, + "") + test("VAR1=\t\\$T", + "VAR1=\t\\$T", + tokens(text("VAR1=\t\\"), varuseText("$T", "T")), + "", + "", + false, + "") + test("VAR2=\t\\\\$T", + "VAR2=\t\\\\$T", + tokens(text("VAR2=\t\\\\"), varuseText("$T", "T")), + "", + "", + false, + "") + + // To escape a dollar, write it twice. + test("$$shellvar $${shellvar} \\${MKVAR} [] \\x", + "$$shellvar $${shellvar} \\${MKVAR} [] \\x", + tokens(text("$$shellvar $${shellvar} \\"), varuse("MKVAR"), text(" [] \\x")), + "", + "", + false, + "") + + // Parse errors are recorded in the rest return value. + test("${UNCLOSED", + "", + tokens(), + "${UNCLOSED", + "", + false, + "") + + // When a parse error occurs, the comment is not parsed and the main text + // is not trimmed to the right, to keep as much original information as + // possible. + test("text before ${UNCLOSED # comment", + "text before ", + tokens(text("text before ")), + "${UNCLOSED ", // FIXME: put the space into spaceBeforeComment + "", // FIXME: the space is missing here + true, + " comment") + + // The dollar-space refers to a normal Make variable named " ". + // The lonely dollar at the very end refers to the variable named "", + // which is specially protected in bmake to always contain the empty string. + // It is heavily used in .for loops in the form ${:Uvalue}. + // + // TODO: The rest of pkglint assumes that the empty string is not a valid + // variable name, mainly because the empty variable name is not visible + // outside of the bmake debugging mode. + test("Lonely $ character $", + "Lonely $ character ", + tokens( + text("Lonely "), + varuseText("$ " /* instead of "${ }" */, " "), + text("character ")), + "$", + "", + false, + "") + + // The character [ prevents the following # from starting a comment, even + // outside of variable modifiers. + test("COMMENT=\t[#] $$\\# $$# comment", + "COMMENT=\t[#] $$# $$", + tokens(text("COMMENT=\t[#] $$# $$")), + "", + "", + true, + " comment") + + test("VAR2=\t\\\\#comment", + "VAR2=\t\\\\", + tokens(text("VAR2=\t\\\\")), + "", + "", + true, + "comment") +} + func (s *Suite) Test_matchMkDirective(c *check.C) { test := func(input, expectedIndent, expectedDirective, expectedArgs, expectedComment string) { @@ -1340,11 +1827,19 @@ func (s *Suite) Test_matchMkDirective(c *check.C) { []interface{}{true, expectedIndent, expectedDirective, expectedArgs, expectedComment}) } + testFail := func(input string) { + m, indent, directive, args, comment := matchMkDirective(input) + if m { + c.Errorf("The line %q could be parsed as directive (%q, %q, %q, %q) but shouldn't.", + indent, directive, args, comment) + } + } + test(".if ${VAR} == value", "", "if", "${VAR} == value", "") test(".\tendif # comment", - "\t", "endif", "", "comment") + "\t", "endif", "", " comment") test(".if ${VAR} == \"#\"", "", "if", "${VAR} == \"", "\"") @@ -1354,6 +1849,9 @@ func (s *Suite) Test_matchMkDirective(c *check.C) { test(".if ${VAR} == \\", "", "if", "${VAR} == \\", "") + + // Unclosed variable + testFail(".if ${VAR") } func (s *Suite) Test_MatchMkInclude(c *check.C) { diff --git a/pkgtools/pkglint/files/mklinechecker.go b/pkgtools/pkglint/files/mklinechecker.go index 111eab92c9a..f9edd9c5cae 100644 --- a/pkgtools/pkglint/files/mklinechecker.go +++ b/pkgtools/pkglint/files/mklinechecker.go @@ -423,7 +423,7 @@ func (ck MkLineChecker) CheckVaruse(varuse *MkVarUse, vuc *VarUseContext) { if G.Opts.WarnQuoting && vuc.quoting != VucQuotUnknown && needsQuoting != unknown { // FIXME: Why "Shellword" when there's no indication that this is actually a shell type? // It's for splitting the value into tokens, taking "double" and 'single' quotes into account. - ck.CheckVaruseShellword(varname, vartype, vuc, varuse.Mod(), needsQuoting) + ck.CheckVaruseShellword(varname, vartype, vuc, varuse.Mod(), needsQuoting == yes) } if G.Pkgsrc.UserDefinedVars.Defined(varname) && !G.Pkgsrc.IsBuildDef(varname) { @@ -544,17 +544,16 @@ func (ck MkLineChecker) checkVarusePermissions(varname string, vartype *Vartype, mkline := ck.MkLine effPerms := vartype.EffectivePermissions(mkline.Basename) + if effPerms == aclpUnknown { + return + } + if effPerms.Contains(aclpUseLoadtime) { + return + } // Is the variable used at load time although that is not allowed? - directly := false - indirectly := false - if !effPerms.Contains(aclpUseLoadtime) { // May not be used at load time. - if vuc.time == vucTimeParse { - directly = true - } else if vuc.vartype != nil && vuc.vartype.Union().Contains(aclpUseLoadtime) { - indirectly = true - } - } + directly := vuc.time == vucTimeParse + indirectly := !directly && vuc.vartype != nil && vuc.vartype.Union().Contains(aclpUseLoadtime) if (directly || indirectly) && !vartype.guessed { if tool := G.ToolByVarname(varname); tool != nil { @@ -566,17 +565,21 @@ func (ck MkLineChecker) checkVarusePermissions(varname string, vartype *Vartype, } } - if !effPerms.Contains(aclpUseLoadtime) && !effPerms.Contains(aclpUse) { + if !effPerms.Contains(aclpUse) { needed := aclpUse if directly || indirectly { needed = aclpUseLoadtime } alternativeFiles := vartype.AllowedFiles(needed) if alternativeFiles != "" { - mkline.Warnf("%s may not be used in this file; it would be ok in %s.", - varname, alternativeFiles) + if G.Mk == nil || G.Mk.FirstTimeSlice("don't-use", varname, mkline.Filename) { + mkline.Warnf("%s may not be used in this file; it would be ok in %s.", + varname, alternativeFiles) + } } else { - mkline.Warnf("%s may not be used in any file; it is a write-only variable.", varname) + if G.Mk == nil || G.Mk.FirstTimeSlice("write-only", varname) { + mkline.Warnf("%s may not be used in any file; it is a write-only variable.", varname) + } } ck.explainPermissions(varname, vartype) @@ -654,7 +657,7 @@ func (ck MkLineChecker) warnVaruseLoadTime(varname string, isIndirect bool) { // CheckVaruseShellword checks whether a variable use of the form ${VAR} // or ${VAR:modifiers} is allowed in a certain context. -func (ck MkLineChecker) CheckVaruseShellword(varname string, vartype *Vartype, vuc *VarUseContext, mod string, needsQuoting YesNoUnknown) { +func (ck MkLineChecker) CheckVaruseShellword(varname string, vartype *Vartype, vuc *VarUseContext, mod string, needsQuoting bool) { if trace.Tracing { defer trace.Call(varname, vartype, vuc, mod, needsQuoting)() } @@ -672,7 +675,7 @@ func (ck MkLineChecker) CheckVaruseShellword(varname string, vartype *Vartype, v if mod == ":M*:Q" && !needMstar { mkline.Notef("The :M* modifier is not needed here.") - } else if needsQuoting == yes { + } else if needsQuoting { modNoQ := strings.TrimSuffix(mod, ":Q") modNoM := strings.TrimSuffix(modNoQ, ":M*") correctMod := modNoM + ifelseStr(needMstar, ":M*:Q", ":Q") @@ -744,14 +747,12 @@ func (ck MkLineChecker) CheckVaruseShellword(varname string, vartype *Vartype, v } } - if hasSuffix(mod, ":Q") && needsQuoting != yes { + if hasSuffix(mod, ":Q") && !needsQuoting { bad := "${" + varname + mod + "}" good := "${" + varname + strings.TrimSuffix(mod, ":Q") + "}" fix := mkline.Line.Autofix() - if needsQuoting == no { - fix.Notef("The :Q operator isn't necessary for ${%s} here.", varname) - } + fix.Notef("The :Q operator isn't necessary for ${%s} here.", varname) fix.Explain( "Many variables in pkgsrc do not need the :Q operator since they", "are not expected to contain whitespace or other special characters.", @@ -780,17 +781,13 @@ func (ck MkLineChecker) checkVaruseDeprecated(varuse *MkVarUse) { } func (ck MkLineChecker) checkVarassignDecreasingVersions() { - if trace.Tracing { - defer trace.Call0()() - } - mkline := ck.MkLine strVersions := mkline.Fields() intVersions := make([]int, len(strVersions)) for i, strVersion := range strVersions { iver, err := strconv.Atoi(strVersion) if err != nil || !(iver > 0) { - mkline.Errorf("All values for %s must be positive integers.", mkline.Varname()) + mkline.Errorf("Value %q for %s must be a positive integer.", strVersion, mkline.Varname()) return } intVersions[i] = iver @@ -798,7 +795,8 @@ func (ck MkLineChecker) checkVarassignDecreasingVersions() { for i, ver := range intVersions { if i > 0 && ver >= intVersions[i-1] { - mkline.Warnf("The values for %s should be in decreasing order.", mkline.Varname()) + mkline.Warnf("The values for %s should be in decreasing order (%d before %d).", + mkline.Varname(), ver, intVersions[i-1]) G.Explain( "If they aren't, it may be possible that needless versions of", "packages are installed.") @@ -858,27 +856,37 @@ func (ck MkLineChecker) checkVarassignLeftDeprecated() { } } +// checkVarassignLeftNotUsed checks whether the left-hand side of a variable +// assignment is not used. If it is unused and also doesn't have a predefined +// data type, it may be a spelling mistake. func (ck MkLineChecker) checkVarassignLeftNotUsed() { varname := ck.MkLine.Varname() varcanon := varnameCanon(varname) - // If the variable is not used and is untyped, it may be a spelling mistake. if ck.MkLine.Op() == opAssignEval && varname == strings.ToLower(varname) { if trace.Tracing { trace.Step1("%s might be unused unless it is an argument to a procedure file.", varname) } + return + } - } else if !varIsUsedSimilar(varname) { - if vartypes := G.Pkgsrc.vartypes; vartypes[varname] != nil || vartypes[varcanon] != nil { - // Ok - } else if deprecated := G.Pkgsrc.Deprecated; deprecated[varname] != "" || deprecated[varcanon] != "" { - // Ok - } else if G.Mk != nil && !G.Mk.FirstTimeSlice("defined but not used: ", varname) { - // Skip - } else { - ck.MkLine.Warnf("%s is defined but not used.", varname) - } + if varIsUsedSimilar(varname) { + return + } + + if vartypes := G.Pkgsrc.vartypes; vartypes[varname] != nil || vartypes[varcanon] != nil { + return + } + + if deprecated := G.Pkgsrc.Deprecated; deprecated[varname] != "" || deprecated[varcanon] != "" { + return + } + + if G.Mk != nil && !G.Mk.FirstTimeSlice("defined but not used: ", varname) { + return } + + ck.MkLine.Warnf("%s is defined but not used.", varname) } // checkVarassignRightVaruse checks that in a variable assignment, @@ -1070,7 +1078,7 @@ func (ck MkLineChecker) checkVartype(varname string, op MkOperator, value, comme case value == "": break - case vartype.kindOfList == lkShell: + default: words, _ := splitIntoMkWords(mkline.Line, value) for _, word := range words { ck.CheckVartypeBasic(varname, vartype.basicType, op, word, comment, vartype.guessed) @@ -1146,26 +1154,34 @@ func (ck MkLineChecker) checkDirectiveCond() { return } - checkCompareVarStr := func(varuse *MkVarUse, op string, value string) { + checkCompareVarStr := func(varuse *MkVarUse, op string, str string) { varname := varuse.varname varmods := varuse.modifiers switch len(varmods) { case 0: - ck.checkCompareVarStr(varname, op, value) + ck.checkCompareVarStr(varname, op, str) case 1: - if m, _, _ := varmods[0].MatchMatch(); m && value != "" { - ck.checkVartype(varname, opUseMatch, value, "") + if m, _, pattern := varmods[0].MatchMatch(); m { + ck.checkVartype(varname, opUseMatch, pattern, "") + + // After applying the :M or :N modifier, every expression may end up empty, + // regardless of its data type. Therefore there's no point in type-checking that case. + if str != "" { + ck.checkVartype(varname, opUseCompare, str, "") + } } default: // This case covers ${VAR:Mfilter:O:u} or similar uses in conditions. - // To check these properly, pkglint first needs to know the most common modifiers and how they interact. - // As of November 2018, the modifiers are not modeled. + // To check these properly, pkglint first needs to know the most common + // modifiers and how they interact. + // As of March 2019, the modifiers are not modeled. // The following tracing statement makes it easy to discover these cases, // in order to decide whether checking them is worthwhile. if trace.Tracing { - trace.Stepf("checkCompareVarStr ${%s%s} %s %s", varuse.varname, varuse.Mod(), op, value) + trace.Stepf("checkCompareVarStr ${%s%s} %s %s", + varuse.varname, varuse.Mod(), op, str) } } } @@ -1285,10 +1301,12 @@ func (ck MkLineChecker) CheckRelativePath(relativePath string, mustExist bool) { return } - abs := resolvedPath - if !filepath.IsAbs(abs) { - abs = path.Dir(mkline.Filename) + "/" + abs + if filepath.IsAbs(resolvedPath) { + mkline.Errorf("The path %q must be relative.", resolvedPath) + return } + + abs := path.Dir(mkline.Filename) + "/" + resolvedPath if _, err := os.Stat(abs); err != nil { if mustExist && !(G.Mk != nil && G.Mk.indentation.IsCheckedFile(resolvedPath)) { mkline.Errorf("Relative path %q does not exist.", resolvedPath) @@ -1305,6 +1323,7 @@ func (ck MkLineChecker) CheckRelativePath(relativePath string, mustExist bool) { // From a package to another package. case hasPrefix(relativePath, "../mk/") && relpath(path.Dir(mkline.Filename), G.Pkgsrc.File(".")) == "..": // For category Makefiles. + // TODO: Or from a pkgsrc wip package to wip/mk. default: mkline.Warnf("Invalid relative path %q.", relativePath) // TODO: Explain this warning. diff --git a/pkgtools/pkglint/files/mklinechecker_test.go b/pkgtools/pkglint/files/mklinechecker_test.go index cd86056879a..57a5b34d7db 100644 --- a/pkgtools/pkglint/files/mklinechecker_test.go +++ b/pkgtools/pkglint/files/mklinechecker_test.go @@ -1,6 +1,9 @@ package pkglint -import "gopkg.in/check.v1" +import ( + "gopkg.in/check.v1" + "runtime" +) func (s *Suite) Test_MkLineChecker_checkVarassignLeft(c *check.C) { t := s.Init(c) @@ -15,6 +18,48 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeft(c *check.C) { "WARN: module.mk:123: _VARNAME is defined but not used.") } +func (s *Suite) Test_MkLineChecker_checkVarassignLeftNotUsed__procedure_call(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("mk/pkg-build-options.mk") + mklines := t.SetUpFileMkLines("category/package/filename.mk", + MkRcsID, + "", + "pkgbase := glib2", + ".include \"../../mk/pkg-build-options.mk\"", + "", + "VAR=\tvalue") + + mklines.Check() + + // There is no warning for pkgbase although it looks unused as well. + // The file pkg-build-options.mk is essentially a procedure call, + // and pkgbase is its parameter. + // + // To distinguish these parameters from ordinary variables, they are + // usually written with the := operator instead of the = operator. + // This has the added benefit that the parameter is only evaluated + // once, especially if it contains references to other variables. + t.CheckOutputLines( + "WARN: ~/category/package/filename.mk:6: VAR is defined but not used.") +} + +// Files from the pkgsrc infrastructure may define and use variables +// whose name starts with an underscore. +func (s *Suite) Test_MkLineChecker_checkVarassignLeft__infrastructure(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("mk/infra.mk", + MkRcsID, + "_VARNAME=\tvalue") + + G.Check(t.File("mk/infra.mk")) + + t.CheckOutputLines( + "WARN: ~/mk/infra.mk:2: _VARNAME is defined but not used.") +} + func (s *Suite) Test_MkLineChecker_Check__url2pkg(c *check.C) { t := s.Init(c) @@ -286,7 +331,7 @@ func (s *Suite) Test_MkLineChecker_checkVartype(c *check.C) { t := s.Init(c) t.SetUpVartypes() - mkline := t.NewMkLine("filename", 1, "DISTNAME=gcc-${GCC_VERSION}") + mkline := t.NewMkLine("filename.mk", 1, "DISTNAME=gcc-${GCC_VERSION}") MkLineChecker{mkline}.checkVartype("DISTNAME", opAssign, "gcc-${GCC_VERSION}", "") @@ -318,11 +363,14 @@ func (s *Suite) Test_MkLineChecker_checkVarassign__URL_with_shell_special_charac G.Pkg = NewPackage(t.File("graphics/gimp-fix-ca")) t.SetUpVartypes() - mkline := t.NewMkLine("filename", 10, "MASTER_SITES=http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=") + mkline := t.NewMkLine("filename.mk", 10, "MASTER_SITES=http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=") MkLineChecker{mkline}.checkVarassign() - t.CheckOutputEmpty() + t.CheckOutputLines( + "WARN: filename.mk:10: The variable MASTER_SITES may not be set " + + "(only given a default value, or appended to) in this file; " + + "it would be ok in Makefile, Makefile.common or options.mk.") } func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { @@ -331,7 +379,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { t.SetUpVartypes() test := func(cond string, output ...string) { - MkLineChecker{t.NewMkLine("filename", 1, cond)}.checkDirectiveCond() + MkLineChecker{t.NewMkLine("filename.mk", 1, cond)}.checkDirectiveCond() if len(output) > 0 { t.CheckOutputLines(output...) } else { @@ -340,50 +388,43 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { } test(".if !empty(PKGSRC_COMPILER:Mmycc)", - "WARN: filename:1: The pattern \"mycc\" cannot match any of "+ + "WARN: filename.mk:1: The pattern \"mycc\" cannot match any of "+ "{ ccache ccc clang distcc f2c gcc hp icc ido "+ "mipspro mipspro-ucode pcc sunpro xlc } for PKGSRC_COMPILER.") test(".elif ${A} != ${B}", - "WARN: filename:1: A is used but not defined.", - "WARN: filename:1: B is used but not defined.") + "WARN: filename.mk:1: A is used but not defined.", + "WARN: filename.mk:1: B is used but not defined.") test(".if ${HOMEPAGE} == \"mailto:someone@example.org\"", - "WARN: filename:1: \"mailto:someone@example.org\" is not a valid URL.", - "WARN: filename:1: HOMEPAGE should not be evaluated at load time.", - "WARN: filename:1: HOMEPAGE may not be used in any file; it is a write-only variable.") + "WARN: filename.mk:1: \"mailto:someone@example.org\" is not a valid URL.", + "WARN: filename.mk:1: HOMEPAGE should not be evaluated at load time.") test(".if !empty(PKGSRC_RUN_TEST:M[Y][eE][sS])", - "WARN: filename:1: PKGSRC_RUN_TEST should be matched "+ + "WARN: filename.mk:1: PKGSRC_RUN_TEST should be matched "+ "against \"[yY][eE][sS]\" or \"[nN][oO]\", not \"[Y][eE][sS]\".") - test(".if !empty(IS_BUILTIN.Xfixes:M[yY][eE][sS])", - "WARN: filename:1: IS_BUILTIN.Xfixes should not be evaluated at load time.", - "WARN: filename:1: IS_BUILTIN.Xfixes may not be used in this file; it would be ok in builtin.mk.") + test(".if !empty(IS_BUILTIN.Xfixes:M[yY][eE][sS])") test(".if !empty(${IS_BUILTIN.Xfixes:M[yY][eE][sS]})", - "WARN: filename:1: The empty() function takes a variable name as parameter, "+ - "not a variable expression.", - "WARN: filename:1: IS_BUILTIN.Xfixes should not be evaluated at load time.", - "WARN: filename:1: IS_BUILTIN.Xfixes may not be used in this file; it would be ok in builtin.mk.") + "WARN: filename.mk:1: The empty() function takes a variable name as parameter, "+ + "not a variable expression.") test(".if ${PKGSRC_COMPILER} == \"msvc\"", - "WARN: filename:1: \"msvc\" is not valid for PKGSRC_COMPILER. "+ + "WARN: filename.mk:1: \"msvc\" is not valid for PKGSRC_COMPILER. "+ "Use one of { ccache ccc clang distcc f2c gcc hp icc ido mipspro mipspro-ucode pcc sunpro xlc } instead.", - "WARN: filename:1: Use ${PKGSRC_COMPILER:Mmsvc} instead of the == operator.") + "WARN: filename.mk:1: Use ${PKGSRC_COMPILER:Mmsvc} instead of the == operator.") test(".if ${PKG_LIBTOOL:Mlibtool}", - "NOTE: filename:1: PKG_LIBTOOL should be compared using == instead of matching against \":Mlibtool\".", - "WARN: filename:1: PKG_LIBTOOL should not be evaluated at load time.", - "WARN: filename:1: PKG_LIBTOOL may not be used in any file; it is a write-only variable.") + "NOTE: filename.mk:1: PKG_LIBTOOL should be compared using == instead of matching against \":Mlibtool\".") test(".if ${MACHINE_PLATFORM:MUnknownOS-*-*} || ${MACHINE_ARCH:Mx86}", - "WARN: filename:1: "+ + "WARN: filename.mk:1: "+ "The pattern \"UnknownOS\" cannot match any of "+ "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ "} for the operating system part of MACHINE_PLATFORM.", - "WARN: filename:1: "+ + "WARN: filename.mk:1: "+ "The pattern \"x86\" cannot match any of "+ "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm "+ "earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb "+ @@ -391,11 +432,13 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { "m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax "+ "powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ "} for MACHINE_ARCH.", - "NOTE: filename:1: MACHINE_ARCH should be compared using == instead of matching against \":Mx86\".") + "NOTE: filename.mk:1: MACHINE_ARCH should be compared using == instead of matching against \":Mx86\".") test(".if ${MASTER_SITES:Mftp://*} == \"ftp://netbsd.org/\"", - "WARN: filename:1: MASTER_SITES should not be evaluated at load time.", - "WARN: filename:1: MASTER_SITES may not be used in any file; it is a write-only variable.") + // FIXME: Indeed, indeed, the :M modifier ends at the colon. + // Why doesn't pkglint complain loudly about the unknown "//*" modifier? + "WARN: filename.mk:1: \"ftp\" is not a valid URL.", + "WARN: filename.mk:1: MASTER_SITES should not be evaluated at load time.") // The only interesting line from the below tracing output is the one // containing "checkCompareVarStr". @@ -405,17 +448,17 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { "TRACE: 1 + (*MkParser).mkCondAtom(\"${VAR:Mpattern1:Mpattern2} == comparison\")", "TRACE: 1 - (*MkParser).mkCondAtom(\"${VAR:Mpattern1:Mpattern2} == comparison\")", "TRACE: 1 checkCompareVarStr ${VAR:Mpattern1:Mpattern2} == comparison", - "TRACE: 1 + MkLineChecker.CheckVaruse(filename:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:parse quoting:plain wordpart:false))", + "TRACE: 1 + MkLineChecker.CheckVaruse(filename.mk:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:parse quoting:plain wordpart:false))", "TRACE: 1 2 + (*Pkgsrc).VariableType(\"VAR\")", "TRACE: 1 2 3 No type definition found for \"VAR\".", "TRACE: 1 2 - (*Pkgsrc).VariableType(\"VAR\", \"=>\", (*pkglint.Vartype)(nil))", - "WARN: filename:1: VAR is used but not defined.", + "WARN: filename.mk:1: VAR is used but not defined.", "TRACE: 1 2 + MkLineChecker.checkVarusePermissions(\"VAR\", (no-type time:parse quoting:plain wordpart:false))", "TRACE: 1 2 3 No type definition found for \"VAR\".", "TRACE: 1 2 - MkLineChecker.checkVarusePermissions(\"VAR\", (no-type time:parse quoting:plain wordpart:false))", "TRACE: 1 2 + (*MkLineImpl).VariableNeedsQuoting(\"VAR\", (*pkglint.Vartype)(nil), (no-type time:parse quoting:plain wordpart:false))", "TRACE: 1 2 - (*MkLineImpl).VariableNeedsQuoting(\"VAR\", (*pkglint.Vartype)(nil), (no-type time:parse quoting:plain wordpart:false), \"=>\", unknown)", - "TRACE: 1 - MkLineChecker.CheckVaruse(filename:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:parse quoting:plain wordpart:false))", + "TRACE: 1 - MkLineChecker.CheckVaruse(filename.mk:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:parse quoting:plain wordpart:false))", "TRACE: - MkLineChecker.checkDirectiveCond(\"${VAR:Mpattern1:Mpattern2} == comparison\")") t.EnableSilentTracing() } @@ -471,6 +514,40 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__no_tracing(c * mklines.Check() } +// Setting a default license is typical for big software projects +// like GNOME or KDE that consist of many packages, or for programming +// languages like Perl or Python that suggest certain licenses. +// +// The default license is typically set in a Makefile.common or module.mk. +func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__license_default(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mkline := t.NewMkLine("filename.mk", 123, "LICENSE?=\tgnu-gpl-v2") + + MkLineChecker{mkline}.checkVarassignLeftPermissions() + + t.CheckOutputEmpty() +} + +// Setting a default license doesn't make sense in a package Makefile +// since that Makefile is only used for a single package. +// It only makes sense to set the license unconditionally there. +func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__license_default_Makefile(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mkline := t.NewMkLine("Makefile", 123, "LICENSE?=\tgnu-gpl-v2") + + MkLineChecker{mkline}.checkVarassignLeftPermissions() + + t.CheckOutputLines( + "WARN: Makefile:123: " + + "The variable LICENSE may not be given a default value " + + "(only set, or appended to) in this file; " + + "it would be ok in *.") +} + // Don't check the permissions for infrastructure files since they have their own rules. func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__infrastructure(c *check.C) { t := s.Init(c) @@ -651,12 +728,13 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__PKGREVISION(c *check. t.SetUpVartypes() mklines := t.NewMkLines("any.mk", MkRcsID, - // PKGREVISION may only be set in Makefile, not used at load time; see vardefs.go. ".if defined(PKGREVISION)", ".endif") mklines.Check() + // Since PKGREVISION may only be set in the package Makefile directly, + // there is no other file that could be mentioned as "it would be ok in". t.CheckOutputLines( "WARN: any.mk:2: PKGREVISION should not be evaluated at load time.", "WARN: any.mk:2: PKGREVISION may not be used in any file; it is a write-only variable.") @@ -677,6 +755,127 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__indirectly(c *check.C "WARN: file.mk:2: ONLY_FOR_UNPRIVILEGED should not be evaluated indirectly at load time.") } +// This test is only here for branch coverage. +// It does not demonstrate anything useful. +func (s *Suite) Test_MkLineChecker_checkVarusePermissions__indirectly_tool(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("file.mk", + MkRcsID, + "USE_TOOLS+=\t${PKGREVISION}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: file.mk:2: PKGREVISION should not be evaluated indirectly at load time.", + "WARN: file.mk:2: PKGREVISION may not be used in any file; it is a write-only variable.") +} + +func (s *Suite) Test_MkLineChecker_checkVarusePermissions__write_only_usable_in_other_file(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("buildlink3.mk", + MkRcsID, + "VAR=\t${VAR} ${AUTO_MKDIRS}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: buildlink3.mk:2: " + + "AUTO_MKDIRS may not be used in this file; " + + "it would be ok in Makefile, Makefile.* or *.mk.") +} + +func (s *Suite) Test_MkLineChecker_checkVarusePermissions__multiple_times_per_file(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("buildlink3.mk", + MkRcsID, + "VAR=\t${VAR} ${AUTO_MKDIRS} ${AUTO_MKDIRS} ${PKGREVISION} ${PKGREVISION}", + "VAR=\t${VAR} ${AUTO_MKDIRS} ${AUTO_MKDIRS} ${PKGREVISION} ${PKGREVISION}") + + mklines.Check() + + // Since these warnings are valid for the whole file, duplicates are suppressed. + t.CheckOutputLines( + "WARN: buildlink3.mk:2: "+ + "AUTO_MKDIRS may not be used in this file; "+ + "it would be ok in Makefile, Makefile.* or *.mk.", + "WARN: buildlink3.mk:2: "+ + "PKGREVISION may not be used in any file; "+ + "it is a write-only variable.") +} + +// In some pkglint tests, the method is called directly without G.Mk being set. +// In practice this doesn't happen. +func (s *Suite) Test_MkLineChecker_checkVarusePermissions__without_mklines(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mkline := t.NewMkLine("buildlink3.mk", 123, + "VAR=\t${VAR} ${AUTO_MKDIRS} ${AUTO_MKDIRS} ${PKGREVISION} ${PKGREVISION}") + + MkLineChecker{mkline}.Check() + + // Since G.Mk is not set, the duplicates are not suppressed. + // Therefore in this case there are more warnings than in realistic situations. + t.CheckOutputLines( + "WARN: buildlink3.mk:123: VAR is defined but not used.", + "WARN: buildlink3.mk:123: VAR is used but not defined.", + "WARN: buildlink3.mk:123: "+ + "AUTO_MKDIRS may not be used in this file; "+ + "it would be ok in Makefile, Makefile.* or *.mk.", + "WARN: buildlink3.mk:123: "+ + "AUTO_MKDIRS may not be used in this file; "+ + "it would be ok in Makefile, Makefile.* or *.mk.", + "WARN: buildlink3.mk:123: "+ + "PKGREVISION may not be used in any file; "+ + "it is a write-only variable.", + "WARN: buildlink3.mk:123: "+ + "PKGREVISION may not be used in any file; "+ + "it is a write-only variable.") +} + +func (s *Suite) Test_MkLineChecker_checkVarassignDecreasingVersions(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("Makefile", + MkRcsID, + "PYTHON_VERSIONS_ACCEPTED=\t36 __future__", + "PYTHON_VERSIONS_ACCEPTED=\t36 -13", + "PYTHON_VERSIONS_ACCEPTED=\t36 ${PKGVERSION_NOREV}", + "PYTHON_VERSIONS_ACCEPTED=\t36 37", + "PYTHON_VERSIONS_ACCEPTED=\t37 36 27 25") + + // TODO: All but the last of the above assignments should be flagged as + // redundant by RedundantScope; as of March 2019, that check is only + // implemented for package Makefiles, not for individual files. + + mklines.Check() + + // Half of these warnings are from VartypeCheck.Version, the + // other half are from checkVarassignDecreasingVersions. + // Strictly speaking some of them are redundant, but that would + // mean to reject only variable references in checkVarassignDecreasingVersions. + // This is probably ok. + // TODO: Fix this. + t.CheckOutputLines( + "WARN: Makefile:2: Invalid version number \"__future__\".", + "ERROR: Makefile:2: Value \"__future__\" for "+ + "PYTHON_VERSIONS_ACCEPTED must be a positive integer.", + "WARN: Makefile:3: Invalid version number \"-13\".", + "ERROR: Makefile:3: Value \"-13\" for "+ + "PYTHON_VERSIONS_ACCEPTED must be a positive integer.", + "ERROR: Makefile:4: Value \"${PKGVERSION_NOREV}\" for "+ + "PYTHON_VERSIONS_ACCEPTED must be a positive integer.", + "WARN: Makefile:5: The values for PYTHON_VERSIONS_ACCEPTED "+ + "should be in decreasing order (37 before 36).") +} + func (s *Suite) Test_MkLineChecker_warnVaruseToolLoadTime(c *check.C) { t := s.Init(c) @@ -707,6 +906,33 @@ func (s *Suite) Test_MkLineChecker_warnVaruseToolLoadTime(c *check.C) { "WARN: Makefile:6: _TOOLS_VARNAME.mk-tool is defined but not used.") } +// This somewhat unrealistic case demonstrates how there can be a tool in a +// Makefile that is not known to the global pkgsrc. +// +// This test covers the "pkgsrcTool != nil" condition. +func (s *Suite) Test_MkLineChecker_warnVaruseToolLoadTime__local_tool(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.CreateFileLines("mk/bsd.prefs.mk") + mklines := t.SetUpFileMkLines("category/package/Makefile", + MkRcsID, + ".include \"../../mk/bsd.prefs.mk\"", + "", + "TOOLS_CREATE+=\t\tmk-tool", + "_TOOLS_VARNAME.mk-tool=\tMK_TOOL", + "", + "TOOL_OUTPUT!=\t${MK_TOOL}") + + mklines.Check() + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:5: Variable names starting with an underscore (_TOOLS_VARNAME.mk-tool) are reserved for internal pkgsrc use.", + "WARN: ~/category/package/Makefile:5: _TOOLS_VARNAME.mk-tool is defined but not used.", + "WARN: ~/category/package/Makefile:7: TOOL_OUTPUT is defined but not used.", + "WARN: ~/category/package/Makefile:7: The tool ${MK_TOOL} cannot be used at load time.") +} + func (s *Suite) Test_MkLineChecker_Check__warn_varuse_LOCALBASE(c *check.C) { t := s.Init(c) @@ -802,6 +1028,32 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparison_with_shell_com "WARN: security/openssl/Makefile:2: Use ${PKGSRC_COMPILER:Mgcc} instead of the == operator.") } +// The :N modifier filters unwanted values. After this filter, any variable value +// may be compared with the empty string, regardless of the variable type. +// Effectively, the :N modifier changes the type from T to Option(T). +func (s *Suite) Test_MkLineChecker_checkDirectiveCond__compare_pattern_with_empty(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", + MkRcsID, + ".if ${X11BASE:Npattern} == \"\"", + ".endif", + "", + ".if ${X11BASE:N<>} == \"*\"", + ".endif", + "", + ".if !(${OPSYS:M*BSD} != \"\")", + ".endif") + + mklines.Check() + + // TODO: There should be a warning about "<>" containing invalid + // characters for a path. See VartypeCheck.Pathname + t.CheckOutputLines( + "WARN: filename.mk:5: \"*\" is not a valid pathname.") +} + func (s *Suite) Test_MkLineChecker_checkDirectiveCondEmpty(c *check.C) { t := s.Init(c) @@ -840,15 +1092,17 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparing_PKGSRC_COMPILER t := s.Init(c) t.SetUpVartypes() - G.Mk = t.NewMkLines("audio/pulseaudio/Makefile", + G.Mk = t.NewMkLines("Makefile", MkRcsID, - ".if ${OPSYS} == \"Darwin\" && ${PKGSRC_COMPILER} == \"clang\"", + ".if ${PKGSRC_COMPILER} == \"clang\"", + ".elif ${PKGSRC_COMPILER} != \"gcc\"", ".endif") G.Mk.Check() t.CheckOutputLines( - "WARN: audio/pulseaudio/Makefile:2: Use ${PKGSRC_COMPILER:Mclang} instead of the == operator.") + "WARN: Makefile:2: Use ${PKGSRC_COMPILER:Mclang} instead of the == operator.", + "WARN: Makefile:3: Use ${PKGSRC_COMPILER:Ngcc} instead of the != operator.") } func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS_with_backticks(c *check.C) { @@ -1298,35 +1552,54 @@ func (s *Suite) Test_MkLineChecker_checkVarassignMisc(c *check.C) { mklines := t.SetUpFileMkLines("module.mk", MkRcsID, "EGDIR= ${PREFIX}/etc/rc.d", + "RPMIGNOREPATH+= ${PREFIX}/etc/rc.d", "_TOOLS_VARNAME.sed= SED", "DIST_SUBDIR= ${PKGNAME}", "WRKSRC= ${PKGNAME}", - "SITES_distfile.tar.gz= ${MASTER_SITE_GITHUB:=user/}", - // TODO: The first of the below assignments should be flagged as redundant by RedundantScope; - // as of January 2019, that check is only implemented for package Makefiles, not for other files. - "PYTHON_VERSIONS_ACCEPTED= -13", - "PYTHON_VERSIONS_ACCEPTED= 27 36") + "SITES_distfile.tar.gz= ${MASTER_SITE_GITHUB:=user/}") mklines.Check() // TODO: Split this test into several, one for each topic. t.CheckOutputLines( "WARN: ~/module.mk:2: Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.", - "WARN: ~/module.mk:3: Variable names starting with an underscore (_TOOLS_VARNAME.sed) are reserved for internal pkgsrc use.", - "WARN: ~/module.mk:3: _TOOLS_VARNAME.sed is defined but not used.", - "WARN: ~/module.mk:4: PKGNAME should not be used in DIST_SUBDIR as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", - "WARN: ~/module.mk:5: PKGNAME should not be used in WRKSRC as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", - "WARN: ~/module.mk:6: SITES_distfile.tar.gz is defined but not used.", - "WARN: ~/module.mk:6: SITES_* is deprecated. Please use SITES.* instead.", - "WARN: ~/module.mk:7: The variable PYTHON_VERSIONS_ACCEPTED may not be set "+ - "(only given a default value, or appended to) in this file; "+ - "it would be ok in Makefile, Makefile.common or options.mk.", - "WARN: ~/module.mk:7: Invalid version number \"-13\".", - "ERROR: ~/module.mk:7: All values for PYTHON_VERSIONS_ACCEPTED must be positive integers.", - "WARN: ~/module.mk:8: The variable PYTHON_VERSIONS_ACCEPTED may not be set "+ - "(only given a default value, or appended to) in this file; "+ - "it would be ok in Makefile, Makefile.common or options.mk.", - "WARN: ~/module.mk:8: The values for PYTHON_VERSIONS_ACCEPTED should be in decreasing order.") + "WARN: ~/module.mk:4: Variable names starting with an underscore (_TOOLS_VARNAME.sed) are reserved for internal pkgsrc use.", + "WARN: ~/module.mk:4: _TOOLS_VARNAME.sed is defined but not used.", + "WARN: ~/module.mk:5: PKGNAME should not be used in DIST_SUBDIR as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", + "WARN: ~/module.mk:6: PKGNAME should not be used in WRKSRC as it includes the PKGREVISION. Please use PKGNAME_NOREV instead.", + "WARN: ~/module.mk:7: SITES_distfile.tar.gz is defined but not used.", + "WARN: ~/module.mk:7: SITES_* is deprecated. Please use SITES.* instead.") +} + +func (s *Suite) Test_MkLineChecker_checkVarassignMisc__multiple_inclusion_guards(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.SetUpVartypes() + t.CreateFileLines("filename.mk", + MkRcsID, + ".if !defined(FILENAME_MK)", + "FILENAME_MK=\t# defined", + ".endif") + t.CreateFileLines("Makefile.common", + MkRcsID, + ".if !defined(MAKEFILE_COMMON)", + "MAKEFILE_COMMON=\t# defined", + "", + ".endif") + t.CreateFileLines("other.mk", + MkRcsID, + "COMMENT=\t# defined") + + G.Check(t.File("filename.mk")) + G.Check(t.File("Makefile.common")) + G.Check(t.File("other.mk")) + + // For multiple-inclusion guards, the meaning of the variable value + // is clear, therefore they are exempted from the warnings. + t.CheckOutputLines( + "NOTE: ~/other.mk:2: Please use \"# empty\", \"# none\" or \"# yes\" " + + "instead of \"# defined\".") } func (s *Suite) Test_MkLineChecker_checkText(c *check.C) { @@ -1409,3 +1682,55 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePath(c *check.C) { // TODO: This warning is unspecific, there is also a pkglint warning "should be ../../category/package". "WARN: ~/category/package/module.mk:11: Invalid relative path \"../package/module.mk\".") } + +func (s *Suite) Test_MkLineChecker_CheckRelativePath__absolute_path(c *check.C) { + t := s.Init(c) + + absDir := ifelseStr(runtime.GOOS == "windows", "C:/", "/") + // Just a random UUID, to really guarantee that the file does not exist. + absPath := absDir + "0f5c2d56-8a7a-4c9d-9caa-859b52bbc8c7" + + t.SetUpPkgsrc() + G.Pkgsrc.LoadInfrastructure() + mklines := t.SetUpFileMkLines("category/package/module.mk", + MkRcsID, + "DISTINFO_FILE=\t"+absPath) + + mklines.Check() + + t.CheckOutputLines( + "ERROR: ~/category/package/module.mk:2: The path \"" + absPath + "\" must be relative.") +} + +func (s *Suite) Test_MkLineChecker_CheckRelativePath__include_if_exists(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("filename.mk", + MkRcsID, + ".include \"included.mk\"", + ".sinclude \"included.mk\"") + + mklines.Check() + + // There is no warning for line 3 because of the "s" in "sinclude". + t.CheckOutputLines( + "ERROR: ~/filename.mk:2: Relative path \"included.mk\" does not exist.") +} + +func (s *Suite) Test_MkLineChecker_CheckRelativePath__wip_mk(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("wip/mk/git-package.mk", + MkRcsID) + t.SetUpPackage("wip/package", + ".include \"../mk/git-package.mk\"") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("wip/package")) + + t.CheckOutputLines( + "WARN: ~/wip/package/Makefile:20: "+ + "References to the pkgsrc-wip infrastructure should look like \"../../wip/mk\", "+ + "not \"../mk\".", + "WARN: ~/wip/package/Makefile:20: Invalid relative path \"../mk/git-package.mk\".") +} diff --git a/pkgtools/pkglint/files/mklines.go b/pkgtools/pkglint/files/mklines.go index 1f875d68698..987b9cb9cd6 100644 --- a/pkgtools/pkglint/files/mklines.go +++ b/pkgtools/pkglint/files/mklines.go @@ -115,7 +115,6 @@ func (mklines *MkLinesImpl) checkAll() { "pre-install": true, "do-install": true, "post-install": true, "pre-package": true, "do-package": true, "post-package": true, "pre-clean": true, "do-clean": true, "post-clean": true} - G.Assertf(len(allowedTargets) == 33, "Error in allowedTargets initialization") mklines.lines.CheckRcsID(0, `#[\t ]+`, "# ") @@ -378,7 +377,13 @@ func (mklines *MkLinesImpl) collectDocumentedVariables() { parser := NewMkParser(nil, words[1], false) varname := parser.Varname() - if hasSuffix(varname, ".") && parser.lexer.SkipRegexp(G.res.Compile(`^<\w+>`)) { + if len(varname) < 3 { + break + } + if hasSuffix(varname, ".") { + if !parser.lexer.SkipRegexp(G.res.Compile(`^<\w+>`)) { + break + } varname += "*" } parser.lexer.SkipByte(':') @@ -389,7 +394,7 @@ func (mklines *MkLinesImpl) collectDocumentedVariables() { scope.Use(varcanon, mkline) } - if 1 < len(words) && words[1] == "Copyright" { + if words[1] == "Copyright" { relevant = false } @@ -401,37 +406,6 @@ func (mklines *MkLinesImpl) collectDocumentedVariables() { finish() } -func (mklines *MkLinesImpl) CheckRedundantAssignments() { - scope := NewRedundantScope() - - isRelevant := func(old, new MkLine) bool { - if new.Op() == opAssignEval { - return false - } - return true - } - - scope.OnRedundant = func(old, new MkLine) { - if isRelevant(old, new) && old.Value() == new.Value() { - new.Notef("Definition of %s is redundant because of %s.", old.Varname(), new.RefTo(old)) - } - } - - scope.OnOverwrite = func(old, new MkLine) { - if isRelevant(old, new) { - old.Warnf("Variable %s is overwritten in %s.", new.Varname(), old.RefTo(new)) - G.Explain( - "The variable definition in this line does not have an effect since", - "it is overwritten elsewhere.", - "This typically happens because of a typo (writing = instead of +=)", - "or because the line that overwrites", - "is in another file that is used by several packages.") - } - } - - mklines.ForEach(scope.Handle) -} - // CheckForUsedComment checks that this file (a Makefile.common) has the given // relativeName in one of the "# used by" comments at the beginning of the file. func (mklines *MkLinesImpl) CheckForUsedComment(relativeName string) { @@ -550,8 +524,8 @@ func (va *VaralignBlock) processVarassign(mkline MkLine) { if mkline.IsMultiline() { // Parsing the continuation marker as variable value is cheating but works well. text := strings.TrimSuffix(mkline.raw[0].orignl, "\n") - m, _, _, _, _, _, value, _, _ := MatchVarassign(text) - continuation = m && value == "\\" + m, a := MatchVarassign(text) + continuation = m && a.value == "\\" } valueAlign := mkline.ValueAlign() diff --git a/pkgtools/pkglint/files/mklines_test.go b/pkgtools/pkglint/files/mklines_test.go index 9bf8fb574f8..5fe9a1c8046 100644 --- a/pkgtools/pkglint/files/mklines_test.go +++ b/pkgtools/pkglint/files/mklines_test.go @@ -335,12 +335,7 @@ func (s *Suite) Test_MkLines_collectDefinedVariables(c *check.C) { // The tools autoreconf and autoheader213 are known at this point because of the USE_TOOLS line. // The SUV variable is used implicitly by the SUBST framework, therefore no warning. // The OSV.NetBSD variable is used implicitly via the OSV variable, therefore no warning. - t.CheckOutputLines( - // FIXME: the below warning is wrong; it's ok to have SUBST blocks in all files, - // maybe except buildlink3.mk. - "WARN: determine-defined-variables.mk:12: The variable SUBST_VARS.subst may not be set " + - "(only given a default value, or appended to) in this file; " + - "it would be ok in Makefile, Makefile.common or options.mk.") + t.CheckOutputEmpty() } func (s *Suite) Test_MkLines_collectDefinedVariables__BUILTIN_FIND_FILES_VAR(c *check.C) { @@ -371,7 +366,7 @@ func (s *Suite) Test_MkLines_collectDefinedVariables__BUILTIN_FIND_FILES_VAR(c * func (s *Suite) Test_MkLines_collectUsedVariables__simple(c *check.C) { t := s.Init(c) - mklines := t.NewMkLines("filename", + mklines := t.NewMkLines("filename.mk", "\t${VAR}") mkline := mklines.mklines[0] G.Mk = mklines @@ -410,7 +405,7 @@ func (s *Suite) Test_MkLines__private_tool_undefined(c *check.C) { t := s.Init(c) t.SetUpVartypes() - mklines := t.NewMkLines("filename", + mklines := t.NewMkLines("filename.mk", MkRcsID, "", "\tmd5sum filename") @@ -418,14 +413,14 @@ func (s *Suite) Test_MkLines__private_tool_undefined(c *check.C) { mklines.Check() t.CheckOutputLines( - "WARN: filename:3: Unknown shell command \"md5sum\".") + "WARN: filename.mk:3: Unknown shell command \"md5sum\".") } func (s *Suite) Test_MkLines__private_tool_defined(c *check.C) { t := s.Init(c) t.SetUpVartypes() - mklines := t.NewMkLines("filename", + mklines := t.NewMkLines("filename.mk", MkRcsID, "TOOLS_CREATE+=\tmd5sum", "", @@ -435,7 +430,7 @@ func (s *Suite) Test_MkLines__private_tool_defined(c *check.C) { // TODO: Is it necessary to add the tool to USE_TOOLS? If not, why not? t.CheckOutputLines( - "WARN: filename:4: The \"md5sum\" tool is used but not added to USE_TOOLS.") + "WARN: filename.mk:4: The \"md5sum\" tool is used but not added to USE_TOOLS.") } func (s *Suite) Test_MkLines_Check__indentation(c *check.C) { @@ -644,6 +639,10 @@ func (s *Suite) Test_MkLines_collectDocumentedVariables(c *check.C) { "# VARIABLE", "#\tA paragraph of a single line is not enough to be recognized as \"relevant\".", "", + "# PARAGRAPH", + "#\tA paragraph may end in a", + "#\tPARA_END_VARNAME.", + "", "# VARBASE1.<param1>", "# VARBASE2.*", "# VARBASE3.${id}") @@ -659,11 +658,12 @@ func (s *Suite) Test_MkLines_collectDocumentedVariables(c *check.C) { sort.Strings(varnames) expected := []string{ + "PARAGRAPH (line 23)", "PKG_DEBUG_LEVEL (line 11)", "PKG_VERBOSE (line 16)", - "VARBASE1.* (line 23)", - "VARBASE2.* (line 24)", - "VARBASE3.* (line 25)"} + "VARBASE1.* (line 27)", + "VARBASE2.* (line 28)", + "VARBASE3.* (line 29)"} c.Check(varnames, deepEquals, expected) } @@ -704,237 +704,6 @@ func (s *Suite) Test_MkLines__unknown_options(c *check.C) { "WARN: options.mk:4: Unknown option \"unknown\".") } -func (s *Suite) Test_MkLines_CheckRedundantAssignments__override_in_mk(c *check.C) { - t := s.Init(c) - included := t.NewMkLines("included.mk", - "OVERRIDE=\tprevious value", - "REDUNDANT=\tredundant") - including := t.NewMkLines("including.mk", - ".include \"included.mk\"", - "OVERRIDE=\toverridden value", - "REDUNDANT=\tredundant") - - var allLines []Line - allLines = append(allLines, including.lines.Lines[:1]...) - allLines = append(allLines, included.lines.Lines...) - allLines = append(allLines, including.lines.Lines[1:]...) - mklines := NewMkLines(NewLines(included.lines.FileName, allLines)) - - // XXX: The warnings from here are not in the same order as the other warnings. - // XXX: There may be some warnings for the same file separated by warnings for other files. - mklines.CheckRedundantAssignments() - - t.CheckOutputLines( - "NOTE: including.mk:3: Definition of REDUNDANT is redundant because of included.mk:2.") -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__override_in_Makefile(c *check.C) { - t := s.Init(c) - included := t.NewMkLines("module.mk", - "VAR=\tvalue ${OTHER}", - "VAR?=\tvalue ${OTHER}", - "VAR=\tnew value") - including := t.NewMkLines("Makefile", - ".include \"module.mk\"", - "VAR=\tthe package may overwrite variables from other files") - - var allLines []Line - allLines = append(allLines, including.lines.Lines[:1]...) - allLines = append(allLines, included.lines.Lines...) - allLines = append(allLines, including.lines.Lines[1:]...) - mklines := NewMkLines(NewLines(including.lines.FileName, allLines)) - - // XXX: The warnings from here are not in the same order as the other warnings. - // XXX: There may be some warnings for the same file separated by warnings for other files. - mklines.CheckRedundantAssignments() - - // No warning for VAR=... in Makefile since it makes sense to have common files - // with default values for variables, overriding some of them in each package. - t.CheckOutputLines( - "NOTE: module.mk:2: Definition of VAR is redundant because of line 1.", - "WARN: module.mk:1: Variable VAR is overwritten in line 3.") -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__default_value_definitely_unused(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR=\tvalue ${OTHER}", - "VAR?=\tdifferent value") - - mklines.CheckRedundantAssignments() - - // FIXME: A default assignment after an unconditional assignment is redundant. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__default_value_overridden(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR?=\tdefault value", - "VAR=\toverridden value") - - mklines.CheckRedundantAssignments() - - t.CheckOutputLines( - "WARN: module.mk:1: Variable VAR is overwritten in line 2.") -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__overwrite_same_value(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR=\tvalue ${OTHER}", - "VAR=\tvalue ${OTHER}") - - mklines.CheckRedundantAssignments() - - t.CheckOutputLines( - "NOTE: module.mk:2: Definition of VAR is redundant because of line 1.") -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__conditional_overwrite(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR=\tdefault", - ".if ${OPSYS} == NetBSD", - "VAR=\topsys", - ".endif") - - mklines.CheckRedundantAssignments() - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__conditional_default(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR=\tdefault", - ".if ${OPSYS} == NetBSD", - "VAR?=\topsys", - ".endif") - - mklines.CheckRedundantAssignments() - - // TODO: WARN: module.mk:3: The value \"opsys\" will never be assigned to VAR because it is defined unconditionally in line 1. - t.CheckOutputEmpty() -} - -// These warnings are precise and accurate since the value of VAR is not used between line 2 and 4. -func (s *Suite) Test_MkLines_CheckRedundantAssignments__overwrite_same_variable_different_value(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "OTHER=\tvalue before", - "VAR=\tvalue ${OTHER}", - "OTHER=\tvalue after", - "VAR=\tvalue ${OTHER}") - - mklines.CheckRedundantAssignments() - - t.CheckOutputLines( - "WARN: module.mk:1: Variable OTHER is overwritten in line 3.", - "NOTE: module.mk:4: Definition of VAR is redundant because of line 2.") -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__overwrite_different_value_used_between(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "OTHER=\tvalue before", - "VAR=\tvalue ${OTHER}", - - // VAR is used here at load time, therefore it must be defined at this point. - // At this point, VAR uses the \"before\" value of OTHER. - "RESULT1:=\t${VAR}", - - "OTHER=\tvalue after", - - // VAR is used here again at load time, this time using the \"after\" value of OTHER. - "RESULT2:=\t${VAR}", - - // Still this definition is redundant. - "VAR=\tvalue ${OTHER}") - - mklines.CheckRedundantAssignments() - - t.CheckOutputLines( - "WARN: module.mk:1: Variable OTHER is overwritten in line 4.", - "NOTE: module.mk:6: Definition of VAR is redundant because of line 2.") -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__procedure_call(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("mk/pthread.buildlink3.mk", - "CHECK_BUILTIN.pthread:=\tyes", - ".include \"../../mk/pthread.builtin.mk\"", - "CHECK_BUILTIN.pthread:=\tno") - - mklines.CheckRedundantAssignments() - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__shell_and_eval(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR:=\tvalue ${OTHER}", - "VAR!=\tvalue ${OTHER}") - - mklines.CheckRedundantAssignments() - - // As of November 2018, pkglint doesn't check redundancies that involve the := or != operators. - // - // What happens here is: - // - // Line 1 evaluates OTHER at load time. - // Line 1 assigns its value to VAR. - // Line 2 evaluates OTHER at load time. - // Line 2 passes its value through the shell and assigns the result to VAR. - // - // Since VAR is defined in line 1, not used afterwards and overwritten in line 2, it is redundant. - // Well, not quite, because evaluating ${OTHER} might have side-effects from :sh or ::= modifiers, - // but these are so rare that they are frowned upon and are not considered by pkglint. - // - // Expected result: - // WARN: module.mk:2: Previous definition of VAR in line 1 is unused. - - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__shell_and_eval_literal(c *check.C) { - t := s.Init(c) - mklines := t.NewMkLines("module.mk", - "VAR:=\tvalue", - "VAR!=\tvalue") - - mklines.CheckRedundantAssignments() - - // Even when := is used with a literal value (which is usually - // only done for procedure calls), the shell evaluation can have - // so many different side effects that pkglint cannot reliably - // help in this situation. - // - // TODO: Why not? The evaluation in line 1 is trivial to analyze. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_CheckRedundantAssignments__included_OPSYS_variable(c *check.C) { - t := s.Init(c) - - t.SetUpPackage("category/package", - ".include \"../../category/dependency/buildlink3.mk\"", - "CONFIGURE_ARGS+=\tone", - "CONFIGURE_ARGS=\ttwo", - "CONFIGURE_ARGS+=\tthree") - t.SetUpPackage("category/dependency") - t.CreateFileDummyBuildlink3("category/dependency/buildlink3.mk") - t.CreateFileLines("category/dependency/builtin.mk", - MkRcsID, - "CONFIGURE_ARGS.Darwin+=\tdarwin") - - G.Check(t.File("category/package")) - - t.CheckOutputLines( - "WARN: ~/category/package/Makefile:21: Variable CONFIGURE_ARGS is overwritten in line 22.") -} - func (s *Suite) Test_MkLines_Check__PLIST_VARS(c *check.C) { t := s.Init(c) @@ -1101,7 +870,8 @@ func (s *Suite) Test_MkLines_Check__hacks_mk(c *check.C) { mklines.Check() // No warning about including bsd.prefs.mk before using the ?= operator. - // FIXME: Why not? + // This is because the hacks.mk files are included implicitly by the + // pkgsrc infrastructure right after bsd.prefs.mk. t.CheckOutputEmpty() } @@ -1167,8 +937,8 @@ func (s *Suite) Test_MkLines_Check__extra_warnings(c *check.C) { G.Pkg = NewPackage(t.File("category/pkgbase")) G.Mk = t.NewMkLines("options.mk", MkRcsID, + "", ".for word in ${PKG_FAIL_REASON}", - "PYTHON_VERSIONS_ACCEPTED=\t27 35 30", "CONFIGURE_ARGS+=\t--sharedir=${PREFIX}/share/kde", "COMMENT=\t# defined", ".endfor", @@ -1181,7 +951,6 @@ func (s *Suite) Test_MkLines_Check__extra_warnings(c *check.C) { G.Mk.Check() t.CheckOutputLines( - "WARN: options.mk:3: The values for PYTHON_VERSIONS_ACCEPTED should be in decreasing order.", "NOTE: options.mk:5: Please use \"# empty\", \"# none\" or \"# yes\" instead of \"# defined\".", "WARN: options.mk:7: Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".", "WARN: options.mk:11: Building the package should take place entirely inside ${WRKSRC}, not \"${WRKSRC}/..\".", diff --git a/pkgtools/pkglint/files/mkparser.go b/pkgtools/pkglint/files/mkparser.go index b10888013ea..4cbcb8877f3 100644 --- a/pkgtools/pkglint/files/mkparser.go +++ b/pkgtools/pkglint/files/mkparser.go @@ -27,6 +27,7 @@ func (p *MkParser) Rest() string { // // The text argument is assumed to be after unescaping the # character, // which means the # is a normal character and does not introduce a Makefile comment. +// For VarUse, this distinction is irrelevant. func NewMkParser(line Line, text string, emitWarnings bool) *MkParser { G.Assertf((line != nil) == emitWarnings, "line must be given iff emitWarnings is set") return &MkParser{line, textproc.NewLexer(text), emitWarnings} @@ -120,13 +121,7 @@ func (p *MkParser) VarUse() *MkVarUse { } lexer.Reset(mark) - } - - if lexer.SkipByte('@') { - return &MkVarUse{"@", nil} - } - if lexer.SkipByte('<') { - return &MkVarUse{"<", nil} + return nil } varname := lexer.NextByteSet(textproc.AlnumU) @@ -152,6 +147,26 @@ func (p *MkParser) VarUse() *MkVarUse { return &MkVarUse{sprintf("%c", varname), nil} } + if !lexer.EOF() { + symbol := lexer.Rest()[:1] + switch symbol { + case "$": + // This is an escaped dollar character and not a variable use. + + case "@", "<", " ": + // These variable names are known to exist. + // + // Many others are also possible but not used in practice. + // In particular, when parsing the :C or :S modifier, + // the $ must not be interpreted as a variable name, + // even when it looks like $/ could refer to the "/" variable. + // + // TODO: Find out whether $" is a variable use when it appears in the :M modifier. + lexer.Skip(1) + return &MkVarUse{symbol, nil} + } + } + lexer.Reset(mark) return nil } @@ -466,6 +481,29 @@ func (p *MkParser) mkCondAtom() MkCond { } } lexer.Reset(mark) + + lexer.Skip(1) + var rhsText strings.Builder + loop: + for { + m := lexer.Mark() + switch { + case p.VarUse() != nil, + lexer.NextBytesSet(textproc.Alnum) != "", + lexer.NextBytesFunc(func(b byte) bool { return b != '"' && b != '\\' }) != "": + rhsText.WriteString(lexer.Since(m)) + + case lexer.SkipString("\\\""), + lexer.SkipString("\\\\"): + rhsText.WriteByte(lexer.Since(m)[1]) + + case lexer.SkipByte('"'): + return &mkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, rhsText.String()}} + default: + break loop + } + } + lexer.Reset(mark) } } @@ -524,9 +562,14 @@ func (p *MkParser) mkCondFunc() *mkCond { func (p *MkParser) Varname() string { lexer := p.lexer + // TODO: duplicated code in MatchVarassign mark := lexer.Mark() lexer.SkipByte('.') - for p.VarUse() != nil || lexer.NextBytesSet(VarnameBytes) != "" { + for lexer.NextBytesSet(VarbaseBytes) != "" || p.VarUse() != nil { + } + if lexer.SkipByte('.') || hasPrefix(lexer.Since(mark), "SITES_") { + for lexer.NextBytesSet(VarparamBytes) != "" || p.VarUse() != nil { + } } return lexer.Since(mark) } @@ -799,6 +842,7 @@ func (w *MkCondWalker) Walk(cond MkCond, callback *MkCondCallback) { if callback.VarUse != nil { callback.VarUse(cond.CompareVarStr.Var) } + w.walkStr(cond.CompareVarStr.Str, callback) case cond.CompareVarNum != nil: if callback.CompareVarNum != nil { @@ -814,5 +858,17 @@ func (w *MkCondWalker) Walk(cond MkCond, callback *MkCondCallback) { call := cond.Call callback.Call(call.Name, call.Arg) } + w.walkStr(cond.Call.Arg, callback) + } +} + +func (w *MkCondWalker) walkStr(str string, callback *MkCondCallback) { + if callback.VarUse != nil { + tokens := NewMkParser(nil, str, false).MkTokens() + for _, token := range tokens { + if token.Varuse != nil { + callback.VarUse(token.Varuse) + } + } } } diff --git a/pkgtools/pkglint/files/mkparser_test.go b/pkgtools/pkglint/files/mkparser_test.go index ae3b03a73d1..e3a4b8aafae 100644 --- a/pkgtools/pkglint/files/mkparser_test.go +++ b/pkgtools/pkglint/files/mkparser_test.go @@ -192,6 +192,11 @@ func (s *Suite) Test_MkParser_VarUse(c *check.C) { test("${PKGNAME:C/-[0-9].*$/-[0-9]*/}", varuse("PKGNAME", "C/-[0-9].*$/-[0-9]*/")) + // TODO: Does the $@ refer to ${.TARGET}, or is it just an unmatchable + // regular expression? + test("${PKGNAME:C/$@/target?/}", + varuse("PKGNAME", "C/$@/target?/")) + test("${PKGNAME:S/py${_PYTHON_VERSION}/py${i}/:C/-[0-9].*$/-[0-9]*/}", varuse("PKGNAME", "S/py${_PYTHON_VERSION}/py${i}/", "C/-[0-9].*$/-[0-9]*/")) @@ -331,6 +336,12 @@ func (s *Suite) Test_MkParser_VarUse(c *check.C) { t.CheckOutputLines( "WARN: Test_MkParser_VarUse.mk:1: Modifier ${PLIST_SUBST_VARS:@var@...@} is missing the final \"@\".") + + // Unfinished variable use + testRest("${", nil, "${") + + // Unfinished nested variable use + testRest("${${", nil, "${${") } func (s *Suite) Test_MkParser_VarUse__ambiguous(c *check.C) { @@ -412,6 +423,13 @@ func (s *Suite) Test_MkParser_MkCond(c *check.C) { test("\"${pkg}\" == \"${name}\"", &mkCond{CompareVarVar: &MkCondCompareVarVar{varuse("pkg"), "==", varuse("name")}}) + // The right-hand side is not analyzed further to keep the data types simple. + test("${ABC} == \"${A}B${C}\"", + &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("ABC"), "==", "${A}B${C}"}}) + + test("${ABC} == \"${A}\\\"${B}\\\\${C}$${shellvar}${D}\"", + &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("ABC"), "==", "${A}\"${B}\\${C}$${shellvar}${D}"}}) + test("exists(/etc/hosts)", &mkCond{Call: &MkCondCall{"exists", "/etc/hosts"}}) @@ -776,9 +794,11 @@ func (s *Suite) Test_MkCondWalker_Walk(c *check.C) { mkline := t.NewMkLine("Makefile", 4, ""+ ".if ${VAR:Mmatch} == ${OTHER} || "+ "${STR} == Str || "+ + "${VAR} == \"${PRE}text${POST}\" || "+ "${NUM} == 3 && "+ "defined(VAR) && "+ "!exists(file.mk) && "+ + "exists(${FILE}) && "+ "(((${NONEMPTY})))") var events []string @@ -830,11 +850,17 @@ func (s *Suite) Test_MkCondWalker_Walk(c *check.C) { " varUse OTHER", " compareVarStr STR, Str", " varUse STR", + " compareVarStr VAR, ${PRE}text${POST}", + " varUse VAR", + " varUse PRE", + " varUse POST", " compareVarNum NUM, 3", " varUse NUM", " defined VAR", " varUse VAR", " call exists, file.mk", + " call exists, ${FILE}", + " varUse FILE", " var NONEMPTY", " varUse NONEMPTY"}) } diff --git a/pkgtools/pkglint/files/mktokenslexer.go b/pkgtools/pkglint/files/mktokenslexer.go new file mode 100644 index 00000000000..0330d52480a --- /dev/null +++ b/pkgtools/pkglint/files/mktokenslexer.go @@ -0,0 +1,89 @@ +package pkglint + +import ( + "netbsd.org/pkglint/textproc" + "strings" +) + +// MkTokensLexer parses a sequence of variable uses (like ${VAR:Mpattern}) +// interleaved with other text that is uninterpreted by bmake. +type MkTokensLexer struct { + // The lexer for the current text-only token. + // If the current token is a variable use, the lexer will always return + // EOF internally. That is not visible from the outside though, as EOF is + // overridden in this type. + *textproc.Lexer + + // The remaining tokens. + tokens []*MkToken +} + +func NewMkTokensLexer(tokens []*MkToken) *MkTokensLexer { + lexer := &MkTokensLexer{nil, tokens} + lexer.next() + return lexer +} + +func (m *MkTokensLexer) next() { + if len(m.tokens) > 0 && m.tokens[0].Varuse == nil { + m.Lexer = textproc.NewLexer(m.tokens[0].Text) + m.tokens = m.tokens[1:] + } else { + m.Lexer = textproc.NewLexer("") + } +} + +// EOF returns whether the whole input has been consumed. +func (m *MkTokensLexer) EOF() bool { return m.Lexer.EOF() && len(m.tokens) == 0 } + +// Rest returns the string concatenation of the tokens that have not yet been consumed. +func (m *MkTokensLexer) Rest() string { + var sb strings.Builder + sb.WriteString(m.Lexer.Rest()) + for _, token := range m.tokens { + sb.WriteString(token.Text) + } + return sb.String() +} + +// NextVarUse returns the next varuse token, unless there is some plain text +// before it. In that case or at EOF, it returns nil. +func (m *MkTokensLexer) NextVarUse() *MkVarUse { + if m.Lexer.EOF() && len(m.tokens) > 0 && m.tokens[0].Varuse != nil { + token := m.tokens[0] + m.tokens = m.tokens[1:] + m.next() + return token.Varuse + } + return nil +} + +// Mark remembers the current position of the lexer. +// The lexer can later be reset to that position by calling Reset. +func (m *MkTokensLexer) Mark() MkTokensLexerMark { + return MkTokensLexerMark{m.Lexer.Rest(), m.tokens} +} + +// Since returns the text between the given mark and the current position +// of the lexer. +func (m *MkTokensLexer) Since(mark MkTokensLexerMark) string { + early := (&MkTokensLexer{textproc.NewLexer(mark.rest), mark.tokens}).Rest() + late := m.Rest() + + return strings.TrimSuffix(early, late) +} + +// Reset sets the lexer back to the given position. +// The lexer may be reset to the same mark multiple times, +// that is, the mark is not destroyed. +func (m *MkTokensLexer) Reset(mark MkTokensLexerMark) { + m.Lexer = textproc.NewLexer(mark.rest) + m.tokens = mark.tokens +} + +// MkTokensLexerMark remembers a position of a token lexer, +// to be restored later. +type MkTokensLexerMark struct { + rest string + tokens []*MkToken +} diff --git a/pkgtools/pkglint/files/mktokenslexer_test.go b/pkgtools/pkglint/files/mktokenslexer_test.go new file mode 100644 index 00000000000..00926c1e369 --- /dev/null +++ b/pkgtools/pkglint/files/mktokenslexer_test.go @@ -0,0 +1,291 @@ +package pkglint + +import ( + "gopkg.in/check.v1" + "netbsd.org/pkglint/textproc" +) + +func (s *Suite) Test_MkTokensLexer__empty_slice_returns_EOF(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer(nil) + + t.Check(lexer.EOF(), equals, true) +} + +// A slice of a single token behaves like textproc.Lexer. +func (s *Suite) Test_MkTokensLexer__single_plain_text_token(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{{"\\# $$ [#] $V", nil}}) + + t.Check(lexer.SkipByte('\\'), equals, true) + t.Check(lexer.Rest(), equals, "# $$ [#] $V") + t.Check(lexer.SkipByte('#'), equals, true) + t.Check(lexer.NextHspace(), equals, " ") + t.Check(lexer.NextBytesSet(textproc.Space.Inverse()), equals, "$$") + t.Check(lexer.Skip(len(lexer.Rest())), equals, true) + t.Check(lexer.EOF(), equals, true) +} + +// If the first element of the slice is a variable use, none of the plain +// text patterns matches. +// +// The code that uses the MkTokensLexer needs to distinguish these cases +// anyway, therefore it doesn't make sense to treat variable uses as plain +// text. +func (s *Suite) Test_MkTokensLexer__single_varuse_token(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{{"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}}) + + t.Check(lexer.EOF(), equals, false) + t.Check(lexer.PeekByte(), equals, -1) + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR", "Mpattern")) +} + +func (s *Suite) Test_MkTokensLexer__plain_then_varuse(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"plain text", nil}, + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}}) + + t.Check(lexer.NextBytesSet(textproc.Digit.Inverse()), equals, "plain text") + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR", "Mpattern")) + t.Check(lexer.EOF(), equals, true) +} + +func (s *Suite) Test_MkTokensLexer__varuse_varuse_varuse(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"${dirs:O:u}", NewMkVarUse("dirs", "O", "u")}, + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}, + {"${.TARGET}", NewMkVarUse(".TARGET")}}) + + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("dirs", "O", "u")) + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR", "Mpattern")) + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse(".TARGET")) + t.Check(lexer.NextVarUse(), check.IsNil) +} + +func (s *Suite) Test_MkTokensLexer__mark_reset_since_in_initial_state(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"${dirs:O:u}", NewMkVarUse("dirs", "O", "u")}, + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}, + {"${.TARGET}", NewMkVarUse(".TARGET")}}) + + start := lexer.Mark() + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("dirs", "O", "u")) + middle := lexer.Mark() + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}${.TARGET}") + lexer.Reset(start) + t.Check(lexer.Rest(), equals, "${dirs:O:u}${VAR:Mpattern}${.TARGET}") + lexer.Reset(middle) + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}${.TARGET}") +} + +func (s *Suite) Test_MkTokensLexer__mark_reset_since_inside_plain_text(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"plain text", nil}, + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}, + {"rest", nil}}) + + start := lexer.Mark() + t.Check(lexer.NextBytesSet(textproc.Alpha), equals, "plain") + middle := lexer.Mark() + t.Check(lexer.Rest(), equals, " text${VAR:Mpattern}rest") + lexer.Reset(start) + t.Check(lexer.Rest(), equals, "plain text${VAR:Mpattern}rest") + lexer.Reset(middle) + t.Check(lexer.Rest(), equals, " text${VAR:Mpattern}rest") +} + +func (s *Suite) Test_MkTokensLexer__mark_reset_since_after_plain_text(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"plain text", nil}, + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}, + {"rest", nil}}) + + start := lexer.Mark() + t.Check(lexer.SkipString("plain text"), equals, true) + end := lexer.Mark() + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}rest") + lexer.Reset(start) + t.Check(lexer.Rest(), equals, "plain text${VAR:Mpattern}rest") + lexer.Reset(end) + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}rest") +} + +func (s *Suite) Test_MkTokensLexer__mark_reset_since_after_varuse(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}, + {"rest", nil}}) + + start := lexer.Mark() + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR", "Mpattern")) + end := lexer.Mark() + t.Check(lexer.Rest(), equals, "rest") + lexer.Reset(start) + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}rest") + lexer.Reset(end) + t.Check(lexer.Rest(), equals, "rest") +} + +func (s *Suite) Test_MkTokensLexer__multiple_marks_in_same_plain_text(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"plain text", nil}, + {"${VAR:Mpattern}", NewMkVarUse("VAR", "Mpattern")}, + {"rest", nil}}) + + start := lexer.Mark() + t.Check(lexer.NextString("plain "), equals, "plain ") + middle := lexer.Mark() + t.Check(lexer.NextString("text"), equals, "text") + end := lexer.Mark() + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}rest") + lexer.Reset(start) + t.Check(lexer.Rest(), equals, "plain text${VAR:Mpattern}rest") + lexer.Reset(middle) + t.Check(lexer.Rest(), equals, "text${VAR:Mpattern}rest") + lexer.Reset(end) + t.Check(lexer.Rest(), equals, "${VAR:Mpattern}rest") +} + +func (s *Suite) Test_MkTokensLexer__multiple_marks_in_varuse(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"${VAR1}", NewMkVarUse("VAR1")}, + {"${VAR2}", NewMkVarUse("VAR2")}, + {"${VAR3}", NewMkVarUse("VAR3")}}) + + start := lexer.Mark() + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR1")) + middle := lexer.Mark() + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR2")) + further := lexer.Mark() + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR3")) + end := lexer.Mark() + t.Check(lexer.Rest(), equals, "") + lexer.Reset(middle) + t.Check(lexer.Rest(), equals, "${VAR2}${VAR3}") + lexer.Reset(further) + t.Check(lexer.Rest(), equals, "${VAR3}") + lexer.Reset(start) + t.Check(lexer.Rest(), equals, "${VAR1}${VAR2}${VAR3}") + lexer.Reset(end) + t.Check(lexer.Rest(), equals, "") +} + +func (s *Suite) Test_MkTokensLexer__EOF_before_plain_text(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{{"rest", nil}}) + + t.Check(lexer.EOF(), equals, false) +} + +func (s *Suite) Test_MkTokensLexer__EOF_before_varuse(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{{"${VAR}", NewMkVarUse("VAR")}}) + + t.Check(lexer.EOF(), equals, false) +} + +// When the MkTokensLexer is constructed, it gets a copy of the tokens array. +// In theory it would be possible to change the tokens after starting lexing, +// but there is no practical case where that would be useful. +// +// Since each slice is a separate view on the underlying array, modifying the +// size of the outside slice does not affect parsing. This is also only a +// theoretical case. +// +// Because all these cases are only theoretical, the MkTokensLexer doesn't +// bother to make this unnecessary copy and works on the shared slice. +func (s *Suite) Test_MkTokensLexer__constructor_uses_shared_array(c *check.C) { + t := s.Init(c) + + tokens := []*MkToken{{"${VAR}", NewMkVarUse("VAR")}} + lexer := NewMkTokensLexer(tokens) + + t.Check(lexer.Rest(), equals, "${VAR}") + + tokens[0].Text = "modified text" + tokens[0].Varuse = NewMkVarUse("MODIFIED", "Mpattern") + tokens = tokens[0:0] + + t.Check(lexer.Rest(), equals, "modified text") +} + +func (s *Suite) Test_MkTokensLexer__peek_after_varuse(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"${VAR}", NewMkVarUse("VAR")}, + {"${VAR}", NewMkVarUse("VAR")}, + {"text", nil}}) + + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR")) + t.Check(lexer.PeekByte(), equals, -1) + + t.Check(lexer.NextVarUse(), deepEquals, NewMkVarUse("VAR")) + t.Check(lexer.PeekByte(), equals, int('t')) +} + +func (s *Suite) Test_MkTokensLexer__varuse_when_plain_text(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{{"text", nil}}) + + t.Check(lexer.NextVarUse(), check.IsNil) + t.Check(lexer.NextString("te"), equals, "te") + t.Check(lexer.NextVarUse(), check.IsNil) + t.Check(lexer.NextString("xt"), equals, "xt") + t.Check(lexer.NextVarUse(), check.IsNil) +} + +// The code that creates the tokens for the lexer never puts two +// plain text MkTokens besides each other. There's no point in doing +// that since they could have been combined into a single token from +// the beginning. +func (s *Suite) Test_MkTokensLexer__adjacent_plain_text(c *check.C) { + t := s.Init(c) + + lexer := NewMkTokensLexer([]*MkToken{ + {"text1", nil}, + {"text2", nil}}) + + // Returns false since the string is distributed over two separate tokens. + t.Check(lexer.SkipString("text1text2"), equals, false) + + t.Check(lexer.SkipString("text1"), equals, true) + + // This returns false since the internal lexer is not advanced to the + // next text token. To do that, all methods from the internal lexer + // would have to be redefined by MkTokensLexer in order to advance the + // internal lexer to the next token. + // + // Since this situation doesn't occur in practice, there's no point in + // implementing it. + t.Check(lexer.SkipString("text2"), equals, false) + + // Just for covering the "Varuse != nil" branch in MkTokensLexer.NextVarUse. + t.Check(lexer.NextVarUse(), check.IsNil) + + // The string is still not found since the next token is only consumed + // by the NextVarUse above if it is indeed a VarUse. + t.Check(lexer.SkipString("text2"), equals, false) +} diff --git a/pkgtools/pkglint/files/package.go b/pkgtools/pkglint/files/package.go index 7e7a3a07199..7f7ec79cd56 100644 --- a/pkgtools/pkglint/files/package.go +++ b/pkgtools/pkglint/files/package.go @@ -28,10 +28,21 @@ type Package struct { EffectivePkgnameLine MkLine // The origin of the three Effective* values Plist PlistContent // Files and directories mentioned in the PLIST files - vars Scope - bl3 map[string]MkLine // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. - included map[string]MkLine // filename => line - seenMakefileCommon bool // Does the package have any .includes? + vars Scope + bl3 map[string]MkLine // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. + + // Remembers the Makefile fragments that have already been included. + // The key to the map is the filename relative to the package directory. + // Typical keys are "../../category/package/buildlink3.mk". + // + // TODO: Include files with multiple-inclusion guard only once. + // + // TODO: Include files without multiple-inclusion guard as often as needed. + // + // TODO: Set an upper limit, to prevent denial of service. + included Once + + seenMakefileCommon bool // Does the package have any .includes? // Files from .include lines that are nested inside .if. // They often depend on OPSYS or on the existence of files in the build environment. @@ -61,7 +72,7 @@ func NewPackage(dir string) *Package { Plist: NewPlistContent(), vars: NewScope(), bl3: make(map[string]MkLine), - included: make(map[string]MkLine), + included: Once{}, conditionalIncludes: make(map[string]MkLine), unconditionalIncludes: make(map[string]MkLine), } @@ -152,7 +163,7 @@ func (pkg *Package) checkLinesBuildlink3Inclusion(mklines MkLines) { } } -func (pkg *Package) loadPackageMakefile() MkLines { +func (pkg *Package) loadPackageMakefile() (MkLines, MkLines) { filename := pkg.File("Makefile") if trace.Tracing { defer trace.Call1(filename)() @@ -162,7 +173,7 @@ func (pkg *Package) loadPackageMakefile() MkLines { allLines := NewMkLines(NewLines("", nil)) if _, result := pkg.readMakefile(filename, mainLines, allLines, ""); !result { LoadMk(filename, NotEmpty|LogErrors) // Just for the LogErrors. - return nil + return nil, nil } // TODO: Is this still necessary? This code is 20 years old and was introduced @@ -183,7 +194,6 @@ func (pkg *Package) loadPackageMakefile() MkLines { } allLines.collectUsedVariables() - allLines.CheckRedundantAssignments() pkg.Pkgdir = pkg.vars.LastValue("PKGDIR") pkg.DistinfoFile = pkg.vars.LastValue("DISTINFO_FILE") @@ -212,7 +222,7 @@ func (pkg *Package) loadPackageMakefile() MkLines { trace.Step1("PKGDIR=%s", pkg.Pkgdir) } - return mainLines + return mainLines, allLines } // TODO: What is allLines used for, is it still necessary? Would it be better as a field in Package? @@ -225,71 +235,92 @@ func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines Mk if fileMklines == nil { return false, false } + exists = true + result = true isMainMakefile := len(mainLines.mklines) == 0 - result = true - lineAction := func(mkline MkLine) bool { - if isMainMakefile { - mainLines.mklines = append(mainLines.mklines, mkline) - mainLines.lines.Lines = append(mainLines.lines.Lines, mkline.Line) + handleIncludeLine := func(mkline MkLine) YesNoUnknown { + includedFile, incDir, incBase := pkg.findIncludedFile(mkline, filename) + + if includedFile == "" { + return unknown } - allLines.mklines = append(allLines.mklines, mkline) - allLines.lines.Lines = append(allLines.lines.Lines, mkline.Line) - includedFile, incDir, incBase := pkg.findIncludedFile(mkline, filename) + dirname, _ := path.Split(filename) + dirname = cleanpath(dirname) + fullIncluded := dirname + "/" + includedFile + relIncludedFile := relpath(pkg.dir, fullIncluded) - if includedFile != "" && pkg.included[includedFile] == nil { - pkg.included[includedFile] = mkline + if !pkg.diveInto(filename, includedFile) { + return unknown + } + + if !pkg.included.FirstTime(relIncludedFile) { + return unknown + } - // TODO: "../../../.." also matches but shouldn't. - if matches(includedFile, `^\.\./[^./][^/]*/[^/]+`) { + if matches(includedFile, `^\.\./[^./][^/]*/[^/]+`) { + if G.Wip && contains(includedFile, "/mk/") { + mkline.Warnf("References to the pkgsrc-wip infrastructure should look like \"../../wip/mk\", not \"../mk\".") + } else { mkline.Warnf("References to other packages should look like \"../../category/package\", not \"../package\".") - mkline.ExplainRelativeDirs() } + mkline.ExplainRelativeDirs() + } - pkg.collectUsedBy(mkline, incDir, incBase, includedFile) + pkg.collectUsedBy(mkline, incDir, incBase, includedFile) - skip := contains(filename, "/mk/") || hasSuffix(includedFile, "/bsd.pkg.mk") || IsPrefs(includedFile) - if !skip { - dirname, _ := path.Split(filename) - dirname = cleanpath(dirname) + if trace.Tracing { + trace.Step1("Including %q.", fullIncluded) + } + fullIncluding := ifelseStr(incBase == "Makefile.common" && incDir != "", filename, "") + innerExists, innerResult := pkg.readMakefile(fullIncluded, mainLines, allLines, fullIncluding) - fullIncluded := dirname + "/" + includedFile - if trace.Tracing { - trace.Step1("Including %q.", fullIncluded) - } - fullIncluding := ifelseStr(incBase == "Makefile.common" && incDir != "", filename, "") - innerExists, innerResult := pkg.readMakefile(fullIncluded, mainLines, allLines, fullIncluding) + if !innerExists { + if fileMklines.indentation.IsCheckedFile(includedFile) { + return yes // See https://github.com/rillig/pkglint/issues/1 + } - if !innerExists { - if fileMklines.indentation.IsCheckedFile(includedFile) { - return true // See https://github.com/rillig/pkglint/issues/1 - } + // Only look in the directory relative to the + // current file and in the package directory. + // Make(1) has a list of include directories, but pkgsrc + // doesn't make use of that, so pkglint also doesn't + // need this extra complexity. + pkgBasedir := pkg.File(".") + if dirname != pkgBasedir { // Prevent unnecessary syscalls + dirname = pkgBasedir + + fullIncludedFallback := dirname + "/" + includedFile + innerExists, innerResult = pkg.readMakefile(fullIncludedFallback, mainLines, allLines, fullIncluding) + } - // Only look in the directory relative to the - // current file and in the package directory. - // Make(1) has a list of include directories, but pkgsrc - // doesn't make use of that, so pkglint also doesn't - // need this extra complexity. - pkgBasedir := pkg.File(".") - if dirname != pkgBasedir { // Prevent unnecessary syscalls - dirname = pkgBasedir - - fullIncludedFallback := dirname + "/" + includedFile - innerExists, innerResult = pkg.readMakefile(fullIncludedFallback, mainLines, allLines, fullIncluding) - } + if !innerExists { + mkline.Errorf("Cannot read %q.", includedFile) + } + } - if !innerExists { - mkline.Errorf("Cannot read %q.", includedFile) - } - } + if !innerResult { + result = false + return no + } - if !innerResult { - result = false - return false - } + return unknown + } + + lineAction := func(mkline MkLine) bool { + if isMainMakefile { + mainLines.mklines = append(mainLines.mklines, mkline) + mainLines.lines.Lines = append(mainLines.lines.Lines, mkline.Line) + } + allLines.mklines = append(allLines.mklines, mkline) + allLines.lines.Lines = append(allLines.lines.Lines, mkline.Line) + + if mkline.IsInclude() { + includeResult := handleIncludeLine(mkline) + if includeResult != unknown { + return includeResult == yes } } @@ -305,6 +336,7 @@ func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines Mk } return true } + atEnd := func(mkline MkLine) {} fileMklines.ForEachEnd(lineAction, atEnd) @@ -315,8 +347,9 @@ func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines Mk // For every included buildlink3.mk, include the corresponding builtin.mk // automatically since the pkgsrc infrastructure does the same. if path.Base(filename) == "buildlink3.mk" { - builtin := path.Join(path.Dir(filename), "builtin.mk") - if fileExists(builtin) { + builtin := cleanpath(path.Dir(filename) + "/builtin.mk") + builtinRel := relpath(pkg.dir, builtin) + if pkg.included.FirstTime(builtinRel) && fileExists(builtin) { pkg.readMakefile(builtin, mainLines, allLines, "") } } @@ -324,6 +357,17 @@ func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines Mk return } +func (pkg *Package) diveInto(includingFile string, includedFile string) bool { + skip := hasSuffix(includedFile, "/bsd.pkg.mk") || IsPrefs(includedFile) + if !skip && contains(includingFile, "/mk/") { + skip = true + if contains(includingFile, "buildlink3.mk") && contains(includedFile, "builtin.mk") { + skip = false + } + } + return !skip +} + func (pkg *Package) collectUsedBy(mkline MkLine, incDir string, incBase string, includedFile string) { switch { case @@ -343,19 +387,17 @@ func (pkg *Package) collectUsedBy(mkline MkLine, incDir string, incBase string, func (pkg *Package) findIncludedFile(mkline MkLine, includingFilename string) (includedFile, incDir, incBase string) { - if mkline.IsInclude() { - // TODO: resolveVariableRefs uses G.Pkg implicitly. It should be made explicit. - // TODO: Try to combine resolveVariableRefs and ResolveVarsInRelativePath. - includedFile = resolveVariableRefs(mkline.ResolveVarsInRelativePath(mkline.IncludedFile())) - if containsVarRef(includedFile) { - if trace.Tracing && !contains(includingFilename, "/mk/") { - trace.Stepf("%s:%s: Skipping include file %q. This may result in false warnings.", - mkline.Filename, mkline.Linenos(), includedFile) - } - includedFile = "" + // TODO: resolveVariableRefs uses G.Pkg implicitly. It should be made explicit. + // TODO: Try to combine resolveVariableRefs and ResolveVarsInRelativePath. + includedFile = resolveVariableRefs(mkline.ResolveVarsInRelativePath(mkline.IncludedFile())) + if containsVarRef(includedFile) { + if trace.Tracing && !contains(includingFilename, "/mk/") { + trace.Stepf("%s:%s: Skipping include file %q. This may result in false warnings.", + mkline.Filename, mkline.Linenos(), includedFile) } - incDir, incBase = path.Split(includedFile) + includedFile = "" } + incDir, incBase = path.Split(includedFile) if includedFile != "" { if mkline.Basename != "buildlink3.mk" { @@ -371,7 +413,7 @@ func (pkg *Package) findIncludedFile(mkline MkLine, includingFilename string) (i return } -func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines) { +func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines, allLines MkLines) { if trace.Tracing { defer trace.Call1(filename)() } @@ -408,11 +450,18 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines) { } if !vars.Defined("LICENSE") && !vars.Defined("META_PACKAGE") && pkg.once.FirstTime("LICENSE") { - NewLineWhole(filename).Errorf("Each package must define its LICENSE.") + line := NewLineWhole(filename) + line.Errorf("Each package must define its LICENSE.") // TODO: Explain why the LICENSE is necessary. + line.Explain( + "To take a good guess on the license of a package,", + sprintf("run %q.", bmake("guess-license"))) } - pkg.checkGnuConfigureUseLanguages() + scope := NewRedundantScope() + scope.Check(allLines) // Updates the variables in the scope + pkg.checkGnuConfigureUseLanguages(scope) + pkg.determineEffectivePkgVars() pkg.checkPossibleDowngrade() @@ -434,28 +483,45 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines) { SaveAutofixChanges(mklines.lines) } -func (pkg *Package) checkGnuConfigureUseLanguages() { - vars := pkg.vars +func (pkg *Package) checkGnuConfigureUseLanguages(s *RedundantScope) { - if gnuLine := vars.FirstDefinition("GNU_CONFIGURE"); gnuLine != nil { + gnuConfigure := s.vars["GNU_CONFIGURE"] + if gnuConfigure == nil || !gnuConfigure.vari.Constant() { + return + } - // FIXME: Instead of using the first definition here, a better approach - // is probably to use all the definitions except those from mk/compiler.mk. - // In real pkgsrc, the last definition is typically from mk/compiler.mk - // and only contains c++. - if useLine := vars.FirstDefinition("USE_LANGUAGES"); useLine != nil { + useLanguages := s.vars["USE_LANGUAGES"] + if useLanguages == nil || !useLanguages.vari.Constant() { + return + } - if matches(useLine.VarassignComment(), `(?-i)\b(?:c|empty|none)\b`) { - // Don't emit a warning since the comment probably contains a - // statement that C is really not needed. + var wrongLines []MkLine + for _, mkline := range useLanguages.vari.WriteLocations() { - } else if !matches(useLine.Value(), `(?:^|[\t ]+)(?:c|c99|objc)(?:[\t ]+|$)`) { - gnuLine.Warnf( - "GNU_CONFIGURE almost always needs a C compiler, "+ - "but \"c\" is not added to USE_LANGUAGES in %s.", - gnuLine.RefTo(useLine)) - } + if G.Pkgsrc.IsInfra(mkline.Line.Filename) { + continue } + + if matches(mkline.VarassignComment(), `(?-i)\b(?:c|empty|none)\b`) { + // Don't emit a warning since the comment probably contains a + // statement that C is really not needed. + return + } + + languages := mkline.Value() + if matches(languages, `(?:^|[\t ]+)(?:c|c99|objc)(?:[\t ]+|$)`) { + return + } + + wrongLines = append(wrongLines, mkline) + } + + gnuLine := gnuConfigure.vari.WriteLocations()[0] + for _, useLine := range wrongLines { + gnuLine.Warnf( + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in %s.", + gnuLine.RefTo(useLine)) } } diff --git a/pkgtools/pkglint/files/package_test.go b/pkgtools/pkglint/files/package_test.go index 6162c4c4c80..3dae5b5bd82 100644 --- a/pkgtools/pkglint/files/package_test.go +++ b/pkgtools/pkglint/files/package_test.go @@ -28,7 +28,8 @@ func (s *Suite) Test_Package_checkLinesBuildlink3Inclusion__package_but_not_file t.CreateFileLines("category/dependency/buildlink3.mk") G.Pkg = NewPackage(t.File("category/package")) - G.Pkg.bl3["../../category/dependency/buildlink3.mk"] = t.NewMkLine("filename", 1, "") + G.Pkg.bl3["../../category/dependency/buildlink3.mk"] = + t.NewMkLine("../../category/dependency/buildlink3.mk", 1, "") mklines := t.NewMkLines("category/package/buildlink3.mk", MkRcsID) @@ -212,7 +213,6 @@ func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { t.CreateFileLines("mk/bsd.pkg.mk", "# dummy") t.CreateFileLines("x11/Makefile", MkRcsID) t.CreateFileLines("x11/9term/PLIST", PlistRcsID, "bin/9term") - t.CreateFileLines("x11/9term/distinfo", RcsID) t.CreateFileLines("x11/9term/Makefile", MkRcsID, "", @@ -221,6 +221,8 @@ func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { "", "COMMENT=\tTerminal", "", + "NO_CHECKSUM=\tyes", + "", ".include \"../../mk/bsd.pkg.mk\"") t.SetUpVartypes() @@ -228,6 +230,7 @@ func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { G.Check(t.File("x11/9term")) // Since the error is grave enough, the warning about the correct position is suppressed. + // TODO: Knowing the correct position helps, though. t.CheckOutputLines( "ERROR: ~/x11/9term/Makefile: Each package must define its LICENSE.") } @@ -524,13 +527,39 @@ func (s *Suite) Test_Package_loadPackageMakefile(c *check.C) { // Including a package Makefile directly is an error (in the last line), // but that is checked later. - // A file including itself does not lead to an endless loop while parsing - // but may still produce unexpected warnings, such as redundant definitions. + // This test demonstrates that a file including itself does not lead to an + // endless loop while parsing. It might trigger an error in the future. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package__relative_included_filenames_in_same_directory(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "PKGNAME=\tpkgname-1.67", + "DISTNAME=\tdistfile_1_67", + ".include \"../../category/package/other.mk\"") + t.CreateFileLines("category/package/other.mk", + MkRcsID, + "PKGNAME=\tpkgname-1.67", + "DISTNAME=\tdistfile_1_67", + ".include \"../../category/package/other.mk\"") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + // TODO: Since other.mk is referenced via "../../category/package", + // it would be nice if this relative path would be reflected in the output + // instead of referring just to "other.mk". + // This needs to be fixed somewhere near relpath. + // + // The notes are in reverse order because they are produced when checking + // other.mk, and there the relative order is correct (line 2 before line 3). t.CheckOutputLines( - "NOTE: ~/category/package/Makefile:3: "+ - "Definition of PKGNAME is redundant because of ../../category/package/Makefile:3.", "NOTE: ~/category/package/Makefile:4: "+ - "Definition of DISTNAME is redundant because of ../../category/package/Makefile:4.") + "Definition of PKGNAME is redundant because of other.mk:2.", + "NOTE: ~/category/package/Makefile:3: "+ + "Definition of DISTNAME is redundant because of other.mk:3.") } func (s *Suite) Test_Package_loadPackageMakefile__PECL_VERSION(c *check.C) { @@ -668,14 +697,16 @@ func (s *Suite) Test_Package__redundant_master_sites(c *check.C) { G.checkdirPackage(t.File("math/R-date")) // The definition in Makefile:6 is redundant because the same definition - // occurs later in Makefile.extension:4. Usually the later definition gets - // the note. In this case though, it would be wrong to mark the - // definition in Makefile.extension as redundant because that definition is - // probably used by other packages as well. Therefore in this case the roles - // of the two lines are swapped; see RedundantScope.Handle, the ".includes" line. + // occurs later in Makefile.extension:4. + // + // When a file includes another file, it's always the including file that + // is marked as redundant since the included file typically provides the + // generally useful value for several packages; + // see RedundantScope.handleVarassign, keyword includePath. t.CheckOutputLines( "NOTE: ~/math/R-date/Makefile:6: " + - "Definition of MASTER_SITES is redundant because of ../../math/R/Makefile.extension:4.") + "Definition of MASTER_SITES is redundant " + + "because of ../../math/R/Makefile.extension:4.") } func (s *Suite) Test_Package_checkUpdate(c *check.C) { @@ -828,6 +859,131 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__USE_IMAKE_and_USE_X11(c * "NOTE: ~/category/package/Makefile:21: USE_IMAKE makes USE_X11 in line 20 redundant.") } +func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__no_C(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "USE_LANGUAGES=\tfortran77", + "USE_LANGUAGES+=\tc++14", + "USE_LANGUAGES+=\tada", + "GNU_CONFIGURE=\tyes") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:23: "+ + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in line 20.", + "WARN: ~/category/package/Makefile:23: "+ + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in line 21.", + "WARN: ~/category/package/Makefile:23: "+ + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in line 22.") +} + +func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__C_in_the_middle(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "USE_LANGUAGES=\tfortran77", + "USE_LANGUAGES+=\tc99", + "USE_LANGUAGES+=\tada", + "GNU_CONFIGURE=\tyes") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + // Until March 2019 pkglint wrongly warned that USE_LANGUAGES would not + // include c or c99, although c99 was added. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__realistic_compiler_mk(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "USE_LANGUAGES=\tfortran77", + "USE_LANGUAGES+=\tc++", + "USE_LANGUAGES+=\tada", + "GNU_CONFIGURE=\tyes", + "", + ".include \"../../mk/compiler.mk\"") + t.CreateFileLines("mk/compiler.mk", + MkRcsID, + ".include \"bsd.prefs.mk\"", + "", + "USE_LANGUAGES?=\tc", + "USE_LANGUAGES+=\tc", + "USE_LANGUAGES+=\tc++") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + // The package defines several languages it needs, but C is not one of them. + // When the package is loaded, the included files are read in recursively, even + // when they come from the pkgsrc infrastructure. + // + // Up to March 2019, the USE_LANGUAGES definitions from mk/compiler.mk were + // loaded as if they were defined by the package, without taking the conditionals + // into account. Thereby "c" was added unconditionally to USE_LANGUAGES. + // + // Since March 2019 the infrastructure files are ignored when determining the value + // of USE_LANGUAGES. + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:23: "+ + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in line 20.", + "WARN: ~/category/package/Makefile:23: "+ + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in line 21.", + "WARN: ~/category/package/Makefile:23: "+ + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in line 22.") +} + +func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__only_GNU_CONFIGURE(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "GNU_CONFIGURE=\tyes") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__ok(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "GNU_CONFIGURE=\tyes", + "USE_LANGUAGES=\tc++ objc") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package__USE_LANGUAGES_too_late(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../mk/compiler.mk\"", + "USE_LANGUAGES=\tc c99 fortran ada c++14") + t.CreateFileLines("mk/compiler.mk", + MkRcsID) + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + // FIXME: There must be a warning "USE_LANGUAGES must be added before including compiler.mk." + t.CheckOutputEmpty() +} + func (s *Suite) Test_Package_readMakefile__skipping(c *check.C) { t := s.Init(c) @@ -914,6 +1070,43 @@ func (s *Suite) Test_Package_readMakefile__builtin_mk(c *check.C) { "WARN: ~/category/package/Makefile:23: OTHER_VAR is used but not defined.") } +// Ensures that the paths in Package.included are indeed relative to the +// package directory. This hadn't been the case until March 2019. +func (s *Suite) Test_Package_readMakefile__included(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../devel/library/buildlink3.mk\"", + ".include \"../../lang/language/module.mk\"") + t.SetUpPackage("devel/library") + t.CreateFileDummyBuildlink3("devel/library/buildlink3.mk") + t.CreateFileLines("devel/library/builtin.mk", + MkRcsID) + t.CreateFileLines("lang/language/module.mk", + MkRcsID, + ".include \"version.mk\"") + t.CreateFileLines("lang/language/version.mk", + MkRcsID) + pkg := NewPackage(t.File("category/package")) + + pkg.loadPackageMakefile() + + expected := []string{ + "../../devel/library/buildlink3.mk", + "../../devel/library/builtin.mk", + "../../lang/language/module.mk", + "../../lang/language/version.mk", + "suppress-varorder.mk"} + + seen := pkg.included + for _, filename := range expected { + if !seen.Seen(filename) { + c.Errorf("File %q is not seen.", filename) + } + } + t.Check(seen.seen, check.HasLen, 5) +} + func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { t := s.Init(c) @@ -1040,3 +1233,61 @@ func (s *Suite) Test_Package__redundant_variable_in_unrelated_files(c *check.C) // PY_PATCHPLIST is not redundant in these files. t.CheckOutputEmpty() } + +// Pkglint loads some files from the pkgsrc infrastructure and skips others. +// +// When a buildlink3.mk file from the infrastructure is included, it should +// be allowed to include its corresponding builtin.mk file in turn. +// +// This is necessary to load the correct variable assignments for the +// redundancy check, in particular variable assignments that serve as +// arguments to "procedure calls", such as mk/find-files.mk. +func (s *Suite) Test_Package_readMakefile__include_infrastructure(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("--dumpmakefile") + t.SetUpPackage("category/package", + ".include \"../../mk/dlopen.buildlink3.mk\"", + ".include \"../../mk/pthread.buildlink3.mk\"") + t.CreateFileLines("mk/dlopen.buildlink3.mk", + ".include \"dlopen.builtin.mk\"") + t.CreateFileLines("mk/dlopen.builtin.mk", + ".include \"pthread.builtin.mk\"") + t.CreateFileLines("mk/pthread.buildlink3.mk", + ".include \"pthread.builtin.mk\"") + t.CreateFileLines("mk/pthread.builtin.mk", + "# This should be included by pthread.buildlink3.mk") + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "Whole Makefile (with all included files) follows:", + "~/category/package/Makefile:1: "+MkRcsID, + "~/category/package/Makefile:2: ", + "~/category/package/Makefile:3: DISTNAME=\tdistname-1.0", + "~/category/package/Makefile:4: #PKGNAME=\tpackage-1.0", + "~/category/package/Makefile:5: CATEGORIES=\tcategory", + "~/category/package/Makefile:6: MASTER_SITES=\t# none", + "~/category/package/Makefile:7: ", + "~/category/package/Makefile:8: MAINTAINER=\tpkgsrc-users@NetBSD.org", + "~/category/package/Makefile:9: HOMEPAGE=\t# none", + "~/category/package/Makefile:10: COMMENT=\tDummy package", + "~/category/package/Makefile:11: LICENSE=\t2-clause-bsd", + "~/category/package/Makefile:12: ", + "~/category/package/Makefile:13: .include \"suppress-varorder.mk\"", + "~/category/package/suppress-varorder.mk:1: "+MkRcsID, + "~/category/package/Makefile:14: # empty", + "~/category/package/Makefile:15: # empty", + "~/category/package/Makefile:16: # empty", + "~/category/package/Makefile:17: # empty", + "~/category/package/Makefile:18: # empty", + "~/category/package/Makefile:19: # empty", + "~/category/package/Makefile:20: .include \"../../mk/dlopen.buildlink3.mk\"", + "~/category/package/../../mk/dlopen.buildlink3.mk:1: .include \"dlopen.builtin.mk\"", + "~/mk/dlopen.builtin.mk:1: .include \"pthread.builtin.mk\"", + "~/category/package/Makefile:21: .include \"../../mk/pthread.buildlink3.mk\"", + "~/category/package/../../mk/pthread.buildlink3.mk:1: .include \"pthread.builtin.mk\"", + "~/mk/pthread.builtin.mk:1: # This should be included by pthread.buildlink3.mk", + "~/category/package/Makefile:22: ", + "~/category/package/Makefile:23: .include \"../../mk/bsd.pkg.mk\"") +} diff --git a/pkgtools/pkglint/files/pkglint.1 b/pkgtools/pkglint/files/pkglint.1 index 566d72a1236..85e02e03a0b 100644 --- a/pkgtools/pkglint/files/pkglint.1 +++ b/pkgtools/pkglint/files/pkglint.1 @@ -1,4 +1,4 @@ -.\" $NetBSD: pkglint.1,v 1.54 2019/02/21 23:44:55 rillig Exp $ +.\" $NetBSD: pkglint.1,v 1.55 2019/03/10 19:01:50 rillig Exp $ .\" From FreeBSD: portlint.1,v 1.8 1997/11/25 14:53:14 itojun Exp .\" .\" Copyright (c) 1997 by Jun-ichiro Itoh <itojun@itojun.org>. diff --git a/pkgtools/pkglint/files/pkglint.go b/pkgtools/pkglint/files/pkglint.go index e5096bc488c..71fe509c840 100644 --- a/pkgtools/pkglint/files/pkglint.go +++ b/pkgtools/pkglint/files/pkglint.go @@ -380,7 +380,7 @@ func (pkglint *Pkglint) checkdirPackage(dir string) { // Load the package Makefile and all included files, // to collect all used and defined variables and similar data. - mklines := pkg.loadPackageMakefile() + mklines, allLines := pkg.loadPackageMakefile() if mklines == nil { return } @@ -437,7 +437,7 @@ func (pkglint *Pkglint) checkdirPackage(dir string) { case path.Base(filename) == "Makefile": pkglint.checkExecutable(filename, st.Mode()) - pkg.checkfilePackageMakefile(filename, mklines) + pkg.checkfilePackageMakefile(filename, mklines, allLines) default: pkglint.checkDirent(filename, st.Mode()) diff --git a/pkgtools/pkglint/files/pkglint_test.go b/pkgtools/pkglint/files/pkglint_test.go index ea11ec91aea..567f772fec3 100644 --- a/pkgtools/pkglint/files/pkglint_test.go +++ b/pkgtools/pkglint/files/pkglint_test.go @@ -419,7 +419,7 @@ func (s *Suite) Test_Pkglint_Check(c *check.C) { func (s *Suite) Test_resolveVariableRefs__circular_reference(c *check.C) { t := s.Init(c) - mkline := t.NewMkLine("filename", 1, "GCC_VERSION=${GCC_VERSION}") + mkline := t.NewMkLine("filename.mk", 1, "GCC_VERSION=${GCC_VERSION}") G.Pkg = NewPackage(t.File("category/pkgbase")) G.Pkg.vars.Define("GCC_VERSION", mkline) @@ -433,9 +433,9 @@ func (s *Suite) Test_resolveVariableRefs__circular_reference(c *check.C) { func (s *Suite) Test_resolveVariableRefs__multilevel(c *check.C) { t := s.Init(c) - mkline1 := t.NewMkLine("filename", 10, "FIRST=\t${SECOND}") - mkline2 := t.NewMkLine("filename", 11, "SECOND=\t${THIRD}") - mkline3 := t.NewMkLine("filename", 12, "THIRD=\tgot it") + mkline1 := t.NewMkLine("filename.mk", 10, "FIRST=\t${SECOND}") + mkline2 := t.NewMkLine("filename.mk", 11, "SECOND=\t${THIRD}") + mkline3 := t.NewMkLine("filename.mk", 12, "THIRD=\tgot it") G.Pkg = NewPackage(t.File("category/pkgbase")) defineVar(mkline1, "FIRST") defineVar(mkline2, "SECOND") @@ -455,7 +455,7 @@ func (s *Suite) Test_resolveVariableRefs__multilevel(c *check.C) { func (s *Suite) Test_resolveVariableRefs__special_chars(c *check.C) { t := s.Init(c) - mkline := t.NewMkLine("filename", 10, "_=x11") + mkline := t.NewMkLine("filename.mk", 10, "_=x11") G.Pkg = NewPackage(t.File("category/pkg")) G.Pkg.vars.Define("GST_PLUGINS0.10_TYPE", mkline) @@ -551,6 +551,15 @@ func (s *Suite) Test_CheckLinesMessage__autofix(c *check.C) { "===========================================================================") } +func (s *Suite) Test_CheckLinesMessage__common(c *check.C) { + t := s.Init(c) + + // FIXME: If there is a MESSAGE.common, it is combined with MESSAGE. + // See meta-pkgs/ruby-redmine-plugins for an example. + + t.CheckOutputEmpty() +} + // Demonstrates that an ALTERNATIVES file can be tested individually, // without any dependencies on a whole package or a PLIST file. func (s *Suite) Test_Pkglint_checkReg__alternatives(c *check.C) { diff --git a/pkgtools/pkglint/files/pkgsrc.go b/pkgtools/pkglint/files/pkgsrc.go index 2af34590b56..f8f58cbf642 100644 --- a/pkgtools/pkglint/files/pkgsrc.go +++ b/pkgtools/pkglint/files/pkgsrc.go @@ -814,6 +814,13 @@ func (src *Pkgsrc) ToRel(filename string) string { return relpath(src.topdir, filename) } +// IsInfra returns whether the given filename (relative to the pkglint +// working directory) is part of the pkgsrc infrastructure. +func (src *Pkgsrc) IsInfra(filename string) bool { + rel := src.ToRel(filename) + return hasPrefix(rel, "mk/") || hasPrefix(rel, "wip/mk/") +} + func (src *Pkgsrc) addBuildDefs(varnames ...string) { for _, varname := range varnames { src.buildDefs[varname] = true @@ -953,6 +960,8 @@ func (src *Pkgsrc) guessVariableType(varname string) (vartype *Vartype) { case hasSuffix(varbase, "_MK"): // TODO: Add BtGuard for inclusion guards, since these variables may only be checked using defined(). gtype = &Vartype{lkNone, BtUnknown, allowAll, true} + case hasSuffix(varbase, "_SKIP"): + gtype = &Vartype{lkShell, BtPathmask, allowRuntime, true} } if gtype == nil { diff --git a/pkgtools/pkglint/files/pkgsrc_test.go b/pkgtools/pkglint/files/pkgsrc_test.go index 3a5201bc3dd..ca54cf82981 100644 --- a/pkgtools/pkglint/files/pkgsrc_test.go +++ b/pkgtools/pkglint/files/pkgsrc_test.go @@ -659,3 +659,26 @@ func (s *Suite) Test_Pkgsrc_VariableType__from_mk(c *check.C) { "WARN: ~/category/package/Makefile:21: ABCPATH is used but not defined.", "0 errors and 2 warnings found.") } + +func (s *Suite) Test_Pkgsrc_guessVariableType__SKIP(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("filename.mk", + MkRcsID, + "MY_CHECK_SKIP=\t*.c \"bad*pathname\"", + "MY_CHECK_SKIP+=\t*.cpp", + ".if ${MY_CHECK_SKIP}", + ".endif") + + mklines.Check() + + // FIXME: The permissions in guessVariableType say allRuntime, which excludes + // aclpUseLoadtime. Therefore there should be a warning about the VarUse in + // the .if line. + // The check in MkLineChecker.checkVarusePermissions is disabled for guessed types. + // + // There is no warning for the += operator in line 3 since the variable type + // (although guessed) is a list of things, and lists may be appended to. + t.CheckOutputLines( + "WARN: filename.mk:2: \"\\\"bad*pathname\\\"\" is not a valid pathname mask.") +} diff --git a/pkgtools/pkglint/files/plist.go b/pkgtools/pkglint/files/plist.go index 0f5f602dd3c..09630961103 100644 --- a/pkgtools/pkglint/files/plist.go +++ b/pkgtools/pkglint/files/plist.go @@ -344,7 +344,7 @@ func (ck *PlistChecker) checkPathShare(pline *PlistLine) { case hasPrefix(text, "share/icons/") && G.Pkg != nil: if hasPrefix(text, "share/icons/hicolor/") && G.Pkg.Pkgpath != "graphics/hicolor-icon-theme" { f := "../../graphics/hicolor-icon-theme/buildlink3.mk" - if G.Pkg.included[f] == nil && ck.once.FirstTime("hicolor-icon-theme") { + if !G.Pkg.included.Seen(f) && ck.once.FirstTime("hicolor-icon-theme") { pline.Errorf("Packages that install hicolor icons must include %q in the Makefile.", f) } } @@ -359,7 +359,7 @@ func (ck *PlistChecker) checkPathShare(pline *PlistLine) { if hasPrefix(text, "share/icons/gnome") && G.Pkg.Pkgpath != "graphics/gnome-icon-theme" { f := "../../graphics/gnome-icon-theme/buildlink3.mk" - if G.Pkg.included[f] == nil { + if !G.Pkg.included.Seen(f) { pline.Errorf("The package Makefile must include %q.", f) G.Explain( "Packages that install GNOME icons must maintain the icon theme", diff --git a/pkgtools/pkglint/files/plist_test.go b/pkgtools/pkglint/files/plist_test.go index 469b1aef6c6..68dfc06f18f 100644 --- a/pkgtools/pkglint/files/plist_test.go +++ b/pkgtools/pkglint/files/plist_test.go @@ -566,6 +566,34 @@ func (s *Suite) Test_PlistChecker_checkPathShare(c *check.C) { "WARN: ~/PLIST:6: Man pages should be installed into man/, not share/man/.") } +func (s *Suite) Test_PlistChecker_checkPathShare__gnome_icon_theme(c *check.C) { + t := s.Init(c) + + t.CreateFileDummyBuildlink3("graphics/gnome-icon-theme/buildlink3.mk") + t.SetUpPackage("graphics/gnome-icon-theme-extras", + "ICON_THEMES=\tyes", + ".include \"../../graphics/gnome-icon-theme/buildlink3.mk\"") + t.CreateFileLines("graphics/gnome-icon-theme-extras/PLIST", + PlistRcsID, + "share/icons/gnome/16x16/devices/media-optical-cd-audio.png", + "share/icons/gnome/16x16/devices/media-optical-dvd.png") + G.Pkgsrc.LoadInfrastructure() + t.Chdir(".") + + // This variant is typically run interactively. + G.Check("graphics/gnome-icon-theme-extras") + + t.CheckOutputEmpty() + + // Note the leading "./". + // This variant is typical for recursive runs of pkglint. + G.Check("./graphics/gnome-icon-theme-extras") + + // Up to March 2019, a bug in relpath produced different behavior + // depending on the leading dot. + t.CheckOutputEmpty() +} + func (s *Suite) Test_PlistLine_CheckTrailingWhitespace(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/redundantscope.go b/pkgtools/pkglint/files/redundantscope.go new file mode 100644 index 00000000000..ae0dbdea83f --- /dev/null +++ b/pkgtools/pkglint/files/redundantscope.go @@ -0,0 +1,279 @@ +package pkglint + +// RedundantScope checks for redundant variable definitions and for variables +// that are accidentally overwritten. It tries to be as correct as possible +// by not flagging anything that is defined conditionally. +// +// There may be some edge cases though like defining PKGNAME, then evaluating +// it using :=, then defining it again. This pattern is so error-prone that +// it should not appear in pkgsrc at all, thus pkglint doesn't even expect it. +// (Well, except for the PKGNAME case, but that's deep in the infrastructure +// and only affects the "nb13" extension.) +// +// TODO: This scope is not only used for detecting redundancies. It also +// provides information about whether the variables are constant or depend on +// other variables. Therefore the name may change soon. +type RedundantScope struct { + vars map[string]*redundantScopeVarinfo + includePath includePath +} +type redundantScopeVarinfo struct { + vari *Var + includePaths []includePath + lastAction uint8 // 0 = none, 1 = read, 2 = write +} + +func NewRedundantScope() *RedundantScope { + return &RedundantScope{vars: make(map[string]*redundantScopeVarinfo)} +} + +func (s *RedundantScope) Check(mklines MkLines) { + mklines.ForEach(func(mkline MkLine) { + s.Handle(mkline, mklines.indentation) + }) +} + +func (s *RedundantScope) Handle(mkline MkLine, ind *Indentation) { + s.updateIncludePath(mkline) + + switch { + case mkline.IsVarassign(): + s.handleVarassign(mkline, ind) + } + + s.handleVarUse(mkline) +} + +func (s *RedundantScope) updateIncludePath(mkline MkLine) { + if mkline.firstLine == 1 { + s.includePath.push(mkline.Location.Filename) + } else { + s.includePath.popUntil(mkline.Location.Filename) + } +} + +func (s *RedundantScope) handleVarassign(mkline MkLine, ind *Indentation) { + varname := mkline.Varname() + info := s.get(varname) + + defer func() { + info.vari.Write(mkline, ind.Depth("") > 0, ind.Varnames()...) + info.lastAction = 2 + s.access(varname) + }() + + // In the very first assignment, no redundancy can occur. + prevWrites := info.vari.WriteLocations() + if len(prevWrites) == 0 { + return + } + + // TODO: Just being conditional is only half the truth. + // To be precise, the "conditional path" must differ between + // this variable assignment and the/any? previous one. + // See Test_RedundantScope__overwrite_inside_conditional. + // Anyway, too few warnings are better than wrong warnings. + if info.vari.Conditional() || ind.Depth("") > 0 { + return + } + + // When the variable has been read after the previous write, + // it is not redundant. + if info.lastAction == 1 { + return + } + + effOp := mkline.Op() + value := mkline.Value() + + // FIXME: Skip the whole redundancy check if the value is not known to be constant. + if effOp == opAssign && info.vari.Value() == value { + effOp = opAssignDefault + } + + if effOp == opAssignEval && value == mkline.WithoutMakeVariables(value) { + // Maybe add support for VAR:= ${OTHER} later. This involves evaluating + // the OTHER variable though using the appropriate scope. Oh, wait, + // there _is_ a scope here. So if OTHER doesn't refer to further + // variables it's all possible. + // + // TODO: The above idea seems possible and useful. + effOp = opAssign + } + + switch effOp { + + case opAssign: // with a different value than before + if s.includePath.includedByOrEqualsAll(info.includePaths) { + + // The situation is: + // + // including.mk: VAR= initial value + // included.mk: VAR= overwriting <-- you are here + // + // Because the included files is never wrong (by definition), + // the including file gets the warning in this case. + s.onOverwrite(prevWrites[len(prevWrites)-1], mkline) + } + + case opAssignDefault: // or opAssign with the same value as before + switch { + + case s.includePath.includesOrEqualsAll(info.includePaths): + + // The situation is: + // + // included.mk: VAR= value + // including.mk: VAR= value <-- you are here + // including.mk: VAR?= value <-- or here + // + // After including one or more files, the variable is either + // overwritten or defaulted with the same value as its + // guaranteed current value. All previous accesses to the + // variable were either in this file or in an included file. + s.onRedundant(mkline, prevWrites[len(prevWrites)-1]) + + case s.includePath.includedByOrEqualsAll(info.includePaths): + + // The situation is: + // + // including.mk: VAR= value + // included.mk: VAR?= value <-- you are here + // included.mk: VAR= value <-- or here + // + // A variable has been defined in an including file. + // The current line either has a default assignment or an + // unconditional assignment. This is common and fine. + // + // Except when this line has the same value as the guaranteed + // current value of the variable. Then it is redundant. + if info.vari.Constant() && info.vari.ConstantValue() == mkline.Value() { + s.onRedundant(prevWrites[len(prevWrites)-1], mkline) + } + } + } +} + +func (s *RedundantScope) handleVarUse(mkline MkLine) { + switch { + case mkline.IsVarassign(), mkline.IsCommentedVarassign(): + for _, varname := range mkline.DetermineUsedVariables() { + info := s.get(varname) + info.vari.Read(mkline) + info.lastAction = 1 + s.access(varname) + } + + case mkline.IsDirective(): + // TODO: Handle varuse for conditions and loops. + break + + case mkline.IsInclude(), mkline.IsSysinclude(): + // TODO: Handle VarUse for includes, which may reference variables. + break + + case mkline.IsDependency(): + // TODO: Handle VarUse for this case. + } +} + +// access returns the info for the given variable, creating it if necessary. +func (s *RedundantScope) get(varname string) *redundantScopeVarinfo { + info := s.vars[varname] + if info == nil { + v := NewVar(varname) + info = &redundantScopeVarinfo{v, nil, 0} + s.vars[varname] = info + } + return info +} + +// access records the current file location, to be used in later inclusion checks. +func (s *RedundantScope) access(varname string) { + info := s.vars[varname] + info.includePaths = append(info.includePaths, s.includePath.copy()) +} + +func (s *RedundantScope) onRedundant(redundant MkLine, because MkLine) { + if redundant.Op() == opAssignDefault { + redundant.Notef("Default assignment of %s has no effect because of %s.", + because.Varname(), redundant.RefTo(because)) + } else { + redundant.Notef("Definition of %s is redundant because of %s.", + because.Varname(), redundant.RefTo(because)) + } +} + +func (s *RedundantScope) onOverwrite(overwritten MkLine, by MkLine) { + overwritten.Warnf("Variable %s is overwritten in %s.", + overwritten.Varname(), overwritten.RefTo(by)) + G.Explain( + "The variable definition in this line does not have an effect since", + "it is overwritten elsewhere.", + "This typically happens because of a typo (writing = instead of +=)", + "or because the line that overwrites", + "is in another file that is used by several packages.") +} + +// includePath remembers the whole sequence of included files, +// such as Makefile includes ../../a/b/buildlink3.mk includes ../../c/d/buildlink3.mk. +// +// This information is used by the RedundantScope to decide whether +// one of two variable assignments is redundant. Two assignments can +// only be redundant if one location includes the other. +type includePath struct { + files []string +} + +func (p *includePath) push(filename string) { + p.files = append(p.files, filename) +} + +func (p *includePath) popUntil(filename string) { + for p.files[len(p.files)-1] != filename { + p.files = p.files[:len(p.files)-1] + } +} + +func (p *includePath) includes(other includePath) bool { + for i, filename := range p.files { + if i >= len(other.files) || other.files[i] != filename { + return false + } + } + return len(p.files) < len(other.files) +} + +func (p *includePath) includesOrEqualsAll(others []includePath) bool { + for _, other := range others { + if !(p.includes(other) || p.equals(other)) { + return false + } + } + return true +} + +func (p *includePath) includedByOrEqualsAll(others []includePath) bool { + for _, other := range others { + if !(other.includes(*p) || p.equals(other)) { + return false + } + } + return true +} + +func (p *includePath) equals(other includePath) bool { + if len(p.files) != len(other.files) { + return false + } + for i, filename := range p.files { + if other.files[i] != filename { + return false + } + } + return true +} + +func (p *includePath) copy() includePath { + return includePath{append([]string(nil), p.files...)} +} diff --git a/pkgtools/pkglint/files/redundantscope_test.go b/pkgtools/pkglint/files/redundantscope_test.go new file mode 100644 index 00000000000..ebe18bacfae --- /dev/null +++ b/pkgtools/pkglint/files/redundantscope_test.go @@ -0,0 +1,1284 @@ +package pkglint + +import "gopkg.in/check.v1" + +// In a single file, five variables get a default value and are later overridden +// with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_default(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT?=\tvalue", + "ASSIGN?=\tvalue", + "APPEND?=\tvalue", + "EVAL?=\tvalue", + "SHELL?=\tvalue", + "", + "DEFAULT?=\tvalue", + "ASSIGN=\tvalue", + "APPEND+=\tvalue", + "EVAL:=\tvalue", + "SHELL!=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "NOTE: file.mk:8: Definition of ASSIGN is redundant because of line 2.", + "WARN: file.mk:4: Variable EVAL is overwritten in line 10.") + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get assigned are value and are later overridden +// with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_assign(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT=\tvalue", + "ASSIGN=\tvalue", + "APPEND=\tvalue", + "EVAL=\tvalue", + "SHELL=\tvalue", + "", + "DEFAULT?=\tvalue", + "ASSIGN=\tvalue", + "APPEND+=\tvalue", + "EVAL:=\tvalue", + "SHELL!=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "NOTE: file.mk:8: Definition of ASSIGN is redundant because of line 2.", + "WARN: file.mk:4: Variable EVAL is overwritten in line 10.") + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get appended a value and are later overridden +// with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_append(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT+=\tvalue", + "ASSIGN+=\tvalue", + "APPEND+=\tvalue", + "EVAL+=\tvalue", + "SHELL+=\tvalue", + "", + "DEFAULT?=\tvalue", + "ASSIGN=\tvalue", + "APPEND+=\tvalue", + "EVAL:=\tvalue", + "SHELL!=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "WARN: file.mk:2: Variable ASSIGN is overwritten in line 8.", + "WARN: file.mk:4: Variable EVAL is overwritten in line 10.") + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get assigned a value using the := operator, +// which in this simple case is equivalent to the = operator. The variables are +// later overridden with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_eval(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT:=\tvalue", + "ASSIGN:=\tvalue", + "APPEND:=\tvalue", + "EVAL:=\tvalue", + "SHELL:=\tvalue", + "", + "DEFAULT?=\tvalue", + "ASSIGN=\tvalue", + "APPEND+=\tvalue", + "EVAL:=\tvalue", + "SHELL!=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "NOTE: file.mk:8: Definition of ASSIGN is redundant because of line 2.", + "WARN: file.mk:4: Variable EVAL is overwritten in line 10.") + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get assigned a value using the != operator, +// which runs a shell command. As of March 2019 pkglint doesn't try to evaluate +// the shell commands, therefore the variable values are unknown. The variables +// are later overridden using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_shell(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT!=\tvalue", + "ASSIGN!=\tvalue", + "APPEND!=\tvalue", + "EVAL!=\tvalue", + "SHELL!=\tvalue", + "", + "DEFAULT?=\tvalue", + "ASSIGN=\tvalue", + "APPEND+=\tvalue", + "EVAL:=\tvalue", + "SHELL!=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "WARN: file.mk:2: Variable ASSIGN is overwritten in line 8.", + "WARN: file.mk:4: Variable EVAL is overwritten in line 10.") + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get a default value and are later overridden +// with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_default_ref(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT?=\t${OTHER}", + "ASSIGN?=\t${OTHER}", + "APPEND?=\t${OTHER}", + "EVAL?=\t${OTHER}", + "SHELL?=\t${OTHER}", + "", + "DEFAULT?=\t${OTHER}", + "ASSIGN=\t${OTHER}", + "APPEND+=\t${OTHER}", + "EVAL:=\t${OTHER}", + "SHELL!=\t${OTHER}") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "NOTE: file.mk:8: Definition of ASSIGN is redundant because of line 2.") + // TODO: "4: is overwritten later", + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get assigned are value and are later overridden +// with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_assign_ref(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT=\t${OTHER}", + "ASSIGN=\t${OTHER}", + "APPEND=\t${OTHER}", + "EVAL=\t${OTHER}", + "SHELL=\t${OTHER}", + "", + "DEFAULT?=\t${OTHER}", + "ASSIGN=\t${OTHER}", + "APPEND+=\t${OTHER}", + "EVAL:=\t${OTHER}", + "SHELL!=\t${OTHER}") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "NOTE: file.mk:8: Definition of ASSIGN is redundant because of line 2.") + // TODO: "4: is overwritten later", + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get appended a value and are later overridden +// with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_append_ref(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT+=\t${OTHER}", + "ASSIGN+=\t${OTHER}", + "APPEND+=\t${OTHER}", + "EVAL+=\t${OTHER}", + "SHELL+=\t${OTHER}", + "", + "DEFAULT?=\t${OTHER}", + "ASSIGN=\t${OTHER}", + "APPEND+=\t${OTHER}", + "EVAL:=\t${OTHER}", + "SHELL!=\t${OTHER}") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "WARN: file.mk:2: Variable ASSIGN is overwritten in line 8.") + // TODO: "4: is overwritten later", + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get assigned a value using the := operator, +// which in this simple case is equivalent to the = operator. The variables are +// later overridden with the same value using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_eval_ref(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT:=\t${OTHER}", + "ASSIGN:=\t${OTHER}", + "APPEND:=\t${OTHER}", + "EVAL:=\t${OTHER}", + "SHELL:=\t${OTHER}", + "", + "DEFAULT?=\t${OTHER}", + "ASSIGN=\t${OTHER}", + "APPEND+=\t${OTHER}", + "EVAL:=\t${OTHER}", + "SHELL!=\t${OTHER}") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "NOTE: file.mk:8: Definition of ASSIGN is redundant because of line 2.") + // TODO: "4: is overwritten later", + // TODO: "5: is overwritten later" +} + +// In a single file, five variables get assigned a value using the != operator, +// which runs a shell command. As of March 2019 pkglint doesn't try to evaluate +// the shell commands, therefore the variable values are unknown. The variables +// are later overridden using the five different assignments operators. +func (s *Suite) Test_RedundantScope__single_file_shell_ref(c *check.C) { + t := s.Init(c) + + mklines := t.NewMkLines("file.mk", + "DEFAULT!=\t${OTHER}", + "ASSIGN!=\t${OTHER}", + "APPEND!=\t${OTHER}", + "EVAL!=\t${OTHER}", + "SHELL!=\t${OTHER}", + "", + "DEFAULT?=\t${OTHER}", + "ASSIGN=\t${OTHER}", + "APPEND+=\t${OTHER}", + "EVAL:=\t${OTHER}", + "SHELL!=\t${OTHER}") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: file.mk:7: Default assignment of DEFAULT has no effect because of line 1.", + "WARN: file.mk:2: Variable ASSIGN is overwritten in line 8.") + // TODO: "4: is overwritten later", + // TODO: "5: is overwritten later" +} + +func (s *Suite) Test_RedundantScope__after_including_same_value(c *check.C) { + t := s.Init(c) + + // Only test the ?=, = and += operators since the others are ignored, + // as of March 2019. + include, get := t.SetUpHierarchy() + include("including.mk", + include("included.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg?= ${OTHER}", + "VAR.def.app?= ${OTHER}", + "VAR.asg.def= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app= ${OTHER}", + "VAR.app.def+= ${OTHER}", + "VAR.app.asg+= ${OTHER}", + "VAR.app.app+= ${OTHER}"), + "VAR.def.def?= ${OTHER}", + "VAR.def.asg= ${OTHER}", + "VAR.def.app+= ${OTHER}", + "VAR.asg.def?= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app+= ${OTHER}", + "VAR.app.def?= ${OTHER}", + "VAR.app.asg= ${OTHER}", + "VAR.app.app+= ${OTHER}") + mklines := get("including.mk") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: including.mk:2: Default assignment of VAR.def.def has no effect because of included.mk:1.", + "NOTE: including.mk:3: Definition of VAR.def.asg is redundant because of included.mk:2.", + // VAR.def.app defines a default value and then appends to it. This is a common pattern. + // Appending the same value feels redundant but probably doesn't happen in practice. + // If it does, there should be a note for it. + "NOTE: including.mk:5: Default assignment of VAR.asg.def has no effect because of included.mk:4.", + "NOTE: including.mk:6: Definition of VAR.asg.asg is redundant because of included.mk:5.", + // VAR.asg.app defines a variable and later appends to it. This is a common pattern. + // Appending the same value feels redundant but probably doesn't happen in practice. + // If it does, there should be a note for it. + "NOTE: including.mk:8: Default assignment of VAR.app.def has no effect because of included.mk:7.", + // VAR.app.asg first appends and then overwrites. This might be a mistake. + // TODO: Find out whether this case happens in actual pkgsrc and if it's accidental. + // VAR.app.app first appends and then appends one more. This is a common pattern. + ) +} + +func (s *Suite) Test_RedundantScope__after_including_different_value(c *check.C) { + t := s.Init(c) + + // Only test the ?=, = and += operators since the others are ignored, + // as of March 2019. + include, get := t.SetUpHierarchy() + include("including.mk", + include("included.mk", + "VAR.def.def?= ${VALUE}", + "VAR.def.asg?= ${VALUE}", + "VAR.def.app?= ${VALUE}", + "VAR.asg.def= ${VALUE}", + "VAR.asg.asg= ${VALUE}", + "VAR.asg.app= ${VALUE}", + "VAR.app.def+= ${VALUE}", + "VAR.app.asg+= ${VALUE}", + "VAR.app.app+= ${VALUE}"), + "VAR.def.def?= ${OTHER}", + "VAR.def.asg= ${OTHER}", + "VAR.def.app+= ${OTHER}", + "VAR.asg.def?= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app+= ${OTHER}", + "VAR.app.def?= ${OTHER}", + "VAR.app.asg= ${OTHER}", + "VAR.app.app+= ${OTHER}") + mklines := get("including.mk") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: including.mk:2: Default assignment of VAR.def.def has no effect because of included.mk:1.", + "NOTE: including.mk:5: Default assignment of VAR.asg.def has no effect because of included.mk:4.", + "NOTE: including.mk:8: Default assignment of VAR.app.def has no effect because of included.mk:7.") +} + +func (s *Suite) Test_RedundantScope__before_including_same_value(c *check.C) { + t := s.Init(c) + + // Only test the ?=, = and += operators since the others are ignored, + // as of March 2019. + include, get := t.SetUpHierarchy() + include("including.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg?= ${OTHER}", + "VAR.def.app?= ${OTHER}", + "VAR.asg.def= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app= ${OTHER}", + "VAR.app.def+= ${OTHER}", + "VAR.app.asg+= ${OTHER}", + "VAR.app.app+= ${OTHER}", + include("included.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg= ${OTHER}", + "VAR.def.app+= ${OTHER}", + "VAR.asg.def?= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app+= ${OTHER}", + "VAR.app.def?= ${OTHER}", + "VAR.app.asg= ${OTHER}", + "VAR.app.app+= ${OTHER}")) + mklines := get("including.mk") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: including.mk:1: Default assignment of VAR.def.def has no effect because of included.mk:1.", + "NOTE: including.mk:2: Default assignment of VAR.def.asg has no effect because of included.mk:2.", + "NOTE: including.mk:4: Definition of VAR.asg.def is redundant because of included.mk:4.", + "NOTE: including.mk:5: Definition of VAR.asg.asg is redundant because of included.mk:5.", + "WARN: including.mk:8: Variable VAR.app.asg is overwritten in included.mk:8.") +} + +func (s *Suite) Test_RedundantScope__before_including_different_value(c *check.C) { + t := s.Init(c) + + // Only test the ?=, = and += operators since the others are ignored, + // as of March 2019. + include, get := t.SetUpHierarchy() + include("including.mk", + "VAR.def.def?= ${VALUE}", + "VAR.def.asg?= ${VALUE}", + "VAR.def.app?= ${VALUE}", + "VAR.asg.def= ${VALUE}", + "VAR.asg.asg= ${VALUE}", + "VAR.asg.app= ${VALUE}", + "VAR.app.def+= ${VALUE}", + "VAR.app.asg+= ${VALUE}", + "VAR.app.app+= ${VALUE}", + include("included.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg= ${OTHER}", + "VAR.def.app+= ${OTHER}", + "VAR.asg.def?= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app+= ${OTHER}", + "VAR.app.def?= ${OTHER}", + "VAR.app.asg= ${OTHER}", + "VAR.app.app+= ${OTHER}")) + mklines := get("including.mk") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "WARN: including.mk:2: Variable VAR.def.asg is overwritten in included.mk:2.", + "WARN: including.mk:5: Variable VAR.asg.asg is overwritten in included.mk:5.", + "WARN: including.mk:8: Variable VAR.app.asg is overwritten in included.mk:8.") +} + +func (s *Suite) Test_RedundantScope__independent_same_value(c *check.C) { + t := s.Init(c) + + // Only test the ?=, = and += operators since the others are ignored, + // as of March 2019. + include, get := t.SetUpHierarchy() + include("including.mk", + include("included1.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg?= ${OTHER}", + "VAR.def.app?= ${OTHER}", + "VAR.asg.def= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app= ${OTHER}", + "VAR.app.def+= ${OTHER}", + "VAR.app.asg+= ${OTHER}", + "VAR.app.app+= ${OTHER}"), + include("included2.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg= ${OTHER}", + "VAR.def.app+= ${OTHER}", + "VAR.asg.def?= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app+= ${OTHER}", + "VAR.app.def?= ${OTHER}", + "VAR.app.asg= ${OTHER}", + "VAR.app.app+= ${OTHER}")) + mklines := get("including.mk") + + NewRedundantScope().Check(mklines) + + // Since the two included files are independent, there cannot be any + // redundancies between them. These redundancies can only be discovered + // when one of them includes the other. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__independent_different_value(c *check.C) { + t := s.Init(c) + + // Only test the ?=, = and += operators since the others are ignored, + // as of March 2019. + include, get := t.SetUpHierarchy() + include("including.mk", + include("included1.mk", + "VAR.def.def?= ${VALUE}", + "VAR.def.asg?= ${VALUE}", + "VAR.def.app?= ${VALUE}", + "VAR.asg.def= ${VALUE}", + "VAR.asg.asg= ${VALUE}", + "VAR.asg.app= ${VALUE}", + "VAR.app.def+= ${VALUE}", + "VAR.app.asg+= ${VALUE}", + "VAR.app.app+= ${VALUE}"), + include("included2.mk", + "VAR.def.def?= ${OTHER}", + "VAR.def.asg= ${OTHER}", + "VAR.def.app+= ${OTHER}", + "VAR.asg.def?= ${OTHER}", + "VAR.asg.asg= ${OTHER}", + "VAR.asg.app+= ${OTHER}", + "VAR.app.def?= ${OTHER}", + "VAR.app.asg= ${OTHER}", + "VAR.app.app+= ${OTHER}")) + mklines := get("including.mk") + + NewRedundantScope().Check(mklines) + + // Since the two included files are independent, there cannot be any + // redundancies between them. Redundancies can only be discovered + // when one of them includes the other. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__file_hierarchy(c *check.C) { + t := s.Init(c) + + include, get := t.SetUpHierarchy() + + include("including.mk", + include("other.mk", + "VAR= other"), + include("module.mk", + "VAR= module", + include("version.mk", + "VAR= version"), + include("env.mk", + "VAR= env"))) + + NewRedundantScope().Check(get("including.mk")) + + // No output since the included files are independent. + t.CheckOutputEmpty() + + NewRedundantScope().Check(get("other.mk")) + + // No output since the file by itself in neither redundant nor + // does it include any other file. + t.CheckOutputEmpty() + + NewRedundantScope().Check(get("module.mk")) + + // No warning about env.mk because it is independent from version.mk. + // Pkglint only produces warnings when it is very sure that the variable + // definition is really redundant in all cases. + // + // One reason to not warn is that at the point where env.mk is evaluated, + // version.mk had last written to the variable. Since version.mk is + // independent from env.mk, there is nothing redundant here. + // Pkglint doesn't do this, but it could. + // + // Another reason not to warn is that all locations where the variable has + // ever been accessed are saved. And if the current location neither includes + // all of the others nor is included by all of the others, there is at least + // one access that is in an unrelated file. This is what pkglint does. + t.CheckOutputLines( + "WARN: module.mk:1: Variable VAR is overwritten in version.mk:1.") +} + +// FIXME: Continue the systematic redundancy tests. +// +// A test where the operators = and += define a variable that afterwards +// is assigned the same value using the ?= operator. +// +// Tests where the variables refer to other variables. These variables may +// be read and written between the relevant assignments. +// +// Tests where the variables are defined conditionally using .if, .else, .endif. +// +// Tests where the variables are defined in a .for loop that might not be +// evaluated at all. +// +// Tests where files are included conditionally and additionally have conditional +// sections, arbitrarily nested. +// +// Tests that show how to suppress the notes about redundant assignments +// and overwritten variables. The explanation must be helpful. +// +// Tests for dynamic variable assignments. For example BUILD_DIRS.NetBSD may +// be modified by any assignment of the form BUILD_DIRS.${var} or even ${var}. +// Without further analysis, pkglint cannot report redundancy warnings for any +// package that uses such variable assignments. + +func (s *Suite) Test_RedundantScope__override_after_including(c *check.C) { + t := s.Init(c) + t.CreateFileLines("included.mk", + "OVERRIDE=\tprevious value", + "REDUNDANT=\tredundant") + t.CreateFileLines("including.mk", + ".include \"included.mk\"", + "OVERRIDE=\toverridden value", + "REDUNDANT=\tredundant") + t.Chdir(".") + mklines := t.LoadMkInclude("including.mk") + + // XXX: The warnings from here are not in the same order as the other warnings. + // XXX: There may be some warnings for the same file separated by warnings for other files. + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: including.mk:3: Definition of REDUNDANT is redundant because of included.mk:2.") +} + +func (s *Suite) Test_RedundantScope__redundant_assign_after_including(c *check.C) { + t := s.Init(c) + t.CreateFileLines("included.mk", + "REDUNDANT=\tredundant") + t.CreateFileLines("including.mk", + ".include \"included.mk\"", + "REDUNDANT=\tredundant") + t.Chdir(".") + mklines := t.LoadMkInclude("including.mk") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: including.mk:2: Definition of REDUNDANT is redundant because of included.mk:1.") +} + +func (s *Suite) Test_RedundantScope__override_in_Makefile_after_including(c *check.C) { + t := s.Init(c) + t.CreateFileLines("module.mk", + "VAR=\tvalue ${OTHER}", + "VAR?=\tvalue ${OTHER}", + "VAR=\tnew value") + t.CreateFileLines("Makefile", + ".include \"module.mk\"", + "VAR=\tthe package may overwrite variables from other files") + t.Chdir(".") + + mklines := t.LoadMkInclude("Makefile") + + // XXX: The warnings from here are not in the same order as the other warnings. + // XXX: There may be some warnings for the same file separated by warnings for other files. + NewRedundantScope().Check(mklines) + + // No warning for VAR=... in Makefile since it makes sense to have common files + // with default values for variables, overriding some of them in each package. + t.CheckOutputLines( + "NOTE: module.mk:2: Default assignment of VAR has no effect because of line 1.", + "WARN: module.mk:2: Variable VAR is overwritten in line 3.") +} + +func (s *Suite) Test_RedundantScope__default_value_definitely_unused(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR=\tvalue ${OTHER}", + "VAR?=\tdifferent value") + + NewRedundantScope().Check(mklines) + + // A default assignment after an unconditional assignment is redundant. + // Even more so when the variable is not used between the two assignments. + t.CheckOutputLines( + "NOTE: module.mk:2: Default assignment of VAR has no effect because of line 1.") +} + +func (s *Suite) Test_RedundantScope__default_value_overridden(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR?=\tdefault value", + "VAR=\toverridden value") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "WARN: module.mk:1: Variable VAR is overwritten in line 2.") +} + +func (s *Suite) Test_RedundantScope__overwrite_same_value(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR=\tvalue ${OTHER}", + "VAR=\tvalue ${OTHER}") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: module.mk:2: Definition of VAR is redundant because of line 1.") +} + +func (s *Suite) Test_RedundantScope__conditional_overwrite(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR=\tdefault", + ".if ${OPSYS} == NetBSD", + "VAR=\topsys", + ".endif") + + NewRedundantScope().Check(mklines) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__overwrite_inside_conditional(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR=\tgeneric", + ".if ${OPSYS} == NetBSD", + "VAR=\tignored", + "VAR=\toverwritten", + ".endif") + + NewRedundantScope().Check(mklines) + + // TODO: expected a warning "WARN: module.mk:4: line 3 is ignored" + // Since line 3 and line 4 are in the same basic block, line 3 is definitely ignored. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__conditionally_include(c *check.C) { + t := s.Init(c) + t.CreateFileLines("module.mk", + "VAR=\tgeneric", + ".if ${OPSYS} == NetBSD", + ". include \"included.mk\"", + ".endif") + t.CreateFileLines("included.mk", + "VAR=\tignored", + "VAR=\toverwritten") + mklines := t.LoadMkInclude("module.mk") + + NewRedundantScope().Check(mklines) + + // TODO: expected a warning "WARN: module.mk:4: line 3 is ignored" + // Since line 3 and line 4 are in the same basic block, line 3 is definitely ignored. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__conditional_default(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR=\tdefault", + ".if ${OPSYS} == NetBSD", + "VAR?=\topsys", + ".endif") + + NewRedundantScope().Check(mklines) + + // TODO: WARN: module.mk:3: The value \"opsys\" will never be assigned to VAR because it is defined unconditionally in line 1. + t.CheckOutputEmpty() +} + +// These warnings are precise and accurate since the value of VAR is not used between line 2 and 4. +func (s *Suite) Test_RedundantScope__overwrite_same_variable_different_value(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "OTHER=\tvalue before", + "VAR=\tvalue ${OTHER}", + "OTHER=\tvalue after", + "VAR=\tvalue ${OTHER}") + + NewRedundantScope().Check(mklines) + + // Strictly speaking, line 1 is redundant because OTHER is not evaluated + // at load time and then immediately overwritten in line 3. If the operator + // in line 2 were a := instead of a =, the situation would be clear. + // Pkglint doesn't warn about the redundancy in line 1 because it prefers + // to omit warnings instead of giving wrong advice. + t.CheckOutputLines( + "NOTE: module.mk:4: Definition of VAR is redundant because of line 2.") +} + +func (s *Suite) Test_RedundantScope__overwrite_different_value_used_between(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "OTHER=\tvalue before", + "VAR=\tvalue ${OTHER}", + + // VAR is used here at load time, therefore it must be defined at this point. + // At this point, VAR uses the \"before\" value of OTHER. + "RESULT1:=\t${VAR}", + + "OTHER=\tvalue after", + + // VAR is used here again at load time, this time using the \"after\" value of OTHER. + "RESULT2:=\t${VAR}", + + // Still this definition is redundant. + "VAR=\tvalue ${OTHER}") + + NewRedundantScope().Check(mklines) + + // There is nothing redundant here. Each write is followed by a + // corresponding read, except for the last one. That is ok though + // because in pkgsrc the last action of a package is to include + // bsd.pkg.mk, which reads almost all variables. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__procedure_call_to_noop(c *check.C) { + t := s.Init(c) + + include, get := t.SetUpHierarchy() + include("mk/pthread.buildlink3.mk", + "CHECK_BUILTIN.pthread:= yes", + include("pthread.builtin.mk", + "# Nothing happens here."), + "CHECK_BUILTIN.pthread:= no") + + NewRedundantScope().Check(get("mk/pthread.buildlink3.mk")) + + t.CheckOutputLines( + "WARN: mk/pthread.buildlink3.mk:1: Variable CHECK_BUILTIN.pthread is overwritten in line 3.") +} + +func (s *Suite) Test_RedundantScope__procedure_call_implemented(c *check.C) { + t := s.Init(c) + + include, get := t.SetUpHierarchy() + include("mk/pthread.buildlink3.mk", + "CHECK_BUILTIN.pthread:= yes", + include("pthread.builtin.mk", + "CHECK_BUILTIN.pthread?= no", + ".if !empty(CHECK_BUILTIN.pthread:M[Nn][Oo])", + ".endif"), + "CHECK_BUILTIN.pthread:= no") + + NewRedundantScope().Check(get("mk/pthread.buildlink3.mk")) + + // This test is a bit unrealistic. It wrongly assumes that all files from + // an .include directive are actually included by pkglint. + // + // See Package.readMakefile/handleIncludeLine/skip. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__procedure_call_implemented_package(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.SetUpPackage("devel/gettext-lib") + t.SetUpPackage("x11/Xaos", + ".include \"../../devel/gettext-lib/buildlink3.mk\"") + t.CreateFileLines("devel/gettext-lib/builtin.mk", + MkRcsID, + "", + ".include \"../../mk/bsd.fast.prefs.mk\"", + "", + "CHECK_BUILTIN.gettext?=\tno", + ".if !empty(CHECK_BUILTIN.gettext:M[nN][oO])", + ".endif") + t.CreateFileLines("devel/gettext-lib/buildlink3.mk", + MkRcsID, + "CHECK_BUILTIN.gettext:=\tyes", + ".include \"builtin.mk\"", + "CHECK_BUILTIN.gettext:=\tno") + G.Pkgsrc.LoadInfrastructure() + + // Checking x11/Xaos instead of devel/gettext-lib avoids warnings + // about the minimal buildlink3.mk file. + G.Check(t.File("x11/Xaos")) + + // There is nothing redundant here. + // Up to March 2019, pkglint didn't pass the correct pathnames to Package.included, + // which triggered a wrong note here. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__procedure_call_infrastructure(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("x11/alacarte", + ".include \"../../mk/pthread.buildlink3.mk\"") + t.CreateFileLines("mk/pthread.buildlink3.mk", + MkRcsID, + "CHECK_BUILTIN.gettext:=\tyes", + ".include \"pthread.builtin.mk\"", + "CHECK_BUILTIN.gettext:=\tno") + t.CreateFileLines("mk/pthread.builtin.mk", + MkRcsID, + "CHECK_BUILTIN.gettext?=\tno", + ".if !empty(CHECK_BUILTIN.gettext:M[nN][oO])", + ".endif") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("x11/alacarte")) + + // There is nothing redundant here. + // + // 1. pthread.buildlink3.mk sets the variable + // 2. pthread.builtin.mk assigns it a default value + // (which is common practice) + // 3. pthread.builtin.mk then reads it + // (which marks the next write as non-redundant) + // 4. pthread.buildlink3.mk sets the variable again + // (this is considered neither overwriting nor redundant) + // + // Up to March 2019, pkglint complained: + // + // WARN: ~/mk/pthread.buildlink3.mk:2: + // Variable CHECK_BUILTIN.gettext is overwritten in line 4. + // + // The cause for the warning is that when including files from the + // infrastructure, pkglint only includes the outermost level of files. + // If an infrastructure file includes another infrastructure file, + // pkglint skips that, for performance reasons. + // + // This optimization effectively made the .include for pthread.builtin.mk + // a no-op, therefore it was correct to issue a warning here. + // + // Since this warning is wrong, in March 2019 another special rule has + // been added to Package.readMakefile.handleIncludeLine.skip saying that + // including a buildlink3.mk file also includes the corresponding + // builtin.mk file. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__shell_and_eval(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR:=\tvalue ${OTHER}", + "VAR!=\tvalue ${OTHER}") + + NewRedundantScope().Check(mklines) + + // As of November 2018, pkglint doesn't check redundancies that involve the := or != operators. + // + // What happens here is: + // + // Line 1 evaluates OTHER at load time. + // Line 1 assigns its value to VAR. + // Line 2 evaluates OTHER at load time. + // Line 2 passes its value through the shell and assigns the result to VAR. + // + // Since VAR is defined in line 1, not used afterwards and overwritten in line 2, it is redundant. + // Well, not quite, because evaluating ${OTHER} might have side-effects from :sh or ::= modifiers, + // but these are so rare that they are frowned upon and are not considered by pkglint. + // + // Expected result: + // WARN: module.mk:2: Previous definition of VAR in line 1 is unused. + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__shell_and_eval_literal(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR:=\tvalue", + "VAR!=\tvalue") + + NewRedundantScope().Check(mklines) + + // Even when := is used with a literal value (which is usually + // only done for procedure calls), the shell evaluation can have + // so many different side effects that pkglint cannot reliably + // help in this situation. + // + // TODO: Why not? The evaluation in line 1 is trivial to analyze. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__included_OPSYS_variable(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + ".include \"../../category/dependency/buildlink3.mk\"", + "CONFIGURE_ARGS+=\tone", + "CONFIGURE_ARGS=\ttwo", + "CONFIGURE_ARGS+=\tthree") + t.SetUpPackage("category/dependency") + t.CreateFileDummyBuildlink3("category/dependency/buildlink3.mk") + t.CreateFileLines("category/dependency/builtin.mk", + MkRcsID, + "CONFIGURE_ARGS.Darwin+=\tdarwin") + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile:21: Variable CONFIGURE_ARGS is overwritten in line 22.") +} + +func (s *Suite) Test_RedundantScope__if_then_else(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("if-then-else.mk", + MkRcsID, + ".if exists(${FILE})", + "OS=\tNetBSD", + ".else", + "OS=\tOTHER", + ".endif") + + NewRedundantScope().Check(mklines) + + // These two definitions are of course not redundant since they happen in + // different branches of the same .if statement. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__if_then_else_without_variable(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("if-then-else.mk", + MkRcsID, + ".if exists(/nonexistent)", + "IT=\texists", + ".else", + "IT=\tdoesn't exist", + ".endif") + + NewRedundantScope().Check(mklines) + + // These two definitions are of course not redundant since they happen in + // different branches of the same .if statement. + // Even though the .if condition does not refer to any variables, + // this still means that the variable assignments are conditional. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__append_then_default(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("append-then-default.mk", + MkRcsID, + "VAR+=\tvalue", + "VAR?=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: ~/append-then-default.mk:3: Default assignment of VAR has no effect because of line 2.") +} + +func (s *Suite) Test_RedundantScope__assign_then_default_in_same_file(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("assign-then-default.mk", + MkRcsID, + "VAR=\tvalue", + "VAR?=\tvalue") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "NOTE: ~/assign-then-default.mk:3: " + + "Default assignment of VAR has no effect because of line 2.") +} + +func (s *Suite) Test_RedundantScope__eval_then_eval(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("filename.mk", + MkRcsID, + "VAR:=\tvalue", + "VAR:=\tvalue", + "VAR:=\tother") + + NewRedundantScope().Check(mklines) + + t.CheckOutputLines( + "WARN: ~/filename.mk:2: Variable VAR is overwritten in line 3.", + "WARN: ~/filename.mk:3: Variable VAR is overwritten in line 4.") +} + +func (s *Suite) Test_RedundantScope__shell_then_assign(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("filename.mk", + MkRcsID, + "VAR!=\techo echo", + "VAR=\techo echo") + + NewRedundantScope().Check(mklines) + + // Although the two variable assignments look very similar, they do + // something entirely different. The first executes the echo command, + // and the second just assigns a string. Therefore the actual variable + // values are different, and the second assignment is not redundant. + // It assigns a different value. Nevertheless, the shell command is + // redundant and can be removed since its result is never used. + t.CheckOutputLines( + "WARN: ~/filename.mk:2: Variable VAR is overwritten in line 3.") +} + +func (s *Suite) Test_RedundantScope__shell_then_read_then_assign(c *check.C) { + t := s.Init(c) + + mklines := t.SetUpFileMkLines("filename.mk", + MkRcsID, + "VAR!=\techo echo", + "OUTPUT:=${VAR}", + "VAR=\techo echo") + + NewRedundantScope().Check(mklines) + + // No warning since the value is used in-between. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__assign_then_default_in_included_file(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("assign-then-default.mk", + MkRcsID, + "VAR=\tvalue", + ".include \"included.mk\"") + t.CreateFileLines("included.mk", + MkRcsID, + "VAR?=\tvalue") + mklines := t.LoadMkInclude("assign-then-default.mk") + + NewRedundantScope().Check(mklines) + + // If assign-then-default.mk:2 is deleted, VAR still has the same value. + t.CheckOutputLines( + "NOTE: ~/assign-then-default.mk:2: Definition of VAR is redundant because of included.mk:2.") +} + +func (s *Suite) Test_RedundantScope__conditionally_included_file(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("including.mk", + MkRcsID, + "VAR=\tvalue", + ".if ${COND}", + ". include \"included.mk\"", + ".endif") + t.CreateFileLines("included.mk", + MkRcsID, + "VAR?=\tvalue") + mklines := t.LoadMkInclude("including.mk") + + NewRedundantScope().Check(mklines) + + // The assignment in including.mk:2 is only redundant if included.mk is actually included. + // Therefore both included.mk:2 nor including.mk:2 are relevant. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__procedure_parameters(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("mk/pkg-build-options.mk", + MkRcsID, + "USED:=\t${pkgbase}") + t.CreateFileLines("including.mk", + MkRcsID, + "pkgbase=\tpackage1", + ".include \"mk/pkg-build-options.mk\"", + "", + "pkgbase=\tpackage2", + ".include \"mk/pkg-build-options.mk\"", + "", + "pkgbase=\tpackage3", + ".include \"mk/pkg-build-options.mk\"") + mklines := t.LoadMkInclude("including.mk") + + NewRedundantScope().Check(mklines) + + // This variable is not overwritten since it is used in-between + // by the included file. + t.CheckOutputEmpty() +} + +// Branch coverage for info.vari.Constant(). The other tests typically +// make a variable non-constant by adding conditional assignments between +// .if/.endif. But there are other ways. The output of shell commands is +// unpredictable for pkglint (as of March 2019), therefore it treats these +// variables as non-constant. +func (s *Suite) Test_RedundantScope_handleVarassign__shell_followed_by_default(c *check.C) { + t := s.Init(c) + + include, get := t.SetUpHierarchy() + include("including.mk", + "VAR!= echo 'hello, world'", + include("included.mk", + "VAR?= hello world")) + + NewRedundantScope().Check(get("including.mk")) + + // If pkglint should ever learn to interpret simple shell commands, there + // should be a warning for including.mk:2 that the shell command generates + // the default value. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope__overwrite_definition_from_included_file(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("included.mk", + MkRcsID, + "WRKSRC=\t${WRKDIR}/${PKGBASE}") + t.CreateFileLines("including.mk", + MkRcsID, + "SUBDIR=\t${WRKSRC}", + ".include \"included.mk\"", + "WRKSRC=\t${WRKDIR}/overwritten") + mklines := t.LoadMkInclude("including.mk") + + NewRedundantScope().Check(mklines) + + // Before pkglint 5.7.2 (2019-03-10), the above setup generated a warning: + // + // WARN: ~/included.mk:2: Variable WRKSRC is overwritten in including.mk:4. + // + // This warning is obviously wrong since the included file must never + // receive a warning. Of course this default definition may be overridden + // by the including file. + // + // The warning was generated because in including.mk:2 the variable WRKSRC + // was used for the first time. Back then, each variable had only a single + // include path. That include path marks where the variable is used and + // defined. + // + // The variable definition at included.mk didn't modify this include path. + // Therefore pkglint wrongly assumed that this variable was only ever + // accessed in including.mk and issued a warning. + // + // To fix this, the RedundantScope now remembers every access to the + // variable, and the redundancy warnings are only issued in cases where + // either all variable accesses are in files including the current file, + // or all variable accesses are in files included by the current file. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_RedundantScope_handleVarassign__conditional(c *check.C) { + t := s.Init(c) + + scope := NewRedundantScope() + mklines := t.NewMkLines("filename.mk", + MkRcsID, + "VAR=\tvalue", + ".if 1", + "VAR=\tconditional", + ".endif") + + mklines.ForEach(func(mkline MkLine) { + scope.Handle(mkline, mklines.indentation) + }) + + t.Check( + scope.get("VAR").vari.WriteLocations(), + deepEquals, + []MkLine{mklines.mklines[1], mklines.mklines[3]}) +} + +func (s *Suite) Test_includePath_includes(c *check.C) { + t := s.Init(c) + + path := func(locations ...string) includePath { + return includePath{locations} + } + + var ( + m = path("Makefile") + mc = path("Makefile", "Makefile.common") + mco = path("Makefile", "Makefile.common", "other.mk") + mo = path("Makefile", "other.mk") + ) + + t.Check(m.includes(m), equals, false) + + t.Check(m.includes(mc), equals, true) + t.Check(m.includes(mco), equals, true) + t.Check(mc.includes(mco), equals, true) + + t.Check(mc.includes(m), equals, false) + t.Check(mc.includes(mo), equals, false) + t.Check(mo.includes(mc), equals, false) +} + +func (s *Suite) Test_includePath_equals(c *check.C) { + t := s.Init(c) + + path := func(locations ...string) includePath { + return includePath{locations} + } + + var ( + m = path("Makefile") + mc = path("Makefile", "Makefile.common") + mco = path("Makefile", "Makefile.common", "other.mk") + mo = path("Makefile", "other.mk") + ) + + t.Check(m.equals(m), equals, true) + + t.Check(m.equals(mc), equals, false) + t.Check(m.equals(mco), equals, false) + t.Check(mc.equals(mco), equals, false) + + t.Check(mc.equals(m), equals, false) + t.Check(mc.equals(mo), equals, false) + t.Check(mo.equals(mc), equals, false) +} diff --git a/pkgtools/pkglint/files/shell.go b/pkgtools/pkglint/files/shell.go index 5315865f42b..b9892dc043d 100644 --- a/pkgtools/pkglint/files/shell.go +++ b/pkgtools/pkglint/files/shell.go @@ -530,14 +530,6 @@ func (scc *SimpleCommandChecker) handleCommandVariable() bool { if varuse := parser.VarUse(); varuse != nil && parser.EOF() { varname := varuse.varname - if tool := G.ToolByVarname(varname); tool != nil { - if tool.Validity == Nowhere { - scc.shline.mkline.Warnf("The %q tool is used but not added to USE_TOOLS.", tool.Name) - } - scc.shline.checkInstallCommand(shellword) - return true - } - if vartype := G.Pkgsrc.VariableType(varname); vartype != nil && vartype.basicType.name == "ShellCommand" { scc.shline.checkInstallCommand(shellword) return true diff --git a/pkgtools/pkglint/files/shell_test.go b/pkgtools/pkglint/files/shell_test.go index bc9177d5b29..dd2add2ef2c 100644 --- a/pkgtools/pkglint/files/shell_test.go +++ b/pkgtools/pkglint/files/shell_test.go @@ -154,7 +154,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { t.SetUpTool("unzip", "UNZIP_CMD", AtRunTime) test := func(shellCommand string) { - G.Mk = t.NewMkLines("filename", + G.Mk = t.NewMkLines("filename.mk", "\t"+shellCommand) shline := NewShellLine(G.Mk.mklines[0]) @@ -170,8 +170,8 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { test("uname=`uname`; echo $$uname; echo; ${PREFIX}/bin/command") t.CheckOutputLines( - "WARN: filename:1: Unknown shell command \"uname\".", - "WARN: filename:1: Please switch to \"set -e\" mode "+ + "WARN: filename.mk:1: Unknown shell command \"uname\".", + "WARN: filename.mk:1: Please switch to \"set -e\" mode "+ "before using a semicolon (after \"uname=`uname`\") to separate commands.") t.SetUpTool("echo", "", AtRunTime) @@ -180,35 +180,30 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { test("echo ${PKGNAME:Q}") // VucQuotPlain t.CheckOutputLines( - "WARN: filename:1: PKGNAME may not be used in this file; "+ - "it would be ok in Makefile, Makefile.* or *.mk.", - "NOTE: filename:1: The :Q operator isn't necessary for ${PKGNAME} here.") + "NOTE: filename.mk:1: The :Q operator isn't necessary for ${PKGNAME} here.") test("echo \"${CFLAGS:Q}\"") // VucQuotDquot t.CheckOutputLines( - "WARN: filename:1: The :Q modifier should not be used inside double quotes.", - "WARN: filename:1: CFLAGS may not be used in this file; "+ - "it would be ok in Makefile, Makefile.common, options.mk or *.mk.", - "WARN: filename:1: Please use ${CFLAGS:M*:Q} instead of ${CFLAGS:Q} "+ + "WARN: filename.mk:1: The :Q modifier should not be used inside double quotes.", + "WARN: filename.mk:1: Please use ${CFLAGS:M*:Q} instead of ${CFLAGS:Q} "+ "and make sure the variable appears outside of any quoting characters.") test("echo '${COMMENT:Q}'") // VucQuotSquot t.CheckOutputLines( - "WARN: filename:1: COMMENT may not be used in any file; it is a write-only variable.", - "WARN: filename:1: Please move ${COMMENT:Q} outside of any quoting characters.") + "WARN: filename.mk:1: Please move ${COMMENT:Q} outside of any quoting characters.") test("echo target=$@ exitcode=$$? '$$' \"\\$$\"") t.CheckOutputLines( - "WARN: filename:1: Please use \"${.TARGET}\" instead of \"$@\".", - "WARN: filename:1: The $? shell variable is often not available in \"set -e\" mode.") + "WARN: filename.mk:1: Please use \"${.TARGET}\" instead of \"$@\".", + "WARN: filename.mk:1: The $? shell variable is often not available in \"set -e\" mode.") test("echo $$@") t.CheckOutputLines( - "WARN: filename:1: The $@ shell variable should only be used in double quotes.") + "WARN: filename.mk:1: The $@ shell variable should only be used in double quotes.") test("echo \"$$\"") // As seen by make(1); the shell sees: echo "$" @@ -233,8 +228,8 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { test("${RUN} subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\"") t.CheckOutputLines( - "WARN: filename:1: Double quotes inside backticks inside double quotes are error prone.", - "WARN: filename:1: The exitcode of \"unzip\" at the left of the | operator is ignored.") + "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.", + "WARN: filename.mk:1: The exitcode of \"unzip\" at the left of the | operator is ignored.") // From mail/thunderbird/Makefile, rev. 1.159 test("" + @@ -247,9 +242,9 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { "done") t.CheckOutputLines( - "WARN: filename:1: XPI_FILES is used but not defined.", - "WARN: filename:1: Double quotes inside backticks inside double quotes are error prone.", - "WARN: filename:1: The exitcode of \"${UNZIP_CMD}\" at the left of the | operator is ignored.") + "WARN: filename.mk:1: XPI_FILES is used but not defined.", + "WARN: filename.mk:1: Double quotes inside backticks inside double quotes are error prone.", + "WARN: filename.mk:1: The exitcode of \"${UNZIP_CMD}\" at the left of the | operator is ignored.") // From x11/wxGTK28/Makefile test("" + @@ -262,25 +257,23 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { // TODO: Why is TOOLS_PATH.msgfmt not recognized? // At least, the warning should be more specific, mentioning USE_TOOLS. t.CheckOutputLines( - "WARN: filename:1: WRKSRC may not be used in this file; "+ - "it would be ok in Makefile, Makefile.* or *.mk.", - "WARN: filename:1: Unknown shell command \"[\".", - "WARN: filename:1: Unknown shell command \"${TOOLS_PATH.msgfmt}\".") + "WARN: filename.mk:1: Unknown shell command \"[\".", + "WARN: filename.mk:1: Unknown shell command \"${TOOLS_PATH.msgfmt}\".") test("@cp from to") t.CheckOutputLines( - "WARN: filename:1: The shell command \"cp\" should not be hidden.") + "WARN: filename.mk:1: The shell command \"cp\" should not be hidden.") test("-cp from to") t.CheckOutputLines( - "WARN: filename:1: Using a leading \"-\" to suppress errors is deprecated.") + "WARN: filename.mk:1: Using a leading \"-\" to suppress errors is deprecated.") test("-${MKDIR} deeply/nested/subdir") t.CheckOutputLines( - "WARN: filename:1: Using a leading \"-\" to suppress errors is deprecated.") + "WARN: filename.mk:1: Using a leading \"-\" to suppress errors is deprecated.") G.Pkg = NewPackage(t.File("category/pkgbase")) G.Pkg.Plist.Dirs["share/pkgbase"] = true @@ -292,15 +285,15 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { // the note should not appear then. t.CheckOutputLines( - "NOTE: filename:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" "+ + "NOTE: filename.mk:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" "+ "instead of \"${INSTALL_DATA_DIR}\".", - "WARN: filename:1: The INSTALL_*_DIR commands can only handle one directory at a time.") + "WARN: filename.mk:1: The INSTALL_*_DIR commands can only handle one directory at a time.") // A directory that is not found in the PLIST. test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/share/other") t.CheckOutputLines( - "NOTE: filename:1: You can use \"INSTALLATION_DIRS+= share/other\" instead of \"${INSTALL_DATA_DIR}\".") + "NOTE: filename.mk:1: You can use \"INSTALLATION_DIRS+= share/other\" instead of \"${INSTALL_DATA_DIR}\".") G.Pkg = nil @@ -315,7 +308,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__strip(c *check.C) { t := s.Init(c) test := func(shellCommand string) { - G.Mk = t.NewMkLines("filename", + G.Mk = t.NewMkLines("filename.mk", "\t"+shellCommand) G.Mk.ForEach(func(mkline MkLine) { @@ -327,8 +320,8 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__strip(c *check.C) { test("${STRIP} executable") t.CheckOutputLines( - "WARN: filename:1: Unknown shell command \"${STRIP}\".", - "WARN: filename:1: STRIP is used but not defined.") + "WARN: filename.mk:1: Unknown shell command \"${STRIP}\".", + "WARN: filename.mk:1: STRIP is used but not defined.") t.SetUpVartypes() @@ -430,7 +423,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__implementation(c *check.C) t := s.Init(c) t.SetUpVartypes() - G.Mk = t.NewMkLines("filename", + G.Mk = t.NewMkLines("filename.mk", "# dummy") shline := NewShellLine(G.Mk.mklines[0]) @@ -445,13 +438,13 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__implementation(c *check.C) G.Mk.ForEach(func(mkline MkLine) { shline.CheckWord(text, false, RunTime) }) t.CheckOutputLines( - "WARN: filename:1: Unknown shell command \"echo\".") + "WARN: filename.mk:1: Unknown shell command \"echo\".") G.Mk.ForEach(func(mkline MkLine) { shline.CheckShellCommandLine(text) }) // No parse errors t.CheckOutputLines( - "WARN: filename:1: Unknown shell command \"echo\".") + "WARN: filename.mk:1: Unknown shell command \"echo\".") } func (s *Suite) Test_ShellLine_CheckShellCommandLine__dollar_without_variable(c *check.C) { @@ -459,7 +452,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__dollar_without_variable(c t.SetUpVartypes() t.SetUpTool("pax", "", AtRunTime) - G.Mk = t.NewMkLines("filename", + G.Mk = t.NewMkLines("filename.mk", "# dummy") shline := NewShellLine(G.Mk.mklines[0]) @@ -516,8 +509,7 @@ func (s *Suite) Test_ShellLine_CheckWord(c *check.C) { test("${COMMENT:Q}", true) - t.CheckOutputLines( - "WARN: dummy.mk:1: COMMENT may not be used in any file; it is a write-only variable.") + t.CheckOutputEmpty() test("\"${DISTINFO_FILE:Q}\"", true) @@ -541,7 +533,7 @@ func (s *Suite) Test_ShellLine_CheckWord(c *check.C) { func (s *Suite) Test_ShellLine_CheckWord__dollar_without_variable(c *check.C) { t := s.Init(c) - shline := t.NewShellLine("filename", 1, "# dummy") + shline := t.NewShellLine("filename.mk", 1, "# dummy") shline.CheckWord("/.*~$$//g", false, RunTime) // Typical argument to pax(1). @@ -552,7 +544,7 @@ func (s *Suite) Test_ShellLine_CheckWord__backslash_plus(c *check.C) { t := s.Init(c) t.SetUpTool("find", "FIND", AtRunTime) - shline := t.NewShellLine("filename", 1, "\tfind . -exec rm -rf {} \\+") + shline := t.NewShellLine("filename.mk", 1, "\tfind . -exec rm -rf {} \\+") shline.CheckShellCommandLine(shline.mkline.ShellCommand()) @@ -563,21 +555,21 @@ func (s *Suite) Test_ShellLine_CheckWord__backslash_plus(c *check.C) { func (s *Suite) Test_ShellLine_CheckWord__squot_dollar(c *check.C) { t := s.Init(c) - shline := t.NewShellLine("filename", 1, "\t'$") + shline := t.NewShellLine("filename.mk", 1, "\t'$") shline.CheckWord(shline.mkline.ShellCommand(), false, RunTime) // FIXME: Should be parsed correctly. Make passes the dollar through (probably), // and the shell parser should complain about the unfinished string literal. t.CheckOutputLines( - "WARN: filename:1: Internal pkglint error in ShTokenizer.ShAtom at \"$\" (quoting=s).", - "WARN: filename:1: Internal pkglint error in ShellLine.CheckWord at \"'$\" (quoting=s), rest: $") + "WARN: filename.mk:1: Internal pkglint error in ShTokenizer.ShAtom at \"$\" (quoting=s).", + "WARN: filename.mk:1: Internal pkglint error in ShellLine.CheckWord at \"'$\" (quoting=s), rest: $") } func (s *Suite) Test_ShellLine_CheckWord__dquot_dollar(c *check.C) { t := s.Init(c) - shline := t.NewShellLine("filename", 1, "\t\"$") + shline := t.NewShellLine("filename.mk", 1, "\t\"$") shline.CheckWord(shline.mkline.ShellCommand(), false, RunTime) @@ -589,12 +581,12 @@ func (s *Suite) Test_ShellLine_CheckWord__dquot_dollar(c *check.C) { func (s *Suite) Test_ShellLine_CheckWord__dollar_subshell(c *check.C) { t := s.Init(c) - shline := t.NewShellLine("filename", 1, "\t$$(echo output)") + shline := t.NewShellLine("filename.mk", 1, "\t$$(echo output)") shline.CheckWord(shline.mkline.ShellCommand(), false, RunTime) t.CheckOutputLines( - "WARN: filename:1: Invoking subshells via $(...) is not portable enough.") + "WARN: filename.mk:1: Invoking subshells via $(...) is not portable enough.") } func (s *Suite) Test_ShellLine_CheckWord__PKGMANDIR(c *check.C) { @@ -709,9 +701,9 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__echo(c *check.C) { echo := t.SetUpTool("echo", "ECHO", AtRunTime) echo.MustUseVarForm = true - G.Mk = t.NewMkLines("filename", + G.Mk = t.NewMkLines("filename.mk", "# dummy") - mkline := t.NewMkLine("filename", 3, "# dummy") + mkline := t.NewMkLine("filename.mk", 3, "# dummy") MkLineChecker{mkline}.checkText("echo \"hello, world\"") @@ -720,7 +712,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__echo(c *check.C) { NewShellLine(mkline).CheckShellCommandLine("echo \"hello, world\"") t.CheckOutputLines( - "WARN: filename:3: Please use \"${ECHO}\" instead of \"echo\".") + "WARN: filename.mk:3: Please use \"${ECHO}\" instead of \"echo\".") } func (s *Suite) Test_ShellLine_CheckShellCommandLine__shell_variables(c *check.C) { @@ -762,21 +754,21 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__shell_variables(c *check.C func (s *Suite) Test_ShellLine_checkInstallCommand(c *check.C) { t := s.Init(c) - G.Mk = t.NewMkLines("filename", + G.Mk = t.NewMkLines("filename.mk", "# dummy") G.Mk.target = "do-install" - shline := t.NewShellLine("filename", 1, "\tdummy") + shline := t.NewShellLine("filename.mk", 1, "\tdummy") shline.checkInstallCommand("sed") t.CheckOutputLines( - "WARN: filename:1: The shell command \"sed\" should not be used in the install phase.") + "WARN: filename.mk:1: The shell command \"sed\" should not be used in the install phase.") shline.checkInstallCommand("cp") t.CheckOutputLines( - "WARN: filename:1: ${CP} should not be used to install files.") + "WARN: filename.mk:1: ${CP} should not be used to install files.") } func (s *Suite) Test_splitIntoMkWords(c *check.C) { diff --git a/pkgtools/pkglint/files/substcontext.go b/pkgtools/pkglint/files/substcontext.go index 45fa1ac3327..5f30908c4e6 100644 --- a/pkgtools/pkglint/files/substcontext.go +++ b/pkgtools/pkglint/files/substcontext.go @@ -283,19 +283,27 @@ func (ctx *SubstContext) suggestSubstVars(mkline MkLine) { continue } + varop := sprintf("SUBST_VARS.%s%s%s", + ctx.id, + ifelseStr(hasSuffix(ctx.id, "+"), " ", ""), + ifelseStr(ctx.curr.seenVars, "+=", "=")) + fix := mkline.Autofix() - fix.Notef("The substitution command %q can be replaced with \"SUBST_VARS.%s+= %s\".", token, ctx.id, varname) + fix.Notef("The substitution command %q can be replaced with \"%s %s\".", + token, varop, varname) fix.Explain( "Replacing @VAR@ with ${VAR} is such a typical pattern that pkgsrc has built-in support for it,", "requiring only the variable name instead of the full sed command.") if mkline.VarassignComment() == "" && len(tokens) == 2 && tokens[0] == "-e" { // TODO: Extract the alignment computation somewhere else, so that it is generally available. alignBefore := tabWidth(mkline.ValueAlign()) - alignAfter := tabWidth(sprintf("SUBST_VARS.%s+=\t", ctx.id)) + alignAfter := tabWidth(varop + "\t") tabs := strings.Repeat("\t", imax((alignAfter-alignBefore)/8, 0)) - fix.Replace(mkline.Text, sprintf("SUBST_VARS.%s+=\t%s%s", ctx.id, tabs, varname)) + fix.Replace(mkline.Text, varop+"\t"+tabs+varname) } fix.Anyway() fix.Apply() + + ctx.curr.seenVars = true } } diff --git a/pkgtools/pkglint/files/substcontext_test.go b/pkgtools/pkglint/files/substcontext_test.go index f0eca45ee8c..3fb53eb3ea1 100644 --- a/pkgtools/pkglint/files/substcontext_test.go +++ b/pkgtools/pkglint/files/substcontext_test.go @@ -31,7 +31,7 @@ func (s *Suite) Test_SubstContext__incomplete(c *check.C) { t.CheckOutputLines( "NOTE: Makefile:13: The substitution command \"s,@PREFIX@,${PREFIX},g\" "+ - "can be replaced with \"SUBST_VARS.interp+= PREFIX\".", + "can be replaced with \"SUBST_VARS.interp= PREFIX\".", "WARN: Makefile:14: Incomplete SUBST block: SUBST_STAGE.interp missing.") } @@ -56,7 +56,7 @@ func (s *Suite) Test_SubstContext__complete(c *check.C) { t.CheckOutputLines( "NOTE: Makefile:13: The substitution command \"s,@PREFIX@,${PREFIX},g\" " + - "can be replaced with \"SUBST_VARS.p+= PREFIX\".") + "can be replaced with \"SUBST_VARS.p= PREFIX\".") } func (s *Suite) Test_SubstContext__OPSYSVARS(c *check.C) { @@ -78,7 +78,7 @@ func (s *Suite) Test_SubstContext__OPSYSVARS(c *check.C) { t.CheckOutputLines( "NOTE: Makefile:14: The substitution command \"s,@PREFIX@,${PREFIX},g\" " + - "can be replaced with \"SUBST_VARS.prefix+= PREFIX\".") + "can be replaced with \"SUBST_VARS.prefix= PREFIX\".") } func (s *Suite) Test_SubstContext__no_class(c *check.C) { @@ -390,7 +390,7 @@ func (s *Suite) Test_SubstContext_suggestSubstVars(c *check.C) { t.CheckOutputLines( "WARN: subst.mk:6: Please use ${SH:Q} instead of ${SH}.", "NOTE: subst.mk:6: The substitution command \"s,@SH@,${SH},g\" "+ - "can be replaced with \"SUBST_VARS.test+= SH\".", + "can be replaced with \"SUBST_VARS.test= SH\".", "NOTE: subst.mk:7: The substitution command \"s,@SH@,${SH:Q},g\" "+ "can be replaced with \"SUBST_VARS.test+= SH\".", "WARN: subst.mk:8: Please use ${SH:T:Q} instead of ${SH:T}.", @@ -416,9 +416,9 @@ func (s *Suite) Test_SubstContext_suggestSubstVars(c *check.C) { t.CheckOutputLines( "NOTE: subst.mk:6: The substitution command \"s,@SH@,${SH},g\" "+ - "can be replaced with \"SUBST_VARS.test+= SH\".", + "can be replaced with \"SUBST_VARS.test= SH\".", "AUTOFIX: subst.mk:6: Replacing \"SUBST_SED.test+=\\t-e s,@SH@,${SH},g\" "+ - "with \"SUBST_VARS.test+=\\tSH\".", + "with \"SUBST_VARS.test=\\tSH\".", "NOTE: subst.mk:7: The substitution command \"s,@SH@,${SH:Q},g\" "+ "can be replaced with \"SUBST_VARS.test+= SH\".", "AUTOFIX: subst.mk:7: Replacing \"SUBST_SED.test+=\\t-e s,@SH@,${SH:Q},g\" "+ @@ -441,6 +441,46 @@ func (s *Suite) Test_SubstContext_suggestSubstVars(c *check.C) { "with \"SUBST_VARS.test+=\\tSH\".") } +// If the SUBST_CLASS identifier ends with a plus, the generated code must +// use the correct assignment operator and be nicely formatted. +func (s *Suite) Test_SubstContext_suggestSubstVars__plus(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + t.SetUpTool("sh", "SH", AtRunTime) + + mklines := t.NewMkLines("subst.mk", + MkRcsID, + "", + "SUBST_CLASSES+=\t\tgtk+", + "SUBST_STAGE.gtk+ =\tpre-configure", + "SUBST_FILES.gtk+ =\tfilename", + "SUBST_SED.gtk+ +=\t-e s,@SH@,${SH:Q},g", + "SUBST_SED.gtk+ +=\t-e s,@SH@,${SH:Q},g") + + mklines.Check() + + t.CheckOutputLines( + "NOTE: subst.mk:6: The substitution command \"s,@SH@,${SH:Q},g\" "+ + "can be replaced with \"SUBST_VARS.gtk+ = SH\".", + "NOTE: subst.mk:7: The substitution command \"s,@SH@,${SH:Q},g\" "+ + "can be replaced with \"SUBST_VARS.gtk+ += SH\".") + + t.SetUpCommandLine("--show-autofix") + + mklines.Check() + + t.CheckOutputLines( + "NOTE: subst.mk:6: The substitution command \"s,@SH@,${SH:Q},g\" "+ + "can be replaced with \"SUBST_VARS.gtk+ = SH\".", + "AUTOFIX: subst.mk:6: Replacing \"SUBST_SED.gtk+ +=\\t-e s,@SH@,${SH:Q},g\" "+ + "with \"SUBST_VARS.gtk+ =\\tSH\".", + "NOTE: subst.mk:7: The substitution command \"s,@SH@,${SH:Q},g\" "+ + "can be replaced with \"SUBST_VARS.gtk+ += SH\".", + "AUTOFIX: subst.mk:7: Replacing \"SUBST_SED.gtk+ +=\\t-e s,@SH@,${SH:Q},g\" "+ + "with \"SUBST_VARS.gtk+ +=\\tSH\".") +} + // simulateSubstLines only tests some of the inner workings of SubstContext. // It is not realistic for all cases. If in doubt, use MkLines.Check. func simulateSubstLines(t *Tester, texts ...string) { diff --git a/pkgtools/pkglint/files/textproc/lexer.go b/pkgtools/pkglint/files/textproc/lexer.go index 3d22338be95..9687e101bb2 100644 --- a/pkgtools/pkglint/files/textproc/lexer.go +++ b/pkgtools/pkglint/files/textproc/lexer.go @@ -226,6 +226,9 @@ func (l *Lexer) Copy() *Lexer { return &Lexer{l.rest} } func (l *Lexer) Commit(other *Lexer) bool { l.rest = other.rest; return true } // NewByteSet creates a bit mask out of a string like "0-9A-Za-z_". +// To add an actual hyphen to the bit mask, write it as "---" +// (a range from hyphen to hyphen). +// // The bit mask can be used with Lexer.NextBytesSet. func NewByteSet(chars string) *ByteSet { var set ByteSet diff --git a/pkgtools/pkglint/files/trace/tracing.go b/pkgtools/pkglint/files/trace/tracing.go index 0f0556041d0..28b678d6459 100644 --- a/pkgtools/pkglint/files/trace/tracing.go +++ b/pkgtools/pkglint/files/trace/tracing.go @@ -131,6 +131,8 @@ func (t *Tracer) traceCall(args ...interface{}) func() { } // Result marks an argument as a result and is only logged when the function returns. +// +// Usage: defer trace.Call(arg1, arg2, tracing.Result(&result1), tracing.Result(&result2))() func (t *Tracer) Result(rv interface{}) Result { if reflect.ValueOf(rv).Kind() != reflect.Ptr { panic(fmt.Sprintf("Result must be called with a pointer to the result, not %#v.", rv)) diff --git a/pkgtools/pkglint/files/util.go b/pkgtools/pkglint/files/util.go index 296843f73df..049bde05263 100644 --- a/pkgtools/pkglint/files/util.go +++ b/pkgtools/pkglint/files/util.go @@ -349,32 +349,68 @@ func mkopSubst(s string, left bool, from string, right bool, to string, flags st // relpath returns the relative path from the directory "from" // to the filesystem entry "to". -func relpath(from, to string) string { +// +// The relative path is built by going from the "from" directory via the +// pkgsrc root to the "to" filename. This produces the form +// "../../category/package" that is found in DEPENDS and .include lines. +// +// Both from and to are interpreted relative to the current working directory, +// unless they are absolute paths. +// +// This function should only be used if the relative path from one file to +// another cannot be computed in another way. The preferred way is to take +// the relative filenames directly from the .include or exists() where they +// appear. +// +// TODO: Invent data types for all kinds of relative paths that occur in pkgsrc +// and pkglint. Make sure that these paths cannot be accidentally mixed. +func relpath(from, to string) (result string) { + + if trace.Tracing { + defer trace.Call(from, to, trace.Result(&result))() + } + + cfrom := cleanpath(from) + cto := cleanpath(to) - // From "dir" to "dir/subdir/...". - if hasPrefix(to, from) && len(to) > len(from)+1 && to[len(from)] == '/' { - return path.Clean(to[len(from)+1:]) + if cfrom == cto { + return "." } - // Take a shortcut for the most common variant in a complete pkgsrc scan, - // which is to resolve the relative path from a package to the pkgsrc root. - // This avoids unnecessary calls to the filesystem API. - if to == "." { - fromParts := strings.FieldsFunc(from, func(r rune) bool { return r == '/' }) - if len(fromParts) == 3 && !hasPrefix(fromParts[0], ".") && !hasPrefix(fromParts[1], ".") && fromParts[2] == "." { + // Take a shortcut for the common case from "dir" to "dir/subdir/...". + if hasPrefix(cto, cfrom) && len(cto) > len(cfrom)+1 && cto[len(cfrom)] == '/' { + return cleanpath(cto[len(cfrom)+1:]) + } + + // Take a shortcut for the common case from "category/package" to ".". + // This is the most common variant in a complete pkgsrc scan. + if cto == "." { + fromParts := strings.FieldsFunc(cfrom, func(r rune) bool { return r == '/' }) + if len(fromParts) == 2 && !hasPrefix(fromParts[0], ".") && !hasPrefix(fromParts[1], ".") { return "../.." } } - absFrom := abspath(from) - absTo := abspath(to) - rel, err := filepath.Rel(absFrom, absTo) - G.AssertNil(err, "relpath %q %q", from, to) - result := filepath.ToSlash(rel) + if cfrom == "." && !filepath.IsAbs(cto) { + return path.Clean(cto) + } + + absFrom := abspath(cfrom) + absTopdir := abspath(G.Pkgsrc.topdir) + absTo := abspath(cto) + + toTop, err := filepath.Rel(absFrom, absTopdir) + G.AssertNil(err, "relpath from %q to topdir %q", absFrom, absTopdir) + + fromTop, err := filepath.Rel(absTopdir, absTo) + G.AssertNil(err, "relpath from topdir %q to %q", absTopdir, absTo) + + result = cleanpath(filepath.ToSlash(toTop) + "/" + filepath.ToSlash(fromTop)) + if trace.Tracing { - trace.Stepf("relpath from %q to %q = %q", from, to, result) + trace.Stepf("relpath from %q to %q = %q", cfrom, cto, result) } - return result + return } func abspath(filename string) string { @@ -401,6 +437,10 @@ func cleanpath(filename string) string { } } + for len(parts) > 1 && parts[len(parts)-1] == "." { + parts = parts[:len(parts)-1] + } + for i := 2; i+3 < len(parts); /* nothing */ { if parts[i] != ".." && parts[i+1] != ".." && parts[i+2] == ".." && parts[i+3] == ".." { if i+4 == len(parts) || parts[i+4] != ".." { @@ -441,6 +481,11 @@ func (o *Once) FirstTimeSlice(whats ...string) bool { return o.check(crc.Sum64()) } +func (o *Once) Seen(what string) bool { + _, seen := o.seen[crc64.Checksum([]byte(what), crc64.MakeTable(crc64.ECMA))] + return seen +} + func (o *Once) check(key uint64) bool { if _, ok := o.seen[key]; ok { return false @@ -454,6 +499,13 @@ func (o *Once) check(key uint64) bool { // Scope remembers which variables are defined and which are used // in a certain scope, such as a package or a file. +// +// TODO: Decide whether the scope should consider variable assignments +// from the pkgsrc infrastructure. For Package.checkGnuConfigureUseLanguages +// it would be better to ignore them completely. +// +// TODO: Merge this code with Var, which defines essentially the +// same features. type Scope struct { firstDef map[string]MkLine // TODO: Can this be removed? lastDef map[string]MkLine @@ -579,8 +631,9 @@ func (s *Scope) FirstDefinition(varname string) MkLine { mkline := s.firstDef[varname] if mkline != nil && mkline.IsVarassign() { lastLine := s.LastDefinition(varname) - if lastLine != mkline { - //mkline.Notef("FirstDefinition differs from LastDefinition in %s.", mkline.RefTo(lastLine)) + if trace.Tracing && lastLine != mkline { + trace.Stepf("%s: FirstDefinition differs from LastDefinition in %s.", + mkline.String(), mkline.RefTo(lastLine)) } return mkline } @@ -722,148 +775,6 @@ func naturalLess(str1, str2 string) bool { return len1 < len2 } -// RedundantScope checks for redundant variable definitions and for variables -// that are accidentally overwritten. It tries to be as correct as possible -// by not flagging anything that is defined conditionally. -// -// There may be some edge cases though like defining PKGNAME, then evaluating -// it using :=, then defining it again. This pattern is so error-prone that -// it should not appear in pkgsrc at all, thus pkglint doesn't even expect it. -// (Well, except for the PKGNAME case, but that's deep in the infrastructure -// and only affects the "nb13" extension.) -type RedundantScope struct { - vars map[string]*redundantScopeVarinfo - dirLevel int // The number of enclosing directives (.if, .for). - includePath includePath - OnRedundant func(old, new MkLine) - OnOverwrite func(old, new MkLine) -} -type redundantScopeVarinfo struct { - mkline MkLine - includePath includePath - value string -} - -func NewRedundantScope() *RedundantScope { - return &RedundantScope{vars: make(map[string]*redundantScopeVarinfo)} -} - -func (s *RedundantScope) Handle(mkline MkLine) { - if mkline.firstLine == 1 { - s.includePath.push(mkline.Location.Filename) - } else { - s.includePath.popUntil(mkline.Location.Filename) - } - - switch { - case mkline.IsVarassign(): - varname := mkline.Varname() - if s.dirLevel != 0 { - // Since the variable is defined or assigned conditionally, - // it becomes too complicated for pkglint to check all possible - // code paths. Therefore ignore the variable from now on. - s.vars[varname] = nil - break - } - - op := mkline.Op() - value := mkline.Value() - valueNovar := mkline.WithoutMakeVariables(value) - if op == opAssignEval && value == valueNovar { - op = /* effectively */ opAssign - } - - existing, found := s.vars[varname] - if !found { - if op == opAssignShell || op == opAssignEval { - s.vars[varname] = nil // Won't be checked further. - } else { - if op == opAssignAppend { - value = " " + value - } - s.vars[varname] = &redundantScopeVarinfo{mkline, s.includePath.copy(), value} - } - - } else if existing != nil { - if op == opAssign && existing.value == value { - op = /* effectively */ opAssignDefault - } - - switch op { - case opAssign: - if s.includePath.includes(existing.includePath) { - // This is the usual pattern of including a file and - // then overwriting some of them. Although technically - // this overwrites the previous definition, it is not - // worth a warning since this is used a lot and - // intentionally. - } else { - s.OnOverwrite(existing.mkline, mkline) - } - existing.value = value - case opAssignAppend: - existing.value += " " + value - case opAssignDefault: - if existing.includePath.includes(s.includePath) { - s.OnRedundant(mkline, existing.mkline) - } else if s.includePath.includes(existing.includePath) || s.includePath.equals(existing.includePath) { - s.OnRedundant(existing.mkline, mkline) - } - case opAssignShell, opAssignEval: - s.vars[varname] = nil // Won't be checked further. - } - } - - case mkline.IsDirective(): - switch mkline.Directive() { - case "for", "if", "ifdef", "ifndef": - s.dirLevel++ - case "endfor", "endif": - s.dirLevel-- - } - } -} - -type includePath struct { - files []string -} - -func (p *includePath) push(filename string) { - p.files = append(p.files, filename) -} - -func (p *includePath) popUntil(filename string) { - for p.files[len(p.files)-1] != filename { - p.files = p.files[:len(p.files)-1] - } -} - -func (p *includePath) includes(other includePath) bool { - for i, filename := range p.files { - if i < len(other.files) && other.files[i] == filename { - continue - } - return false - } - return len(p.files) < len(other.files) -} - -func (p *includePath) equals(other includePath) bool { - if len(p.files) != len(other.files) { - return false - } - for i, filename := range p.files { - if other.files[i] != filename { - return false - } - } - return true -} - -func (p *includePath) copy() includePath { - return includePath{append([]string(nil), p.files...)} -} - // IsPrefs returns whether the given file, when included, loads the user // preferences. func IsPrefs(filename string) bool { @@ -1004,7 +915,6 @@ func seeGuide(sectionName, sectionID string) string { func wrap(max int, lines ...string) []string { var wrapped []string var sb strings.Builder - nonSpace := textproc.Space.Inverse() for _, line := range lines { @@ -1024,7 +934,7 @@ func wrap(max int, lines ...string) []string { for !lexer.EOF() { bol := len(lexer.Rest()) == len(line) space := lexer.NextBytesSet(textproc.Space) - word := lexer.NextBytesSet(nonSpace) + word := lexer.NextBytesSet(notSpace) if bol && sb.Len() > 0 { space = " " @@ -1169,3 +1079,26 @@ func (si *StringInterner) Intern(str string) string { si.strs[key] = key return key } + +// StringSets stores unique strings in insertion order. +type StringSet struct { + Elements []string + seen map[string]struct{} +} + +func NewStringSet() StringSet { + return StringSet{nil, make(map[string]struct{})} +} + +func (s *StringSet) Add(element string) { + if _, found := s.seen[element]; !found { + s.seen[element] = struct{}{} + s.Elements = append(s.Elements, element) + } +} + +func (s *StringSet) AddAll(elements []string) { + for _, element := range elements { + s.Add(element) + } +} diff --git a/pkgtools/pkglint/files/util_test.go b/pkgtools/pkglint/files/util_test.go index 743a66c95ca..9bc99956c96 100644 --- a/pkgtools/pkglint/files/util_test.go +++ b/pkgtools/pkglint/files/util_test.go @@ -73,39 +73,84 @@ func (s *Suite) Test_tabWidth(c *check.C) { } func (s *Suite) Test_cleanpath(c *check.C) { - c.Check(cleanpath("simple/path"), equals, "simple/path") - c.Check(cleanpath("/absolute/path"), equals, "/absolute/path") + test := func(from, to string) { + c.Check(cleanpath(from), equals, to) + } + + test("simple/path", "simple/path") + test("/absolute/path", "/absolute/path") // Single dot components are removed, unless it's the only component of the path. - c.Check(cleanpath("./././."), equals, ".") - c.Check(cleanpath("./././"), equals, ".") - c.Check(cleanpath("dir/multi/././/file"), equals, "dir/multi/file") - c.Check(cleanpath("dir/"), equals, "dir") + test("./././.", ".") + test("./././", ".") + test("dir/multi/././/file", "dir/multi/file") + test("dir/", "dir") + + test("dir/", "dir") // Components like aa/bb/../.. are removed, but not in the initial part of the path, // and only if they are not followed by another "..". - c.Check(cleanpath("dir/../dir/../dir/../dir/subdir/../../Makefile"), equals, "dir/../dir/../dir/../Makefile") - c.Check(cleanpath("111/222/../../333/444/../../555/666/../../777/888/9"), equals, "111/222/../../777/888/9") - c.Check(cleanpath("1/2/3/../../4/5/6/../../7/8/9/../../../../10"), equals, "1/2/3/../../4/7/8/9/../../../../10") - c.Check(cleanpath("cat/pkg.v1/../../cat/pkg.v2/Makefile"), equals, "cat/pkg.v1/../../cat/pkg.v2/Makefile") - c.Check(cleanpath("aa/../../../../../a/b/c/d"), equals, "aa/../../../../../a/b/c/d") - c.Check(cleanpath("aa/bb/../../../../a/b/c/d"), equals, "aa/bb/../../../../a/b/c/d") - c.Check(cleanpath("aa/bb/cc/../../../a/b/c/d"), equals, "aa/bb/cc/../../../a/b/c/d") - c.Check(cleanpath("aa/bb/cc/dd/../../a/b/c/d"), equals, "aa/bb/a/b/c/d") - c.Check(cleanpath("aa/bb/cc/dd/ee/../a/b/c/d"), equals, "aa/bb/cc/dd/ee/../a/b/c/d") - c.Check(cleanpath("../../../../../a/b/c/d"), equals, "../../../../../a/b/c/d") - c.Check(cleanpath("aa/../../../../a/b/c/d"), equals, "aa/../../../../a/b/c/d") - c.Check(cleanpath("aa/bb/../../../a/b/c/d"), equals, "aa/bb/../../../a/b/c/d") - c.Check(cleanpath("aa/bb/cc/../../a/b/c/d"), equals, "aa/bb/cc/../../a/b/c/d") - c.Check(cleanpath("aa/bb/cc/dd/../a/b/c/d"), equals, "aa/bb/cc/dd/../a/b/c/d") - c.Check(cleanpath("aa/../cc/../../a/b/c/d"), equals, "aa/../cc/../../a/b/c/d") + test("dir/../dir/../dir/../dir/subdir/../../Makefile", "dir/../dir/../dir/../Makefile") + test("111/222/../../333/444/../../555/666/../../777/888/9", "111/222/../../777/888/9") + test("1/2/3/../../4/5/6/../../7/8/9/../../../../10", "1/2/3/../../4/7/8/9/../../../../10") + test("cat/pkg.v1/../../cat/pkg.v2/Makefile", "cat/pkg.v1/../../cat/pkg.v2/Makefile") + test("aa/../../../../../a/b/c/d", "aa/../../../../../a/b/c/d") + test("aa/bb/../../../../a/b/c/d", "aa/bb/../../../../a/b/c/d") + test("aa/bb/cc/../../../a/b/c/d", "aa/bb/cc/../../../a/b/c/d") + test("aa/bb/cc/dd/../../a/b/c/d", "aa/bb/a/b/c/d") + test("aa/bb/cc/dd/ee/../a/b/c/d", "aa/bb/cc/dd/ee/../a/b/c/d") + test("../../../../../a/b/c/d", "../../../../../a/b/c/d") + test("aa/../../../../a/b/c/d", "aa/../../../../a/b/c/d") + test("aa/bb/../../../a/b/c/d", "aa/bb/../../../a/b/c/d") + test("aa/bb/cc/../../a/b/c/d", "aa/bb/cc/../../a/b/c/d") + test("aa/bb/cc/dd/../a/b/c/d", "aa/bb/cc/dd/../a/b/c/d") + test("aa/../cc/../../a/b/c/d", "aa/../cc/../../a/b/c/d") // The initial 2 components of the path are typically category/package, when // pkglint is called from the pkgsrc top-level directory. // This path serves as the context and therefore is always kept. - c.Check(cleanpath("aa/bb/../../cc/dd/../../ee/ff"), equals, "aa/bb/../../ee/ff") - c.Check(cleanpath("aa/bb/../../cc/dd/../.."), equals, "aa/bb/../..") - c.Check(cleanpath("aa/bb/cc/dd/../.."), equals, "aa/bb") + test("aa/bb/../../cc/dd/../../ee/ff", "aa/bb/../../ee/ff") + test("aa/bb/../../cc/dd/../..", "aa/bb/../..") + test("aa/bb/cc/dd/../..", "aa/bb") + test("aa/bb/../../cc/dd/../../ee/ff/buildlink3.mk", "aa/bb/../../ee/ff/buildlink3.mk") + test("./aa/bb/../../cc/dd/../../ee/ff/buildlink3.mk", "aa/bb/../../ee/ff/buildlink3.mk") + + test("../.", "..") + test("../././././././.", "..") + test(".././././././././", "..") +} + +func (s *Suite) Test_relpath(c *check.C) { + t := s.Init(c) + + t.Chdir(".") + t.Check(G.Pkgsrc.topdir, equals, t.tmpdir) + + test := func(from, to, result string) { + c.Check(relpath(from, to), equals, result) + } + + test("some/dir", "some/directory", "../../some/directory") + + test("category/package/.", ".", "../..") + + // This case is handled by one of the shortcuts that avoid file system access. + test( + "./.", + "x11/frameworkintegration/../../meta-pkgs/kde/kf5.mk", + "meta-pkgs/kde/kf5.mk") + + // This happens when "pkglint -r x11" is run. + G.Pkgsrc.topdir = "x11/.." + + test( + "./.", + "x11/frameworkintegration/../../meta-pkgs/kde/kf5.mk", + "meta-pkgs/kde/kf5.mk") + test( + "x11/..", + "x11/frameworkintegration/../../meta-pkgs/kde/kf5.mk", + "meta-pkgs/kde/kf5.mk") } // Relpath is called so often that handling the most common calls @@ -119,9 +164,9 @@ func (s *Suite) Test_relpath__quick(c *check.C) { test("some/dir", "some/dir/../..", "../..") test("some/dir", "some/dir/./././../..", "../..") test("some/dir", "some/dir/", ".") - test("some/dir", "some/directory", "../directory") - test("category/package/.", ".", "../..") + test("some/dir", ".", "../..") + test("some/dir/.", ".", "../..") } // This is not really an internal error but won't happen in practice anyway. @@ -129,10 +174,14 @@ func (s *Suite) Test_relpath__quick(c *check.C) { func (s *Suite) Test_relpath__failure_on_Windows(c *check.C) { t := s.Init(c) - if runtime.GOOS == "windows" { + if runtime.GOOS == "windows" && hasPrefix(t.tmpdir, "C:/") { t.ExpectPanic( func() { relpath("c:/", "d:/") }, - "Pkglint internal error: relpath \"c:/\" \"d:/\": Rel: can't make d:/ relative to c:/") + sprintf( + "Pkglint internal error: "+ + "relpath from topdir %q to %q: "+ + "Rel: can't make %s relative to %s", + t.tmpdir, "D:/", "D:/", t.tmpdir)) } } @@ -710,28 +759,3 @@ func (s *Suite) Test_StringInterner(c *check.C) { t.Check(si.Intern("Hello, world"), equals, "Hello, world") t.Check(si.Intern("Hello, world"[0:5]), equals, "Hello") } - -func (s *Suite) Test_includePath_includes(c *check.C) { - t := s.Init(c) - - path := func(locations ...string) includePath { - return includePath{locations} - } - - var ( - m = path("Makefile") - mc = path("Makefile", "Makefile.common") - mco = path("Makefile", "Makefile.common", "other.mk") - mo = path("Makefile", "other.mk") - ) - - t.Check(m.includes(m), equals, false) - - t.Check(m.includes(mc), equals, true) - t.Check(m.includes(mco), equals, true) - t.Check(mc.includes(mco), equals, true) - - t.Check(mc.includes(m), equals, false) - t.Check(mc.includes(mo), equals, false) - t.Check(mo.includes(mc), equals, false) -} diff --git a/pkgtools/pkglint/files/var.go b/pkgtools/pkglint/files/var.go index 9050bb62597..553555c2997 100644 --- a/pkgtools/pkglint/files/var.go +++ b/pkgtools/pkglint/files/var.go @@ -2,23 +2,275 @@ package pkglint // Var describes a variable in a Makefile snippet. // -// TODO: Remove this type in June 2019 if it is still a stub. +// It keeps track of all places where the variable is accessed or modified (see +// ReadLocations, WriteLocations) and provides information for further static +// analysis, such as: +// +// * Whether the variable value is constant, and if so, what the constant value +// is (see Constant, ConstantValue). +// +// * What its (approximated) value is, either including values from the pkgsrc +// infrastructure (see ValueInfra) or excluding them (Value). +// +// * On which other variables this variable depends (see Conditional, +// ConditionalVars). +// +// TODO: Decide how to handle OPSYS-specific variables, such as LDFLAGS.SunOS. +// +// TODO: Decide how to handle parameterized variables, such as SUBST_MESSAGE.doc. type Var struct { Name string - Type *Vartype + + // 0 = neither written nor read + // 1 = constant + // 2 = constant and read; further writes will make it non-constant + // 3 = not constant anymore + constantState uint8 + constantValue string + + value string + valueInfra string + + readLocations []MkLine + writeLocations []MkLine + + conditional bool + conditionalVars StringSet + + refs StringSet } -func NewVar(name string) *Var { return &Var{name, nil} } +func NewVar(name string) *Var { + return &Var{name, 0, "", "", "", nil, nil, false, NewStringSet(), NewStringSet()} +} -// Constant returns whether the variable is only ever assigned a single value, -// without being dependent on any other variable. +// Conditional returns whether the variable value depends on other variables. +func (v *Var) Conditional() bool { + return v.conditional +} + +// ConditionalVars returns all variables in conditions on which the value of +// this variable depends. // -// Multiple assignments (such as VAR=1, VAR+=2, VAR+=3) are considered constant -// as well, as long as the variable is not used in-between these assignments. -// That is, no .include or .if may appear there, and none of the ::= modifiers may -// be involved. +// The returned slice must not be modified. +func (v *Var) ConditionalVars() []string { + return v.conditionalVars.Elements +} + +// Refs returns all variables on which this variable depends. These are: +// +// Variables that are referenced in the value, such as in VAR=${OTHER}. // -// Simple .for loops that append to the variable are ok though. -func (v *Var) Constant() bool { return false } +// Variables that are used in conditions that enclose one of the assignments +// to this variable, such as .if ${OPSYS} == NetBSD. +// +// Variables that are used in .for loops in which this variable is assigned +// a value, such as DIRS in: +// .for dir in ${DIRS} +// VAR+=${dir} +// .endfor +func (v *Var) Refs() []string { + return v.refs.Elements +} -func (v *Var) ConstantValue() string { return "" } +// AddRef marks this variable as being dependent on the given variable name. +// This can be used for the .for loops mentioned in Refs. +func (v *Var) AddRef(varname string) { + v.refs.Add(varname) +} + +// Constant returns whether the variable's value is a constant. +// It may reference other variables since these references are evaluated +// lazily, when the variable value is actually needed. +// +// Multiple assignments (such as VAR=1, VAR+=2, VAR+=3) are considered to +// form a single constant as well, as long as the variable is not read before +// or in-between these assignments. The definition of "read" is very strict +// here since every mention of the variable counts. This may prevent some +// essentially constant values from being detected as such, but these special +// cases may be implemented later. +// +// TODO: Simple .for loops that append to the variable are ok as well. +// (This needs to be worded more precisely since that part potentially +// adds a lot of complexity to the whole data structure.) +// +// Variable assignments in the pkgsrc infrastructure are taken into account +// for determining whether a variable is constant. +func (v *Var) Constant() bool { + return v.constantState == 1 || v.constantState == 2 +} + +// ConstantValue returns the constant value of the variable. +// It is only allowed when Constant() returns true. +// +// Variable assignments in the pkgsrc infrastructure are taken into account +// for determining the constant value. +func (v *Var) ConstantValue() string { + G.Assertf(v.Constant(), "Variable must be constant.") + return v.constantValue +} + +// Value returns the (approximated) value of the variable, taking into account +// all variable assignments that happen outside the pkgsrc infrastructure. +// +// For variables that are conditionally assigned (as in .if/.else), the +// returned value is not reliable. It may be the value from either branch, or +// even the combined value of both branches. +// +// See Constant and ConstantValue for more reliable information. +func (v *Var) Value() string { + return v.value +} + +// ValueInfra returns the (approximated) value of the variable, taking into +// account all variable assignments from the package, the user and the pkgsrc +// infrastructure. +// +// For variables that are conditionally assigned (as in .if/.else), the +// returned value is not reliable. It may be the value from either branch, or +// even the combined value of both branches. +// +// See Constant and ConstantValue for more reliable information, but these +// ignore assignments from the infrastructure. +func (v *Var) ValueInfra() string { + return v.valueInfra +} + +// ReadLocations returns the locations where the variable is read, such as +// in ${VAR} or defined(VAR) or empty(VAR). +// +// Uses inside conditionals are included, no matter whether they are actually +// reachable in practice. +// +// Indirect uses through other variables (such as VAR2=${VAR}, VAR3=${VAR2}) +// are not listed. +// +// Variable uses in the pkgsrc infrastructure are taken into account. +func (v *Var) ReadLocations() []MkLine { + return v.readLocations +} + +// WriteLocations returns the locations where the variable is modified. +// +// Assignments inside conditionals are included, no matter whether they are actually +// reachable in practice. +// +// Variable assignments in the pkgsrc infrastructure are taken into account. +func (v *Var) WriteLocations() []MkLine { + return v.writeLocations +} + +func (v *Var) Read(mkline MkLine) { + v.readLocations = append(v.readLocations, mkline) + v.constantState = [...]uint8{3, 2, 2, 3}[v.constantState] +} + +// Write marks the variable as being assigned in the given line. +// Only standard assignments (VAR=value) are handled. +// Side-effect assignments (${VAR::=value}) are not handled here since +// they don't occur in practice. +func (v *Var) Write(mkline MkLine, conditional bool, conditionVarnames ...string) { + G.Assertf(mkline.Varname() == v.Name, "wrong variable name") + + v.writeLocations = append(v.writeLocations, mkline) + + if conditional || len(conditionVarnames) > 0 { + v.conditional = true + } + v.conditionalVars.AddAll(conditionVarnames) + + v.refs.AddAll(mkline.DetermineUsedVariables()) + v.refs.AddAll(conditionVarnames) + + v.update(mkline, &v.valueInfra) + if !G.Pkgsrc.IsInfra(mkline.Line.Filename) { + v.update(mkline, &v.value) + } + + v.updateConstantValue(mkline) +} + +func (v *Var) update(mkline MkLine, update *string) { + firstWrite := len(v.writeLocations) == 1 + if v.Conditional() && !firstWrite { + return + } + + value := mkline.Value() + switch mkline.Op() { + case opAssign, opAssignEval: + *update = value + + case opAssignDefault: + if firstWrite { + *update = value + } + + case opAssignAppend: + *update += " " + value + + case opAssignShell: + // Ignore these for now. + // Later it might be useful to parse the shell commands to + // evaluate simple commands like "test && echo yes || echo no". + } +} + +func (v *Var) updateConstantValue(mkline MkLine) { + if v.constantState == 3 { + return + } + + // Even if the variable references other variables, this does not + // influence whether the variable is considered constant. (Except + // for the := operator.) + // + // Strictly speaking, the referenced variables must be still + // be constant at the end of loading the complete package. + // (And even after that, because of the ::= modifier. But luckily + // almost no one knows that modifier.) + + if v.Conditional() { + v.constantState = 3 + v.constantValue = "" + return + } + + value := mkline.Value() + switch mkline.Op() { + case opAssign: + v.constantValue = value + + case opAssignEval: + if value != mkline.WithoutMakeVariables(value) { + // To leave the variable in the constant state, the current value + // of the referenced variables would need to be resolved. + // + // This in turn requires the proper scope for resolving variable + // references. Furthermore, the referenced variables must be + // constant at this point. Later changes to these variables + // can be ignored though. + // + // Because this sounds complicated to implement, the variable + // is marked as non-constant for now. + v.constantState = 3 + v.constantValue = "" + } else { + v.constantValue = value + } + + case opAssignDefault: + if v.constantState == 0 { + v.constantValue = value + } + + case opAssignAppend: + v.constantValue += " " + value + + case opAssignShell: + v.constantState = 3 + v.constantValue = "" + } + + v.constantState = [...]uint8{1, 1, 3, 3}[v.constantState] +} diff --git a/pkgtools/pkglint/files/var_test.go b/pkgtools/pkglint/files/var_test.go index eb37ff58616..a11fa4aa145 100644 --- a/pkgtools/pkglint/files/var_test.go +++ b/pkgtools/pkglint/files/var_test.go @@ -2,18 +2,336 @@ package pkglint import "gopkg.in/check.v1" -func (s *Suite) Test_Var_Constant(c *check.C) { +func (s *Suite) Test_Var_ConstantValue__assign(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME=\toverwritten"), false) + + t.Check(v.ConstantValue(), equals, "overwritten") +} + +// Variables that reference other variable are considered constants. +// Even if these referenced variables change their value in-between, +// this does not affect the constant-ness of this variable, since the +// references are resolved lazily. +func (s *Suite) Test_Var_ConstantValue__assign_reference(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME=\t${OTHER}"), false) + + t.Check(v.Constant(), equals, true) +} + +func (s *Suite) Test_Var_ConstantValue__assign_eval_reference(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME:=\t${OTHER}"), false) + + // To analyze this case correctly, pkglint would have to know + // the current value of ${OTHER} in line 124. For that it would + // need the complete scope including all other variables. + // + // As of March 2019 this is not implemented, therefore pkglint + // doesn't treat the variable as constant, to prevent wrong warnings. + t.Check(v.Constant(), equals, false) +} + +func (s *Suite) Test_Var_ConstantValue__assign_conditional(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + t.Check(v.ConditionalVars(), check.IsNil) + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tconditional"), true, "OPSYS") + + t.Check(v.Constant(), equals, false) +} + +func (s *Suite) Test_Var_ConstantValue__default(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME?=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME?=\tignored"), false) + + t.Check(v.ConstantValue(), equals, "value") +} + +func (s *Suite) Test_Var_ConstantValue__eval_then_default(c *check.C) { + t := s.Init(c) + v := NewVar("VARNAME") - // FIXME: Replace this test with an actual use case. + v.Write(t.NewMkLine("buildlink3.mk", 123, "VARNAME:=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("builtin.mk", 124, "VARNAME?=\tignored"), false) - c.Check(v.Constant(), equals, false) + t.Check(v.ConstantValue(), equals, "value") } -func (s *Suite) Test_Var_ConstantValue(c *check.C) { +func (s *Suite) Test_Var_ConstantValue__append(c *check.C) { + t := s.Init(c) + v := NewVar("VARNAME") - // FIXME: Replace this test with an actual use case. + v.Write(t.NewMkLine("write.mk", 123, "VARNAME+=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, " value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME+=\tappended"), false) + + t.Check(v.ConstantValue(), equals, " value appended") +} + +func (s *Suite) Test_Var_ConstantValue__eval(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME:=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME:=\toverwritten"), false) + + t.Check(v.ConstantValue(), equals, "overwritten") +} + +// Variables that are based on running shell commands are never constant. +func (s *Suite) Test_Var_ConstantValue__shell(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME!=\techo hello"), false) + + t.Check(v.Constant(), equals, false) +} + +func (s *Suite) Test_Var_ConstantValue__referenced_before(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + // Since the value of VARNAME escapes here, the value is not + // guaranteed to be the same in all evaluations of ${VARNAME}. + // For example, OTHER may be used at load time in an .if + // condition. + v.Read(t.NewMkLine("readwrite.mk", 123, "OTHER=\t${VARNAME}")) + + t.Check(v.Constant(), equals, false) + + v.Write(t.NewMkLine("readwrite.mk", 124, "VARNAME=\tvalue"), false) + + t.Check(v.Constant(), equals, false) +} + +func (s *Suite) Test_Var_ConstantValue__referenced_in_between(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("readwrite.mk", 123, "VARNAME=\tvalue"), false) + + t.Check(v.ConstantValue(), equals, "value") + + // Since the value of VARNAME escapes here, the value is not + // guaranteed to be the same in all evaluations of ${VARNAME}. + // For example, OTHER may be used at load time in an .if + // condition. + v.Read(t.NewMkLine("readwrite.mk", 124, "OTHER=\t${VARNAME}")) + + t.Check(v.ConstantValue(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 125, "VARNAME=\toverwritten"), false) + + t.Check(v.Constant(), equals, false) +} + +func (s *Suite) Test_Var_ConditionalVars(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + t.Check(v.Conditional(), equals, false) + t.Check(v.ConditionalVars(), check.IsNil) + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tconditional"), true, "OPSYS") + + t.Check(v.Constant(), equals, false) + t.Check(v.Conditional(), equals, true) + t.Check(v.ConditionalVars(), deepEquals, []string{"OPSYS"}) + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME=\tconditional"), true, "OPSYS") + + t.Check(v.Conditional(), equals, true) + t.Check(v.ConditionalVars(), deepEquals, []string{"OPSYS"}) +} + +func (s *Suite) Test_Var_Value__initial_conditional_write(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME:=\toverwritten conditionally"), true, "OPSYS") + + // Since there is no previous value, the simplest choice is to just + // take the first seen value, no matter if that value is conditional + // or not. + t.Check(v.Conditional(), equals, true) + t.Check(v.Constant(), equals, false) + t.Check(v.Value(), equals, "overwritten conditionally") +} + +func (s *Suite) Test_Var_Value__conditional_write_after_unconditional(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine("write.mk", 123, "VARNAME=\tvalue"), false) + + t.Check(v.Value(), equals, "value") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME+=\tappended"), false) + + t.Check(v.Value(), equals, "value appended") + + v.Write(t.NewMkLine("write.mk", 124, "VARNAME:=\toverwritten conditionally"), true, "OPSYS") + + // When there is a previous value, it's probably best to keep + // that value since this way the following code results in the + // most generic value: + // VAR= generic + // .if ${OPSYS} == NetBSD + // VAR= specific + // .endif + // The value stays the same, still it is marked as conditional and therefore + // not constant anymore. + t.Check(v.Conditional(), equals, true) + t.Check(v.Constant(), equals, false) + t.Check(v.Value(), equals, "value appended") +} + +func (s *Suite) Test_Var_Value__infrastructure(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine(t.File("write.mk"), 123, "VARNAME=\tvalue"), false) + + t.Check(v.Value(), equals, "value") + + v.Write(t.NewMkLine(t.File("mk/write.mk"), 123, "VARNAME=\tinfra"), false) + + t.Check(v.Value(), equals, "value") + + v.Write(t.NewMkLine(t.File("wip/mk/write.mk"), 123, "VARNAME=\twip infra"), false) + + t.Check(v.Value(), equals, "value") +} + +func (s *Suite) Test_Var_ValueInfra(c *check.C) { + t := s.Init(c) + + v := NewVar("VARNAME") + + v.Write(t.NewMkLine(t.File("write.mk"), 123, "VARNAME=\tvalue"), false) + + t.Check(v.ValueInfra(), equals, "value") + + v.Write(t.NewMkLine(t.File("mk/write.mk"), 123, "VARNAME=\tinfra"), false) + + t.Check(v.ValueInfra(), equals, "infra") + + v.Write(t.NewMkLine(t.File("wip/mk/write.mk"), 123, "VARNAME=\twip infra"), false) + + t.Check(v.ValueInfra(), equals, "wip infra") +} + +func (s *Suite) Test_Var_ReadLocations(c *check.C) { + t := s.Init(c) + + v := NewVar("VAR") + + t.Check(v.ReadLocations(), check.IsNil) + + mkline123 := t.NewMkLine("read.mk", 123, "OTHER=\t${VAR}") + v.Read(mkline123) + + t.Check(v.ReadLocations(), deepEquals, []MkLine{mkline123}) + + mkline124 := t.NewMkLine("read.mk", 124, "OTHER=\t${VAR} ${VAR}") + v.Read(mkline124) + v.Read(mkline124) + + // For now, count every read of the variable. I'm not yet sure + // whether that's the best way or whether to make the lines unique. + t.Check(v.ReadLocations(), deepEquals, []MkLine{mkline123, mkline124, mkline124}) +} + +func (s *Suite) Test_Var_WriteLocations(c *check.C) { + t := s.Init(c) + + v := NewVar("VAR") + + t.Check(v.WriteLocations(), check.IsNil) + + mkline123 := t.NewMkLine("write.mk", 123, "VAR=\tvalue") + v.Write(mkline123, false) + + t.Check(v.WriteLocations(), deepEquals, []MkLine{mkline123}) + + // Multiple writes from the same line may happen because of a .for loop. + mkline125 := t.NewMkLine("write.mk", 125, "VAR+=\t${var}") + v.Write(mkline125, false) + v.Write(mkline125, false) + + // For now, count every write of the variable. I'm not yet sure + // whether that's the best way or whether to make the lines unique. + t.Check(v.WriteLocations(), deepEquals, []MkLine{mkline123, mkline125, mkline125}) +} + +func (s *Suite) Test_Var_Refs(c *check.C) { + t := s.Init(c) + + v := NewVar("VAR") + + t.Check(v.Refs(), check.IsNil) + + // The referenced variables are taken from the mkline. + // They don't need to be passed separately. + v.Write(t.NewMkLine("write.mk", 123, "VAR=${OTHER} ${${OPSYS} == NetBSD :? ${THEN} : ${ELSE}}"), true, "COND") + + v.AddRef("FOR") - c.Check(v.ConstantValue(), equals, "") + t.Check(v.Refs(), deepEquals, []string{"OTHER", "OPSYS", "THEN", "ELSE", "COND", "FOR"}) } diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go index 9a9d8c8f585..eb9e53f0584 100644 --- a/pkgtools/pkglint/files/vardefs.go +++ b/pkgtools/pkglint/files/vardefs.go @@ -37,7 +37,7 @@ func (src *Pkgsrc) InitVartypes() { pkg := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, ""+ "Makefile: set, use; "+ - "buildlink3.mk, builtin.mk:; "+ + "buildlink3.mk, builtin.mk: none; "+ "Makefile.*, *.mk: default, set, use") } @@ -45,7 +45,7 @@ func (src *Pkgsrc) InitVartypes() { pkgload := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, ""+ "Makefile: set, use, use-loadtime; "+ - "buildlink3.mk, builtin.mk:; "+ + "buildlink3.mk, builtin.mk: none; "+ "Makefile.*, *.mk: default, set, use, use-loadtime") } @@ -54,21 +54,29 @@ func (src *Pkgsrc) InitVartypes() { pkglist := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, ""+ "Makefile, Makefile.common, options.mk: append, default, set, use; "+ - "buildlink3.mk, builtin.mk:; "+ + "buildlink3.mk, builtin.mk: none; "+ "*.mk: append, default, use") } + // Some package-defined lists may also be appended in buildlink3.mk files, + // for example platform-specific CFLAGS and LDFLAGS. + pkglistbl3 := func(varname string, kindOfList KindOfList, checker *BasicType) { + acl(varname, kindOfList, checker, ""+ + "Makefile, Makefile.common, options.mk: append, default, set, use; "+ + "buildlink3.mk, builtin.mk, *.mk: append, default, use") + } + // sys declares a user-defined or system-defined variable that must not be modified by packages. // // It also must not be used in buildlink3.mk and builtin.mk files or at load-time, // since the system/user preferences may not have been loaded when these files are included. sys := func(varname string, kindOfList KindOfList, checker *BasicType) { - acl(varname, kindOfList, checker, "buildlink3.mk:; *: use") + acl(varname, kindOfList, checker, "buildlink3.mk: none; *: use") } // usr declares a user-defined variable that must not be modified by packages. usr := func(varname string, kindOfList KindOfList, checker *BasicType) { - acl(varname, kindOfList, checker, "buildlink3.mk:; *: use-loadtime, use") + acl(varname, kindOfList, checker, "buildlink3.mk: none; *: use-loadtime, use") } // sysload declares a system-provided variable that may already be used at load time. @@ -81,7 +89,7 @@ func (src *Pkgsrc) InitVartypes() { } cmdline := func(varname string, kindOfList KindOfList, checker *BasicType) { - acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk:; *: use-loadtime, use") + acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk: none; *: use-loadtime, use") } compilerLanguages := enum( @@ -100,7 +108,10 @@ func (src *Pkgsrc) InitVartypes() { } } } - for _, language := range [...]string{"ada", "c", "c99", "c++", "c++11", "fortran", "fortran77", "java", "objc", "obj-c++"} { + alwaysAvailable := [...]string{ + "ada", "c", "c99", "c++", "c++11", "c++14", + "fortran", "fortran77", "java", "objc", "obj-c++"} + for _, language := range alwaysAvailable { languages[language] = true } @@ -153,8 +164,8 @@ func (src *Pkgsrc) InitVartypes() { return enum(defval) } - // enumFromDirs reads the directories from category, takes all - // that have a single number in them and ranks them from earliest + // enumFromDirs reads the directories from category, takes all that have + // a single number in them (such as php72) and ranks them from earliest // to latest. // // If the directories cannot be found, the allowed values are taken @@ -232,7 +243,7 @@ func (src *Pkgsrc) InitVartypes() { usr("CROSSBASE", lkNone, BtPathname) usr("VARBASE", lkNone, BtPathname) acl("X11_TYPE", lkNone, enum("modular native"), "*: use-loadtime, use") - usr("X11BASE", lkNone, BtPathname) + acl("X11BASE", lkNone, BtPathname, "*: use-loadtime, use") usr("MOTIFBASE", lkNone, BtPathname) usr("PKGINFODIR", lkNone, BtPathname) usr("PKGMANDIR", lkNone, BtPathname) @@ -294,7 +305,7 @@ func (src *Pkgsrc) InitVartypes() { usrpkg := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, ""+ "Makefile: default, set, use, use-loadtime; "+ - "buildlink3.mk, builtin.mk:; "+ + "buildlink3.mk, builtin.mk: none; "+ "Makefile.*, *.mk: default, set, use, use-loadtime; "+ "*: use-loadtime, use") } @@ -488,10 +499,10 @@ func (src *Pkgsrc) InitVartypes() { // some other variables, sorted alphabetically - acl(".CURDIR", lkNone, BtPathname, "buildlink3.mk:; *: use, use-loadtime") - acl(".IMPSRC", lkShell, BtPathname, "buildlink3.mk:; *: use, use-loadtime") - acl(".TARGET", lkNone, BtPathname, "buildlink3.mk:; *: use, use-loadtime") - acl("@", lkNone, BtPathname, "buildlink3.mk:; *: use, use-loadtime") + acl(".CURDIR", lkNone, BtPathname, "buildlink3.mk: none; *: use, use-loadtime") + acl(".IMPSRC", lkShell, BtPathname, "buildlink3.mk: none; *: use, use-loadtime") + acl(".TARGET", lkNone, BtPathname, "buildlink3.mk: none; *: use, use-loadtime") + acl("@", lkNone, BtPathname, "buildlink3.mk: none; *: use, use-loadtime") acl("ALL_ENV", lkShell, BtShellWord, "") acl("ALTERNATIVES_FILE", lkNone, BtFileName, "") acl("ALTERNATIVES_SRC", lkShell, BtPathname, "") @@ -578,8 +589,8 @@ func (src *Pkgsrc) InitVartypes() { acl("CATEGORIES", lkShell, BtCategory, "Makefile: set, append; Makefile.common: set, default, append") sysload("CC_VERSION", lkNone, BtMessage) sysload("CC", lkNone, BtShellCommand) - pkglist("CFLAGS", lkShell, BtCFlag) // may also be changed by the user - pkglist("CFLAGS.*", lkShell, BtCFlag) // may also be changed by the user + pkglistbl3("CFLAGS", lkShell, BtCFlag) // may also be changed by the user + pkglistbl3("CFLAGS.*", lkShell, BtCFlag) // may also be changed by the user acl("CHECK_BUILTIN", lkNone, BtYesNo, "builtin.mk: default; Makefile: set") acl("CHECK_BUILTIN.*", lkNone, BtYesNo, "Makefile, options.mk, buildlink3.mk: set; builtin.mk: default, use-loadtime; *: use-loadtime") acl("CHECK_FILES_SKIP", lkShell, BtBasicRegularExpression, "Makefile, Makefile.common: append") @@ -751,7 +762,7 @@ func (src *Pkgsrc) InitVartypes() { pkg("GITHUB_TYPE", lkNone, enum("tag release")) pkg("GMAKE_REQD", lkNone, BtVersion) acl("GNU_ARCH", lkNone, enum("mips"), "") - acl("GNU_ARCH.*", lkNone, BtIdentifier, "buildlink3.mk:; *: set, use") + acl("GNU_ARCH.*", lkNone, BtIdentifier, "buildlink3.mk: none; *: set, use") acl("GNU_CONFIGURE", lkNone, BtYes, "Makefile, Makefile.common: set") acl("GNU_CONFIGURE_INFODIR", lkNone, BtPathname, "Makefile, Makefile.common: set") acl("GNU_CONFIGURE_LIBDIR", lkNone, BtPathname, "Makefile, Makefile.common: set") @@ -807,8 +818,8 @@ func (src *Pkgsrc) InitVartypes() { usr("KRB5_DEFAULT", lkNone, enum("heimdal mit-krb5")) sys("KRB5_TYPE", lkNone, BtIdentifier) sys("LD", lkNone, BtShellCommand) - pkglist("LDFLAGS", lkShell, BtLdFlag) - pkglist("LDFLAGS.*", lkShell, BtLdFlag) + pkglistbl3("LDFLAGS", lkShell, BtLdFlag) // May also be changed by the user. + pkglistbl3("LDFLAGS.*", lkShell, BtLdFlag) // May also be changed by the user. sysload("LIBABISUFFIX", lkNone, BtIdentifier) // Can also be empty. sys("LIBGRP", lkNone, BtUserGroupName) sys("LIBMODE", lkNone, BtFileMode) @@ -819,8 +830,8 @@ func (src *Pkgsrc) InitVartypes() { sys("LIBTOOL", lkNone, BtShellCommand) acl("LIBTOOL_OVERRIDE", lkShell, BtPathmask, "Makefile: set, append") pkglist("LIBTOOL_REQD", lkShell, BtVersion) - acl("LICENCE", lkNone, BtLicense, "Makefile, Makefile.common, options.mk: set, append") - acl("LICENSE", lkNone, BtLicense, "Makefile, Makefile.common, options.mk: set, append") + acl("LICENCE", lkNone, BtLicense, "buildlink3.mk, builtin.mk: none; Makefile: set, append; *: default, set, append") + acl("LICENSE", lkNone, BtLicense, "buildlink3.mk, builtin.mk: none; Makefile: set, append; *: default, set, append") pkg("LICENSE_FILE", lkNone, BtPathname) sys("LINK.*", lkNone, BtShellCommand) sys("LINKER_RPATH_FLAG", lkNone, BtShellWord) @@ -937,7 +948,7 @@ func (src *Pkgsrc) InitVartypes() { acl("PATCH_ARGS", lkShell, BtShellWord, "") acl("PATCH_DIST_ARGS", lkShell, BtShellWord, "Makefile: set, append") acl("PATCH_DIST_CAT", lkNone, BtShellCommand, "") - acl("PATCH_DIST_STRIP*", lkNone, BtShellWord, "buildlink3.mk, builtin.mk:; Makefile, Makefile.common, *.mk: set") + acl("PATCH_DIST_STRIP*", lkNone, BtShellWord, "buildlink3.mk, builtin.mk: none; Makefile, Makefile.common, *.mk: set") acl("PATCH_SITES", lkShell, BtFetchURL, "Makefile, Makefile.common, options.mk: set") acl("PATCH_STRIP", lkNone, BtShellWord, "") sys("PATH", lkNone, BtPathlist) // From the PATH environment variable. @@ -986,7 +997,7 @@ func (src *Pkgsrc) InitVartypes() { sys("PKGNAME_NOREV", lkNone, BtPkgName) sysload("PKGPATH", lkNone, BtPathname) acl("PKGREPOSITORY", lkNone, BtUnknown, "") - acl("PKGREVISION", lkNone, BtPkgRevision, "Makefile: set") + acl("PKGREVISION", lkNone, BtPkgRevision, "Makefile: set; *: none") sys("PKGSRCDIR", lkNone, BtPathname) acl("PKGSRCTOP", lkNone, BtYes, "Makefile: set") sys("PKGSRC_SETENV", lkNone, BtShellCommand) @@ -1109,6 +1120,7 @@ func (src *Pkgsrc) InitVartypes() { pkg("RESTRICTED", lkNone, BtMessage) usr("ROOT_USER", lkNone, BtUserGroupName) usr("ROOT_GROUP", lkNone, BtUserGroupName) + pkglist("RPMIGNOREPATH", lkShell, BtPathmask) acl("RUBY_BASE", lkNone, enumFromDirs("lang", `^ruby(\d+)$`, "ruby$1", "ruby22 ruby23 ruby24 ruby25"), ""+ "special:rubyversion.mk: set; "+ "*: use-loadtime, use") @@ -1142,15 +1154,15 @@ func (src *Pkgsrc) InitVartypes() { pkglist("SPECIAL_PERMS", lkShell, BtPerms) sys("STEP_MSG", lkNone, BtShellCommand) sys("STRIP", lkNone, BtShellCommand) // see mk/tools/strip.mk - acl("SUBDIR", lkShell, BtFileName, "Makefile: append; *:") + acl("SUBDIR", lkShell, BtFileName, "Makefile: append; *: none") acl("SUBST_CLASSES", lkShell, BtIdentifier, "Makefile: set, append; *: append") - acl("SUBST_CLASSES.*", lkShell, BtIdentifier, "Makefile: set, append; *: append") + acl("SUBST_CLASSES.*", lkShell, BtIdentifier, "Makefile: set, append; *: append") // OPSYS-specific acl("SUBST_FILES.*", lkShell, BtPathmask, "Makefile, Makefile.*, *.mk: set, append") acl("SUBST_FILTER_CMD.*", lkNone, BtShellCommand, "Makefile, Makefile.*, *.mk: set") acl("SUBST_MESSAGE.*", lkNone, BtMessage, "Makefile, Makefile.*, *.mk: set") acl("SUBST_SED.*", lkNone, BtSedCommands, "Makefile, Makefile.*, *.mk: set, append") pkg("SUBST_STAGE.*", lkNone, BtStage) - pkglist("SUBST_VARS.*", lkShell, BtVariableName) + acl("SUBST_VARS.*", lkShell, BtVariableName, "Makefile, Makefile.*, *.mk: set, append") pkglist("SUPERSEDES", lkShell, BtDependency) acl("TEST_DEPENDS", lkShell, BtDependencyWithPath, "Makefile, Makefile.common, *.mk: append") pkglist("TEST_DIRS", lkShell, BtWrksrcSubdirectory) @@ -1162,7 +1174,7 @@ func (src *Pkgsrc) InitVartypes() { sys("TOOLS_BROKEN", lkShell, BtTool) sys("TOOLS_CMD.*", lkNone, BtPathname) acl("TOOLS_CREATE", lkShell, BtTool, "Makefile, Makefile.common, options.mk: append") - acl("TOOLS_DEPENDS.*", lkShell, BtDependencyWithPath, "buildlink3.mk:; Makefile, Makefile.*: set, default; *: use") + acl("TOOLS_DEPENDS.*", lkShell, BtDependencyWithPath, "buildlink3.mk: none; Makefile, Makefile.*: set, default; *: use") sys("TOOLS_GNU_MISSING", lkShell, BtTool) sys("TOOLS_NOOP", lkShell, BtTool) sys("TOOLS_PATH.*", lkNone, BtPathname) @@ -1181,7 +1193,7 @@ func (src *Pkgsrc) InitVartypes() { pkg("USE_CMAKE", lkNone, BtYes) usr("USE_DESTDIR", lkNone, BtYes) pkglist("USE_FEATURES", lkShell, BtIdentifier) - acl("USE_GAMESGROUP", lkNone, BtYesNo, "buildlink3.mk, builtin.mk:; *: set, default, use") + acl("USE_GAMESGROUP", lkNone, BtYesNo, "buildlink3.mk, builtin.mk: none; *: set, default, use") pkg("USE_GCC_RUNTIME", lkNone, BtYesNo) pkg("USE_GNU_CONFIGURE_HOST", lkNone, BtYesNo) acl("USE_GNU_ICONV", lkNone, BtYes, "Makefile, Makefile.common, options.mk: set") @@ -1245,12 +1257,10 @@ func parseACLEntries(varname string, aclEntries string) []ACLEntry { var result []ACLEntry prevperms := "(first)" for _, arg := range strings.Split(aclEntries, "; ") { - var globs, perms string - if fields := strings.SplitN(arg, ": ", 2); len(fields) == 2 { - globs, perms = fields[0], fields[1] - } else { - globs = strings.TrimSuffix(arg, ":") - } + fields := strings.SplitN(arg, ": ", 2) + G.Assertf(len(fields) == 2, "Invalid ACL entry %q", arg) + globs, perms := fields[0], ifelseStr(fields[1] == "none", "", fields[1]) + G.Assertf(perms != prevperms, "Repeated permissions %q for %q.", perms, varname) prevperms = perms diff --git a/pkgtools/pkglint/files/vartype.go b/pkgtools/pkglint/files/vartype.go index b53fe35d6aa..417a84bf567 100644 --- a/pkgtools/pkglint/files/vartype.go +++ b/pkgtools/pkglint/files/vartype.go @@ -38,6 +38,10 @@ const ( aclpAppend // VAR += value aclpUseLoadtime // OTHER := ${VAR}, OTHER != ${VAR} aclpUse // OTHER = ${VAR} + + // TODO: Try what happens if this constant is removed. + // All variables should have proper permission definitions for all files. + // Missing permission definitions could also count as "none". aclpUnknown aclpAllWrite = aclpSet | aclpSetDefault | aclpAppend aclpAllRead = aclpUseLoadtime | aclpUse diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go index f5bd5e885ce..28cd5eb1a3d 100644 --- a/pkgtools/pkglint/files/vartypecheck.go +++ b/pkgtools/pkglint/files/vartypecheck.go @@ -878,7 +878,7 @@ func (cv *VartypeCheck) PkgOptionsVar() { // // Despite its name, it is more similar to RelativePkgDir than to RelativePkgPath. func (cv *VartypeCheck) PkgPath() { - pkgsrcdir := relpath(path.Dir(cv.MkLine.Filename), G.Pkgsrc.File(".")) + pkgsrcdir := cv.MkLine.PathToFile(G.Pkgsrc.File(".")) MkLineChecker{cv.MkLine}.CheckRelativePkgdir(pkgsrcdir + "/" + cv.Value) } diff --git a/pkgtools/pkglint/files/vartypecheck_test.go b/pkgtools/pkglint/files/vartypecheck_test.go index 2c9913dc4e7..a7be8db0029 100644 --- a/pkgtools/pkglint/files/vartypecheck_test.go +++ b/pkgtools/pkglint/files/vartypecheck_test.go @@ -22,9 +22,9 @@ func (s *Suite) Test_VartypeCheck_AwkCommand(c *check.C) { // The warning should be adjusted to reflect this. vt.Output( - "WARN: filename:1: $0 is ambiguous. "+ + "WARN: filename.mk:1: $0 is ambiguous. "+ "Use ${0} if you mean a Make variable or $$0 if you mean a shell variable.", - "WARN: filename:3: $0 is ambiguous. "+ + "WARN: filename.mk:3: $0 is ambiguous. "+ "Use ${0} if you mean a Make variable or $$0 if you mean a shell variable.") } @@ -42,8 +42,8 @@ func (s *Suite) Test_VartypeCheck_BasicRegularExpression(c *check.C) { ".*\\.pl$$") vt.Output( - "WARN: filename:1: Internal pkglint error in MkLine.Tokenize at \"$\".", - "WARN: filename:3: Internal pkglint error in MkLine.Tokenize at \"$\".") + "WARN: filename.mk:1: Internal pkglint error in MkLine.Tokenize at \"$\".", + "WARN: filename.mk:3: Internal pkglint error in MkLine.Tokenize at \"$\".") } @@ -58,7 +58,7 @@ func (s *Suite) Test_VartypeCheck_BuildlinkDepmethod(c *check.C) { "${BUILDLINK_DEPMETHOD.kernel}") vt.Output( - "WARN: filename:2: Invalid dependency method \"unknown\". Valid methods are \"build\" or \"full\".") + "WARN: filename.mk:2: Invalid dependency method \"unknown\". Valid methods are \"build\" or \"full\".") } func (s *Suite) Test_VartypeCheck_Category(c *check.C) { @@ -78,8 +78,8 @@ func (s *Suite) Test_VartypeCheck_Category(c *check.C) { "wip") vt.Output( - "ERROR: filename:2: Invalid category \"arabic\".", - "ERROR: filename:4: Invalid category \"wip\".") + "ERROR: filename.mk:2: Invalid category \"arabic\".", + "ERROR: filename.mk:4: Invalid category \"wip\".") } func (s *Suite) Test_VartypeCheck_CFlag(c *check.C) { @@ -103,10 +103,10 @@ func (s *Suite) Test_VartypeCheck_CFlag(c *check.C) { "`pkg-config`_plus") vt.Output( - "WARN: filename:2: Compiler flag \"/W3\" should start with a hyphen.", - "WARN: filename:3: Compiler flag \"target:sparc64\" should start with a hyphen.", - "WARN: filename:5: Unknown compiler flag \"-XX:+PrintClassHistogramAfterFullGC\".", - "WARN: filename:11: Compiler flag \"`pkg-config`_plus\" should start with a hyphen.") + "WARN: filename.mk:2: Compiler flag \"/W3\" should start with a hyphen.", + "WARN: filename.mk:3: Compiler flag \"target:sparc64\" should start with a hyphen.", + "WARN: filename.mk:5: Unknown compiler flag \"-XX:+PrintClassHistogramAfterFullGC\".", + "WARN: filename.mk:11: Compiler flag \"`pkg-config`_plus\" should start with a hyphen.") vt.Op(opUseMatch) vt.Values( @@ -139,19 +139,19 @@ func (s *Suite) Test_VartypeCheck_Comment(c *check.C) { "'SQL injection fuzzer") vt.Output( - "ERROR: filename:2: COMMENT must be set.", - "WARN: filename:3: COMMENT should not begin with \"A\".", - "WARN: filename:3: COMMENT should not end with a period.", - "WARN: filename:4: COMMENT should start with a capital letter.", - "WARN: filename:4: COMMENT should not be longer than 70 characters.", - "WARN: filename:5: COMMENT should not be enclosed in quotes.", - "WARN: filename:6: COMMENT should not be enclosed in quotes.", - "WARN: filename:7: COMMENT should not contain \"is a\".", - "WARN: filename:8: COMMENT should not contain \"is an\".", - "WARN: filename:9: COMMENT should not contain \"is a\".", - "WARN: filename:10: COMMENT should not start with the package name.", - "WARN: filename:11: COMMENT should not start with the package name.", - "WARN: filename:11: COMMENT should not contain \"is a\".") + "ERROR: filename.mk:2: COMMENT must be set.", + "WARN: filename.mk:3: COMMENT should not begin with \"A\".", + "WARN: filename.mk:3: COMMENT should not end with a period.", + "WARN: filename.mk:4: COMMENT should start with a capital letter.", + "WARN: filename.mk:4: COMMENT should not be longer than 70 characters.", + "WARN: filename.mk:5: COMMENT should not be enclosed in quotes.", + "WARN: filename.mk:6: COMMENT should not be enclosed in quotes.", + "WARN: filename.mk:7: COMMENT should not contain \"is a\".", + "WARN: filename.mk:8: COMMENT should not contain \"is an\".", + "WARN: filename.mk:9: COMMENT should not contain \"is a\".", + "WARN: filename.mk:10: COMMENT should not start with the package name.", + "WARN: filename.mk:11: COMMENT should not start with the package name.", + "WARN: filename.mk:11: COMMENT should not contain \"is a\".") } func (s *Suite) Test_VartypeCheck_ConfFiles(c *check.C) { @@ -167,9 +167,9 @@ func (s *Suite) Test_VartypeCheck_ConfFiles(c *check.C) { "share/etc/bootrc /etc/bootrc") vt.Output( - "WARN: filename:1: Values for CONF_FILES should always be pairs of paths.", - "WARN: filename:3: Values for CONF_FILES should always be pairs of paths.", - "WARN: filename:5: The destination file \"/etc/bootrc\" should start with a variable reference.") + "WARN: filename.mk:1: Values for CONF_FILES should always be pairs of paths.", + "WARN: filename.mk:3: Values for CONF_FILES should always be pairs of paths.", + "WARN: filename.mk:5: The destination file \"/etc/bootrc\" should start with a variable reference.") } func (s *Suite) Test_VartypeCheck_Dependency(c *check.C) { @@ -206,21 +206,21 @@ func (s *Suite) Test_VartypeCheck_Dependency(c *check.C) { "package-1.0>=1.0.3") vt.Output( - "WARN: filename:1: Invalid dependency pattern \"Perl\".", - "WARN: filename:3: Please use \"perl5-[0-9]*\" instead of \"perl5-*\".", - "WARN: filename:5: Only [0-9]* is allowed in the numeric part of a dependency.", - "WARN: filename:5: The version pattern \"[5.10-5.22]*\" should not contain a hyphen.", - "WARN: filename:6: Invalid dependency pattern \"py-docs\".", - "WARN: filename:10: Please use \"5.22{,nb*}\" instead of \"5.22\" as the version pattern.", - "WARN: filename:11: Please use \"5.*\" instead of \"5*\" as the version pattern.", - "WARN: filename:12: The version pattern \"2.0-[0-9]*\" should not contain a hyphen.", - "WARN: filename:21: Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.", - "WARN: filename:22: Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.", - "WARN: filename:23: Invalid dependency pattern \"package-1.0|garbage\".", + "WARN: filename.mk:1: Invalid dependency pattern \"Perl\".", + "WARN: filename.mk:3: Please use \"perl5-[0-9]*\" instead of \"perl5-*\".", + "WARN: filename.mk:5: Only [0-9]* is allowed in the numeric part of a dependency.", + "WARN: filename.mk:5: The version pattern \"[5.10-5.22]*\" should not contain a hyphen.", + "WARN: filename.mk:6: Invalid dependency pattern \"py-docs\".", + "WARN: filename.mk:10: Please use \"5.22{,nb*}\" instead of \"5.22\" as the version pattern.", + "WARN: filename.mk:11: Please use \"5.*\" instead of \"5*\" as the version pattern.", + "WARN: filename.mk:12: The version pattern \"2.0-[0-9]*\" should not contain a hyphen.", + "WARN: filename.mk:21: Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.", + "WARN: filename.mk:22: Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.", + "WARN: filename.mk:23: Invalid dependency pattern \"package-1.0|garbage\".", // TODO: Mention that the path should be removed. - "WARN: filename:25: Invalid dependency pattern \"package>=1.0:../../category/package\".", + "WARN: filename.mk:25: Invalid dependency pattern \"package>=1.0:../../category/package\".", // TODO: Mention that version numbers in a pkgbase must be appended directly, without hyphen. - "WARN: filename:26: Invalid dependency pattern \"package-1.0>=1.0.3\".") + "WARN: filename.mk:26: Invalid dependency pattern \"package-1.0>=1.0.3\".") } func (s *Suite) Test_VartypeCheck_DependencyWithPath(c *check.C) { @@ -282,7 +282,7 @@ func (s *Suite) Test_VartypeCheck_DistSuffix(c *check.C) { ".tar.gz # overrides a definition from a Makefile.common") vt.Output( - "NOTE: filename:1: EXTRACT_SUFX is \".tar.gz\" by default, so this definition may be redundant.") + "NOTE: filename.mk:1: EXTRACT_SUFX is \".tar.gz\" by default, so this definition may be redundant.") } func (s *Suite) Test_VartypeCheck_EmulPlatform(c *check.C) { @@ -295,12 +295,12 @@ func (s *Suite) Test_VartypeCheck_EmulPlatform(c *check.C) { "${LINUX}") vt.Output( - "WARN: filename:2: \"nextbsd\" is not valid for the operating system part of EMUL_PLATFORM. "+ + "WARN: filename.mk:2: \"nextbsd\" is not valid for the operating system part of EMUL_PLATFORM. "+ "Use one of "+ "{ bitrig bsdos cygwin darwin dragonfly freebsd haiku hpux "+ "interix irix linux mirbsd netbsd openbsd osf1 solaris sunos "+ "} instead.", - "WARN: filename:2: \"8087\" is not valid for the hardware architecture part of EMUL_PLATFORM. "+ + "WARN: filename.mk:2: \"8087\" is not valid for the hardware architecture part of EMUL_PLATFORM. "+ "Use one of { "+ "aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 "+ "cobalt coldfire convex dreamcast "+ @@ -313,7 +313,7 @@ func (s *Suite) Test_VartypeCheck_EmulPlatform(c *check.C) { "mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 "+ "s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ "} instead.", - "WARN: filename:3: \"${LINUX}\" is not a valid emulation platform.") + "WARN: filename.mk:3: \"${LINUX}\" is not a valid emulation platform.") } func (s *Suite) Test_VartypeCheck_Enum(c *check.C) { @@ -329,8 +329,8 @@ func (s *Suite) Test_VartypeCheck_Enum(c *check.C) { "[") vt.Output( - "WARN: filename:3: The pattern \"sun-jdk*\" cannot match any of { jdk1 jdk2 jdk4 } for JDK.", - "WARN: filename:5: Invalid match pattern \"[\".") + "WARN: filename.mk:3: The pattern \"sun-jdk*\" cannot match any of { jdk1 jdk2 jdk4 } for JDK.", + "WARN: filename.mk:5: Invalid match pattern \"[\".") } func (s *Suite) Test_VartypeCheck_Enum__use_match(c *check.C) { @@ -385,13 +385,13 @@ func (s *Suite) Test_VartypeCheck_FetchURL(c *check.C) { "${MASTER_SITE_INVALID:=subdir/}") vt.Output( - "WARN: filename:1: Please use ${MASTER_SITE_GITHUB:=example/} "+ + "WARN: filename.mk:1: Please use ${MASTER_SITE_GITHUB:=example/} "+ "instead of \"https://github.com/example/project/\" "+ "and run \""+confMake+" help topic=github\" for further tips.", - "WARN: filename:2: Please use ${MASTER_SITE_GNU:=bison} "+ + "WARN: filename.mk:2: Please use ${MASTER_SITE_GNU:=bison} "+ "instead of \"http://ftp.gnu.org/pub/gnu/bison\".", - "ERROR: filename:3: The subdirectory in MASTER_SITE_GNU must end with a slash.", - "ERROR: filename:4: The site MASTER_SITE_INVALID does not exist.") + "ERROR: filename.mk:3: The subdirectory in MASTER_SITE_GNU must end with a slash.", + "ERROR: filename.mk:4: The site MASTER_SITE_INVALID does not exist.") // PR 46570, keyword gimp-fix-ca vt.Values( @@ -405,7 +405,7 @@ func (s *Suite) Test_VartypeCheck_FetchURL(c *check.C) { "http://example.org/download?filename=<distfile>;version=<version>") vt.Output( - "WARN: filename:8: \"http://example.org/download?filename=<distfile>;version=<version>\" is not a valid URL.") + "WARN: filename.mk:8: \"http://example.org/download?filename=<distfile>;version=<version>\" is not a valid URL.") } func (s *Suite) Test_VartypeCheck_Filename(c *check.C) { @@ -417,8 +417,8 @@ func (s *Suite) Test_VartypeCheck_Filename(c *check.C) { "OS/2-manual.txt") vt.Output( - "WARN: filename:1: \"Filename with spaces.docx\" is not a valid filename.", - "WARN: filename:2: A filename should not contain a slash.") + "WARN: filename.mk:1: \"Filename with spaces.docx\" is not a valid filename.", + "WARN: filename.mk:2: A filename should not contain a slash.") vt.Op(opUseMatch) vt.Values( @@ -438,8 +438,8 @@ func (s *Suite) Test_VartypeCheck_FileMask(c *check.C) { "OS/2-manual.txt") vt.Output( - "WARN: filename:1: \"FileMask with spaces.docx\" is not a valid filename mask.", - "WARN: filename:2: A filename mask should not contain a slash.") + "WARN: filename.mk:1: \"FileMask with spaces.docx\" is not a valid filename mask.", + "WARN: filename.mk:2: A filename mask should not contain a slash.") vt.Op(opUseMatch) vt.Values( @@ -463,9 +463,9 @@ func (s *Suite) Test_VartypeCheck_FileMode(c *check.C) { "") vt.Output( - "WARN: filename:1: Invalid file mode \"u+rwx\".", - "WARN: filename:4: Invalid file mode \"12345\".", - "WARN: filename:6: Invalid file mode \"\".") + "WARN: filename.mk:1: Invalid file mode \"u+rwx\".", + "WARN: filename.mk:4: Invalid file mode \"12345\".", + "WARN: filename.mk:6: Invalid file mode \"\".") vt.Op(opUseMatch) vt.Values( @@ -474,7 +474,7 @@ func (s *Suite) Test_VartypeCheck_FileMode(c *check.C) { // There's no guarantee that a filename only contains [A-Za-z0-9.]. // Therefore there are no useful checks in this situation. vt.Output( - "WARN: filename:11: Invalid file mode \"u+rwx\".") + "WARN: filename.mk:11: Invalid file mode \"u+rwx\".") } func (s *Suite) Test_VartypeCheck_GccReqd(c *check.C) { @@ -491,8 +491,8 @@ func (s *Suite) Test_VartypeCheck_GccReqd(c *check.C) { "6", "7.3") vt.Output( - "WARN: filename:5: GCC version numbers should only contain the major version (5).", - "WARN: filename:7: GCC version numbers should only contain the major version (7).") + "WARN: filename.mk:5: GCC version numbers should only contain the major version (5).", + "WARN: filename.mk:7: GCC version numbers should only contain the major version (7).") } func (s *Suite) Test_VartypeCheck_Homepage(c *check.C) { @@ -506,7 +506,7 @@ func (s *Suite) Test_VartypeCheck_Homepage(c *check.C) { "${MASTER_SITES}") vt.Output( - "WARN: filename:3: HOMEPAGE should not be defined in terms of MASTER_SITEs.") + "WARN: filename.mk:3: HOMEPAGE should not be defined in terms of MASTER_SITEs.") G.Pkg = NewPackage(t.File("category/package")) @@ -517,7 +517,7 @@ func (s *Suite) Test_VartypeCheck_Homepage(c *check.C) { // doesn't define MASTER_SITES, that variable cannot be expanded, which means // the warning cannot refer to its value. vt.Output( - "WARN: filename:4: HOMEPAGE should not be defined in terms of MASTER_SITEs.") + "WARN: filename.mk:4: HOMEPAGE should not be defined in terms of MASTER_SITEs.") delete(G.Pkg.vars.firstDef, "MASTER_SITES") delete(G.Pkg.vars.lastDef, "MASTER_SITES") @@ -528,7 +528,7 @@ func (s *Suite) Test_VartypeCheck_Homepage(c *check.C) { "${MASTER_SITES}") vt.Output( - "WARN: filename:5: HOMEPAGE should not be defined in terms of MASTER_SITEs. " + + "WARN: filename.mk:5: HOMEPAGE should not be defined in terms of MASTER_SITEs. " + "Use https://cdn.NetBSD.org/pub/pkgsrc/distfiles/ directly.") delete(G.Pkg.vars.firstDef, "MASTER_SITES") @@ -542,7 +542,7 @@ func (s *Suite) Test_VartypeCheck_Homepage(c *check.C) { // When MASTER_SITES itself makes use of another variable, pkglint doesn't // resolve that variable and just outputs the simple variant of this warning. vt.Output( - "WARN: filename:6: HOMEPAGE should not be defined in terms of MASTER_SITEs.") + "WARN: filename.mk:6: HOMEPAGE should not be defined in terms of MASTER_SITEs.") } @@ -559,9 +559,9 @@ func (s *Suite) Test_VartypeCheck_Identifier(c *check.C) { "") vt.Output( - "WARN: filename:2: Invalid identifier \"identifiers cannot contain spaces\".", - "WARN: filename:3: Invalid identifier \"id/cannot/contain/slashes\".", - "WARN: filename:5: Invalid identifier \"\".") + "WARN: filename.mk:2: Invalid identifier \"identifiers cannot contain spaces\".", + "WARN: filename.mk:3: Invalid identifier \"id/cannot/contain/slashes\".", + "WARN: filename.mk:5: Invalid identifier \"\".") vt.Op(opUseMatch) vt.Values( @@ -571,7 +571,7 @@ func (s *Suite) Test_VartypeCheck_Identifier(c *check.C) { "A*B") vt.Output( - "WARN: filename:12: Invalid identifier pattern \"[A-Z.]\" for MYSQL_CHARSET.") + "WARN: filename.mk:12: Invalid identifier pattern \"[A-Z.]\" for MYSQL_CHARSET.") } func (s *Suite) Test_VartypeCheck_Integer(c *check.C) { @@ -586,8 +586,8 @@ func (s *Suite) Test_VartypeCheck_Integer(c *check.C) { "11111111111111111111111111111111111111111111111") vt.Output( - "WARN: filename:1: Invalid integer \"${OTHER_VAR}\".", - "WARN: filename:3: Invalid integer \"-13\".") + "WARN: filename.mk:1: Invalid integer \"${OTHER_VAR}\".", + "WARN: filename.mk:3: Invalid integer \"-13\".") } func (s *Suite) Test_VartypeCheck_LdFlag(c *check.C) { @@ -615,10 +615,10 @@ func (s *Suite) Test_VartypeCheck_LdFlag(c *check.C) { "anything") vt.Output( - "WARN: filename:4: Unknown linker flag \"-unknown\".", - "WARN: filename:5: Linker flag \"no-hyphen\" should start with a hyphen.", - "WARN: filename:6: Please use \"${COMPILER_RPATH_FLAG}\" instead of \"-Wl,--rpath\".", - "WARN: filename:12: Linker flag \"`pkg-config`_plus\" should start with a hyphen.") + "WARN: filename.mk:4: Unknown linker flag \"-unknown\".", + "WARN: filename.mk:5: Linker flag \"no-hyphen\" should start with a hyphen.", + "WARN: filename.mk:6: Please use \"${COMPILER_RPATH_FLAG}\" instead of \"-Wl,--rpath\".", + "WARN: filename.mk:12: Linker flag \"`pkg-config`_plus\" should start with a hyphen.") } func (s *Suite) Test_VartypeCheck_License(c *check.C) { @@ -640,9 +640,9 @@ func (s *Suite) Test_VartypeCheck_License(c *check.C) { "${UNKNOWN_LICENSE}") vt.Output( - "ERROR: filename:2: Parse error for license condition \"AND mit\".", - "WARN: filename:3: License file ~/licenses/artistic does not exist.", - "ERROR: filename:4: Parse error for license condition \"${UNKNOWN_LICENSE}\".") + "ERROR: filename.mk:2: Parse error for license condition \"AND mit\".", + "WARN: filename.mk:3: License file ~/licenses/artistic does not exist.", + "ERROR: filename.mk:4: Parse error for license condition \"${UNKNOWN_LICENSE}\".") vt.Op(opAssignAppend) vt.Values( @@ -650,8 +650,8 @@ func (s *Suite) Test_VartypeCheck_License(c *check.C) { "AND mit") vt.Output( - "ERROR: filename:11: Parse error for appended license condition \"gnu-gpl-v2\".", - "WARN: filename:12: License file ~/licenses/mit does not exist.") + "ERROR: filename.mk:11: Parse error for appended license condition \"gnu-gpl-v2\".", + "WARN: filename.mk:12: License file ~/licenses/mit does not exist.") } func (s *Suite) Test_VartypeCheck_MachineGnuPlatform(c *check.C) { @@ -668,18 +668,18 @@ func (s *Suite) Test_VartypeCheck_MachineGnuPlatform(c *check.C) { "x86_64-pc") // Just for code coverage. vt.Output( - "WARN: filename:2: The pattern \"Cygwin\" cannot match any of "+ + "WARN: filename.mk:2: The pattern \"Cygwin\" cannot match any of "+ "{ aarch64 aarch64_be alpha amd64 arc arm armeb armv4 armv4eb armv6 armv6eb armv7 armv7eb "+ "cobalt convex dreamcast hpcmips hpcsh hppa hppa64 i386 i486 ia64 m5407 m68010 m68k m88k "+ "mips mips64 mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 "+ "rs6000 s390 sh shle sparc sparc64 vax x86_64 "+ "} for the hardware architecture part of MACHINE_GNU_PLATFORM.", - "WARN: filename:2: The pattern \"amd64\" cannot match any of "+ + "WARN: filename.mk:2: The pattern \"amd64\" cannot match any of "+ "{ bitrig bsdos cygwin darwin dragonfly freebsd haiku hpux interix irix linux mirbsd "+ "netbsd openbsd osf1 solaris sunos } "+ "for the operating system part of MACHINE_GNU_PLATFORM.", - "WARN: filename:4: \"*-*-*-*\" is not a valid platform pattern.", - "WARN: filename:6: \"x86_64-pc\" is not a valid platform pattern.") + "WARN: filename.mk:4: \"*-*-*-*\" is not a valid platform pattern.", + "WARN: filename.mk:6: \"x86_64-pc\" is not a valid platform pattern.") } func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { @@ -698,12 +698,12 @@ func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { "NetBSD-[0-1]*-*") vt.Output( - "WARN: filename:1: \"linux-i386\" is not a valid platform pattern.", - "WARN: filename:2: The pattern \"nextbsd\" cannot match any of "+ + "WARN: filename.mk:1: \"linux-i386\" is not a valid platform pattern.", + "WARN: filename.mk:2: The pattern \"nextbsd\" cannot match any of "+ "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ "} for the operating system part of ONLY_FOR_PLATFORM.", - "WARN: filename:2: The pattern \"8087\" cannot match any of "+ + "WARN: filename.mk:2: The pattern \"8087\" cannot match any of "+ "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 "+ "cobalt coldfire convex dreamcast "+ "earm earmeb earmhf earmhfeb earmv4 earmv4eb "+ @@ -714,11 +714,11 @@ func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { "mlrisc ns32k pc532 pmax powerpc powerpc64 "+ "rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ "} for the hardware architecture part of ONLY_FOR_PLATFORM.", - "WARN: filename:3: The pattern \"netbsd\" cannot match any of "+ + "WARN: filename.mk:3: The pattern \"netbsd\" cannot match any of "+ "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ "} for the operating system part of ONLY_FOR_PLATFORM.", - "WARN: filename:3: The pattern \"l*\" cannot match any of "+ + "WARN: filename.mk:3: The pattern \"l*\" cannot match any of "+ "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 "+ "cobalt coldfire convex dreamcast "+ "earm earmeb earmhf earmhfeb earmv4 earmv4eb "+ @@ -729,8 +729,8 @@ func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { "mlrisc ns32k pc532 pmax powerpc powerpc64 "+ "rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ "} for the hardware architecture part of ONLY_FOR_PLATFORM.", - "WARN: filename:5: \"FreeBSD*\" is not a valid platform pattern.", - "WARN: filename:8: Please use \"[0-1].*\" instead of \"[0-1]*\" as the version pattern.") + "WARN: filename.mk:5: \"FreeBSD*\" is not a valid platform pattern.", + "WARN: filename.mk:8: Please use \"[0-1].*\" instead of \"[0-1]*\" as the version pattern.") } func (s *Suite) Test_VartypeCheck_MailAddress(c *check.C) { @@ -744,10 +744,10 @@ func (s *Suite) Test_VartypeCheck_MailAddress(c *check.C) { "user1@example.org,user2@example.org") vt.Output( - "WARN: filename:1: Please write \"NetBSD.org\" instead of \"netbsd.org\".", - "ERROR: filename:2: This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.", - "ERROR: filename:3: This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.", - "WARN: filename:4: \"user1@example.org,user2@example.org\" is not a valid mail address.") + "WARN: filename.mk:1: Please write \"NetBSD.org\" instead of \"netbsd.org\".", + "ERROR: filename.mk:2: This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.", + "ERROR: filename.mk:3: This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.", + "WARN: filename.mk:4: \"user1@example.org,user2@example.org\" is not a valid mail address.") } func (s *Suite) Test_VartypeCheck_Message(c *check.C) { @@ -759,7 +759,7 @@ func (s *Suite) Test_VartypeCheck_Message(c *check.C) { "Correct paths") vt.Output( - "WARN: filename:1: SUBST_MESSAGE.id should not be quoted.") + "WARN: filename.mk:1: SUBST_MESSAGE.id should not be quoted.") } func (s *Suite) Test_VartypeCheck_Option(c *check.C) { @@ -777,9 +777,9 @@ func (s *Suite) Test_VartypeCheck_Option(c *check.C) { "UPPER") vt.Output( - "WARN: filename:3: Unknown option \"unknown\".", - "WARN: filename:4: Use of the underscore character in option names is deprecated.", - "ERROR: filename:5: Invalid option name \"UPPER\". "+ + "WARN: filename.mk:3: Unknown option \"unknown\".", + "WARN: filename.mk:4: Use of the underscore character in option names is deprecated.", + "ERROR: filename.mk:5: Invalid option name \"UPPER\". "+ "Option names must start with a lowercase letter and be all-lowercase.") } @@ -792,10 +792,10 @@ func (s *Suite) Test_VartypeCheck_Pathlist(c *check.C) { "/directory with spaces") vt.Output( - "ERROR: filename:1: The component \".\" of PATH must be an absolute path.", - "ERROR: filename:1: The component \"\" of PATH must be an absolute path.", - "WARN: filename:1: \"${PREFIX}/!!!\" is not a valid pathname.", - "WARN: filename:2: \"/directory with spaces\" is not a valid pathname.") + "ERROR: filename.mk:1: The component \".\" of PATH must be an absolute path.", + "ERROR: filename.mk:1: The component \"\" of PATH must be an absolute path.", + "WARN: filename.mk:1: \"${PREFIX}/!!!\" is not a valid pathname.", + "WARN: filename.mk:2: \"/directory with spaces\" is not a valid pathname.") } func (s *Suite) Test_VartypeCheck_PathMask(c *check.C) { @@ -808,7 +808,7 @@ func (s *Suite) Test_VartypeCheck_PathMask(c *check.C) { "src/*/*") vt.Output( - "WARN: filename:2: \"src/*&*\" is not a valid pathname mask.") + "WARN: filename.mk:2: \"src/*&*\" is not a valid pathname mask.") vt.Op(opUseMatch) vt.Values("any") @@ -831,7 +831,7 @@ func (s *Suite) Test_VartypeCheck_Pathname(c *check.C) { // FIXME: Warn about the absolute pathname in line 4. vt.Output( - "WARN: filename:1: \"${PREFIX}/*\" is not a valid pathname.") + "WARN: filename.mk:1: \"${PREFIX}/*\" is not a valid pathname.") } func (s *Suite) Test_VartypeCheck_Perl5Packlist(c *check.C) { @@ -843,7 +843,7 @@ func (s *Suite) Test_VartypeCheck_Perl5Packlist(c *check.C) { "anything else") vt.Output( - "WARN: filename:1: PERL5_PACKLIST should not depend on other variables.") + "WARN: filename.mk:1: PERL5_PACKLIST should not depend on other variables.") } func (s *Suite) Test_VartypeCheck_Perms(c *check.C) { @@ -860,8 +860,8 @@ func (s *Suite) Test_VartypeCheck_Perms(c *check.C) { "${REAL_ROOT_GROUP}") vt.Output( - "ERROR: filename:2: ROOT_USER must not be used in permission definitions. Use REAL_ROOT_USER instead.", - "ERROR: filename:5: ROOT_GROUP must not be used in permission definitions. Use REAL_ROOT_GROUP instead.") + "ERROR: filename.mk:2: ROOT_USER must not be used in permission definitions. Use REAL_ROOT_USER instead.", + "ERROR: filename.mk:5: ROOT_GROUP must not be used in permission definitions. Use REAL_ROOT_GROUP instead.") } func (s *Suite) Test_VartypeCheck_Pkgname(c *check.C) { @@ -880,7 +880,7 @@ func (s *Suite) Test_VartypeCheck_Pkgname(c *check.C) { "pkgbase-3.1.4.1.5.9.2.6.5.3.5.8.9.7.9") vt.Output( - "WARN: filename:8: \"pkgbase-z1\" is not a valid package name.") + "WARN: filename.mk:8: \"pkgbase-z1\" is not a valid package name.") vt.Op(opUseMatch) vt.Values( @@ -899,8 +899,8 @@ func (s *Suite) Test_VartypeCheck_PkgOptionsVar(c *check.C) { "PKG_OPTS.mc") vt.Output( - "ERROR: filename:1: PKGBASE must not be used in PKG_OPTIONS_VAR.", - "ERROR: filename:3: PKG_OPTIONS_VAR must be "+ + "ERROR: filename.mk:1: PKGBASE must not be used in PKG_OPTIONS_VAR.", + "ERROR: filename.mk:3: PKG_OPTIONS_VAR must be "+ "of the form \"PKG_OPTIONS.*\", not \"PKG_OPTS.mc\".") } @@ -919,10 +919,10 @@ func (s *Suite) Test_VartypeCheck_PkgPath(c *check.C) { "../../invalid/relative") vt.Output( - "ERROR: filename:3: Relative path \"../../invalid\" does not exist.", - "WARN: filename:3: \"../../invalid\" is not a valid relative package directory.", - "ERROR: filename:4: Relative path \"../../../../invalid/relative\" does not exist.", - "WARN: filename:4: \"../../../../invalid/relative\" is not a valid relative package directory.") + "ERROR: filename.mk:3: Relative path \"../../invalid\" does not exist.", + "WARN: filename.mk:3: \"../../invalid\" is not a valid relative package directory.", + "ERROR: filename.mk:4: Relative path \"../../../../invalid/relative\" does not exist.", + "WARN: filename.mk:4: \"../../../../invalid/relative\" is not a valid relative package directory.") } func (s *Suite) Test_VartypeCheck_PkgRevision(c *check.C) { @@ -933,8 +933,8 @@ func (s *Suite) Test_VartypeCheck_PkgRevision(c *check.C) { "3a") vt.Output( - "WARN: filename:1: PKGREVISION must be a positive integer number.", - "ERROR: filename:1: PKGREVISION only makes sense directly in the package Makefile.") + "WARN: filename.mk:1: PKGREVISION must be a positive integer number.", + "ERROR: filename.mk:1: PKGREVISION only makes sense directly in the package Makefile.") vt.File("Makefile") vt.Values( @@ -953,8 +953,8 @@ func (s *Suite) Test_VartypeCheck_PythonDependency(c *check.C) { "cairo,X") vt.Output( - "WARN: filename:2: Python dependencies should not contain variables.", - "WARN: filename:3: Invalid Python dependency \"cairo,X\".") + "WARN: filename.mk:2: Python dependencies should not contain variables.", + "WARN: filename.mk:3: Invalid Python dependency \"cairo,X\".") } func (s *Suite) Test_VartypeCheck_PrefixPathname(c *check.C) { @@ -966,7 +966,7 @@ func (s *Suite) Test_VartypeCheck_PrefixPathname(c *check.C) { "share/locale") vt.Output( - "WARN: filename:1: Please use \"${PKGMANDIR}/man1\" instead of \"man/man1\".") + "WARN: filename.mk:1: Please use \"${PKGMANDIR}/man1\" instead of \"man/man1\".") } func (s *Suite) Test_VartypeCheck_RelativePkgPath(c *check.C) { @@ -985,9 +985,9 @@ func (s *Suite) Test_VartypeCheck_RelativePkgPath(c *check.C) { "../../invalid/relative") vt.Output( - "ERROR: filename:1: Relative path \"category/other-package\" does not exist.", - "ERROR: filename:4: Relative path \"invalid\" does not exist.", - "ERROR: filename:5: Relative path \"../../invalid/relative\" does not exist.") + "ERROR: filename.mk:1: Relative path \"category/other-package\" does not exist.", + "ERROR: filename.mk:4: Relative path \"invalid\" does not exist.", + "ERROR: filename.mk:5: Relative path \"../../invalid/relative\" does not exist.") } func (s *Suite) Test_VartypeCheck_Restricted(c *check.C) { @@ -998,7 +998,7 @@ func (s *Suite) Test_VartypeCheck_Restricted(c *check.C) { "May only be distributed free of charge") vt.Output( - "WARN: filename:1: The only valid value for NO_BIN_ON_CDROM is ${RESTRICTED}.") + "WARN: filename.mk:1: The only valid value for NO_BIN_ON_CDROM is ${RESTRICTED}.") } func (s *Suite) Test_VartypeCheck_SedCommands(c *check.C) { @@ -1019,15 +1019,15 @@ func (s *Suite) Test_VartypeCheck_SedCommands(c *check.C) { "-e s,$${unclosedShellVar") // Just for code coverage. vt.Output( - "NOTE: filename:1: Please always use \"-e\" in sed commands, even if there is only one substitution.", - "NOTE: filename:2: Each sed command should appear in an assignment of its own.", - "WARN: filename:3: The # character starts a Makefile comment.", - "ERROR: filename:3: Invalid shell words \"\\\"s,\" in sed commands.", - "WARN: filename:8: Unknown sed command \"1d\".", - "ERROR: filename:9: The -e option to sed requires an argument.", - "WARN: filename:10: Unknown sed command \"-i\".", - "NOTE: filename:10: Please always use \"-e\" in sed commands, even if there is only one substitution.", - "WARN: filename:11: Unclosed shell variable starting at \"$${unclosedShellVar\".") + "NOTE: filename.mk:1: Please always use \"-e\" in sed commands, even if there is only one substitution.", + "NOTE: filename.mk:2: Each sed command should appear in an assignment of its own.", + "WARN: filename.mk:3: The # character starts a Makefile comment.", + "ERROR: filename.mk:3: Invalid shell words \"\\\"s,\" in sed commands.", + "WARN: filename.mk:8: Unknown sed command \"1d\".", + "ERROR: filename.mk:9: The -e option to sed requires an argument.", + "WARN: filename.mk:10: Unknown sed command \"-i\".", + "NOTE: filename.mk:10: Please always use \"-e\" in sed commands, even if there is only one substitution.", + "WARN: filename.mk:11: Unclosed shell variable starting at \"$${unclosedShellVar\".") } func (s *Suite) Test_VartypeCheck_ShellCommand(c *check.C) { @@ -1057,7 +1057,7 @@ func (s *Suite) Test_VartypeCheck_ShellCommands(c *check.C) { "echo bin/program;") vt.Output( - "WARN: filename:1: This shell command list should end with a semicolon.") + "WARN: filename.mk:1: This shell command list should end with a semicolon.") } func (s *Suite) Test_VartypeCheck_Stage(c *check.C) { @@ -1070,7 +1070,7 @@ func (s *Suite) Test_VartypeCheck_Stage(c *check.C) { "pre-test") vt.Output( - "WARN: filename:2: Invalid stage name \"post-modern\". " + + "WARN: filename.mk:2: Invalid stage name \"post-modern\". " + "Use one of {pre,do,post}-{extract,patch,configure,build,test,install}.") } @@ -1092,10 +1092,10 @@ func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { "unknown") vt.Output( - "ERROR: filename:2: Invalid tool dependency \"unknown\". "+ + "ERROR: filename.mk:2: Invalid tool dependency \"unknown\". "+ "Use one of \"bootstrap\", \"build\", \"pkgsrc\", \"run\" or \"test\".", - "ERROR: filename:4: Invalid tool dependency \"mal:formed:tool\".", - "ERROR: filename:5: Unknown tool \"unknown\".") + "ERROR: filename.mk:4: Invalid tool dependency \"mal:formed:tool\".", + "ERROR: filename.mk:5: Unknown tool \"unknown\".") vt.Varname("USE_TOOLS.NetBSD") vt.Op(opAssignAppend) @@ -1104,7 +1104,7 @@ func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { "tool2:unknown") vt.Output( - "ERROR: filename:12: Invalid tool dependency \"unknown\". " + + "ERROR: filename.mk:12: Invalid tool dependency \"unknown\". " + "Use one of \"bootstrap\", \"build\", \"pkgsrc\", \"run\" or \"test\".") vt.Varname("TOOLS_NOOP") @@ -1118,7 +1118,7 @@ func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { "gmake:run") vt.Output( - "ERROR: filename:31: Unknown tool \"gmake\".") + "ERROR: filename.mk:31: Unknown tool \"gmake\".") vt.Varname("USE_TOOLS") vt.Op(opUseMatch) @@ -1156,17 +1156,17 @@ func (s *Suite) Test_VartypeCheck_URL(c *check.C) { "string with spaces") vt.Output( - "WARN: filename:4: Please write NetBSD.org instead of www.netbsd.org.", - "NOTE: filename:5: For consistency, please add a trailing slash to \"https://www.example.org\".", - "WARN: filename:8: \"\" is not a valid URL.", - "WARN: filename:9: \"ftp://example.org/<\" is not a valid URL.", - "WARN: filename:10: \"gopher://example.org/<\" is not a valid URL.", - "WARN: filename:11: \"http://example.org/<\" is not a valid URL.", - "WARN: filename:12: \"https://example.org/<\" is not a valid URL.", - "WARN: filename:13: \"https://www.example.org/path with spaces\" is not a valid URL.", - "WARN: filename:14: \"httpxs://www.example.org\" is not a valid URL. Only ftp, gopher, http, and https URLs are allowed here.", - "WARN: filename:15: \"mailto:someone@example.org\" is not a valid URL.", - "WARN: filename:16: \"string with spaces\" is not a valid URL.") + "WARN: filename.mk:4: Please write NetBSD.org instead of www.netbsd.org.", + "NOTE: filename.mk:5: For consistency, please add a trailing slash to \"https://www.example.org\".", + "WARN: filename.mk:8: \"\" is not a valid URL.", + "WARN: filename.mk:9: \"ftp://example.org/<\" is not a valid URL.", + "WARN: filename.mk:10: \"gopher://example.org/<\" is not a valid URL.", + "WARN: filename.mk:11: \"http://example.org/<\" is not a valid URL.", + "WARN: filename.mk:12: \"https://example.org/<\" is not a valid URL.", + "WARN: filename.mk:13: \"https://www.example.org/path with spaces\" is not a valid URL.", + "WARN: filename.mk:14: \"httpxs://www.example.org\" is not a valid URL. Only ftp, gopher, http, and https URLs are allowed here.", + "WARN: filename.mk:15: \"mailto:someone@example.org\" is not a valid URL.", + "WARN: filename.mk:16: \"string with spaces\" is not a valid URL.") // Yes, even in 2019, some pkgsrc-wip packages really use a gopher HOMEPAGE. vt.Values( @@ -1187,8 +1187,8 @@ func (s *Suite) Test_VartypeCheck_UserGroupName(c *check.C) { "${OTHER_VAR}") vt.Output( - "WARN: filename:1: Invalid user or group name \"user with spaces\".", - "WARN: filename:4: Invalid user or group name \"domain\\\\user\".") + "WARN: filename.mk:1: Invalid user or group name \"user with spaces\".", + "WARN: filename.mk:4: Invalid user or group name \"domain\\\\user\".") } func (s *Suite) Test_VartypeCheck_VariableName(c *check.C) { @@ -1202,7 +1202,7 @@ func (s *Suite) Test_VartypeCheck_VariableName(c *check.C) { "${INDIRECT}") vt.Output( - "WARN: filename:2: \"VarBase\" is not a valid variable name.") + "WARN: filename.mk:2: \"VarBase\" is not a valid variable name.") } func (s *Suite) Test_VartypeCheck_Version(c *check.C) { @@ -1218,7 +1218,7 @@ func (s *Suite) Test_VartypeCheck_Version(c *check.C) { "4pre7", "${VER}") vt.Output( - "WARN: filename:4: Invalid version number \"4.1-SNAPSHOT\".") + "WARN: filename.mk:4: Invalid version number \"4.1-SNAPSHOT\".") vt.Op(opUseMatch) vt.Values( @@ -1230,10 +1230,10 @@ func (s *Suite) Test_VartypeCheck_Version(c *check.C) { "1.[2-7].*", "[0-9]*") vt.Output( - "WARN: filename:11: Invalid version number pattern \"a*\".", - "WARN: filename:12: Invalid version number pattern \"1.2/456\".", - "WARN: filename:13: Please use \"4.*\" instead of \"4*\" as the version pattern.", - "WARN: filename:15: Please use \"1.[234].*\" instead of \"1.[234]*\" as the version pattern.") + "WARN: filename.mk:11: Invalid version number pattern \"a*\".", + "WARN: filename.mk:12: Invalid version number pattern \"1.2/456\".", + "WARN: filename.mk:13: Please use \"4.*\" instead of \"4*\" as the version pattern.", + "WARN: filename.mk:15: Please use \"1.[234].*\" instead of \"1.[234]*\" as the version pattern.") } func (s *Suite) Test_VartypeCheck_WrapperReorder(c *check.C) { @@ -1246,8 +1246,8 @@ func (s *Suite) Test_VartypeCheck_WrapperReorder(c *check.C) { "reorder:l:first", "omit:first") vt.Output( - "WARN: filename:2: Unknown wrapper reorder command \"reorder:l:first\".", - "WARN: filename:3: Unknown wrapper reorder command \"omit:first\".") + "WARN: filename.mk:2: Unknown wrapper reorder command \"reorder:l:first\".", + "WARN: filename.mk:3: Unknown wrapper reorder command \"omit:first\".") } func (s *Suite) Test_VartypeCheck_WrapperTransform(c *check.C) { @@ -1266,8 +1266,8 @@ func (s *Suite) Test_VartypeCheck_WrapperTransform(c *check.C) { "unknown", "-e 's,-Wall,-Wall -Wextra,'") vt.Output( - "WARN: filename:7: Unknown wrapper transform command \"rpath:/usr/lib\".", - "WARN: filename:8: Unknown wrapper transform command \"unknown\".") + "WARN: filename.mk:7: Unknown wrapper transform command \"rpath:/usr/lib\".", + "WARN: filename.mk:8: Unknown wrapper transform command \"unknown\".") } func (s *Suite) Test_VartypeCheck_WrksrcSubdirectory(c *check.C) { @@ -1287,14 +1287,14 @@ func (s *Suite) Test_VartypeCheck_WrksrcSubdirectory(c *check.C) { "${WRKDIR}/sub", "${SRCDIR}/sub") vt.Output( - "NOTE: filename:1: You can use \".\" instead of \"${WRKSRC}\".", - "NOTE: filename:2: You can use \".\" instead of \"${WRKSRC}/\".", - "NOTE: filename:3: You can use \".\" instead of \"${WRKSRC}/.\".", - "NOTE: filename:4: You can use \"subdir\" instead of \"${WRKSRC}/subdir\".", - "NOTE: filename:6: You can use \"directory\" instead of \"${WRKSRC}/directory\".", - "WARN: filename:8: \"../other\" is not a valid subdirectory of ${WRKSRC}.", - "WARN: filename:9: \"${WRKDIR}/sub\" is not a valid subdirectory of ${WRKSRC}.", - "WARN: filename:10: \"${SRCDIR}/sub\" is not a valid subdirectory of ${WRKSRC}.") + "NOTE: filename.mk:1: You can use \".\" instead of \"${WRKSRC}\".", + "NOTE: filename.mk:2: You can use \".\" instead of \"${WRKSRC}/\".", + "NOTE: filename.mk:3: You can use \".\" instead of \"${WRKSRC}/.\".", + "NOTE: filename.mk:4: You can use \"subdir\" instead of \"${WRKSRC}/subdir\".", + "NOTE: filename.mk:6: You can use \"directory\" instead of \"${WRKSRC}/directory\".", + "WARN: filename.mk:8: \"../other\" is not a valid subdirectory of ${WRKSRC}.", + "WARN: filename.mk:9: \"${WRKDIR}/sub\" is not a valid subdirectory of ${WRKSRC}.", + "WARN: filename.mk:10: \"${SRCDIR}/sub\" is not a valid subdirectory of ${WRKSRC}.") } func (s *Suite) Test_VartypeCheck_Yes(c *check.C) { @@ -1307,8 +1307,8 @@ func (s *Suite) Test_VartypeCheck_Yes(c *check.C) { "${YESVAR}") vt.Output( - "WARN: filename:2: APACHE_MODULE should be set to YES or yes.", - "WARN: filename:3: APACHE_MODULE should be set to YES or yes.") + "WARN: filename.mk:2: APACHE_MODULE should be set to YES or yes.", + "WARN: filename.mk:3: APACHE_MODULE should be set to YES or yes.") vt.Varname("PKG_DEVELOPER") vt.Op(opUseMatch) @@ -1318,9 +1318,9 @@ func (s *Suite) Test_VartypeCheck_Yes(c *check.C) { "${YESVAR}") vt.Output( - "WARN: filename:11: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.", - "WARN: filename:12: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.", - "WARN: filename:13: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.") + "WARN: filename.mk:11: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.", + "WARN: filename.mk:12: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.", + "WARN: filename.mk:13: PKG_DEVELOPER should only be used in a \".if defined(...)\" condition.") } func (s *Suite) Test_VartypeCheck_YesNo(c *check.C) { @@ -1334,8 +1334,8 @@ func (s *Suite) Test_VartypeCheck_YesNo(c *check.C) { "${YESVAR}") vt.Output( - "WARN: filename:3: GNU_CONFIGURE should be set to YES, yes, NO, or no.", - "WARN: filename:4: GNU_CONFIGURE should be set to YES, yes, NO, or no.") + "WARN: filename.mk:3: GNU_CONFIGURE should be set to YES, yes, NO, or no.", + "WARN: filename.mk:4: GNU_CONFIGURE should be set to YES, yes, NO, or no.") } func (s *Suite) Test_VartypeCheck_YesNoIndirectly(c *check.C) { @@ -1349,7 +1349,7 @@ func (s *Suite) Test_VartypeCheck_YesNoIndirectly(c *check.C) { "${YESVAR}") vt.Output( - "WARN: filename:3: GNU_CONFIGURE should be set to YES, yes, NO, or no.") + "WARN: filename.mk:3: GNU_CONFIGURE should be set to YES, yes, NO, or no.") } // VartypeCheckTester helps to test the many different checks in VartypeCheck. @@ -1375,7 +1375,7 @@ func NewVartypeCheckTester(t *Tester, checker func(cv *VartypeCheck)) *VartypeCh return &VartypeCheckTester{ t, checker, - "filename", + "filename.mk", 1, "", opAssign} |