diff options
author | rillig <rillig@pkgsrc.org> | 2018-12-17 00:15:39 +0000 |
---|---|---|
committer | rillig <rillig@pkgsrc.org> | 2018-12-17 00:15:39 +0000 |
commit | e6556331ee09765fb6455f21118582024d11052d (patch) | |
tree | 4fc4a1037677a7ced6b069b69c5e6dd5c44afe43 /pkgtools/pkglint/files | |
parent | 36b175ee4c46e94c1ec95d91a022c222fb2d9f7b (diff) | |
download | pkgsrc-e6556331ee09765fb6455f21118582024d11052d.tar.gz |
pkgtools/pkglint: update to 5.6.9
Changes since 5.6.8:
* In addition to the pkglint binary, the whole pkglint code is installed as
a library, so that other packages can use the code for doing their own
checks on pkgsrc packages, Makefiles, shell programs, or the other file
types from pkgsrc.
* BUILDLINK_*.* may be used in all files.
* Lots of refactorings
Diffstat (limited to 'pkgtools/pkglint/files')
86 files changed, 2282 insertions, 1244 deletions
diff --git a/pkgtools/pkglint/files/alternatives.go b/pkgtools/pkglint/files/alternatives.go index 781bb8507fe..3fd6397a7ec 100644 --- a/pkgtools/pkglint/files/alternatives.go +++ b/pkgtools/pkglint/files/alternatives.go @@ -1,4 +1,4 @@ -package main +package pkglint import "strings" diff --git a/pkgtools/pkglint/files/alternatives_test.go b/pkgtools/pkglint/files/alternatives_test.go index 7d8e60d18b5..59d0021bf7b 100644 --- a/pkgtools/pkglint/files/alternatives_test.go +++ b/pkgtools/pkglint/files/alternatives_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/autofix.go b/pkgtools/pkglint/files/autofix.go index 0855749b898..30b7f18c381 100644 --- a/pkgtools/pkglint/files/autofix.go +++ b/pkgtools/pkglint/files/autofix.go @@ -1,7 +1,6 @@ -package main +package pkglint import ( - "fmt" "io/ioutil" "netbsd.org/pkglint/regex" "os" @@ -182,7 +181,7 @@ func (fix *Autofix) Custom(fixer func(showAutofix, autofix bool)) { // of the actual fix for logging it later when Apply is called. // Describef may be called multiple times before calling Apply. func (fix *Autofix) Describef(lineno int, format string, args ...interface{}) { - fix.actions = append(fix.actions, autofixAction{fmt.Sprintf(format, args...), lineno}) + fix.actions = append(fix.actions, autofixAction{sprintf(format, args...), lineno}) } // InsertBefore prepends a line before the current line. @@ -266,7 +265,7 @@ func (fix *Autofix) Apply() { logFix := G.Logger.IsAutofix() if logDiagnostic { - msg := fmt.Sprintf(fix.diagFormat, fix.diagArgs...) + msg := sprintf(fix.diagFormat, fix.diagArgs...) if !logFix { if fix.diagFormat == AutofixFormat || G.Logger.FirstTime(line.Filename, line.Linenos(), msg) { line.showSource(G.out) diff --git a/pkgtools/pkglint/files/autofix_test.go b/pkgtools/pkglint/files/autofix_test.go index 6f73dce2eeb..529b80ba1a1 100644 --- a/pkgtools/pkglint/files/autofix_test.go +++ b/pkgtools/pkglint/files/autofix_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" @@ -905,15 +905,15 @@ func (s *Suite) Test_Autofix__lonely_source_2(c *check.C) { "", "\t\thttps://mirror1.sf.net/ https://mirror2.sf.net/directory/", "", - "\tThe first URL is missing the directory. To fix this, write", + "\tThe first URL is missing the directory. To fix this, write", "\t\t${MASTER_SITE_SOURCEFORGE:=directory/}.", "", "\tExample: -l${LIBS} expands to", "", "\t\t-llib1 lib2", "", - "\tThe second library is missing the -l. To fix this, write", - "\t${LIBS:@lib@-l${lib}@}.", + "\tThe second library is missing the -l. To fix this, write", + "\t${LIBS:S,^,-l,}.", "") } diff --git a/pkgtools/pkglint/files/buildlink3.go b/pkgtools/pkglint/files/buildlink3.go index 43b446ca1b2..70d8071cb08 100644 --- a/pkgtools/pkglint/files/buildlink3.go +++ b/pkgtools/pkglint/files/buildlink3.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "netbsd.org/pkglint/pkgver" diff --git a/pkgtools/pkglint/files/buildlink3_test.go b/pkgtools/pkglint/files/buildlink3_test.go index fd4def10e7f..5f1da8c782f 100644 --- a/pkgtools/pkglint/files/buildlink3_test.go +++ b/pkgtools/pkglint/files/buildlink3_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/category.go b/pkgtools/pkglint/files/category.go index 68a22ad8702..0ae19f22689 100644 --- a/pkgtools/pkglint/files/category.go +++ b/pkgtools/pkglint/files/category.go @@ -1,9 +1,6 @@ -package main +package pkglint -import ( - "fmt" - "netbsd.org/pkglint/textproc" -) +import "netbsd.org/pkglint/textproc" func CheckdirCategory(dir string) { if trace.Tracing { @@ -34,7 +31,7 @@ func CheckdirCategory(dir string) { _ = lex.NextBytesSet(valid) ch := lex.NextByteSet(invalid) if ch != -1 { - uni += fmt.Sprintf(" %U", ch) + uni += sprintf(" %U", ch) } } diff --git a/pkgtools/pkglint/files/category_test.go b/pkgtools/pkglint/files/category_test.go index 1920a87d788..3ded301ffba 100644 --- a/pkgtools/pkglint/files/category_test.go +++ b/pkgtools/pkglint/files/category_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/check_test.go b/pkgtools/pkglint/files/check_test.go index a6a8956f84f..650a90ec877 100644 --- a/pkgtools/pkglint/files/check_test.go +++ b/pkgtools/pkglint/files/check_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "bytes" @@ -139,7 +139,7 @@ func (t *Tester) SetupCommandLine(args ...string) { defer func() { trace.Tracing = prevTracing }() exitcode := G.ParseCommandLine(append([]string{"pkglint"}, args...)) - if exitcode != nil && *exitcode != 0 { + if exitcode != -1 && exitcode != 0 { t.CheckOutputEmpty() t.c.Fatalf("Cannot parse command line: %#v", args) } @@ -199,7 +199,11 @@ func (t *Tester) SetupFileMkLines(relativeFileName string, lines ...string) MkLi // SetupPkgsrc sets up a minimal but complete pkgsrc installation in the // temporary folder, so that pkglint runs without any errors. // Individual files may be overwritten by calling other Setup* methods. +// // This setup is especially interesting for testing Pkglint.Main. +// +// If the test works on a lower level than Pkglint.Main, +// LoadInfrastructure must be called to actually load the infrastructure files. func (t *Tester) SetupPkgsrc() { // This file is needed to locate the pkgsrc root directory. @@ -317,9 +321,9 @@ func (t *Tester) SetupPackage(pkgpath string, makefileLines ...string) string { line: for _, line := range makefileLines { - if m, prefix := match1(line, `^(\w+=)`); m { + if m, prefix := match1(line, `^#?(\w+=)`); m { for i, existingLine := range mlines { - if hasPrefix(existingLine, prefix) { + if hasPrefix(strings.TrimPrefix(existingLine, "#"), prefix) { mlines[i] = line continue line } @@ -602,6 +606,7 @@ func (t *Tester) Output() string { t.stdout.Reset() t.stderr.Reset() + G.Logger.logged = Once{} output := stdout + stderr if t.tmpdir != "" { @@ -617,6 +622,7 @@ func (t *Tester) Output() string { // See CheckOutputLines. func (t *Tester) CheckOutputEmpty() { output := t.Output() + actualLines := strings.Split(output, "\n") actualLines = actualLines[:len(actualLines)-1] t.Check(emptyToNil(actualLines), deepEquals, emptyToNil(nil)) diff --git a/pkgtools/pkglint/files/cmd/pkglint/pkglint.go b/pkgtools/pkglint/files/cmd/pkglint/pkglint.go new file mode 100644 index 00000000000..4f355a0c0ee --- /dev/null +++ b/pkgtools/pkglint/files/cmd/pkglint/pkglint.go @@ -0,0 +1,10 @@ +package main + +import ( + "netbsd.org/pkglint" + "os" +) + +func main() { + os.Exit(pkglint.Main()) +} diff --git a/pkgtools/pkglint/files/distinfo.go b/pkgtools/pkglint/files/distinfo.go index 614f7511697..c333c446f51 100644 --- a/pkgtools/pkglint/files/distinfo.go +++ b/pkgtools/pkglint/files/distinfo.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "bytes" @@ -118,8 +118,8 @@ func (ck *distinfoLinesChecker) checkAlgorithms(line Line) { ck.currentFirstLine.Warnf("Patch file %q does not exist in directory %q.", filename, pathToPatchdir) G.Explain( "If the patches directory looks correct, the patch may have been", - "removed without updating the distinfo file. In such a case please", - "update the distinfo file.", + "removed without updating the distinfo file.", + "In such a case please update the distinfo file.", "", "If the patches directory looks wrong, pkglint needs to be improved.") diff --git a/pkgtools/pkglint/files/distinfo_test.go b/pkgtools/pkglint/files/distinfo_test.go index 70d7f7872dd..8d4a3ed5364 100644 --- a/pkgtools/pkglint/files/distinfo_test.go +++ b/pkgtools/pkglint/files/distinfo_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/expecter.go b/pkgtools/pkglint/files/expecter.go index 9fde3c5ee2a..98dde6098ce 100644 --- a/pkgtools/pkglint/files/expecter.go +++ b/pkgtools/pkglint/files/expecter.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "netbsd.org/pkglint/regex" @@ -6,6 +6,8 @@ import ( ) // Expecter records the state when checking a list of lines from top to bottom. +// +// TODO: Maybe rename to LineLexer. type Expecter struct { lines Lines index int diff --git a/pkgtools/pkglint/files/expecter_test.go b/pkgtools/pkglint/files/expecter_test.go index 16088fb4473..3222486d607 100644 --- a/pkgtools/pkglint/files/expecter_test.go +++ b/pkgtools/pkglint/files/expecter_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/files.go b/pkgtools/pkglint/files/files.go index c924472df88..b24cbfbe10e 100644 --- a/pkgtools/pkglint/files/files.go +++ b/pkgtools/pkglint/files/files.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "io/ioutil" diff --git a/pkgtools/pkglint/files/files_test.go b/pkgtools/pkglint/files/files_test.go index 09036921422..6f09e214947 100644 --- a/pkgtools/pkglint/files/files_test.go +++ b/pkgtools/pkglint/files/files_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/fuzzer_test.go b/pkgtools/pkglint/files/fuzzer_test.go index 653186861d3..df1284c2a16 100644 --- a/pkgtools/pkglint/files/fuzzer_test.go +++ b/pkgtools/pkglint/files/fuzzer_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/intqa/testnames.go b/pkgtools/pkglint/files/intqa/testnames.go index 7161aaf743a..5ed0683ba3e 100644 --- a/pkgtools/pkglint/files/intqa/testnames.go +++ b/pkgtools/pkglint/files/intqa/testnames.go @@ -263,7 +263,7 @@ func (ck *TestNameChecker) Check() { } } if len(ck.errors) > 0 || (ck.warn && len(ck.warnings) > 0) { - fmt.Printf("%d %s and %d %s.", + ck.c.Errorf("%d %s and %d %s.", len(ck.errors), ifelseStr(len(ck.errors) == 1, "error", "errors"), len(ck.warnings), @@ -287,7 +287,7 @@ func (ck *TestNameChecker) isIgnored(filename string) bool { func newElement(typeName, funcName, filename string) *testeeElement { typeName = strings.TrimSuffix(typeName, "Impl") - e := &testeeElement{File: filename, Type: typeName, Func: funcName} + e := testeeElement{File: filename, Type: typeName, Func: funcName} e.FullName = e.Type + ifelseStr(e.Type != "" && e.Func != "", ".", "") + e.Func @@ -299,7 +299,7 @@ func newElement(typeName, funcName, filename string) *testeeElement { e.Prefix = e.Type + ifelseStr(e.Type != "" && e.Func != "", "_", "") + e.Func } - return e + return &e } func (el *testeeElement) Less(other *testeeElement) bool { diff --git a/pkgtools/pkglint/files/licenses.go b/pkgtools/pkglint/files/licenses.go index 1456b9c7d91..18b51568641 100644 --- a/pkgtools/pkglint/files/licenses.go +++ b/pkgtools/pkglint/files/licenses.go @@ -1,4 +1,4 @@ -package main +package pkglint import "netbsd.org/pkglint/licenses" diff --git a/pkgtools/pkglint/files/licenses_test.go b/pkgtools/pkglint/files/licenses_test.go index e95e7c89214..c6c4c4da33c 100644 --- a/pkgtools/pkglint/files/licenses_test.go +++ b/pkgtools/pkglint/files/licenses_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/line.go b/pkgtools/pkglint/files/line.go index 73569f292ac..c8de51f465c 100644 --- a/pkgtools/pkglint/files/line.go +++ b/pkgtools/pkglint/files/line.go @@ -1,4 +1,4 @@ -package main +package pkglint // When files are read in by pkglint, they are interpreted in terms of // lines. For Makefiles, line continuations are handled properly, allowing diff --git a/pkgtools/pkglint/files/line_test.go b/pkgtools/pkglint/files/line_test.go index 7d0fc9e962e..7093eea884d 100644 --- a/pkgtools/pkglint/files/line_test.go +++ b/pkgtools/pkglint/files/line_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/linechecker.go b/pkgtools/pkglint/files/linechecker.go index dbca8bfa17a..a6e9a12691e 100644 --- a/pkgtools/pkglint/files/linechecker.go +++ b/pkgtools/pkglint/files/linechecker.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "fmt" @@ -117,20 +117,20 @@ func (ck LineChecker) CheckWordAbsolutePathname(word string) { ck.line.Warnf("Found absolute pathname: %s", word) if contains(ck.line.Text, "DESTDIR") { G.Explain( - "Absolute pathnames are often an indicator for unportable code. As", - "pkgsrc aims to be a portable system, absolute pathnames should be", - "avoided whenever possible.", + "Absolute pathnames are often an indicator for unportable code.", + "As pkgsrc aims to be a portable system,", + "absolute pathnames should be avoided whenever possible.", "", - "A special variable in this context is ${DESTDIR}, which is used in", - "GNU projects to specify a different directory for installation than", - "what the programs see later when they are executed. Usually it is", - "empty, so if anything after that variable starts with a slash, it is", - "considered an absolute pathname.") + "A special variable in this context is ${DESTDIR},", + "which is used in GNU projects to specify a different directory", + "for installation than what the programs see later when they are executed.", + "Usually it is empty, so if anything after that variable starts with a slash,", + "it is considered an absolute pathname.") } else { G.Explain( - "Absolute pathnames are often an indicator for unportable code. As", - "pkgsrc aims to be a portable system, absolute pathnames should be", - "avoided whenever possible.") + "Absolute pathnames are often an indicator for unportable code.", + "As pkgsrc aims to be a portable system,", + "absolute pathnames should be avoided whenever possible.") // TODO: Explain how to actually fix this warning properly. } diff --git a/pkgtools/pkglint/files/linechecker_test.go b/pkgtools/pkglint/files/linechecker_test.go index dbb261c7f39..f8db63bf9c3 100644 --- a/pkgtools/pkglint/files/linechecker_test.go +++ b/pkgtools/pkglint/files/linechecker_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" @@ -37,7 +37,7 @@ func (s *Suite) Test_LineChecker_CheckAbsolutePathname(c *check.C) { t.CheckOutputLines( "WARN: Makefile:2: Found absolute pathname: /bin", "", - "\tAbsolute pathnames are often an indicator for unportable code. As", + "\tAbsolute pathnames are often an indicator for unportable code. As", "\tpkgsrc aims to be a portable system, absolute pathnames should be", "\tavoided whenever possible.", "", @@ -53,13 +53,13 @@ func (s *Suite) Test_LineChecker_CheckAbsolutePathname(c *check.C) { "WARN: Makefile:9: The \"/dev/stderr\" file is not portable.", "WARN: Makefile:14: Found absolute pathname: /bin", "", - "\tAbsolute pathnames are often an indicator for unportable code. As", + "\tAbsolute pathnames are often an indicator for unportable code. As", "\tpkgsrc aims to be a portable system, absolute pathnames should be", "\tavoided whenever possible.", "", "\tA special variable in this context is ${DESTDIR}, which is used in", "\tGNU projects to specify a different directory for installation than", - "\twhat the programs see later when they are executed. Usually it is", + "\twhat the programs see later when they are executed. Usually it is", "\tempty, so if anything after that variable starts with a slash, it is", "\tconsidered an absolute pathname.", "") diff --git a/pkgtools/pkglint/files/lines.go b/pkgtools/pkglint/files/lines.go index 769ac200ef0..16f7267b4df 100644 --- a/pkgtools/pkglint/files/lines.go +++ b/pkgtools/pkglint/files/lines.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "netbsd.org/pkglint/regex" @@ -51,11 +51,12 @@ func (ls *LinesImpl) CheckRcsID(index int, prefixRe regex.Pattern, suggestedPref "current version can be traced back later from a binary package.", "This is to ensure reproducible builds, for example for finding bugs.", "", - "These CVS Ids are specific to the CVS version control system, and", - "pkgsrc-wip uses Git instead. Therefore, having the expanded CVS Ids", - "in those files represents the file from which they were originally", - "copied but not their current state. Because of that, these markers", - "should be replaced with the plain, unexpanded string $"+"NetBSD$.", + "These CVS Ids are specific to the CVS version control system,", + "and pkgsrc-wip uses Git instead.", + "Therefore, having the expanded CVS Ids in those files represents", + "the file from which they were originally copied but not their current state.", + "Because of that, these markers should be replaced with the plain,", + "unexpanded string $"+"NetBSD$.", "", "To preserve the history of the CVS Id, should that ever be needed,", "remove the leading $.") diff --git a/pkgtools/pkglint/files/lines_test.go b/pkgtools/pkglint/files/lines_test.go index 8583948fd16..fcdca3aa81f 100644 --- a/pkgtools/pkglint/files/lines_test.go +++ b/pkgtools/pkglint/files/lines_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" @@ -57,8 +57,8 @@ func (s *Suite) Test_Lines_CheckRcsID__wip(c *check.C) { G.CheckDirent(t.File("wip/package")) t.CheckOutputLines( - "AUTOFIX: ~/wip/package/file1.mk:1: Replacing \"# $NetBSD: lines_test.go,v 1.2 2018/12/02 23:12:43 rillig Exp $\" with \"# $NetBSD: lines_test.go,v 1.2 2018/12/02 23:12:43 rillig Exp $\".", - "AUTOFIX: ~/wip/package/file3.mk:1: Inserting a line \"# $NetBSD: lines_test.go,v 1.2 2018/12/02 23:12:43 rillig Exp $\" before this line.", - "AUTOFIX: ~/wip/package/file4.mk:1: Inserting a line \"# $NetBSD: lines_test.go,v 1.2 2018/12/02 23:12:43 rillig Exp $\" before this line.", - "AUTOFIX: ~/wip/package/file5.mk:1: Inserting a line \"# $NetBSD: lines_test.go,v 1.2 2018/12/02 23:12:43 rillig Exp $\" before this line.") + "AUTOFIX: ~/wip/package/file1.mk:1: Replacing \"# $NetBSD: lines_test.go,v 1.3 2018/12/17 00:15:39 rillig Exp $\" with \"# $NetBSD: lines_test.go,v 1.3 2018/12/17 00:15:39 rillig Exp $\".", + "AUTOFIX: ~/wip/package/file3.mk:1: Inserting a line \"# $NetBSD: lines_test.go,v 1.3 2018/12/17 00:15:39 rillig Exp $\" before this line.", + "AUTOFIX: ~/wip/package/file4.mk:1: Inserting a line \"# $NetBSD: lines_test.go,v 1.3 2018/12/17 00:15:39 rillig Exp $\" before this line.", + "AUTOFIX: ~/wip/package/file5.mk:1: Inserting a line \"# $NetBSD: lines_test.go,v 1.3 2018/12/17 00:15:39 rillig Exp $\" before this line.") } diff --git a/pkgtools/pkglint/files/logging.go b/pkgtools/pkglint/files/logging.go index e3e5c00af7e..1120c240bbd 100644 --- a/pkgtools/pkglint/files/logging.go +++ b/pkgtools/pkglint/files/logging.go @@ -1,8 +1,7 @@ -package main +package pkglint import ( "bytes" - "fmt" "io" "netbsd.org/pkglint/histogram" "path" @@ -166,7 +165,7 @@ func (l *Logger) Diag(line Line, level *LogLevel, format string, args ...interfa filename := line.Filename linenos := line.Linenos() - msg := fmt.Sprintf(format, args...) + msg := sprintf(format, args...) if !l.FirstTime(filename, linenos, msg) { l.suppressDiag = false return @@ -208,9 +207,9 @@ func (l *Logger) Logf(level *LogLevel, filename, lineno, format, msg string) { linenoSep := ifelseStr(effLineno != "", ":", "") var diag string if l.Opts.GccOutput { - diag = fmt.Sprintf("%s%s%s%s%s: %s\n", filename, linenoSep, effLineno, filenameSep, level.GccName, msg) + diag = sprintf("%s%s%s%s%s: %s\n", filename, linenoSep, effLineno, filenameSep, level.GccName, msg) } else { - diag = fmt.Sprintf("%s%s%s%s%s: %s\n", level.TraditionalName, filenameSep, filename, linenoSep, effLineno, msg) + diag = sprintf("%s%s%s%s%s: %s\n", level.TraditionalName, filenameSep, filename, linenoSep, effLineno, msg) } out.Write(escapePrintable(diag)) diff --git a/pkgtools/pkglint/files/logging_test.go b/pkgtools/pkglint/files/logging_test.go index 58c4e1c2568..77d0bbe0502 100644 --- a/pkgtools/pkglint/files/logging_test.go +++ b/pkgtools/pkglint/files/logging_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/mkline.go b/pkgtools/pkglint/files/mkline.go index 6f1fd876305..2ea46f0281a 100644 --- a/pkgtools/pkglint/files/mkline.go +++ b/pkgtools/pkglint/files/mkline.go @@ -1,4 +1,4 @@ -package main +package pkglint // Checks concerning single lines in Makefiles. @@ -73,9 +73,8 @@ func NewMkLine(line Line) *MkLineImpl { if hasPrefix(text, " ") && line.Basename != "bsd.buildlink3.mk" { line.Warnf("Makefile lines should not start with space characters.") G.Explain( - "If you want this line to contain a shell program, use a tab", - "character for indentation. Otherwise please remove the leading", - "whitespace.") + "If this line should be a shell command connected to a target, use a tab character for indentation.", + "Otherwise remove the leading whitespace.") } if m, commented, varname, spaceAfterVarname, op, valueAlign, value, spaceAfterValue, comment := MatchVarassign(text); m { @@ -99,7 +98,8 @@ func NewMkLine(line Line) *MkLineImpl { line.Warnf("The # character starts a comment.") G.Explain( "In a variable assignment, an unescaped # starts a comment that", - "continues until the end of the line. To escape the #, write \\#.") + "continues until the end of the line.", + "To escape the #, write \\#.") } return &MkLineImpl{line, &mkLineAssignImpl{ @@ -162,7 +162,7 @@ func NewMkLine(line Line) *MkLineImpl { } func (mkline *MkLineImpl) String() string { - return fmt.Sprintf("%s:%s", mkline.Filename, mkline.Linenos()) + return sprintf("%s:%s", mkline.Filename, mkline.Linenos()) } // IsVarassign returns true for variable assignments of the form VAR=value. @@ -357,7 +357,7 @@ func (mkline *MkLineImpl) Tokenize(s string, warn bool) []*MkToken { p := NewMkParser(mkline.Line, s, true) tokens := p.MkTokens() if warn && p.Rest() != "" { - mkline.Warnf("Pkglint parse error in MkLine.Tokenize at %q.", p.Rest()) + mkline.Warnf("Internal pkglint error in MkLine.Tokenize at %q.", p.Rest()) } return tokens } @@ -588,7 +588,10 @@ func (mkline *MkLineImpl) RefTo(other MkLine) string { return mkline.Line.RefTo(other.Line) } -var AlnumDash = textproc.NewByteSet("a-z---") +var ( + LowerDash = textproc.NewByteSet("a-z---") + AlnumDot = textproc.NewByteSet("A-Za-z0-9_.") +) func matchMkDirective(text string) (m bool, indent, directive, args, comment string) { lexer := textproc.NewLexer(text) @@ -597,7 +600,7 @@ func matchMkDirective(text string) (m bool, indent, directive, args, comment str } indent = lexer.NextHspace() - directive = lexer.NextBytesSet(AlnumDash) + directive = lexer.NextBytesSet(LowerDash) switch directive { case "if", "else", "elif", "endif", "ifdef", "ifndef", @@ -920,7 +923,7 @@ func (vuc *VarUseContext) String() string { if vuc.vartype != nil { typename = vuc.vartype.String() } - return fmt.Sprintf("(%s time:%s quoting:%s wordpart:%v)", typename, vuc.time, vuc.quoting, vuc.IsWordPart) + return sprintf("(%s time:%s quoting:%s wordpart:%v)", typename, vuc.time, vuc.quoting, vuc.IsWordPart) } // Indentation remembers the stack of preprocessing directives and their @@ -1157,7 +1160,12 @@ func (ind *Indentation) CheckFinish(filename string) { } // VarnameBytes contains characters that may be used in variable names. -// The bracket is included here for the tool of the same name, e.g. "TOOLS_PATH.[". +// The bracket is included only for the tool of the same name, e.g. "TOOLS_PATH.[". +// +// This approach differs from the one in devel/bmake/files/parse.c:/^Parse_IsVar, +// but in practice it works equally well. Luckily there aren't many situations +// where a complicated variable name contains unbalanced parentheses or braces, +// which would confuse the devel/bmake parser. var VarnameBytes = textproc.NewByteSet("A-Za-z_0-9*+---.[") func MatchVarassign(text string) (m, commented bool, varname, spaceAfterVarname, op, valueAlign, value, spaceAfterValue, comment string) { diff --git a/pkgtools/pkglint/files/mkline_test.go b/pkgtools/pkglint/files/mkline_test.go index cfcb4a19e33..86389770f57 100644 --- a/pkgtools/pkglint/files/mkline_test.go +++ b/pkgtools/pkglint/files/mkline_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" @@ -229,7 +229,7 @@ func (s *Suite) Test_VarUseContext_String(c *check.C) { t.SetupVartypes() vartype := G.Pkgsrc.VariableType("PKGNAME") - vuc := &VarUseContext{vartype, vucTimeUnknown, vucQuotBackt, false} + vuc := VarUseContext{vartype, vucTimeUnknown, vucQuotBackt, false} c.Check(vuc.String(), equals, "(Pkgname time:unknown quoting:backt wordpart:false)") } @@ -320,8 +320,8 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__unknown_rhs(c *check.C) { mkline := t.NewMkLine("filename", 1, "PKGNAME:= ${UNKNOWN}") t.SetupVartypes() - vuc := &VarUseContext{G.Pkgsrc.VariableType("PKGNAME"), vucTimeParse, vucQuotUnknown, false} - nq := mkline.VariableNeedsQuoting("UNKNOWN", nil, vuc) + vuc := VarUseContext{G.Pkgsrc.VariableType("PKGNAME"), vucTimeParse, vucQuotUnknown, false} + nq := mkline.VariableNeedsQuoting("UNKNOWN", nil, &vuc) c.Check(nq, equals, unknown) } @@ -333,8 +333,8 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__append_URL_to_list_of_URLs(c * t.SetupMasterSite("MASTER_SITE_SOURCEFORGE", "http://downloads.sourceforge.net/sourceforge/") mkline := t.NewMkLine("Makefile", 95, "MASTER_SITES=\t${HOMEPAGE}") - vuc := &VarUseContext{G.Pkgsrc.vartypes["MASTER_SITES"], vucTimeRun, vucQuotPlain, false} - nq := mkline.VariableNeedsQuoting("HOMEPAGE", G.Pkgsrc.vartypes["HOMEPAGE"], vuc) + vuc := VarUseContext{G.Pkgsrc.vartypes["MASTER_SITES"], vucTimeRun, vucQuotPlain, false} + nq := mkline.VariableNeedsQuoting("HOMEPAGE", G.Pkgsrc.vartypes["HOMEPAGE"], &vuc) c.Check(nq, equals, no) diff --git a/pkgtools/pkglint/files/mklinechecker.go b/pkgtools/pkglint/files/mklinechecker.go index 4d7fb69a00f..3121a596284 100644 --- a/pkgtools/pkglint/files/mklinechecker.go +++ b/pkgtools/pkglint/files/mklinechecker.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "netbsd.org/pkglint/regex" @@ -49,10 +49,10 @@ func (ck MkLineChecker) checkShellCommand() { fix := mkline.Autofix() fix.Notef("Shell programs should be indented with a single tab.") fix.Explain( - "The first tab in the line marks the line as a shell command. Since", - "every line of shell commands starts with a completely new shell", - "environment, there is no need to indent some of the commands, or to", - "use more horizontal space than necessary.") + "The first tab in the line marks the line as a shell command.", + "Since every line of shell commands starts with a completely new shell environment,", + "there is no need to indent some of the commands,", + "or to use more horizontal space than necessary.") fix.ReplaceRegex(`^\t\t+`, "\t", 1) fix.Apply() } @@ -83,8 +83,8 @@ func (ck MkLineChecker) checkInclude() { G.Explain( "To include portions of another Makefile, extract the common parts", "and put them into a Makefile.common or a Makefile fragment called", - "module.mk or similar. After that, both this one and the other", - "package should include the newly created file.") + "module.mk or similar.", + "After that, both this one and the other package should include the newly created file.") case IsPrefs(includedFile): if mkline.Basename == "buildlink3.mk" && includedFile == "../../mk/bsd.prefs.mk" { @@ -267,14 +267,14 @@ func (ck MkLineChecker) checkDependencyRule(allowedTargets map[string]bool) { // This is deliberate, see the explanation below. } else if !allowedTargets[target] { - mkline.Warnf("Unusual target %q.", target) + mkline.Warnf("Undeclared target %q.", target) G.Explain( - "If you want to define your own target, declare it like this:", + "To define a custom target in a package, declare it like this:", "", "\t.PHONY: my-target", "", - "In the rare case that you actually want a file-based make(1)", - "target, write it like this:", + "To define a custom target that creates a file (should be rarely needed),", + "declare it like this:", "", "\t${.CURDIR}/my-file:") } @@ -351,10 +351,10 @@ func (ck MkLineChecker) checkVarassignPermissions() { } G.Explain( "The allowed actions for a variable are determined based on the file", - "name in which the variable is used or defined. The exact rules are", + "name in which the variable is used or defined.", // FIXME: List the rules in this very explanation. - "hard-coded into pkglint. If they seem to be incorrect, please ask", - "on the tech-pkg@NetBSD.org mailing list.") + "The exact rules are hard-coded into pkglint.", + "If they seem to be incorrect, please ask on the tech-pkg@NetBSD.org mailing list.") } } @@ -403,9 +403,10 @@ func (ck MkLineChecker) CheckVaruse(varuse *MkVarUse, vuc *VarUseContext) { mkline.Warnf("The user-defined variable %s is used but not added to BUILD_DEFS.", varname) G.Explain( "When a pkgsrc package is built, many things can be configured by the", - "pkgsrc user in the mk.conf file. All these configurations should be", - "recorded in the binary package so the package can be reliably", - "rebuilt. The BUILD_DEFS variable contains a list of all these", + "pkgsrc user in the mk.conf file.", + "All these configurations should be recorded in the binary package", + "so the package can be reliably rebuilt.", + "The BUILD_DEFS variable contains a list of all these", "user-settable variables, so please add your variable to it, too.") } } @@ -558,10 +559,10 @@ func (ck MkLineChecker) checkVarusePermissions(varname string, vartype *Vartype, } G.Explain( "The allowed actions for a variable are determined based on the file", - "name in which the variable is used or defined. The exact rules are", + "name in which the variable is used or defined.", // FIXME: List the rules in this very explanation. - "hard-coded into pkglint. If they seem to be incorrect, please ask", - "on the tech-pkg@NetBSD.org mailing list.") + "The exact rules are hard-coded into pkglint.", + "If they seem to be incorrect, please ask on the tech-pkg@NetBSD.org mailing list.") } } @@ -595,15 +596,17 @@ func (ck MkLineChecker) warnVaruseToolLoadTime(varname string, tool *Tool) { ck.MkLine.Warnf("The tool ${%s} cannot be used at load time.", varname) G.Explain( "To use a tool at load time, it must be declared in the package", - "Makefile by adding it to USE_TOOLS. After that, bsd.prefs.mk must", - "be included. Adding the tool to USE_TOOLS at any later time has", - "no effect, which means that the tool can only be used at run time.", + "Makefile by adding it to USE_TOOLS.", + "After that, bsd.prefs.mk must be included.", + "Adding the tool to USE_TOOLS at any later time has no effect,", + "which means that the tool can only be used at run time.", "That's the rule for the package Makefiles.", "", "Since any other .mk file can be included from anywhere else, there", "is no guarantee that the tool is properly defined for using it at", - "load time (see above for the tricky rules). Therefore the tools can", - "only be used at run time, except in the package Makefile itself.") + "load time (see above for the tricky rules).", + "Therefore the tools can only be used at run time,", + "except in the package Makefile itself.") } func (ck MkLineChecker) warnVaruseLoadTime(varname string, isIndirect bool) { @@ -612,10 +615,10 @@ func (ck MkLineChecker) warnVaruseLoadTime(varname string, isIndirect bool) { if !isIndirect { mkline.Warnf("%s should not be evaluated at load time.", varname) G.Explain( - "Many variables, especially lists of something, get their values", - "incrementally. Therefore it is generally unsafe to rely on their", - "value until it is clear that it will never change again. This", - "point is reached when the whole package Makefile is loaded and", + "Many variables, especially lists of something, get their values incrementally.", + "Therefore it is generally unsafe to rely on their", + "value until it is clear that it will never change again.", + "This point is reached when the whole package Makefile is loaded and", "execution of the shell commands starts; in some cases earlier.", "", "Additionally, when using the \":=\" operator, each $$ is replaced", @@ -627,8 +630,8 @@ func (ck MkLineChecker) warnVaruseLoadTime(varname string, isIndirect bool) { mkline.Warnf("%s should not be evaluated indirectly at load time.", varname) G.Explain( "The variable on the left-hand side may be evaluated at load time,", - "but the variable on the right-hand side may not. Because of the", - "assignment in this line, the variable might be used indirectly", + "but the variable on the right-hand side may not.", + "Because of the assignment in this line, the variable might be used indirectly", "at load time, before it is guaranteed to be properly initialized.") } @@ -667,15 +670,16 @@ func (ck MkLineChecker) CheckVaruseShellword(varname string, vartype *Vartype, v "", "\thttps://mirror1.sf.net/ https://mirror2.sf.net/directory/", "", - "The first URL is missing the directory. To fix this, write", + "The first URL is missing the directory.", + "To fix this, write", "\t${MASTER_SITE_SOURCEFORGE:=directory/}.", "", "Example: -l${LIBS} expands to", "", "\t-llib1 lib2", "", - "The second library is missing the -l. To fix this, write", - "${LIBS:@lib@-l${lib}@}.") + "The second library is missing the -l.", + "To fix this, write ${LIBS:S,^,-l,}.") } else { mkline.Warnf("The variable %s should be quoted as part of a shell word.", varname) mkline.Explain( @@ -952,10 +956,12 @@ func (ck MkLineChecker) checkVarassignSpecific() { mkline.Notef("Please use \"# empty\", \"# none\" or \"# yes\" instead of \"# defined\".") G.Explain( "The value #defined says something about the state of the variable,", - "but not what that _means_. In some cases a variable that is defined", + "but not what that _means_.", + "In some cases a variable that is defined", "means \"yes\", in other cases it is an empty list (which is also", "only the state of the variable), whose meaning could be described", - "with \"none\". It is this meaning that should be described.") + "with \"none\".", + "It is this meaning that should be described.") } if varname == "DIST_SUBDIR" || varname == "WRKSRC" { @@ -971,6 +977,7 @@ func (ck MkLineChecker) checkVarassignSpecific() { } if varname == "PKG_SKIP_REASON" && G.Mk.indentation.DependsOn("OPSYS") { + // TODO: Provide autofix for simple cases, like ".if ${OPSYS} == SunOS". mkline.Notef("Consider setting NOT_FOR_PLATFORM instead of " + "PKG_SKIP_REASON depending on ${OPSYS}.") } @@ -1000,10 +1007,11 @@ func (ck MkLineChecker) checkVarassignBsdPrefs() { G.Explain( "The ?= operator is used to provide a default value to a variable.", "In pkgsrc, many variables can be set by the pkgsrc user in the", - "mk.conf file. This file must be included explicitly. If a ?=", - "operator appears before mk.conf has been included, it will not care", - "about the user's preferences, which can result in unexpected", - "behavior.", + "mk.conf file.", + "This file must be included explicitly.", + "If a ?= operator appears before mk.conf has been included,", + "it will not care about the user's preferences,", + "which can result in unexpected behavior.", "", "The easiest way to include the mk.conf file is by including the", "bsd.prefs.mk file, which will take care of everything.") @@ -1193,14 +1201,12 @@ func (ck MkLineChecker) checkDirectiveCondEmpty(varuse *MkVarUse) { if matches(varname, `^\$.*:[MN]`) { ck.MkLine.Warnf("The empty() function takes a variable name as parameter, not a variable expression.") G.Explain( - "Instead of empty(${VARNAME:Mpattern}), you should write either", - "of the following:", + "Instead of empty(${VARNAME:Mpattern}), you should write either of the following:", "", "\tempty(VARNAME:Mpattern)", "\t${VARNAME:Mpattern} == \"\"", "", - "Instead of !empty(${VARNAME:Mpattern}), you should write either", - "of the following:", + "Instead of !empty(${VARNAME:Mpattern}), you should write either of the following:", "", "\t!empty(VARNAME:Mpattern)", "\t${VARNAME:Mpattern}") @@ -1234,9 +1240,8 @@ func (ck MkLineChecker) checkCompareVarStr(varname, op, value string) { if varname == "PKGSRC_COMPILER" { ck.MkLine.Warnf("Use ${PKGSRC_COMPILER:%s%s} instead of the %s operator.", ifelseStr(op == "==", "M", "N"), value, op) G.Explain( - "The PKGSRC_COMPILER can be a list of chained compilers, e.g. \"ccache", - "distcc clang\". Therefore, comparing it using == or != leads to", - "wrong results in these cases.") + "The PKGSRC_COMPILER can be a list of chained compilers, e.g. \"ccache distcc clang\".", + "Therefore, comparing it using == or != leads to wrong results in these cases.") } } diff --git a/pkgtools/pkglint/files/mklinechecker_test.go b/pkgtools/pkglint/files/mklinechecker_test.go index ccad83a69e6..36e154ce9db 100644 --- a/pkgtools/pkglint/files/mklinechecker_test.go +++ b/pkgtools/pkglint/files/mklinechecker_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" @@ -181,7 +181,7 @@ func (s *Suite) Test_MkLineChecker_checkDependencyRule(c *check.C) { mklines.Check() t.CheckOutputLines( - "WARN: category/package/filename.mk:8: Unusual target \"target-3\".") + "WARN: category/package/filename.mk:8: Undeclared target \"target-3\".") } func (s *Suite) Test_MkLineChecker_checkVartype__simple_type(c *check.C) { @@ -518,7 +518,7 @@ func (s *Suite) Test_MkLineChecker__unclosed_varuse(c *check.C) { "WARN: Makefile:2: EGDIRS is defined but not used.", // XXX: This warning is redundant because of the "Unclosed" warning above. - "WARN: Makefile:2: Pkglint parse error in MkLine.Tokenize at "+ + "WARN: Makefile:2: Internal pkglint error in MkLine.Tokenize at "+ "\"${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".") } @@ -653,9 +653,40 @@ func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS(c *check.C) { "WARN: Makefile:2: Compiler flag \"%s\\\\\\\"\" should start with a hyphen.") } +func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("--autofix", "-Wspace") + lines := t.SetupFileLines("filename.mk", + MkRcsID, + ".if defined(A)", + ".for a in ${A}", + ".if defined(C)", + ".endif", + ".endfor", + ".endif") + mklines := NewMkLines(lines) + + mklines.Check() + + t.CheckOutputLines( + "AUTOFIX: ~/filename.mk:3: Replacing \".\" with \". \".", + "AUTOFIX: ~/filename.mk:4: Replacing \".\" with \". \".", + "AUTOFIX: ~/filename.mk:5: Replacing \".\" with \". \".", + "AUTOFIX: ~/filename.mk:6: Replacing \".\" with \". \".") + t.CheckFileLines("filename.mk", + "# $"+"NetBSD$", + ".if defined(A)", + ". for a in ${A}", + ". if defined(C)", + ". endif", + ". endfor", + ".endif") +} + // Up to 2018-01-28, pkglint applied the autofix also to the continuation // lines, which is incorrect. It replaced the dot in "4.*" with spaces. -func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix_multiline(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall", "--autofix") @@ -915,6 +946,26 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__deprecated_PKG_DEBUG(c *check.C) "WARN: module.mk:123: Use of \"_PKG_DEBUG\" is deprecated. Use RUN (with more error checking) instead.") } +// PR 46570, item "15. net/uucp/Makefile has a make loop" +func (s *Suite) Test_MkLineChecker_checkVaruseUndefined__indirect_variables(c *check.C) { + t := s.Init(c) + + t.SetupTool("echo", "ECHO", AfterPrefsMk) + mkline := t.NewMkLine("net/uucp/Makefile", 123, "\techo ${UUCP_${var}}") + + MkLineChecker{mkline}.Check() + + // No warning about UUCP_${var} being used but not defined. + // + // Normally, parameterized variables use a dot instead of an underscore as separator. + // This is one of the few other cases. Pkglint doesn't warn about dynamic variable + // names like UUCP_${var} or SITES_${distfile}. + // + // It does warn about simple variable names though, like ${var} in this example. + t.CheckOutputLines( + "WARN: net/uucp/Makefile:123: var is used but not defined.") +} + func (s *Suite) Test_MkLineChecker_checkVarassignSpecific(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/mklines.go b/pkgtools/pkglint/files/mklines.go index 873c5e0104b..583b36c1870 100644 --- a/pkgtools/pkglint/files/mklines.go +++ b/pkgtools/pkglint/files/mklines.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "strings" @@ -349,9 +349,14 @@ func (mklines *MkLinesImpl) collectDocumentedVariables() { commentLines := 0 relevant := true + // TODO: Correctly interpret declarations like "package-settable variables:" and + // TODO: "user-settable variables", as well as "default: ...", "allowed: ...", + // TODO: "list of" and other types. + finish := func() { if commentLines >= 3 && relevant { for varname, mkline := range scope.used { + mklines.vars.Define(varname, mkline) mklines.vars.Use(varname, mkline) } } @@ -379,9 +384,10 @@ func (mklines *MkLinesImpl) collectDocumentedVariables() { } parser.lexer.SkipByte(':') - varbase := varnameBase(varname) - if varbase == strings.ToUpper(varbase) && matches(varbase, `[A-Z]`) && parser.EOF() { - scope.Use(varname, mkline) + varcanon := varnameCanon(varname) + if varcanon == strings.ToUpper(varcanon) && matches(varcanon, `[A-Z]`) && parser.EOF() { + scope.Define(varcanon, mkline) + scope.Use(varcanon, mkline) } if 1 < len(words) && words[1] == "Copyright" { @@ -420,8 +426,9 @@ func (mklines *MkLinesImpl) CheckRedundantAssignments() { old.Warnf("Variable %s is overwritten in %s.", new.Varname(), old.RefTo(new)) G.Explain( "The variable definition in this line does not have an effect since", - "it is overwritten elsewhere. This typically happens because of a", - "typo (writing = instead of +=) or because the line that overwrites", + "it is overwritten elsewhere.", + "This typically happens because of a typo (writing = instead of +=)", + "or because the line that overwrites", "is in another file that is used by several packages.") } } @@ -429,6 +436,8 @@ func (mklines *MkLinesImpl) CheckRedundantAssignments() { mklines.ForEach(scope.Handle) } +// CheckForUsedComment checks that this file (a Makefile.common) has the given +// relativeName in one of the "# used by" comments at the beginning of the file. func (mklines *MkLinesImpl) CheckForUsedComment(relativeName string) { lines := mklines.lines if lines.Len() < 3 { @@ -447,17 +456,21 @@ func (mklines *MkLinesImpl) CheckForUsedComment(relativeName string) { i++ } + // TODO: Sort the comments. + // TODO: Discuss whether these comments are actually helpful. + fix := lines.Lines[i].Autofix() fix.Warnf("Please add a line %q here.", expected) fix.Explain( "Since Makefile.common files usually don't have any comments and", - "therefore not a clearly defined interface, they should at least", + "therefore not a clearly defined purpose, they should at least", "contain references to all files that include them, so that it is", "easier to see what effects future changes may have.", "", "If there are more than five packages that use a Makefile.common,", - "you should think about giving it a proper name (maybe plugin.mk) and", - "documenting its interface.") + "that file should have a clearly defined and documented purpose,", + "and the filename should reflect that purpose.", + "Typical names are module.mk, plugin.mk or version.mk.") fix.InsertBefore(expected) fix.Apply() diff --git a/pkgtools/pkglint/files/mklines_test.go b/pkgtools/pkglint/files/mklines_test.go index 4a7b1d02f78..d9f329ef01b 100644 --- a/pkgtools/pkglint/files/mklines_test.go +++ b/pkgtools/pkglint/files/mklines_test.go @@ -1,42 +1,10 @@ -package main +package pkglint import ( - "fmt" "gopkg.in/check.v1" "sort" ) -func (s *Suite) Test_MkLines_Check__autofix_directive_indentation(c *check.C) { - t := s.Init(c) - - t.SetupCommandLine("--autofix", "-Wspace") - lines := t.SetupFileLines("filename.mk", - MkRcsID, - ".if defined(A)", - ".for a in ${A}", - ".if defined(C)", - ".endif", - ".endfor", - ".endif") - mklines := NewMkLines(lines) - - mklines.Check() - - t.CheckOutputLines( - "AUTOFIX: ~/filename.mk:3: Replacing \".\" with \". \".", - "AUTOFIX: ~/filename.mk:4: Replacing \".\" with \". \".", - "AUTOFIX: ~/filename.mk:5: Replacing \".\" with \". \".", - "AUTOFIX: ~/filename.mk:6: Replacing \".\" with \". \".") - t.CheckFileLines("filename.mk", - "# $"+"NetBSD$", - ".if defined(A)", - ". for a in ${A}", - ". if defined(C)", - ". endif", - ". endfor", - ".endif") -} - func (s *Suite) Test_MkLines_Check__unusual_target(c *check.C) { t := s.Init(c) @@ -51,7 +19,7 @@ func (s *Suite) Test_MkLines_Check__unusual_target(c *check.C) { mklines.Check() t.CheckOutputLines( - "WARN: Makefile:3: Unusual target \"echo\".") + "WARN: Makefile:3: Undeclared target \"echo\".") } func (s *Suite) Test_MkLines__quoting_LDFLAGS_for_GNU_configure(c *check.C) { @@ -218,33 +186,11 @@ func (s *Suite) Test_MkLines__PKG_SKIP_REASON_depending_on_OPSYS(c *check.C) { "NOTE: Makefile:4: Consider setting NOT_FOR_PLATFORM instead of PKG_SKIP_REASON depending on ${OPSYS}.") } -// PR 46570, item "15. net/uucp/Makefile has a make loop" -func (s *Suite) Test_MkLines__indirect_variables(c *check.C) { - t := s.Init(c) - - t.SetupTool("echo", "ECHO", AfterPrefsMk) - mklines := t.NewMkLines("net/uucp/Makefile", - MkRcsID, - "", - "post-configure:", - ".for var in MAIL_PROGRAM CMDPATH", - "\t"+`${RUN} ${ECHO} "#define ${var} \""${UUCP_${var}}"\""`, - ".endfor") - - mklines.Check() - - // No warning about UUCP_${var} being used but not defined. - // Normally, parameterized variables use a dot instead of an - // underscore as separator. This is one of the other cases, - // and pkglint just doesn't warn about dynamic variable names - // like UUCP_${var} or SITES_${distfile}. - t.CheckOutputEmpty() -} - -func (s *Suite) Test_MkLines_Check__list_variable_as_part_of_word(c *check.C) { +func (s *Suite) Test_MkLines_Check__use_list_variable_as_part_of_word(c *check.C) { t := s.Init(c) t.SetupVartypes() + t.SetupTool("tr", "", AtRunTime) mklines := t.NewMkLines("converters/chef/Makefile", MkRcsID, "\tcd ${WRKSRC} && tr '\\r' '\\n' < ${DISTDIR}/${DIST_SUBDIR}/${DISTFILES} > chef.l") @@ -252,7 +198,6 @@ func (s *Suite) Test_MkLines_Check__list_variable_as_part_of_word(c *check.C) { mklines.Check() t.CheckOutputLines( - "WARN: converters/chef/Makefile:2: Unknown shell command \"tr\".", "WARN: converters/chef/Makefile:2: The list variable DISTFILES should not be embedded in a word.") } @@ -263,17 +208,19 @@ func (s *Suite) Test_MkLines_Check__absolute_pathname_depending_on_OPSYS(c *chec mklines := t.NewMkLines("games/heretic2-demo/Makefile", MkRcsID, ".if ${OPSYS} == \"DragonFly\"", - "TOOLS_PLATFORM.gtar=\t/usr/bin/bsdtar", + "TAR_CMD=\t/usr/bin/bsdtar", ".endif", - "TOOLS_PLATFORM.gtar=\t/usr/bin/bsdtar") + "TAR_CMD=\t/usr/bin/bsdtar", + "", + "do-extract:", + "\t${TAR_CMD}") mklines.Check() - // No warning about an unknown shell command in line 3, - // since that line depends on OPSYS. + // No warning about an unknown shell command in line 3 since that line depends on OPSYS. + // Shell commands that are specific to an operating system are probably defined + // and used intentionally, so even commands that are not known tools are allowed. t.CheckOutputLines( - "WARN: games/heretic2-demo/Makefile:3: The variable TOOLS_PLATFORM.gtar may not be set by any package.", - "WARN: games/heretic2-demo/Makefile:5: The variable TOOLS_PLATFORM.gtar may not be set by any package.", "WARN: games/heretic2-demo/Makefile:5: Unknown shell command \"/usr/bin/bsdtar\".") } @@ -281,50 +228,77 @@ func (s *Suite) Test_MkLines_CheckForUsedComment(c *check.C) { t := s.Init(c) t.SetupCommandLine("--show-autofix") - t.NewMkLines("Makefile.common", - MkRcsID, - "", - "# used by sysutils/mc", - ).CheckForUsedComment("sysutils/mc") - - t.CheckOutputEmpty() - t.NewMkLines("Makefile.common").CheckForUsedComment("category/package") + test := func(pkgpath string, lines []string, diagnostics []string) { + mklines := t.NewMkLines("Makefile.common", lines...) - t.CheckOutputEmpty() - - t.NewMkLines("Makefile.common", - MkRcsID, - ).CheckForUsedComment("category/package") + mklines.CheckForUsedComment(pkgpath) - t.CheckOutputEmpty() - - t.NewMkLines("Makefile.common", - MkRcsID, - "", - ).CheckForUsedComment("category/package") - - t.CheckOutputEmpty() - - t.NewMkLines("Makefile.common", - MkRcsID, - "", - "VARNAME=\tvalue", - ).CheckForUsedComment("category/package") - - t.CheckOutputLines( - "WARN: Makefile.common:2: Please add a line \"# used by category/package\" here.", - "AUTOFIX: Makefile.common:2: Inserting a line \"# used by category/package\" before this line.") - - t.NewMkLines("Makefile.common", - MkRcsID, - "#", - "#", - ).CheckForUsedComment("category/package") + if len(diagnostics) > 0 { + t.CheckOutputLines(diagnostics...) + } else { + t.CheckOutputEmpty() + } + } - t.CheckOutputLines( - "WARN: Makefile.common:3: Please add a line \"# used by category/package\" here.", - "AUTOFIX: Makefile.common:3: Inserting a line \"# used by category/package\" before this line.") + lines := func(lines ...string) []string { return lines } + diagnostics := func(diagnostics ...string) []string { return diagnostics } + + // This file is too short to be checked. + test( + "category/package", + lines(), + diagnostics()) + + // Still too short. + test( + "category/package", + lines( + MkRcsID), + diagnostics()) + + // Still too short. + test( + "category/package", + lines( + MkRcsID, + ""), + diagnostics()) + + // This file is correctly mentioned. + test( + "sysutils/mc", + lines( + MkRcsID, + "", + "# used by sysutils/mc"), + diagnostics()) + + // This file is not correctly mentioned, therefore the line is inserted. + // TODO: Since the following line is of a different type, an additional empty line should be inserted. + test( + "category/package", + lines( + MkRcsID, + "", + "VARNAME=\tvalue"), + diagnostics( + "WARN: Makefile.common:2: Please add a line \"# used by category/package\" here.", + "AUTOFIX: Makefile.common:2: Inserting a line \"# used by category/package\" before this line.")) + + // The "used by" comments may either start in line 2 or in line 3. + test( + "category/package", + lines( + MkRcsID, + "#", + "#"), + diagnostics( + "WARN: Makefile.common:3: Please add a line \"# used by category/package\" here.", + "AUTOFIX: Makefile.common:3: Inserting a line \"# used by category/package\" before this line.")) + + // TODO: What if there is an introductory comment first? That should stay at the top of the file. + // TODO: What if the "used by" comments appear in the second paragraph, preceded by only comments and empty lines? c.Check(G.autofixAvailable, equals, true) } @@ -353,7 +327,7 @@ func (s *Suite) Test_MkLines_collectDefinedVariables(c *check.C) { "SUV= value for substitution", "", "pre-configure:", - "\t${RUN} autoreconf; autoheader-2.13; unknown-command", + "\t${RUN} autoreconf; autoheader-2.13", "\t${ECHO} ${OSV:Q}") mklines.Check() @@ -362,10 +336,11 @@ func (s *Suite) Test_MkLines_collectDefinedVariables(c *check.C) { // The SUV variable is used implicitly by the SUBST framework, therefore no warning. // The OSV.NetBSD variable is used implicitly via the OSV variable, therefore no warning. t.CheckOutputLines( - // FIXME: the below warning is wrong; it's ok to have SUBST blocks in all files, maybe except buildlink3.mk. - "WARN: determine-defined-variables.mk:12: The variable SUBST_VARS.subst may not be set "+ - "(only given a default value, appended to) in this file; it would be ok in Makefile, Makefile.common, options.mk.", - "WARN: determine-defined-variables.mk:16: Unknown shell command \"unknown-command\".") + // FIXME: the below warning is wrong; it's ok to have SUBST blocks in all files, + // maybe except buildlink3.mk. + "WARN: determine-defined-variables.mk:12: The variable SUBST_VARS.subst may not be set " + + "(only given a default value, appended to) in this file; " + + "it would be ok in Makefile, Makefile.common, options.mk.") } func (s *Suite) Test_MkLines_collectDefinedVariables__BUILTIN_FIND_FILES_VAR(c *check.C) { @@ -403,7 +378,7 @@ func (s *Suite) Test_MkLines_collectUsedVariables__simple(c *check.C) { mklines.collectUsedVariables() - c.Check(len(mklines.vars.used), equals, 1) + c.Check(mklines.vars.used, deepEquals, map[string]MkLine{"VAR": mkline}) c.Check(mklines.vars.FirstUse("VAR"), equals, mkline) } @@ -553,10 +528,10 @@ func (s *Suite) Test_MkLines_Check__endif_comment(c *check.C) { ".endfor # j", // Wrong, should be i. "", ".if ${PKG_OPTIONS:Moption}", - ".endif # option", + ".endif # option", // Correct. "", ".if ${PKG_OPTIONS:Moption}", - ".endif # opti", // This typo gets unnoticed since "opti" is a substring of the condition. + ".endif # opti", // This typo goes unnoticed since "opti" is a substring of the condition. "", ".if ${OPSYS} == NetBSD", ".elif ${OPSYS} == FreeBSD", @@ -573,7 +548,7 @@ func (s *Suite) Test_MkLines_Check__endif_comment(c *check.C) { "WARN: opsys.mk:20: Comment \"NetBSD\" does not match condition \"${OPSYS} == FreeBSD\".") } -func (s *Suite) Test_MkLines_Check__unbalanced_directives(c *check.C) { +func (s *Suite) Test_MkLines_Check__unfinished_directives(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -594,6 +569,28 @@ func (s *Suite) Test_MkLines_Check__unbalanced_directives(c *check.C) { "ERROR: opsys.mk:EOF: .for from line 3 must be closed.") } +func (s *Suite) Test_MkLines_Check__unbalanced_directives(c *check.C) { + t := s.Init(c) + + t.SetupVartypes() + mklines := t.NewMkLines("opsys.mk", + MkRcsID, + "", + ".for i in 1 2 3 4 5", + ". if ${OPSYS} == NetBSD", + ". endfor", + ".endif") + + mklines.Check() + + // As of November 2018 pkglint doesn't find that the inner .if is closed by an .endfor. + // This is checked by bmake, though. + // + // As soon as pkglint starts to analyze .if/.for as regular statements + // like in most programming languages, it will find this inconsistency, too. + t.CheckOutputEmpty() +} + func (s *Suite) Test_MkLines_Check__incomplete_subst_at_end(c *check.C) { t := s.Init(c) @@ -643,14 +640,14 @@ func (s *Suite) Test_MkLines__wip_category_Makefile(c *check.C) { mklines.Check() t.CheckOutputLines( - "WARN: ~/wip/Makefile:14: Unusual target \"clean-tmpdir\".", + "WARN: ~/wip/Makefile:14: Undeclared target \"clean-tmpdir\".", "", - "\tIf you want to define your own target, declare it like this:", + "\tTo define a custom target in a package, declare it like this:", "", "\t\t.PHONY: my-target", "", - "\tIn the rare case that you actually want a file-based make(1) target,", - "\twrite it like this:", + "\tTo define a custom target that creates a file (should be rarely", + "\tneeded), declare it like this:", "", "\t\t${.CURDIR}/my-file:", "") @@ -667,6 +664,8 @@ func (s *Suite) Test_MkLines_collectDocumentedVariables(c *check.C) { "# Copyright 2000-2018", "#", "# This whole comment is ignored, until the next empty line.", + "# Since it contains the word \"copyright\", it's probably legalese", + "# instead of documentation.", "", "# User-settable variables:", "#", @@ -687,22 +686,21 @@ func (s *Suite) Test_MkLines_collectDocumentedVariables(c *check.C) { "# VARBASE3.${id}") // The variables that appear in the documentation are marked as - // used, to prevent the "defined but not used" warnings. + // both used and defined, to prevent the "defined but not used" warnings. mklines.collectDocumentedVariables() var varnames []string for varname, mkline := range mklines.vars.used { - varnames = append(varnames, fmt.Sprintf("%s (line %s)", varname, mkline.Linenos())) + varnames = append(varnames, sprintf("%s (line %s)", varname, mkline.Linenos())) } sort.Strings(varnames) expected := []string{ - "PKG_DEBUG_LEVEL (line 9)", - "PKG_VERBOSE (line 14)", - "VARBASE1.* (line 21)", - "VARBASE2.* (line 22)", - "VARBASE3.${id} (line 23)", - "VARBASE3.* (line 23)"} + "PKG_DEBUG_LEVEL (line 11)", + "PKG_VERBOSE (line 16)", + "VARBASE1.* (line 23)", + "VARBASE2.* (line 24)", + "VARBASE3.* (line 25)"} c.Check(varnames, deepEquals, expected) } @@ -743,7 +741,34 @@ func (s *Suite) Test_MkLines__unknown_options(c *check.C) { "WARN: options.mk:4: Unknown option \"unknown\".") } -func (s *Suite) Test_MkLines_CheckRedundantAssignments(c *check.C) { +func (s *Suite) Test_MkLines_CheckRedundantAssignments__override_in_mk(c *check.C) { + t := s.Init(c) + included := t.NewMkLines("included.mk", + "OVERRIDE=\tprevious value", + "REDUNDANT=\tredundant") + including := t.NewMkLines("including.mk", + "OVERRIDE=\toverridden value", + "REDUNDANT=\tredundant") + + var allLines []Line + allLines = append(allLines, included.lines.Lines...) + allLines = append(allLines, including.lines.Lines...) + mklines := NewMkLines(NewLines(included.lines.FileName, allLines)) + + // XXX: The warnings from here are not in the same order as the other warnings. + // XXX: There may be some warnings for the same file separated by warnings for other files. + mklines.CheckRedundantAssignments() + + // No warning for VAR=... in Makefile since it makes sense to have common files + // with default values for variables, overriding some of them in each package. + t.CheckOutputLines( + // FIXME: The below warning is wrong because overwriting in a different file is ok. + "WARN: included.mk:1: Variable OVERRIDE is overwritten in including.mk:1.", + // FIXME: It's the other way round: including.mk:2 is redundant because of included.mk:2. + "NOTE: included.mk:2: Definition of REDUNDANT is redundant because of including.mk:2.") +} + +func (s *Suite) Test_MkLines_CheckRedundantAssignments__override_in_Makefile(c *check.C) { t := s.Init(c) included := t.NewMkLines("module.mk", "VAR=\tvalue ${OTHER}", @@ -751,19 +776,24 @@ func (s *Suite) Test_MkLines_CheckRedundantAssignments(c *check.C) { "VAR=\tnew value") makefile := t.NewMkLines("Makefile", "VAR=\tthe package may overwrite variables from other files") - allLines := append(append([]Line(nil), included.lines.Lines...), makefile.lines.Lines...) + + var allLines []Line + allLines = append(allLines, included.lines.Lines...) + allLines = append(allLines, makefile.lines.Lines...) mklines := NewMkLines(NewLines(included.lines.FileName, allLines)) // XXX: The warnings from here are not in the same order as the other warnings. // XXX: There may be some warnings for the same file separated by warnings for other files. mklines.CheckRedundantAssignments() + // No warning for VAR=... in Makefile since it makes sense to have common files + // with default values for variables, overriding some of them in each package. t.CheckOutputLines( "NOTE: module.mk:1: Definition of VAR is redundant because of line 2.", "WARN: module.mk:1: Variable VAR is overwritten in line 3.") } -func (s *Suite) Test_MkLines_CheckRedundantAssignments__different_value(c *check.C) { +func (s *Suite) Test_MkLines_CheckRedundantAssignments__default_value_definitely_unused(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("module.mk", "VAR=\tvalue ${OTHER}", @@ -771,9 +801,22 @@ func (s *Suite) Test_MkLines_CheckRedundantAssignments__different_value(c *check mklines.CheckRedundantAssignments() + // FIXME: A default assignment after an unconditional assignment is redundant. t.CheckOutputEmpty() } +func (s *Suite) Test_MkLines_CheckRedundantAssignments__default_value_overridden(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR?=\tdefault value", + "VAR=\toverridden value") + + mklines.CheckRedundantAssignments() + + t.CheckOutputLines( + "WARN: module.mk:1: Variable VAR is overwritten in line 2.") +} + func (s *Suite) Test_MkLines_CheckRedundantAssignments__overwrite_same_value(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("module.mk", @@ -786,6 +829,75 @@ func (s *Suite) Test_MkLines_CheckRedundantAssignments__overwrite_same_value(c * "NOTE: module.mk:1: Definition of VAR is redundant because of line 2.") } +func (s *Suite) Test_MkLines_CheckRedundantAssignments__conditional_overwrite(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR=\tdefault", + ".if ${OPSYS} == NetBSD", + "VAR=\topsys", + ".endif") + + mklines.CheckRedundantAssignments() + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLines_CheckRedundantAssignments__conditional_default(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "VAR=\tdefault", + ".if ${OPSYS} == NetBSD", + "VAR?=\topsys", + ".endif") + + mklines.CheckRedundantAssignments() + + // TODO: WARN: module.mk:3: The value \"opsys\" will never be assigned to VAR because it is defined unconditionally in line 1. + t.CheckOutputEmpty() +} + +// These warnings are precise and accurate since the value of VAR is not used between line 2 and 4. +func (s *Suite) Test_MkLines_CheckRedundantAssignments__overwrite_same_variable_different_value(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "OTHER=\tvalue before", + "VAR=\tvalue ${OTHER}", + "OTHER=\tvalue after", + "VAR=\tvalue ${OTHER}") + + mklines.CheckRedundantAssignments() + + t.CheckOutputLines( + "WARN: module.mk:1: Variable OTHER is overwritten in line 3.", + "NOTE: module.mk:2: Definition of VAR is redundant because of line 4.") +} + +func (s *Suite) Test_MkLines_CheckRedundantAssignments__overwrite_different_value_used_between(c *check.C) { + t := s.Init(c) + mklines := t.NewMkLines("module.mk", + "OTHER=\tvalue before", + "VAR=\tvalue ${OTHER}", + + // VAR is used here at load time, therefore it must be defined at this point. + // At this point, VAR uses the \"before\" value of OTHER. + "RESULT1:=\t${VAR}", + + "OTHER=\tvalue after", + + // VAR is used here again at load time, this time using the \"after\" value of OTHER. + "RESULT2:=\t${VAR}", + + // Still this definition is redundant. + "VAR=\tvalue ${OTHER}") + + mklines.CheckRedundantAssignments() + + t.CheckOutputLines( + "WARN: module.mk:1: Variable OTHER is overwritten in line 4.", + // FIXME: It's the other way round: line 6 is redundant because of line 2. + "NOTE: module.mk:2: Definition of VAR is redundant because of line 6.") +} + func (s *Suite) Test_MkLines_CheckRedundantAssignments__procedure_call(c *check.C) { t := s.Init(c) mklines := t.NewMkLines("mk/pthread.buildlink3.mk", @@ -806,8 +918,22 @@ func (s *Suite) Test_MkLines_CheckRedundantAssignments__shell_and_eval(c *check. mklines.CheckRedundantAssignments() - // Combining := and != is too complicated to be analyzed by pkglint, - // therefore no warning. + // As of November 2018, pkglint doesn't check redundancies that involve the := or != operators. + // + // What happens here is: + // + // Line 1 evaluates OTHER at load time. + // Line 1 assigns its value to VAR. + // Line 2 evaluates OTHER at load time. + // Line 2 passes its value through the shell and assigns the result to VAR. + // + // Since VAR is defined in line 1, not used afterwards and overwritten in line 2, it is redundant. + // Well, not quite, because evaluating ${OTHER} might have side-effects from :sh or ::= modifiers, + // but these are so rare that they are frowned upon and are not considered by pkglint. + // + // Expected result: + // WARN: module.mk:2: Previous definition of VAR in line 1 is unused. + t.CheckOutputEmpty() } @@ -823,6 +949,8 @@ func (s *Suite) Test_MkLines_CheckRedundantAssignments__shell_and_eval_literal(c // only done for procedure calls), the shell evaluation can have // so many different side effects that pkglint cannot reliably // help in this situation. + // + // TODO: Why not? The evaluation in line 1 is trivial to analyze. t.CheckOutputEmpty() } @@ -889,6 +1017,39 @@ func (s *Suite) Test_MkLines_Check__PLIST_VARS_indirect(c *check.C) { mklines.Check() + // As of November 2018, pkglint doesn't analyze the .if 0 block. + // Therefore it doesn't know that the option1 block will never match because of the 0. + // This is ok though since it could be a temporary workaround from the package maintainer. + // + // As of November 2018, pkglint doesn't analyze the .for loop. + // Therefore it doesn't know that an .if block for option3 is missing. + t.CheckOutputEmpty() +} + +func (s *Suite) Test_MkLines_Check__PLIST_VARS_indirect_2(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Wno-space") + t.SetupVartypes() + t.SetupOption("a", "") + t.SetupOption("b", "") + t.SetupOption("c", "") + + mklines := t.NewMkLines("module.mk", + MkRcsID, + "", + "PKG_SUPPORTED_OPTIONS= a b c", + "PLIST_VARS+= ${PKG_SUPPORTED_OPTIONS:S,a,,g}", + "", + "PLIST_VARS+= only-added", + "", + "PLIST.only-defined= yes") + + mklines.Check() + + // If the PLIST_VARS contain complex expressions that involve other variables, + // it becomes too difficult for pkglint to decide whether the IDs can still match. + // Therefore, in such a case, no diagnostics are logged at all. t.CheckOutputEmpty() } @@ -946,33 +1107,6 @@ func (s *Suite) Test_MkLines_Check__defined_and_used_variables(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_MkLines_Check__indirect_PLIST_VARS(c *check.C) { - t := s.Init(c) - - t.SetupCommandLine("-Wno-space") - t.SetupVartypes() - t.SetupOption("a", "") - t.SetupOption("b", "") - t.SetupOption("c", "") - - mklines := t.NewMkLines("module.mk", - MkRcsID, - "", - "PKG_SUPPORTED_OPTIONS= a b c", - "PLIST_VARS+= ${PKG_SUPPORTED_OPTIONS:S,a,,g}", - "", - "PLIST_VARS+= only-added", - "", - "PLIST.only-defined= yes") - - mklines.Check() - - // If the PLIST_VARS contain complex expressions that involve other variables, - // it becomes too difficult for pkglint to decide whether the IDs can still match. - // Therefore, in such a case, no diagnostics are logged at all. - t.CheckOutputEmpty() -} - func (s *Suite) Test_MkLines_Check__hacks_mk(c *check.C) { t := s.Init(c) @@ -986,6 +1120,7 @@ func (s *Suite) Test_MkLines_Check__hacks_mk(c *check.C) { mklines.Check() // No warning about including bsd.prefs.mk before using the ?= operator. + // FIXME: Why not? t.CheckOutputEmpty() } @@ -1012,7 +1147,7 @@ func (s *Suite) Test_MkLines_Check__MASTER_SITE_in_HOMEPAGE(c *check.C) { "WARN: devel/catch/Makefile:5: HOMEPAGE should not be defined in terms of MASTER_SITEs.") } -func (s *Suite) Test_MkLines_Check__VERSION_as_wordpart_in_MASTER_SITES(c *check.C) { +func (s *Suite) Test_MkLines_Check__VERSION_as_word_part_in_MASTER_SITES(c *check.C) { t := s.Init(c) t.SetupVartypes() @@ -1072,6 +1207,8 @@ func (s *Suite) Test_MkLines_Check__extra_warnings(c *check.C) { "NOTE: options.mk:11: You can use \"../build\" instead of \"${WRKSRC}/../build\".") } +// Ensures that during MkLines.ForEach, the conditional variables in +// MkLines.Indentation are correctly updated for each line. func (s *Suite) Test_MkLines_ForEach__conditional_variables(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/mklines_varalign_test.go b/pkgtools/pkglint/files/mklines_varalign_test.go index 87a2964534a..3e144278f09 100755 --- a/pkgtools/pkglint/files/mklines_varalign_test.go +++ b/pkgtools/pkglint/files/mklines_varalign_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" @@ -14,7 +14,7 @@ type VaralignTester struct { diagnostics []string // The expected diagnostics in default mode autofixes []string // The expected diagnostics in --autofix mode fixed []string // The expected fixed lines, with spaces instead of tabs - source bool + ShowSource bool // The --show-source command line option } func NewVaralignTester(s *Suite, c *check.C) *VaralignTester { @@ -34,7 +34,7 @@ func (vt *VaralignTester) Diagnostics(diagnostics ...string) { vt.diagnostics = func (vt *VaralignTester) Autofixes(autofixes ...string) { vt.autofixes = autofixes } // Fixed remembers the expected fixed lines. To make the layout changes -// clearly visible, tabs are replaced with spaces in these expected lines. +// clearly visible, the lines given here use spaces instead of tabs. // The fixed lines that have been written to the file are still using tabs. func (vt *VaralignTester) Fixed(lines ...string) { vt.fixed = lines } @@ -52,7 +52,7 @@ func (vt *VaralignTester) run(autofix bool) { if autofix { cmdline = append(cmdline, "--autofix") } - if vt.source { + if vt.ShowSource { cmdline = append(cmdline, "--source") } t.SetupCommandLine(cmdline...) @@ -143,7 +143,7 @@ func (s *Suite) Test_Varalign__one_var_spaces(c *check.C) { } // Inconsistently aligned lines for variables of the same length are -// replaced with tabs, so that they nicely align. +// replaced with tabs, so that they align nicely. func (s *Suite) Test_Varalign__two_vars__spaces(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -187,7 +187,8 @@ func (s *Suite) Test_Varalign__several_vars__spaces(c *check.C) { vt.Run() } -// Continuation lines may be indented with a single space. +// Lines that are continued my be indented with a single space +// if the first line of the variable definition has no value. func (s *Suite) Test_Varalign__continuation(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -201,7 +202,7 @@ func (s *Suite) Test_Varalign__continuation(c *check.C) { vt.Run() } -// To align these two lines, the first line needs more more tab. +// To align these two lines, the first line needs one more tab. // The second line is further to the right but doesn't count as // an outlier since it is not far enough. // Adding one more tab to the indentation is generally considered ok. @@ -228,11 +229,13 @@ func (s *Suite) Test_Varalign__short_long__tab(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( "BLOCK=\tshort", - "BLOCK_LONGVAR=\tlong") + "BLOCK_LONGVAR=\t\t\t\tlong") vt.Diagnostics( - "NOTE: ~/Makefile:1: This variable value should be aligned to column 17.") + "NOTE: ~/Makefile:1: This variable value should be aligned to column 17.", + "NOTE: ~/Makefile:2: This variable value should be aligned to column 17.") vt.Autofixes( - "AUTOFIX: ~/Makefile:1: Replacing \"\\t\" with \"\\t\\t\".") + "AUTOFIX: ~/Makefile:1: Replacing \"\\t\" with \"\\t\\t\".", + "AUTOFIX: ~/Makefile:2: Replacing \"\\t\\t\\t\\t\" with \"\\t\".") vt.Fixed( "BLOCK= short", "BLOCK_LONGVAR= long") @@ -322,9 +325,9 @@ func (s *Suite) Test_Varalign__aligned_continuation(c *check.C) { vt.Run() } -// Shell commands are assumed to be already nicely indented. +// Shell commands in continuation lines are assumed to be already nicely indented. // This particular example is not, but pkglint cannot decide this as of -// version 5.5.2. +// version 5.5.2 (January 2018). func (s *Suite) Test_Varalign__shell_command(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -343,10 +346,9 @@ func (s *Suite) Test_Varalign__shell_command(c *check.C) { } // The most common pattern for laying out continuation lines is to have all -// values in the continuation lines, one value per line, all indented to the -// same depth. -// The depth is either a single tab or aligns with the other variables in the -// paragraph. +// values in the continuation lines, one value per line, all indented to the same depth. +// The depth is either a single tab (see the test below) or aligns with the other +// variables in the paragraph (this test). func (s *Suite) Test_Varalign__continuation_value_starts_in_second_line(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -371,6 +373,31 @@ func (s *Suite) Test_Varalign__continuation_value_starts_in_second_line(c *check vt.Run() } +// The most common pattern for laying out continuation lines is to have all +// values in the continuation lines, one value per line, all indented to the same depth. +// The depth is either a single tab (this test) or aligns with the other +// variables in the paragraph (see the test above). +func (s *Suite) Test_Varalign__continuation_value_starts_in_second_line_with_single_tab(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "WRKSRC=\t${WRKDIR}", + "DISTFILES=\tdistfile-1.0.0.tar.gz", + "SITES.distfile-1.0.0.tar.gz= \\", + "\t${MASTER_SITES_SOURCEFORGE} \\", + "\t${MASTER_SITES_GITHUB}") + vt.Diagnostics( + "NOTE: ~/Makefile:1: This variable value should be aligned to column 17.") + vt.Autofixes( + "AUTOFIX: ~/Makefile:1: Replacing \"\\t\" with \"\\t\\t\".") + vt.Fixed( + "WRKSRC= ${WRKDIR}", + "DISTFILES= distfile-1.0.0.tar.gz", + "SITES.distfile-1.0.0.tar.gz= \\", + " ${MASTER_SITES_SOURCEFORGE} \\", + " ${MASTER_SITES_GITHUB}") + vt.Run() +} + // Another common pattern is to write the first value in the first line and // subsequent values indented to the same depth as the value in the first // line. @@ -458,9 +485,8 @@ func (s *Suite) Test_Varalign__continuation_mixed_indentation_in_first_line(c *c // When there is an outlier, no matter whether indented using space or tab, // fix the whole block to use the indentation of the second-longest line. -// Since all of the remaining lines have the same indentation (in this case, -// there is only 1 line at all), that existing indentation is used instead of -// the minimum necessary, which would only be a single tab. +// In this case, all of the remaining lines have the same indentation (there is only 1 line at all). +// Therefore this existing indentation is used instead of the minimum necessary, which would only be a single tab. func (s *Suite) Test_Varalign__tab_outlier(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -541,9 +567,10 @@ func (s *Suite) Test_Varalign__single_space(c *check.C) { vt.Run() } -// These variables all look nicely aligned, but they use spaces instead -// of tabs for alignment. The spaces are replaced with tabs, making the -// indentation a little deeper. +// These variables all look nicely aligned, but they use spaces instead of tabs for alignment. +// The spaces are replaced with tabs, which makes the indentation 4 spaces deeper in the first paragraph. +// In the second paragraph it's even 7 additional spaces. +// This is ok though since it is the prevailing indentation style in pkgsrc. func (s *Suite) Test_Varalign__only_space(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -575,8 +602,12 @@ func (s *Suite) Test_Varalign__only_space(c *check.C) { vt.Run() } -// The indentation is deeper than necessary, but all lines agree on -// the same column. Therefore this indentation depth is kept. +// The indentation is deeper than necessary, but all lines agree on the same column. +// Therefore this indentation depth is kept. It looks good and is probably due to +// some other paragraphs in the file that are indented equally deep. +// +// As of December 2018, pkglint only looks at a single paragraph at a time, +// therefore it cannot reliably decide whether this deep indentation is necessary. func (s *Suite) Test_Varalign__mixed_tabs_and_spaces_same_column(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -592,7 +623,7 @@ func (s *Suite) Test_Varalign__mixed_tabs_and_spaces_same_column(c *check.C) { vt.Run() } -// Both lines are indented to the same column. This is a very simple case. +// Both lines are indented to the same column. Therefore none of them is considered an outlier. func (s *Suite) Test_Varalign__outlier_1(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -608,8 +639,7 @@ func (s *Suite) Test_Varalign__outlier_1(c *check.C) { vt.Run() } -// A single space that ends at the same depth as a tab is replaced with a -// tab, for consistency. +// A single space that ends at the same depth as a tab is replaced with a tab, for consistency. func (s *Suite) Test_Varalign__outlier_2(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -627,8 +657,8 @@ func (s *Suite) Test_Varalign__outlier_2(c *check.C) { // A short line that is indented with spaces is aligned to a longer line // that is indented with tabs. This is because space-indented lines are -// only special when their indentation is much deeper than the tab-indented -// ones. +// only allowed when their indentation is much deeper than the tab-indented +// ones (so-called outliers), or as the first line of a continuation line. func (s *Suite) Test_Varalign__outlier_3(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -648,6 +678,7 @@ func (s *Suite) Test_Varalign__outlier_3(c *check.C) { // This space-indented line doesn't count as an outlier yet because it // is only a single tab away. The limit is two tabs. +// Therefore both lines are indented with tabs. func (s *Suite) Test_Varalign__outlier_4(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -667,6 +698,8 @@ func (s *Suite) Test_Varalign__outlier_4(c *check.C) { // This space-indented line is an outlier since it is far enough from the // tab-indented line. The latter would require 2 tabs to align to the former. +// Therefore the short line is not indented to the long line, in order to +// keep the indentation reasonably short for a large amount of the lines. func (s *Suite) Test_Varalign__outlier_5(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -680,7 +713,7 @@ func (s *Suite) Test_Varalign__outlier_5(c *check.C) { vt.Run() } -// Short space-indented lines are expanded to the tab-depth. +// Short space-indented lines do not count as outliers. They are are aligned to the longer tab-indented line. func (s *Suite) Test_Varalign__outlier_6(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -696,8 +729,7 @@ func (s *Suite) Test_Varalign__outlier_6(c *check.C) { vt.Run() } -// The long line is not an outlier but very close. One more space, and -// it would count. +// The long line is not an outlier but very close. One more space, and it would count. func (s *Suite) Test_Varalign__outlier_10(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -863,7 +895,7 @@ func (s *Suite) Test_Varalign__fix_without_diagnostic(c *check.C) { " RUBY_SHLIBMAJOR=${RUBY_SHLIBMAJOR:Q} \\", " RUBY_NOSHLIBMAJOR=${RUBY_NOSHLIBMAJOR} \\", " RUBY_NAME=${RUBY_NAME:Q}") - vt.source = true + vt.ShowSource = true vt.Run() } @@ -878,7 +910,7 @@ func (s *Suite) Test_Varalign__continuation_line_last_empty(c *check.C) { "\tb \\", "\tc \\", "", - "NEXT_VAR=\tmust not be indented") + "NEXT_VAR=\tsecond line") vt.Diagnostics( "NOTE: ~/Makefile:1--5: This variable value should be aligned with tabs, not spaces, to column 17.") vt.Autofixes( @@ -889,13 +921,15 @@ func (s *Suite) Test_Varalign__continuation_line_last_empty(c *check.C) { " b \\", " c \\", "", - "NEXT_VAR= must not be indented") + "NEXT_VAR= second line") vt.Run() } // Commented-out variables take part in the realignment. -// The TZ=UTC below is part of the two-line comment since make(1) -// interprets it in the same way. +// The TZ=UTC below is part of the two-line comment since make(1) interprets it in the same way. +// +// This is one of the few cases where commented variable assignments are treated specially. +// See MkLine.IsCommentedVarassign. func (s *Suite) Test_Varalign__realign_commented_single_lines(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -927,6 +961,9 @@ func (s *Suite) Test_Varalign__realign_commented_single_lines(c *check.C) { vt.Run() } +// Commented variable assignments are realigned, too. +// In this case, the BEFORE and COMMENTED variables are already aligned properly. +// The line starting with "AFTER" is actually part of the comment, therefore it is not changed. func (s *Suite) Test_Varalign__realign_commented_continuation_line(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -935,7 +972,7 @@ func (s *Suite) Test_Varalign__realign_commented_continuation_line(c *check.C) { "#\tvalue1 \\", "#\tvalue2 \\", "#\tvalue3 \\", - "AFTER=\tafter") // This line continues the comment. + "AFTER=\tafter") vt.Diagnostics() vt.Autofixes() vt.Fixed( @@ -950,6 +987,9 @@ func (s *Suite) Test_Varalign__realign_commented_continuation_line(c *check.C) { // The HOMEPAGE is completely ignored. Since its value is empty it doesn't // need any alignment. Whether it is commented out doesn't matter. +// +// If the HOMEPAGE were taken into account, the alignment would differ and +// the COMMENT line would be realigned to column 17, reducing the indentation by one tab. func (s *Suite) Test_Varalign__realign_variable_without_value(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( @@ -965,6 +1005,8 @@ func (s *Suite) Test_Varalign__realign_variable_without_value(c *check.C) { // This commented multiline variable is already perfectly aligned. // Nothing needs to be fixed. +// This is a simple case since a paragraph containing only one line +// is always aligned properly, except when the indentation uses spaces instead of tabs. func (s *Suite) Test_Varalign__realign_commented_multiline(c *check.C) { vt := NewVaralignTester(s, c) vt.Input( diff --git a/pkgtools/pkglint/files/mkparser.go b/pkgtools/pkglint/files/mkparser.go index 4441acff60a..2f07d4b8cb3 100644 --- a/pkgtools/pkglint/files/mkparser.go +++ b/pkgtools/pkglint/files/mkparser.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "netbsd.org/pkglint/regex" @@ -14,11 +14,23 @@ type MkParser struct { // NewMkParser creates a new parser for the given text. // If emitWarnings is false, line may be nil. +// +// TODO: Document what exactly text is. Is it the form taken from the file, or is it after unescaping "\#" to #? +// +// TODO: Remove the emitWarnings argument in order to separate parsing from checking. func NewMkParser(line Line, text string, emitWarnings bool) *MkParser { G.Assertf((line != nil) == emitWarnings, "line must be given iff emitWarnings is set") return &MkParser{NewParser(line, text, emitWarnings)} } +// MkTokens splits a text like in the following example: +// Text${VAR:Mmodifier}${VAR2}more text${VAR3} +// into tokens like these: +// Text +// ${VAR:Mmodifier} +// ${VAR2} +// more text +// ${VAR3} func (p *MkParser) MkTokens() []*MkToken { lexer := p.lexer @@ -27,6 +39,7 @@ func (p *MkParser) MkTokens() []*MkToken { // FIXME: Aren't the comments already gone at this stage? if lexer.SkipByte('#') { lexer.Skip(len(lexer.Rest())) + continue } mark := lexer.Mark() @@ -35,14 +48,7 @@ func (p *MkParser) MkTokens() []*MkToken { continue } - again: - dollar := strings.IndexByte(lexer.Rest(), '$') - if dollar == -1 { - dollar = len(lexer.Rest()) - } - lexer.Skip(dollar) - if lexer.SkipString("$$") { - goto again + for lexer.NextBytesFunc(func(b byte) bool { return b != '$' }) != "" || lexer.SkipString("$$") { } text := lexer.Since(mark) if text != "" { @@ -67,6 +73,7 @@ func (p *MkParser) VarUse() *MkVarUse { if lexer.SkipByte('{') || lexer.SkipByte('(') { usingRoundParen := lexer.Since(mark)[1] == '(' + closing := byte('}') if usingRoundParen { closing = ')' @@ -79,7 +86,11 @@ func (p *MkParser) VarUse() *MkVarUse { if lexer.SkipByte(closing) { if usingRoundParen && p.EmitWarnings { parenVaruse := lexer.Since(mark) - bracesVaruse := "${" + parenVaruse[2:len(parenVaruse)-1] + "}" + edit := []byte(parenVaruse) + edit[1] = '{' + edit[len(edit)-1] = '}' + bracesVaruse := string(edit) + fix := p.Line.Autofix() fix.Warnf("Please use curly braces {} instead of round parentheses () for %s.", varname) fix.Replace(parenVaruse, bracesVaruse) @@ -89,8 +100,13 @@ func (p *MkParser) VarUse() *MkVarUse { } } - for p.VarUse() != nil || lexer.SkipRegexp(G.res.Compile(regex.Pattern(`^([^$:`+string(closing)+`]|\$\$)+`))) { + // This code path parses ${arbitrary text :L} and ${expression :? true-branch : false-branch }. + // The text in front of the :L or :? modifier doesn't have to be a variable name. + + re := G.res.Compile(regex.Pattern(ifelseStr(usingRoundParen, `^(?:[^$:)]|\$\$)+`, `^(?:[^$:}]|\$\$)+`))) + for p.VarUse() != nil || lexer.SkipRegexp(re) { } + rest := p.Rest() if hasPrefix(rest, ":L") || hasPrefix(rest, ":?") { varexpr := lexer.Since(varnameMark) @@ -99,6 +115,7 @@ func (p *MkParser) VarUse() *MkVarUse { return &MkVarUse{varexpr, modifiers} } } + lexer.Reset(mark) } @@ -108,23 +125,46 @@ func (p *MkParser) VarUse() *MkVarUse { if lexer.SkipByte('<') { return &MkVarUse{"<", nil} } - if varname := lexer.NextBytesSet(textproc.AlnumU); varname != "" { + + varname := lexer.NextByteSet(textproc.AlnumU) + if varname != -1 { + if p.EmitWarnings { - p.Line.Warnf("$%[1]s is ambiguous. Use ${%[1]s} if you mean a Makefile variable or $$%[1]s if you mean a shell variable.", varname) + varnameRest := lexer.Copy().NextBytesSet(textproc.AlnumU) + if varnameRest != "" { + p.Line.Errorf("$%[1]s is ambiguous. Use ${%[1]s} if you mean a Make variable or $$%[1]s if you mean a shell variable.", + sprintf("%c%s", varname, varnameRest)) + p.Line.Explain( + "Only the first letter after the dollar is the variable name.", + "Everything following it is normal text, even if it looks like a variable name to human readers.") + } else { + p.Line.Warnf("$%[1]c is ambiguous. Use ${%[1]c} if you mean a Make variable or $$%[1]c if you mean a shell variable.", varname) + p.Line.Explain( + "In its current form, this variable is parsed as a Make variable.", + "For human readers though, $x looks more like a shell variable than a Make variable,", + "since Make variables are usually written using braces (BSD-style) or parentheses (GNU-style).") + } } - return &MkVarUse{varname, nil} + + return &MkVarUse{sprintf("%c", varname), nil} } lexer.Reset(mark) return nil } +// VarUseModifiers parses the modifiers of a variable being used, such as :Q, :Mpattern. +// +// See the bmake manual page. func (p *MkParser) VarUseModifiers(varname string, closing byte) []MkVarUseModifier { lexer := p.lexer var modifiers []MkVarUseModifier appendModifier := func(s string) { modifiers = append(modifiers, MkVarUseModifier{s}) } + + // The :S and :C modifiers may be chained without using the : as separator. mayOmitColon := false + loop: for lexer.SkipByte(':') || mayOmitColon { mayOmitColon = false @@ -132,17 +172,39 @@ loop: switch lexer.PeekByte() { case 'E', 'H', 'L', 'O', 'Q', 'R', 'T', 's', 't', 'u': - if lexer.SkipRegexp(G.res.Compile(`^(E|H|L|Ox?|Q|R|T|sh|tA|tW|tl|tu|tw|u)`)) { - appendModifier(lexer.Since(modifierMark)) + mod := lexer.NextBytesSet(textproc.Alnum) + switch mod { + + case + "E", // Extension, e.g. path/file.suffix => suffix + "H", // Head, e.g. dir/subdir/file.suffix => dir/subdir + "L", // XXX: Shouldn't this be handled specially? + "O", // Order alphabetically + "Ox", // Shuffle + "Q", // Quote shell meta-characters + "R", // Strip the file suffix, e.g. path/file.suffix => file + "T", // Basename, e.g. path/file.suffix => file.suffix + "sh", // Evaluate the variable value as shell command + "tA", // Try to convert to absolute path + "tW", // Causes the value to be treated as a single word + "tl", // To lowercase + "tu", // To uppercase + "tw", // Causes the value to be treated as list of words + "u": // Remove adjacent duplicate words (like uniq(1)) + appendModifier(mod) continue - } - if lexer.SkipString("ts") { + + case "ts": + // See devel/bmake/files/var.c:/case 't' rest := lexer.Rest() - if len(rest) >= 2 && (rest[1] == closing || rest[1] == ':') { + switch { + case len(rest) >= 2 && (rest[1] == closing || rest[1] == ':'): lexer.Skip(1) - } else if len(rest) >= 1 && (rest[0] == closing || rest[0] == ':') { - } else if lexer.SkipRegexp(G.res.Compile(`^\\\d+`)) { - } else { + case len(rest) >= 1 && (rest[0] == closing || rest[0] == ':'): + break + case lexer.SkipRegexp(G.res.Compile(`^\\\d+`)): + break + default: break loop } appendModifier(lexer.Since(modifierMark)) @@ -159,41 +221,14 @@ loop: continue case 'C', 'S': - // bmake allows _any_ separator, even letters. - lexer.Skip(1) - if m := lexer.NextRegexp(G.res.Compile(`^[%,/:;@^|]`)); m != nil { - separator := m[0][0] - lexer.SkipByte('^') - skipOther := func() { - for p.VarUse() != nil || - lexer.SkipString("$$") || - (len(lexer.Rest()) >= 2 && lexer.PeekByte() == '\\' && lexer.Skip(2)) || - lexer.NextBytesFunc(func(b byte) bool { return b != separator && b != '$' && b != closing && b != '\\' }) != "" { - - } - } - skipOther() - lexer.SkipByte('$') - if lexer.SkipByte(separator) { - skipOther() - if lexer.SkipByte(separator) { - lexer.SkipRegexp(G.res.Compile(`^[1gW]`)) // FIXME: Multiple modifiers may be mentioned - appendModifier(lexer.Since(modifierMark)) - mayOmitColon = true - continue - } - } + if p.varUseModifierSubst(lexer, closing) { + appendModifier(lexer.Since(modifierMark)) + mayOmitColon = true + continue } case '@': - if m := lexer.NextRegexp(G.res.Compile(`^@([\w.]+)@`)); m != nil { - loopvar := m[1] - re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^$:@}\\]|\\.)+`, `^([^$:@)\\]|\\.)+`))) - for p.VarUse() != nil || lexer.SkipString("$$") || lexer.SkipRegexp(re) { - } - if !lexer.SkipByte('@') && p.EmitWarnings { - p.Line.Warnf("Modifier ${%s:@%s@...@} is missing the final \"@\".", varname, loopvar) - } + if p.varUseModifierAt(lexer, closing, varname) { appendModifier(lexer.Since(modifierMark)) continue } @@ -206,7 +241,7 @@ loop: case '?': lexer.Skip(1) - re := G.res.Compile(regex.Pattern(`^([^$:` + string(closing) + `]|\$\$)+`)) + re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^$:}]|\$\$)+`, `^([^$:)]|\$\$)+`))) for p.VarUse() != nil || lexer.SkipRegexp(re) { } if lexer.SkipByte(':') { @@ -230,6 +265,65 @@ loop: return modifiers } +func (p *MkParser) varUseModifierSubst(lexer *textproc.Lexer, closing byte) bool { + lexer.Skip(1) + sep := lexer.PeekByte() // bmake allows _any_ separator, even letters. + if sep == -1 { + return false + } + + lexer.Skip(1) + separator := byte(sep) + + isOther := func(b byte) bool { + return b != separator && b != '$' && b != closing && b != '\\' + } + + skipOther := func() { + for p.VarUse() != nil || + lexer.SkipString("$$") || + (len(lexer.Rest()) >= 2 && lexer.PeekByte() == '\\' && lexer.Skip(2)) || + lexer.NextBytesFunc(isOther) != "" { + } + } + + lexer.SkipByte('^') + skipOther() + lexer.SkipByte('$') + + if !lexer.SkipByte(separator) { + return false + } + + skipOther() + + if !lexer.SkipByte(separator) { + return false + } + + lexer.SkipRegexp(G.res.Compile(`^[1gW]`)) // FIXME: Multiple modifiers may be mentioned + + return true +} + +func (p *MkParser) varUseModifierAt(lexer *textproc.Lexer, closing byte, varname string) bool { + lexer.Skip(1) + loopVar := lexer.NextBytesSet(AlnumDot) + if loopVar == "" || !lexer.SkipByte('@') { + return false + } + + re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^$:@}\\]|\\.)+`, `^([^$:@)\\]|\\.)+`))) + for p.VarUse() != nil || lexer.SkipString("$$") || lexer.SkipRegexp(re) { + } + + if !lexer.SkipByte('@') && p.EmitWarnings { + p.Line.Warnf("Modifier ${%s:@%s@...@} is missing the final \"@\".", varname, loopVar) + } + + return true +} + // MkCond parses a condition like ${OPSYS} == "NetBSD". // See devel/bmake/files/cond.c. func (p *MkParser) MkCond() MkCond { @@ -308,7 +402,7 @@ func (p *MkParser) mkCondAtom() MkCond { } } - case 'a' <= lexer.PeekByte() && lexer.PeekByte() <= 'z': + case lexer.TestByteSet(textproc.Lower): return p.mkCondFunc() default: @@ -321,37 +415,46 @@ func (p *MkParser) mkCondAtom() MkCond { lexer.Reset(mark) } } + if lhs != nil { - if m := lexer.NextRegexp(G.res.Compile(`^[\t ]*(<|<=|==|!=|>=|>)[\t ]*(\d+(?:\.\d+)?)`)); m != nil { + if m := lexer.NextRegexp(G.res.Compile(`^[\t ]*(<|<=|==|!=|>=|>)[\t ]*(0x[0-9A-Fa-f]+|\d+(?:\.\d+)?)`)); m != nil { return &mkCond{CompareVarNum: &MkCondCompareVarNum{lhs, m[1], m[2]}} } - if m := lexer.NextRegexp(G.res.Compile(`^[\t ]*(<|<=|==|!=|>=|>)[\t ]*`)); m != nil { - op := m[1] - if op == "==" || op == "!=" { - if mrhs := lexer.NextRegexp(G.res.Compile(`^"([^"\$\\]*)"`)); mrhs != nil { - return &mkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, mrhs[1]}} - } + + m := lexer.NextRegexp(G.res.Compile(`^[\t ]*(<|<=|==|!=|>=|>)[\t ]*`)) + if m == nil { + return &mkCond{Not: &mkCond{Empty: lhs}} // See devel/bmake/files/cond.c:/\* For \.if \$/ + } + + op := m[1] + if op == "==" || op == "!=" { + if mrhs := lexer.NextRegexp(G.res.Compile(`^"([^"\$\\]*)"`)); mrhs != nil { + return &mkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, mrhs[1]}} } - if str := lexer.NextBytesSet(textproc.AlnumU); str != "" { - return &mkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, str}} - } else if rhs := p.VarUse(); rhs != nil { - return &mkCond{CompareVarVar: &MkCondCompareVarVar{lhs, op, rhs}} - } else if lexer.PeekByte() == '"' { - mark := lexer.Mark() + } + + if str := lexer.NextBytesSet(textproc.AlnumU); str != "" { + return &mkCond{CompareVarStr: &MkCondCompareVarStr{lhs, op, str}} + } + + if rhs := p.VarUse(); rhs != nil { + return &mkCond{CompareVarVar: &MkCondCompareVarVar{lhs, op, rhs}} + } + + if lexer.PeekByte() == '"' { + mark := lexer.Mark() + lexer.Skip(1) + if quotedRHS := p.VarUse(); quotedRHS != nil { if lexer.SkipByte('"') { - if quotedRHS := p.VarUse(); quotedRHS != nil { - if lexer.SkipByte('"') { - return &mkCond{CompareVarVar: &MkCondCompareVarVar{lhs, op, quotedRHS}} - } - } + return &mkCond{CompareVarVar: &MkCondCompareVarVar{lhs, op, quotedRHS}} } - lexer.Reset(mark) } - } else { - return &mkCond{Not: &mkCond{Empty: lhs}} // See devel/bmake/files/cond.c:/\* For \.if \$/ + lexer.Reset(mark) } } - if m := lexer.NextRegexp(G.res.Compile(`^\d+(?:\.\d+)?`)); m != nil { + + // See devel/bmake/files/cond.c:/^CondCvtArg + if m := lexer.NextRegexp(G.res.Compile(`^(?:0x[0-9A-Fa-f]+|\d+(?:\.\d+)?)`)); m != nil { return &mkCond{Num: m[0]} } } @@ -363,7 +466,7 @@ func (p *MkParser) mkCondFunc() *mkCond { lexer := p.lexer mark := lexer.Mark() - funcName := lexer.NextBytesFunc(func(b byte) bool { return 'a' <= b && b <= 'z' }) + funcName := lexer.NextBytesSet(textproc.Lower) lexer.SkipHspace() if !lexer.SkipByte('(') { return nil @@ -384,6 +487,10 @@ func (p *MkParser) mkCondFunc() *mkCond { } } + // TODO: Consider suggesting ${VAR} instead of !empty(VAR) since it is shorter and + // avoids unnecessary negation, which makes the expression less confusing. + // This applies especially to the ${VAR:Mpattern} form. + case "commands", "exists", "make", "target": argMark := lexer.Mark() for p.VarUse() != nil || lexer.NextBytesFunc(func(b byte) bool { return b != '$' && b != ')' }) != "" { @@ -408,6 +515,12 @@ func (p *MkParser) Varname() string { return lexer.Since(mark) } +// MkCond is a condition in a Makefile, such as ${OPSYS} == NetBSD. +// +// The representation is somewhere between syntactic and semantic. +// Unnecessary parentheses are omitted in this representation, +// but !empty(VARNAME) is represented differently from ${VARNAME} != "". +// For higher level analysis, a unified representation might be better. type MkCond = *mkCond type mkCond struct { @@ -465,10 +578,12 @@ func (w *MkCondWalker) Walk(cond MkCond, callback *MkCondCallback) { for _, or := range cond.Or { w.Walk(or, callback) } + case cond.And != nil: for _, and := range cond.And { w.Walk(and, callback) } + case cond.Not != nil: w.Walk(cond.Not, callback) @@ -477,8 +592,11 @@ func (w *MkCondWalker) Walk(cond MkCond, callback *MkCondCallback) { callback.Defined(cond.Defined) } if callback.VarUse != nil { + // This is not really a VarUse, it's more a VarUseDefined. + // But in practice they are similar enough to be treated the same. callback.VarUse(&MkVarUse{cond.Defined, nil}) } + case cond.Empty != nil: if callback.Empty != nil { callback.Empty(cond.Empty) @@ -486,6 +604,7 @@ func (w *MkCondWalker) Walk(cond MkCond, callback *MkCondCallback) { if callback.VarUse != nil { callback.VarUse(cond.Empty) } + case cond.CompareVarVar != nil: if callback.CompareVarVar != nil { cvv := cond.CompareVarVar @@ -496,6 +615,7 @@ func (w *MkCondWalker) Walk(cond MkCond, callback *MkCondCallback) { callback.VarUse(cvv.Left) callback.VarUse(cvv.Right) } + case cond.CompareVarStr != nil: if callback.CompareVarStr != nil { cvs := cond.CompareVarStr @@ -504,6 +624,7 @@ func (w *MkCondWalker) Walk(cond MkCond, callback *MkCondCallback) { if callback.VarUse != nil { callback.VarUse(cond.CompareVarStr.Var) } + case cond.CompareVarNum != nil: if callback.CompareVarNum != nil { cvn := cond.CompareVarNum @@ -512,6 +633,7 @@ func (w *MkCondWalker) Walk(cond MkCond, callback *MkCondCallback) { if callback.VarUse != nil { callback.VarUse(cond.CompareVarNum.Var) } + case cond.Call != nil: if callback.Call != nil { call := cond.Call diff --git a/pkgtools/pkglint/files/mkparser_test.go b/pkgtools/pkglint/files/mkparser_test.go index 4654016d851..439d5072bea 100644 --- a/pkgtools/pkglint/files/mkparser_test.go +++ b/pkgtools/pkglint/files/mkparser_test.go @@ -1,7 +1,6 @@ -package main +package pkglint import ( - "fmt" "gopkg.in/check.v1" "strings" ) @@ -36,119 +35,320 @@ func (s *Suite) Test_MkParser_MkTokens(c *check.C) { text += "}" return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)} } + + // Everything except VarUses is passed through unmodified. + + test("literal", + literal("literal")) + + test("\\/share\\/ { print \"share directory\" }", + literal("\\/share\\/ { print \"share directory\" }")) + + test("find . -name \\*.orig -o -name \\*.pre", + literal("find . -name \\*.orig -o -name \\*.pre")) + + test("-e 's|\\$${EC2_HOME.*}|EC2_HOME}|g'", + literal("-e 's|\\$${EC2_HOME.*}|EC2_HOME}|g'")) + + test("$$var1 $$var2 $$? $$", + literal("$$var1 $$var2 $$? $$")) + + testRest("hello, ${W:L:tl}orld", []*MkToken{ + literal("hello, "), + varuse("W", "L", "tl"), + literal("orld")}, + "") + + testRest("ftp://${PKGNAME}/ ${MASTER_SITES:=subdir/}", []*MkToken{ + literal("ftp://"), + varuse("PKGNAME"), + literal("/ "), + varuse("MASTER_SITES", "=subdir/")}, + "") + + // FIXME: Text must match modifiers. + testRest("${VAR:S,a,b,c,d,e,f}", + []*MkToken{{ + Text: "${VAR:S,a,b,c,d,e,f}", + Varuse: NewMkVarUse("VAR", "S,a,b,")}}, + "") + + testRest("Text${VAR:Mmodifier}${VAR2}more text${VAR3}", []*MkToken{ + literal("Text"), + varuse("VAR", "Mmodifier"), + varuse("VAR2"), + literal("more text"), + varuse("VAR3")}, + "") +} + +func (s *Suite) Test_MkParser_VarUse(c *check.C) { + t := s.Init(c) + + testRest := func(input string, expectedTokens []*MkToken, expectedRest string) { + line := t.NewLines("Test_MkParser_VarUse.mk", input).Lines[0] + p := NewMkParser(line, input, true) + actualTokens := p.MkTokens() + c.Check(actualTokens, deepEquals, expectedTokens) + for i, expectedToken := range expectedTokens { + if i < len(actualTokens) { + c.Check(*actualTokens[i], deepEquals, *expectedToken) + c.Check(actualTokens[i].Varuse, deepEquals, expectedToken.Varuse) + } + } + c.Check(p.Rest(), equals, expectedRest) + } + test := func(input string, expectedToken *MkToken) { + testRest(input, []*MkToken{expectedToken}, "") + } + varuse := func(varname string, modifiers ...string) *MkToken { + text := "${" + varname + for _, modifier := range modifiers { + text += ":" + modifier + } + text += "}" + return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)} + } varuseText := func(text, varname string, modifiers ...string) *MkToken { return &MkToken{Text: text, Varuse: NewMkVarUse(varname, modifiers...)} } - test("literal", literal("literal")) - test("\\/share\\/ { print \"share directory\" }", literal("\\/share\\/ { print \"share directory\" }")) - test("find . -name \\*.orig -o -name \\*.pre", literal("find . -name \\*.orig -o -name \\*.pre")) - test("-e 's|\\$${EC2_HOME.*}|EC2_HOME}|g'", literal("-e 's|\\$${EC2_HOME.*}|EC2_HOME}|g'")) - - test("${VARIABLE}", varuse("VARIABLE")) - test("${VARIABLE.param}", varuse("VARIABLE.param")) - test("${VARIABLE.${param}}", varuse("VARIABLE.${param}")) - test("${VARIABLE.hicolor-icon-theme}", varuse("VARIABLE.hicolor-icon-theme")) - test("${VARIABLE.gtk+extra}", varuse("VARIABLE.gtk+extra")) - test("${VARIABLE:S/old/new/}", varuse("VARIABLE", "S/old/new/")) - test("${GNUSTEP_LFLAGS:S/-L//g}", varuse("GNUSTEP_LFLAGS", "S/-L//g")) - test("${SUSE_VERSION:S/.//}", varuse("SUSE_VERSION", "S/.//")) - test("${MASTER_SITE_GNOME:=sources/alacarte/0.13/}", varuse("MASTER_SITE_GNOME", "=sources/alacarte/0.13/")) - test("${INCLUDE_DIRS:H:T}", varuse("INCLUDE_DIRS", "H", "T")) - test("${A.${B.${C.${D}}}}", varuse("A.${B.${C.${D}}}")) - test("${RUBY_VERSION:C/([0-9]+)\\.([0-9]+)\\.([0-9]+)/\\1/}", varuse("RUBY_VERSION", "C/([0-9]+)\\.([0-9]+)\\.([0-9]+)/\\1/")) - test("${PERL5_${_var_}:Q}", varuse("PERL5_${_var_}", "Q")) - test("${PKGNAME_REQD:C/(^.*-|^)py([0-9][0-9])-.*/\\2/}", varuse("PKGNAME_REQD", "C/(^.*-|^)py([0-9][0-9])-.*/\\2/")) - test("${PYLIB:S|/|\\\\/|g}", varuse("PYLIB", "S|/|\\\\/|g")) - test("${PKGNAME_REQD:C/ruby([0-9][0-9]+)-.*/\\1/}", varuse("PKGNAME_REQD", "C/ruby([0-9][0-9]+)-.*/\\1/")) - test("${RUBY_SHLIBALIAS:S/\\//\\\\\\//}", varuse("RUBY_SHLIBALIAS", "S/\\//\\\\\\//")) - test("${RUBY_VER_MAP.${RUBY_VER}:U${RUBY_VER}}", varuse("RUBY_VER_MAP.${RUBY_VER}", "U${RUBY_VER}")) - test("${RUBY_VER_MAP.${RUBY_VER}:U18}", varuse("RUBY_VER_MAP.${RUBY_VER}", "U18")) - test("${CONFIGURE_ARGS:S/ENABLE_OSS=no/ENABLE_OSS=yes/g}", varuse("CONFIGURE_ARGS", "S/ENABLE_OSS=no/ENABLE_OSS=yes/g")) - test("${PLIST_RUBY_DIRS:S,DIR=\"PREFIX/,DIR=\",}", varuse("PLIST_RUBY_DIRS", "S,DIR=\"PREFIX/,DIR=\",")) - test("${LDFLAGS:S/-Wl,//g:Q}", varuse("LDFLAGS", "S/-Wl,//g", "Q")) - test("${_PERL5_REAL_PACKLIST:S/^/${DESTDIR}/}", varuse("_PERL5_REAL_PACKLIST", "S/^/${DESTDIR}/")) - test("${_PYTHON_VERSION:C/^([0-9])/\\1./1}", varuse("_PYTHON_VERSION", "C/^([0-9])/\\1./1")) - test("${PKGNAME:S/py${_PYTHON_VERSION}/py${i}/}", varuse("PKGNAME", "S/py${_PYTHON_VERSION}/py${i}/")) - test("${PKGNAME:C/-[0-9].*$/-[0-9]*/}", varuse("PKGNAME", "C/-[0-9].*$/-[0-9]*/")) - test("${PKGNAME:S/py${_PYTHON_VERSION}/py${i}/:C/-[0-9].*$/-[0-9]*/}", varuse("PKGNAME", "S/py${_PYTHON_VERSION}/py${i}/", "C/-[0-9].*$/-[0-9]*/")) - test("${_PERL5_VARS:tl:S/^/-V:/}", varuse("_PERL5_VARS", "tl", "S/^/-V:/")) + test("${VARIABLE}", + varuse("VARIABLE")) + + test("${VARIABLE.param}", + varuse("VARIABLE.param")) + + test("${VARIABLE.${param}}", + varuse("VARIABLE.${param}")) + + test("${VARIABLE.hicolor-icon-theme}", + varuse("VARIABLE.hicolor-icon-theme")) + + test("${VARIABLE.gtk+extra}", + varuse("VARIABLE.gtk+extra")) + + test("${VARIABLE:S/old/new/}", + varuse("VARIABLE", "S/old/new/")) + + test("${GNUSTEP_LFLAGS:S/-L//g}", + varuse("GNUSTEP_LFLAGS", "S/-L//g")) + + test("${SUSE_VERSION:S/.//}", + varuse("SUSE_VERSION", "S/.//")) + + test("${MASTER_SITE_GNOME:=sources/alacarte/0.13/}", + varuse("MASTER_SITE_GNOME", "=sources/alacarte/0.13/")) + + test("${INCLUDE_DIRS:H:T}", + varuse("INCLUDE_DIRS", "H", "T")) + + test("${A.${B.${C.${D}}}}", + varuse("A.${B.${C.${D}}}")) + + test("${RUBY_VERSION:C/([0-9]+)\\.([0-9]+)\\.([0-9]+)/\\1/}", + varuse("RUBY_VERSION", "C/([0-9]+)\\.([0-9]+)\\.([0-9]+)/\\1/")) + + test("${PERL5_${_var_}:Q}", + varuse("PERL5_${_var_}", "Q")) + + test("${PKGNAME_REQD:C/(^.*-|^)py([0-9][0-9])-.*/\\2/}", + varuse("PKGNAME_REQD", "C/(^.*-|^)py([0-9][0-9])-.*/\\2/")) + + test("${PYLIB:S|/|\\\\/|g}", + varuse("PYLIB", "S|/|\\\\/|g")) + + test("${PKGNAME_REQD:C/ruby([0-9][0-9]+)-.*/\\1/}", + varuse("PKGNAME_REQD", "C/ruby([0-9][0-9]+)-.*/\\1/")) + + test("${RUBY_SHLIBALIAS:S/\\//\\\\\\//}", + varuse("RUBY_SHLIBALIAS", "S/\\//\\\\\\//")) + + test("${RUBY_VER_MAP.${RUBY_VER}:U${RUBY_VER}}", + varuse("RUBY_VER_MAP.${RUBY_VER}", "U${RUBY_VER}")) + + test("${RUBY_VER_MAP.${RUBY_VER}:U18}", + varuse("RUBY_VER_MAP.${RUBY_VER}", "U18")) + + test("${CONFIGURE_ARGS:S/ENABLE_OSS=no/ENABLE_OSS=yes/g}", + varuse("CONFIGURE_ARGS", "S/ENABLE_OSS=no/ENABLE_OSS=yes/g")) + + test("${PLIST_RUBY_DIRS:S,DIR=\"PREFIX/,DIR=\",}", + varuse("PLIST_RUBY_DIRS", "S,DIR=\"PREFIX/,DIR=\",")) + + test("${LDFLAGS:S/-Wl,//g:Q}", + varuse("LDFLAGS", "S/-Wl,//g", "Q")) + + test("${_PERL5_REAL_PACKLIST:S/^/${DESTDIR}/}", + varuse("_PERL5_REAL_PACKLIST", "S/^/${DESTDIR}/")) + + test("${_PYTHON_VERSION:C/^([0-9])/\\1./1}", + varuse("_PYTHON_VERSION", "C/^([0-9])/\\1./1")) + + test("${PKGNAME:S/py${_PYTHON_VERSION}/py${i}/}", + varuse("PKGNAME", "S/py${_PYTHON_VERSION}/py${i}/")) + + test("${PKGNAME:C/-[0-9].*$/-[0-9]*/}", + varuse("PKGNAME", "C/-[0-9].*$/-[0-9]*/")) + + test("${PKGNAME:S/py${_PYTHON_VERSION}/py${i}/:C/-[0-9].*$/-[0-9]*/}", + varuse("PKGNAME", "S/py${_PYTHON_VERSION}/py${i}/", "C/-[0-9].*$/-[0-9]*/")) + + test("${_PERL5_VARS:tl:S/^/-V:/}", + varuse("_PERL5_VARS", "tl", "S/^/-V:/")) + test("${_PERL5_VARS_OUT:M${_var_:tl}=*:S/^${_var_:tl}=${_PERL5_PREFIX:=/}//}", varuse("_PERL5_VARS_OUT", "M${_var_:tl}=*", "S/^${_var_:tl}=${_PERL5_PREFIX:=/}//")) - test("${RUBY${RUBY_VER}_PATCHLEVEL}", varuse("RUBY${RUBY_VER}_PATCHLEVEL")) - test("${DISTFILES:M*.gem}", varuse("DISTFILES", "M*.gem")) - test("${LOCALBASE:S^/^_^}", varuse("LOCALBASE", "S^/^_^")) - test("${SOURCES:%.c=%.o}", varuse("SOURCES", "%.c=%.o")) + + test("${RUBY${RUBY_VER}_PATCHLEVEL}", + varuse("RUBY${RUBY_VER}_PATCHLEVEL")) + + test("${DISTFILES:M*.gem}", + varuse("DISTFILES", "M*.gem")) + + test("${LOCALBASE:S^/^_^}", + varuse("LOCALBASE", "S^/^_^")) + + test("${SOURCES:%.c=%.o}", + varuse("SOURCES", "%.c=%.o")) + test("${GIT_TEMPLATES:@.t.@ ${EGDIR}/${GIT_TEMPLATEDIR}/${.t.} ${PREFIX}/${GIT_CORE_TEMPLATEDIR}/${.t.} @:M*}", varuse("GIT_TEMPLATES", "@.t.@ ${EGDIR}/${GIT_TEMPLATEDIR}/${.t.} ${PREFIX}/${GIT_CORE_TEMPLATEDIR}/${.t.} @", "M*")) - test("${DISTNAME:C:_:-:}", varuse("DISTNAME", "C:_:-:")) - test("${CF_FILES:H:O:u:S@^@${PKG_SYSCONFDIR}/@}", varuse("CF_FILES", "H", "O", "u", "S@^@${PKG_SYSCONFDIR}/@")) - test("${ALT_GCC_RTS:S%${LOCALBASE}%%:S%/%%}", varuse("ALT_GCC_RTS", "S%${LOCALBASE}%%", "S%/%%")) - test("${PREFIX:C;///*;/;g:C;/$;;}", varuse("PREFIX", "C;///*;/;g", "C;/$;;")) - test("${GZIP_CMD:[1]:Q}", varuse("GZIP_CMD", "[1]", "Q")) - test("${RUBY_RAILS_SUPPORTED:[#]}", varuse("RUBY_RAILS_SUPPORTED", "[#]")) - test("${DISTNAME:C/-[0-9]+$$//:C/_/-/}", varuse("DISTNAME", "C/-[0-9]+$$//", "C/_/-/")) - test("${DISTNAME:slang%=slang2%}", varuse("DISTNAME", "slang%=slang2%")) - test("${OSMAP_SUBSTVARS:@v@-e 's,\\@${v}\\@,${${v}},g' @}", varuse("OSMAP_SUBSTVARS", "@v@-e 's,\\@${v}\\@,${${v}},g' @")) - test("${BRANDELF:D${BRANDELF} -t Linux ${LINUX_LDCONFIG}:U${TRUE}}", varuse("BRANDELF", "D${BRANDELF} -t Linux ${LINUX_LDCONFIG}", "U${TRUE}")) - test("${${_var_}.*}", varuse("${_var_}.*")) - - test("${GCONF_SCHEMAS:@.s.@${INSTALL_DATA} ${WRKSRC}/src/common/dbus/${.s.} ${DESTDIR}${GCONF_SCHEMAS_DIR}/@}", - varuse("GCONF_SCHEMAS", "@.s.@${INSTALL_DATA} ${WRKSRC}/src/common/dbus/${.s.} ${DESTDIR}${GCONF_SCHEMAS_DIR}/@")) - /* weird features */ - test("${${EMACS_VERSION_MAJOR}>22:?@comment :}", varuse("${EMACS_VERSION_MAJOR}>22", "?@comment :")) - test("${empty(CFLAGS):?:-cflags ${CFLAGS:Q}}", varuse("empty(CFLAGS)", "?:-cflags ${CFLAGS:Q}")) - test("${${PKGSRC_COMPILER}==gcc:?gcc:cc}", varuse("${PKGSRC_COMPILER}==gcc", "?gcc:cc")) + test("${DISTNAME:C:_:-:}", + varuse("DISTNAME", "C:_:-:")) + + test("${CF_FILES:H:O:u:S@^@${PKG_SYSCONFDIR}/@}", + varuse("CF_FILES", "H", "O", "u", "S@^@${PKG_SYSCONFDIR}/@")) + + test("${ALT_GCC_RTS:S%${LOCALBASE}%%:S%/%%}", + varuse("ALT_GCC_RTS", "S%${LOCALBASE}%%", "S%/%%")) + + test("${PREFIX:C;///*;/;g:C;/$;;}", + varuse("PREFIX", "C;///*;/;g", "C;/$;;")) + + test("${GZIP_CMD:[1]:Q}", + varuse("GZIP_CMD", "[1]", "Q")) - test("${${XKBBASE}/xkbcomp:L:Q}", varuse("${XKBBASE}/xkbcomp", "L", "Q")) - test("${${PKGBASE} ${PKGVERSION}:L}", varuse("${PKGBASE} ${PKGVERSION}", "L")) + test("${RUBY_RAILS_SUPPORTED:[#]}", + varuse("RUBY_RAILS_SUPPORTED", "[#]")) + test("${DISTNAME:C/-[0-9]+$$//:C/_/-/}", + varuse("DISTNAME", "C/-[0-9]+$$//", "C/_/-/")) + + test("${DISTNAME:slang%=slang2%}", + varuse("DISTNAME", "slang%=slang2%")) + + test("${OSMAP_SUBSTVARS:@v@-e 's,\\@${v}\\@,${${v}},g' @}", + varuse("OSMAP_SUBSTVARS", "@v@-e 's,\\@${v}\\@,${${v}},g' @")) + + test("${BRANDELF:D${BRANDELF} -t Linux ${LINUX_LDCONFIG}:U${TRUE}}", + varuse("BRANDELF", "D${BRANDELF} -t Linux ${LINUX_LDCONFIG}", "U${TRUE}")) + + test("${${_var_}.*}", + varuse("${_var_}.*")) + + test("${OPTIONS:@opt@printf 'Option %s is selected\n' ${opt:Q}';@}", + varuse("OPTIONS", "@opt@printf 'Option %s is selected\n' ${opt:Q}';@")) + + /* weird features */ + test("${${EMACS_VERSION_MAJOR}>22:?@comment :}", + varuse("${EMACS_VERSION_MAJOR}>22", "?@comment :")) + + test("${empty(CFLAGS):?:-cflags ${CFLAGS:Q}}", + varuse("empty(CFLAGS)", "?:-cflags ${CFLAGS:Q}")) + + test("${${PKGSRC_COMPILER}==gcc:?gcc:cc}", + varuse("${PKGSRC_COMPILER}==gcc", "?gcc:cc")) + + test("${${XKBBASE}/xkbcomp:L:Q}", + varuse("${XKBBASE}/xkbcomp", "L", "Q")) + + test("${${PKGBASE} ${PKGVERSION}:L}", + varuse("${PKGBASE} ${PKGVERSION}", "L")) + + // This complicated expression returns the major.minor.patch version + // of the package given in ${d}. + // + // The :L modifier interprets the variable name not as a variable name + // but takes it as the variable value. Followed by the :sh modifier, + // this combination evaluates to the output of pkg_info. + // + // In this output, all non-digit characters are replaced with spaces so + // that the remaining value is a space-separated list of version parts. + // From these parts, the first 3 are taken and joined using a dot as separator. test("${${${PKG_INFO} -E ${d} || echo:L:sh}:L:C/[^[0-9]]*/ /g:[1..3]:ts.}", varuse("${${PKG_INFO} -E ${d} || echo:L:sh}", "L", "C/[^[0-9]]*/ /g", "[1..3]", "ts.")) - // For :S and :C, the colon can be left out. + // For :S and :C, the colon can be left out. It's confusing but possible. test("${VAR:S/-//S/.//}", varuseText("${VAR:S/-//S/.//}", "VAR", "S/-//", "S/.//")) - test("${VAR:ts}", varuse("VAR", "ts")) // The separator character can be left out. - test("${VAR:ts\\000012}", varuse("VAR", "ts\\000012")) // The separator character can be a long octal number. - test("${VAR:ts\\124}", varuse("VAR", "ts\\124")) // Or even decimal. + // The :S and :C modifiers accept an arbitrary character as separator. Here it is "a". + test("${VAR:Sahara}", + varuse("VAR", "Sahara")) + + test("${VAR:ts}", + varuse("VAR", "ts")) // The separator character can be left out, which means empty. + + test("${VAR:ts\\000012}", + varuse("VAR", "ts\\000012")) // The separator character can be a long octal number. + + test("${VAR:ts\\124}", + varuse("VAR", "ts\\124")) // Or even decimal. testRest("${VAR:ts---}", nil, "${VAR:ts---}") // The :ts modifier only takes single-character separators. - test("$<", varuseText("$<", "<")) // Same as ${.IMPSRC} + test("$<", + varuseText("$<", "<")) // Same as ${.IMPSRC} + + test("$(GNUSTEP_USER_ROOT)", + varuseText("$(GNUSTEP_USER_ROOT)", "GNUSTEP_USER_ROOT")) - test("$(GNUSTEP_USER_ROOT)", varuseText("$(GNUSTEP_USER_ROOT)", "GNUSTEP_USER_ROOT")) t.CheckOutputLines( - "WARN: Test_MkParser_MkTokens.mk:1: Please use curly braces {} instead of round parentheses () for GNUSTEP_USER_ROOT.") + "WARN: Test_MkParser_VarUse.mk:1: Please use curly braces {} instead of round parentheses () for GNUSTEP_USER_ROOT.") testRest("${VAR)", nil, "${VAR)") // Opening brace, closing parenthesis testRest("$(VAR}", nil, "$(VAR}") // Opening parenthesis, closing brace t.CheckOutputEmpty() // Warnings are only printed for balanced expressions. - test("${PLIST_SUBST_VARS:@var@${var}=${${var}:Q}@}", varuse("PLIST_SUBST_VARS", "@var@${var}=${${var}:Q}@")) - test("${PLIST_SUBST_VARS:@var@${var}=${${var}:Q}}", varuse("PLIST_SUBST_VARS", "@var@${var}=${${var}:Q}")) // Missing @ at the end + test("${PLIST_SUBST_VARS:@var@${var}=${${var}:Q}@}", + varuse("PLIST_SUBST_VARS", "@var@${var}=${${var}:Q}@")) + + test("${PLIST_SUBST_VARS:@var@${var}=${${var}:Q}}", + varuse("PLIST_SUBST_VARS", "@var@${var}=${${var}:Q}")) // Missing @ at the end + t.CheckOutputLines( - "WARN: Test_MkParser_MkTokens.mk:1: Modifier ${PLIST_SUBST_VARS:@var@...@} is missing the final \"@\".") + "WARN: Test_MkParser_VarUse.mk:1: Modifier ${PLIST_SUBST_VARS:@var@...@} is missing the final \"@\".") +} - testRest("hello, ${W:L:tl}orld", []*MkToken{ - literal("hello, "), - varuse("W", "L", "tl"), - literal("orld")}, - "") - testRest("ftp://${PKGNAME}/ ${MASTER_SITES:=subdir/}", []*MkToken{ - literal("ftp://"), - varuse("PKGNAME"), - literal("/ "), - varuse("MASTER_SITES", "=subdir/")}, - "") +func (s *Suite) Test_MkParser_VarUse__ambiguous(c *check.C) { + t := s.Init(c) - // FIXME: Text must match modifiers. - testRest("${VAR:S,a,b,c,d,e,f}", - []*MkToken{{ - Text: "${VAR:S,a,b,c,d,e,f}", - Varuse: NewMkVarUse("VAR", "S,a,b,")}}, + t.SetupCommandLine("--explain") + + mkline := t.NewMkLine("module.mk", 123, "\t$Varname $X") + p := NewMkParser(mkline.Line, mkline.ShellCommand(), true) + + tokens := p.MkTokens() + c.Check(tokens, deepEquals, []*MkToken{ + {"$V", NewMkVarUse("V")}, + {"arname ", nil}, + {"$X", NewMkVarUse("X")}}) + + t.CheckOutputLines( + "ERROR: module.mk:123: $Varname is ambiguous. Use ${Varname} if you mean a Make variable or $$Varname if you mean a shell variable.", + "", + "\tOnly the first letter after the dollar is the variable name.", + "\tEverything following it is normal text, even if it looks like a", + "\tvariable name to human readers.", + "", + "WARN: module.mk:123: $X is ambiguous. Use ${X} if you mean a Make variable or $$X if you mean a shell variable.", + "", + "\tIn its current form, this variable is parsed as a Make variable. For", + "\thuman readers though, $x looks more like a shell variable than a", + "\tMake variable, since Make variables are usually written using braces", + "\t(BSD-style) or parentheses (GNU-style).", "") } @@ -164,44 +364,63 @@ func (s *Suite) Test_MkParser_MkCond(c *check.C) { } varuse := NewMkVarUse + // TODO: Add tests for &&, ||, !. + + // TODO: Add test for !empty(VAR:M}). + test("${OPSYS:MNetBSD}", &mkCond{Not: &mkCond{Empty: varuse("OPSYS", "MNetBSD")}}) + test("defined(VARNAME)", &mkCond{Defined: "VARNAME"}) + test("empty(VARNAME)", &mkCond{Empty: varuse("VARNAME")}) + test("!empty(VARNAME)", &mkCond{Not: &mkCond{Empty: varuse("VARNAME")}}) + test("!empty(VARNAME:M[yY][eE][sS])", &mkCond{Not: &mkCond{Empty: varuse("VARNAME", "M[yY][eE][sS]")}}) + + // Colons are unescaped at this point because they cannot be mistaken for separators anymore. test("!empty(USE_TOOLS:Mautoconf\\:run)", &mkCond{Not: &mkCond{Empty: varuse("USE_TOOLS", "Mautoconf:run")}}) + test("${VARNAME} != \"Value\"", &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME"), "!=", "Value"}}) + test("${VARNAME:Mi386} != \"Value\"", &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME", "Mi386"), "!=", "Value"}}) + test("${VARNAME} != Value", &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME"), "!=", "Value"}}) + test("\"${VARNAME}\" != Value", &mkCond{CompareVarStr: &MkCondCompareVarStr{varuse("VARNAME"), "!=", "Value"}}) + test("${pkg} == \"${name}\"", &mkCond{CompareVarVar: &MkCondCompareVarVar{varuse("pkg"), "==", varuse("name")}}) + test("\"${pkg}\" == \"${name}\"", &mkCond{CompareVarVar: &MkCondCompareVarVar{varuse("pkg"), "==", varuse("name")}}) - test("(defined(VARNAME))", - &mkCond{Defined: "VARNAME"}) + test("exists(/etc/hosts)", &mkCond{Call: &MkCondCall{"exists", "/etc/hosts"}}) + test("exists(${PREFIX}/var)", &mkCond{Call: &MkCondCall{"exists", "${PREFIX}/var"}}) + test("${OPSYS} == \"NetBSD\" || ${OPSYS} == \"OpenBSD\"", &mkCond{Or: []*mkCond{ {CompareVarStr: &MkCondCompareVarStr{varuse("OPSYS"), "==", "NetBSD"}}, {CompareVarStr: &MkCondCompareVarStr{varuse("OPSYS"), "==", "OpenBSD"}}}}) + test("${OPSYS} == \"NetBSD\" && ${MACHINE_ARCH} == \"i386\"", &mkCond{And: []*mkCond{ {CompareVarStr: &MkCondCompareVarStr{varuse("OPSYS"), "==", "NetBSD"}}, {CompareVarStr: &MkCondCompareVarStr{varuse("MACHINE_ARCH"), "==", "i386"}}}}) + test("defined(A) && defined(B) || defined(C) && defined(D)", &mkCond{Or: []*mkCond{ {And: []*mkCond{ @@ -210,55 +429,85 @@ func (s *Suite) Test_MkParser_MkCond(c *check.C) { {And: []*mkCond{ {Defined: "C"}, {Defined: "D"}}}}}) + test("${MACHINE_ARCH:Mi386} || ${MACHINE_OPSYS:MNetBSD}", &mkCond{Or: []*mkCond{ {Not: &mkCond{Empty: varuse("MACHINE_ARCH", "Mi386")}}, {Not: &mkCond{Empty: varuse("MACHINE_OPSYS", "MNetBSD")}}}}) // Exotic cases + + // ".if 0" can be used to skip over a block of code. test("0", &mkCond{Num: "0"}) + + test("0xCAFEBABE", + &mkCond{Num: "0xCAFEBABE"}) + + test("${VAR} == 0xCAFEBABE", + &mkCond{ + CompareVarNum: &MkCondCompareVarNum{ + Var: varuse("VAR"), + Op: "==", + Num: "0xCAFEBABE"}}) + test("! ( defined(A) && empty(VARNAME) )", &mkCond{Not: &mkCond{ And: []*mkCond{ {Defined: "A"}, {Empty: varuse("VARNAME")}}}}) + test("${REQD_MAJOR} > ${MAJOR}", &mkCond{CompareVarVar: &MkCondCompareVarVar{varuse("REQD_MAJOR"), ">", varuse("MAJOR")}}) + test("${OS_VERSION} >= 6.5", &mkCond{CompareVarNum: &MkCondCompareVarNum{varuse("OS_VERSION"), ">=", "6.5"}}) + test("${OS_VERSION} == 5.3", &mkCond{CompareVarNum: &MkCondCompareVarNum{varuse("OS_VERSION"), "==", "5.3"}}) + test("!empty(${OS_VARIANT:MIllumos})", // Probably not intended &mkCond{Not: &mkCond{Empty: varuse("${OS_VARIANT:MIllumos}")}}) - test("defined (VARNAME)", // There may be whitespace before the parenthesis; see devel/bmake/files/cond.c:^compare_function. + + // There may be whitespace before the parenthesis; see devel/bmake/files/cond.c:^compare_function. + test("defined (VARNAME)", &mkCond{Defined: "VARNAME"}) + test("${\"${PKG_OPTIONS:Moption}\":?--enable-option:--disable-option}", &mkCond{Not: &mkCond{Empty: varuse("\"${PKG_OPTIONS:Moption}\"", "?--enable-option:--disable-option")}}) // Errors + testRest("!empty(PKG_OPTIONS:Msndfile) || defined(PKG_OPTIONS:Msamplerate)", &mkCond{Not: &mkCond{Empty: varuse("PKG_OPTIONS", "Msndfile")}}, "|| defined(PKG_OPTIONS:Msamplerate)") + testRest("${LEFT} &&", &mkCond{Not: &mkCond{Empty: varuse("LEFT")}}, "&&") + testRest("\"unfinished string literal", nil, "\"unfinished string literal") + + // Not even the ${VAR} gets through here, although that can be expected. FIXME: Why? testRest("${VAR} == \"unfinished string literal", - nil, // Not even the ${VAR} gets through here, although that can be expected. + nil, "${VAR} == \"unfinished string literal") } -func (s *Suite) Test_MkParser__varuse_parentheses_autofix(c *check.C) { +// Pkglint can replace $(VAR) with ${VAR}. It doesn't look at all components +// of nested variables though because this case is not important enough to +// invest much development time. It occurs so seldom that it is acceptable +// to run pkglint multiple times in such a case. +func (s *Suite) Test_MkParser_VarUse__parentheses_autofix(c *check.C) { t := s.Init(c) t.SetupCommandLine("--autofix") t.SetupVartypes() lines := t.SetupFileLines("Makefile", MkRcsID, - "COMMENT=$(P1) $(P2)) $(P3:Q) ${BRACES}") + "COMMENT=$(P1) $(P2)) $(P3:Q) ${BRACES} $(A.$(B.$(C)))") mklines := NewMkLines(lines) mklines.Check() @@ -266,10 +515,11 @@ func (s *Suite) Test_MkParser__varuse_parentheses_autofix(c *check.C) { t.CheckOutputLines( "AUTOFIX: ~/Makefile:2: Replacing \"$(P1)\" with \"${P1}\".", "AUTOFIX: ~/Makefile:2: Replacing \"$(P2)\" with \"${P2}\".", - "AUTOFIX: ~/Makefile:2: Replacing \"$(P3:Q)\" with \"${P3:Q}\".") + "AUTOFIX: ~/Makefile:2: Replacing \"$(P3:Q)\" with \"${P3:Q}\".", + "AUTOFIX: ~/Makefile:2: Replacing \"$(C)\" with \"${C}\".") t.CheckFileLines("Makefile", MkRcsID, - "COMMENT=${P1} ${P2}) ${P3:Q} ${BRACES}") + "COMMENT=${P1} ${P2}) ${P3:Q} ${BRACES} $(A.$(B.${C}))") } func (s *Suite) Test_MkCondWalker_Walk(c *check.C) { @@ -294,9 +544,12 @@ func (s *Suite) Test_MkCondWalker_Walk(c *check.C) { } addEvent := func(name string, args ...string) { - events = append(events, fmt.Sprintf("%14s %s", name, strings.Join(args, ", "))) + events = append(events, sprintf("%14s %s", name, strings.Join(args, ", "))) } + // TODO: Add callbacks for And, Or, Not if needed. + // Especially Not(Empty(VARNAME)) should be an interesting case. + mkline.Cond().Walk(&MkCondCallback{ Defined: func(varname string) { addEvent("defined", varname) diff --git a/pkgtools/pkglint/files/mkshparser.go b/pkgtools/pkglint/files/mkshparser.go index 28de8d43c0b..f6e04017675 100644 --- a/pkgtools/pkglint/files/mkshparser.go +++ b/pkgtools/pkglint/files/mkshparser.go @@ -1,11 +1,8 @@ -package main +package pkglint -import ( - "fmt" - "strconv" -) +import "strconv" -func parseShellProgram(line Line, program string) (list *MkShList, err error) { +func parseShellProgram(line Line, program string) (*MkShList, error) { if trace.Tracing { defer trace.Call(program)() } @@ -27,16 +24,38 @@ type ParseError struct { } func (e *ParseError) Error() string { - return fmt.Sprintf("parse error at %#v", e.RemainingTokens) + return sprintf("parse error at %#v", e.RemainingTokens) } +// ShellLexer categorizes tokens for shell commands, providing +// the lexer required by the yacc-generated parser. +// +// The main work of tokenizing is done in ShellTokenizer though. +// +// Example: +// while :; do var=$$other; done +// => +// while +// space " " +// word ":" +// semicolon +// space " " +// do +// space " " +// assign "var=$$other" +// semicolon +// space " " +// done +// +// See splitIntoShellTokens and ShellTokenizer. type ShellLexer struct { current string - ioredirect string + ioRedirect string remaining []string atCommandStart bool sinceFor int sinceCase int + inCasePattern bool // true inside (pattern1|pattern2|pattern3); works only for simple cases error string result *MkShList } @@ -44,11 +63,12 @@ type ShellLexer struct { func NewShellLexer(tokens []string, rest string) *ShellLexer { return &ShellLexer{ current: "", - ioredirect: "", + ioRedirect: "", remaining: tokens, atCommandStart: true, error: rest} } + func (lex *ShellLexer) Lex(lval *shyySymType) (ttype int) { if len(lex.remaining) == 0 { return 0 @@ -68,8 +88,8 @@ func (lex *ShellLexer) Lex(lval *shyySymType) (ttype int) { }() } - token := lex.ioredirect - lex.ioredirect = "" + token := lex.ioRedirect + lex.ioRedirect = "" if token == "" { token = lex.remaining[0] lex.current = token @@ -82,6 +102,7 @@ func (lex *ShellLexer) Lex(lval *shyySymType) (ttype int) { return tkSEMI case ";;": lex.atCommandStart = true + lex.inCasePattern = true return tkSEMISEMI case "\n": lex.atCommandStart = true @@ -90,13 +111,14 @@ func (lex *ShellLexer) Lex(lval *shyySymType) (ttype int) { lex.atCommandStart = true return tkBACKGROUND case "|": - lex.atCommandStart = true + lex.atCommandStart = !lex.inCasePattern return tkPIPE case "(": - lex.atCommandStart = true + lex.atCommandStart = !lex.inCasePattern return tkLPAREN case ")": lex.atCommandStart = true + lex.inCasePattern = false return tkRPAREN case "&&": lex.atCommandStart = true @@ -104,6 +126,7 @@ func (lex *ShellLexer) Lex(lval *shyySymType) (ttype int) { case "||": lex.atCommandStart = true return tkOR + case ">": lex.atCommandStart = false return tkGT @@ -136,7 +159,7 @@ func (lex *ShellLexer) Lex(lval *shyySymType) (ttype int) { if m, fdstr, op := match2(token, `^(\d+)(<<-|<<|<>|<&|>>|>&|>\||<|>)$`); m { fd, _ := strconv.Atoi(fdstr) lval.IONum = fd - lex.ioredirect = op + lex.ioRedirect = op return tkIO_NUMBER } @@ -165,6 +188,7 @@ func (lex *ShellLexer) Lex(lval *shyySymType) (ttype int) { case "do": return tkDO case "done": + // TODO: add test that ensures "lex.atCommandStart = false" is required here. return tkDONE case "in": lex.atCommandStart = false @@ -176,6 +200,7 @@ func (lex *ShellLexer) Lex(lval *shyySymType) (ttype int) { case "{": return tkLBRACE case "}": + // TODO: add test that ensures "lex.atCommandStart = false" is required here. return tkRBRACE case "!": return tkEXCLAM @@ -199,16 +224,17 @@ func (lex *ShellLexer) Lex(lval *shyySymType) (ttype int) { case lex.sinceCase == 2 && token == "in": ttype = tkIN lex.atCommandStart = false + lex.inCasePattern = true case (lex.atCommandStart || lex.sinceCase == 3) && token == "esac": ttype = tkESAC lex.atCommandStart = false case lex.atCommandStart && matches(token, `^[A-Za-z_]\w*=`): ttype = tkASSIGNMENT_WORD - p := NewShTokenizer(dummyLine, token, false) + p := NewShTokenizer(dummyLine, token, false) // Just for converting the string to a ShToken lval.Word = p.ShToken() default: ttype = tkWORD - p := NewShTokenizer(dummyLine, token, false) + p := NewShTokenizer(dummyLine, token, false) // Just for converting the string to a ShToken lval.Word = p.ShToken() lex.atCommandStart = false } diff --git a/pkgtools/pkglint/files/mkshparser_test.go b/pkgtools/pkglint/files/mkshparser_test.go index fee13279b79..22ef5064f30 100644 --- a/pkgtools/pkglint/files/mkshparser_test.go +++ b/pkgtools/pkglint/files/mkshparser_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "encoding/json" @@ -6,7 +6,7 @@ import ( "strconv" ) -func (s *Suite) Test_parseShellProgram__parse_error(c *check.C) { +func (s *Suite) Test_parseShellProgram__parse_error_for_unfinished_shell_variable(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("module.mk", 1, "\t$${") @@ -18,7 +18,7 @@ func (s *Suite) Test_parseShellProgram__parse_error(c *check.C) { c.Check(err.Error(), equals, "parse error at []string{\"\"}") t.CheckOutputLines( - "WARN: module.mk:1: Pkglint parse error in ShTokenizer.ShAtom at \"$${\" (quoting=plain).") + "WARN: module.mk:1: Internal pkglint error in ShTokenizer.ShAtom at \"$${\" (quoting=plain).") } type ShSuite struct { @@ -48,7 +48,8 @@ func (s *ShSuite) Test_ShellParser__program(c *check.C) { b.List().AddCommand(b.SimpleCommand("echo"))) s.test(""+ - "cd ${WRKSRC} && ${FIND} ${${_list_}} -type f ! -name '*.orig' 2> /dev/null "+ + "cd ${WRKSRC} "+ + "&& ${FIND} ${${_list_}} -type f ! -name '*.orig' 2> /dev/null "+ "| pax -rw -pm ${DESTDIR}${PREFIX}/${${_dir_}}", b.List().AddAndOr(b.AndOr( b.Pipeline(false, b.SimpleCommand("cd", "${WRKSRC}"))).Add("&&", @@ -256,6 +257,7 @@ func (s *ShSuite) Test_ShellParser__compound_list(c *check.C) { func (s *ShSuite) Test_ShellParser__term(c *check.C) { b := s.init(c) + // TODO _ = b } @@ -342,6 +344,12 @@ func (s *ShSuite) Test_ShellParser__case_clause(c *check.C) { b.Words("pattern"), b.List().AddCommand(b.SimpleCommand("case-item-action")), sepNone)))) + s.test("case $$expr in (if|then|else) ;; esac", + b.List().AddCommand(b.Case( + b.Token("$$expr"), + b.CaseItem( + b.Words("if", "then", "else"), + b.List(), sepNone)))) } func (s *ShSuite) Test_ShellParser__if_clause(c *check.C) { @@ -384,6 +392,7 @@ func (s *ShSuite) Test_ShellParser__until_clause(c *check.C) { func (s *ShSuite) Test_ShellParser__function_definition(c *check.C) { b := s.init(c) + // TODO _ = b } @@ -426,12 +435,15 @@ func (s *ShSuite) Test_ShellParser__simple_command(c *check.C) { // RUN is a special Make variable since it ends with a semicolon; // therefore it needs to be split off before passing the rest of // the command to the shell command parser. + // Otherwise it would be interpreted as a shell command, + // and the real shell command would be its argument. s.test("${RUN} subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\"", b.List().AddCommand(b.SimpleCommand("${RUN}", "subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\""))) s.test("PATH=/nonexistent env PATH=${PATH:Q} true", b.List().AddCommand(b.SimpleCommand("PATH=/nonexistent", "env", "PATH=${PATH:Q}", "true"))) + // The opening curly brace only has its special meaning when it appears as a whole word. s.test("{OpenGrok args", b.List().AddCommand(b.SimpleCommand("{OpenGrok", "args"))) } @@ -442,9 +454,6 @@ func (s *ShSuite) Test_ShellParser__io_redirect(c *check.C) { s.test("echo >> ${PLIST_SRC}", b.List().AddCommand(b.SimpleCommand("echo", ">>${PLIST_SRC}"))) - s.test("echo >> ${PLIST_SRC}", - b.List().AddCommand(b.SimpleCommand("echo", ">>${PLIST_SRC}"))) - s.test("echo 1>output 2>>append 3>|clobber 4>&5 6<input >>append", b.List().AddCommand(&MkShCommand{Simple: &MkShSimpleCommand{ Assignments: nil, @@ -484,6 +493,7 @@ func (s *ShSuite) Test_ShellParser__io_redirect(c *check.C) { func (s *ShSuite) Test_ShellParser__io_here(c *check.C) { b := s.init(c) + // TODO _ = b } @@ -495,18 +505,18 @@ func (s *ShSuite) init(c *check.C) *MkShBuilder { func (s *ShSuite) test(program string, expected *MkShList) { tokens, rest := splitIntoShellTokens(dummyLine, program) s.c.Check(rest, equals, "") - lexer := &ShellLexer{ + lexer := ShellLexer{ current: "", remaining: tokens, atCommandStart: true, error: ""} - parser := ­yParserImpl{} + parser := shyyParserImpl{} - succeeded := parser.Parse(lexer) + succeeded := parser.Parse(&lexer) c := s.c - if ok1, ok2 := c.Check(succeeded, equals, 0), c.Check(lexer.error, equals, ""); ok1 && ok2 { + if c.Check(succeeded, equals, 0) && c.Check(lexer.error, equals, "") { if !c.Check(lexer.result, deepEquals, expected) { actualJSON, actualErr := json.MarshalIndent(lexer.result, "", " ") expectedJSON, expectedErr := json.MarshalIndent(expected, "", " ") @@ -582,14 +592,14 @@ func (b *MkShBuilder) AndOr(pipeline *MkShPipeline) *MkShAndOr { } func (b *MkShBuilder) Pipeline(negated bool, cmds ...*MkShCommand) *MkShPipeline { - return NewMkShPipeline(negated, cmds...) + return NewMkShPipeline(negated, cmds) } func (b *MkShBuilder) SimpleCommand(words ...string) *MkShCommand { - cmd := &MkShSimpleCommand{} + cmd := MkShSimpleCommand{} assignments := true for _, word := range words { - if assignments && matches(word, `^\w+=`) { + if assignments && matches(word, `^[A-Za-z_]\w*=`) { cmd.Assignments = append(cmd.Assignments, b.Token(word)) } else if m, fdstr, op, rest := match3(word, `^(\d*)(<<-|<<|<&|>>|>&|>\||<|>)(.*)$`); m { fd, err := strconv.Atoi(fdstr) @@ -606,29 +616,30 @@ func (b *MkShBuilder) SimpleCommand(words ...string) *MkShCommand { } } } - return &MkShCommand{Simple: cmd} + return &MkShCommand{Simple: &cmd} } func (b *MkShBuilder) If(condActionElse ...*MkShList) *MkShCommand { - ifclause := &MkShIfClause{} + ifClause := MkShIf{} for i, part := range condActionElse { - if i%2 == 0 && i != len(condActionElse)-1 { - ifclause.Conds = append(ifclause.Conds, part) - } else if i%2 == 1 { - ifclause.Actions = append(ifclause.Actions, part) - } else { - ifclause.Else = part + switch { + case i%2 == 0 && i != len(condActionElse)-1: + ifClause.Conds = append(ifClause.Conds, part) + case i%2 == 1: + ifClause.Actions = append(ifClause.Actions, part) + default: + ifClause.Else = part } } - return &MkShCommand{Compound: &MkShCompoundCommand{If: ifclause}} + return &MkShCommand{Compound: &MkShCompoundCommand{If: &ifClause}} } func (b *MkShBuilder) For(varname string, items []*ShToken, action *MkShList) *MkShCommand { - return &MkShCommand{Compound: &MkShCompoundCommand{For: &MkShForClause{varname, items, action}}} + return &MkShCommand{Compound: &MkShCompoundCommand{For: &MkShFor{varname, items, action}}} } func (b *MkShBuilder) Case(selector *ShToken, items ...*MkShCaseItem) *MkShCommand { - return &MkShCommand{Compound: &MkShCompoundCommand{Case: &MkShCaseClause{selector, items}}} + return &MkShCommand{Compound: &MkShCompoundCommand{Case: &MkShCase{selector, items}}} } func (b *MkShBuilder) CaseItem(patterns []*ShToken, action *MkShList, separator MkShSeparator) *MkShCaseItem { @@ -638,14 +649,14 @@ func (b *MkShBuilder) CaseItem(patterns []*ShToken, action *MkShList, separator func (b *MkShBuilder) While(cond, action *MkShList, redirects ...*MkShRedirection) *MkShCommand { return &MkShCommand{ Compound: &MkShCompoundCommand{ - Loop: &MkShLoopClause{cond, action, false}}, + Loop: &MkShLoop{cond, action, false}}, Redirects: redirects} } func (b *MkShBuilder) Until(cond, action *MkShList, redirects ...*MkShRedirection) *MkShCommand { return &MkShCommand{ Compound: &MkShCompoundCommand{ - Loop: &MkShLoopClause{cond, action, true}}, + Loop: &MkShLoop{cond, action, true}}, Redirects: redirects} } @@ -666,6 +677,7 @@ func (b *MkShBuilder) Subshell(list *MkShList) *MkShCommand { func (b *MkShBuilder) Token(mktext string) *ShToken { tokenizer := NewShTokenizer(dummyLine, mktext, false) token := tokenizer.ShToken() + G.Assertf(tokenizer.parser.EOF(), "Invalid token: %q", tokenizer.parser.Rest()) return token } diff --git a/pkgtools/pkglint/files/mkshtypes.go b/pkgtools/pkglint/files/mkshtypes.go index 262113ada29..4046c82b701 100644 --- a/pkgtools/pkglint/files/mkshtypes.go +++ b/pkgtools/pkglint/files/mkshtypes.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "netbsd.org/pkglint/regex" @@ -7,10 +7,14 @@ import ( // MkShList is a list of shell commands, separated by newlines or semicolons. // -// Example: cd $dir && echo "In $dir"; cd ..; ls -l +// Example: +// cd $dir && echo "In $dir"; cd ..; ls -l type MkShList struct { - AndOrs []*MkShAndOr - Separators []MkShSeparator // One less entry than in AndOrs. + AndOrs []*MkShAndOr + + // The separators after each AndOr. + // There may be one less entry than in AndOrs. + Separators []MkShSeparator } func NewMkShList() *MkShList { @@ -30,7 +34,11 @@ func (list *MkShList) AddSeparator(separator MkShSeparator) *MkShList { // MkShAndOr is a group of commands that are connected with && or || // conditions. // -// Example: cd $dir && echo "In $dir" || echo "Cannot cd into $dir" +// The operators && and || have the same precedence and are evaluated +// strictly from left to right. +// +// Example: +// cd $dir && echo "In $dir" || echo "Cannot cd into $dir" type MkShAndOr struct { Pipes []*MkShPipeline Ops []string // Each element is either "&&" or "||" @@ -54,7 +62,7 @@ type MkShPipeline struct { Cmds []*MkShCommand } -func NewMkShPipeline(negated bool, cmds ...*MkShCommand) *MkShPipeline { +func NewMkShPipeline(negated bool, cmds []*MkShCommand) *MkShPipeline { return &MkShPipeline{negated, cmds} } @@ -65,9 +73,10 @@ func (pipe *MkShPipeline) Add(cmd *MkShCommand) *MkShPipeline { // MkShCommand is a simple or compound shell command. // -// Example: LC_ALL=C sort */*.c > sorted -// Example: dir() { ls -l "$@"; } -// Example: { echo "first"; echo "second"; } +// Examples: +// LC_ALL=C sort */*.c > sorted +// dir() { ls -l "$@"; } +// { echo "first"; echo "second"; } type MkShCommand struct { Simple *MkShSimpleCommand Compound *MkShCompoundCommand @@ -77,64 +86,70 @@ type MkShCommand struct { // MkShCompoundCommand is a group of commands. // -// Example: { echo "first"; echo "second"; } -// Example: for f in *.c; do compile "$f"; done -// Example: if [ -f "$file" ]; then echo "It exists"; fi -// Example: while sleep 1; do printf .; done +// Examples: +// { echo "first"; echo "second"; } +// for f in *.c; do compile "$f"; done +// if [ -f "$file" ]; then echo "It exists"; fi +// while sleep 1; do printf .; done type MkShCompoundCommand struct { Brace *MkShList Subshell *MkShList - For *MkShForClause - Case *MkShCaseClause - If *MkShIfClause - Loop *MkShLoopClause + For *MkShFor + Case *MkShCase + If *MkShIf + Loop *MkShLoop } -// MkShForClause is a "for" loop. +// MkShFor is a "for" loop. // -// Example: for f in *.c; do compile "$f"; done -type MkShForClause struct { +// Example: +// for f in *.c; do compile "$f"; done +type MkShFor struct { Varname string Values []*ShToken Body *MkShList } -// MkShCaseClause is a "case" statement, including all its branches. +// MkShCase is a "case" statement, including all its branches. // -// Example: case $filename in *.c) echo "C source" ;; esac -type MkShCaseClause struct { +// Example: +// case $filename in *.c) echo "C source" ;; esac +type MkShCase struct { Word *ShToken Cases []*MkShCaseItem } // MkShCaseItem is one branch of a "case" statement. // -// Example: *.c) echo "C source" ;; +// Example: +// *.c) echo "C source" ;; type MkShCaseItem struct { Patterns []*ShToken Action *MkShList Separator MkShSeparator } -// MkShIfClause is a conditional statement, possibly having +// MkShIf is a conditional statement, possibly having // many branches. // -// Example: if [ -f "$file" ]; then echo "It exists"; fi -type MkShIfClause struct { +// Example: +// if [ -f "$file" ]; then echo "It exists"; fi +type MkShIf struct { Conds []*MkShList Actions []*MkShList Else *MkShList } -func (cl *MkShIfClause) Prepend(cond *MkShList, action *MkShList) { +func (cl *MkShIf) Prepend(cond *MkShList, action *MkShList) { cl.Conds = append([]*MkShList{cond}, cl.Conds...) cl.Actions = append([]*MkShList{action}, cl.Actions...) } -// MkShLoopClause is a "while" or "until" loop. +// MkShLoop is a "while" or "until" loop. // -// Example: while sleep 1; do printf .; done -type MkShLoopClause struct { +// Example: +// while sleep 1; do printf .; done +type MkShLoop struct { Cond *MkShList Action *MkShList Until bool @@ -142,7 +157,8 @@ type MkShLoopClause struct { // MkShFunctionDefinition is the definition of a shell function. // -// Example: dir() { ls -l "$@"; } +// Example: +// dir() { ls -l "$@"; } type MkShFunctionDefinition struct { Name string Body *MkShCompoundCommand @@ -151,7 +167,8 @@ type MkShFunctionDefinition struct { // MkShSimpleCommand is a shell command that does not involve any // pipeline or conditionals. // -// Example: LC_ALL=C sort */*.c > sorted +// Example: +// LC_ALL=C sort */*.c > sorted type MkShSimpleCommand struct { Assignments []*ShToken Name *ShToken @@ -159,6 +176,18 @@ type MkShSimpleCommand struct { Redirections []*MkShRedirection } +// StrCommand is structurally similar to MkShSimpleCommand, but all +// components are converted to strings to allow for simpler checks, +// especially for analyzing command line options. +// +// Example: +// LC_ALL=C sort */*.c > sorted +type StrCommand struct { + Assignments []string + Name string + Args []string +} + func NewStrCommand(cmd *MkShSimpleCommand) *StrCommand { strcmd := StrCommand{ make([]string, len(cmd.Assignments)), @@ -176,17 +205,6 @@ func NewStrCommand(cmd *MkShSimpleCommand) *StrCommand { return &strcmd } -// StrCommand is structurally similar to MkShSimpleCommand, but all -// components are converted to strings to allow for simpler checks, -// especially for analyzing command line options. -// -// Example: LC_ALL=C sort */*.c > sorted -type StrCommand struct { - Assignments []string - Name string - Args []string -} - // HasOption checks whether one of the arguments is exactly the given opt. func (c *StrCommand) HasOption(opt string) bool { for _, arg := range c.Args { @@ -222,14 +240,16 @@ func (c *StrCommand) String() string { // MkShRedirection is a single file descriptor redirection. // -// Example: > sorted -// Example: 2>&1 +// Examples: +// > sorted +// 2>&1 type MkShRedirection struct { Fd int // Or -1 Op string // See io_file in shell.y for possible values Target *ShToken // The filename or &fd } +// MkShSeparator is one of ; & newline. type MkShSeparator uint8 const ( diff --git a/pkgtools/pkglint/files/mkshtypes_test.go b/pkgtools/pkglint/files/mkshtypes_test.go index a6735039cff..3a0b9963ba6 100644 --- a/pkgtools/pkglint/files/mkshtypes_test.go +++ b/pkgtools/pkglint/files/mkshtypes_test.go @@ -1,4 +1,4 @@ -package main +package pkglint func (list *MkShList) AddSemicolon() *MkShList { return list.AddSeparator(sepSemicolon) } func (list *MkShList) AddBackground() *MkShList { return list.AddSeparator(sepBackground) } diff --git a/pkgtools/pkglint/files/mkshwalker.go b/pkgtools/pkglint/files/mkshwalker.go index 0d100229915..dece424ef34 100644 --- a/pkgtools/pkglint/files/mkshwalker.go +++ b/pkgtools/pkglint/files/mkshwalker.go @@ -1,7 +1,6 @@ -package main +package pkglint import ( - "fmt" "reflect" "strings" ) @@ -14,23 +13,48 @@ type MkShWalker struct { Command func(command *MkShCommand) SimpleCommand func(command *MkShSimpleCommand) CompoundCommand func(command *MkShCompoundCommand) - Case func(caseClause *MkShCaseClause) + Case func(caseClause *MkShCase) CaseItem func(caseItem *MkShCaseItem) FunctionDefinition func(funcdef *MkShFunctionDefinition) - If func(ifClause *MkShIfClause) - Loop func(loop *MkShLoopClause) + If func(ifClause *MkShIf) + Loop func(loop *MkShLoop) Words func(words []*ShToken) Word func(word *ShToken) Redirects func(redirects []*MkShRedirection) Redirect func(redirect *MkShRedirection) - For func(forClause *MkShForClause) - Varname func(varname string) + For func(forClause *MkShFor) + + // For variable definition in a for loop. + Varname func(varname string) } + + // Context[0] is the currently visited element, + // Context[1] is its immediate parent element, and so on. + // This is useful when the check for a CaseItem needs to look at the enclosing Case. Context []MkShWalkerPathElement } type MkShWalkerPathElement struct { - Index int + + // For fields that can be repeated, this is the index as seen from the parent element. + // For fields that cannot be repeated, it is -1. + // + // For example, in the SimpleCommand "var=value cmd arg1 arg2", + // there are multiple child elements of type Words. + // + // The first Words are the variable assignments, which have index 0. + // + // The command "cmd" has type Word, therefore it cannot be confused + // with either of the Words lists and has index -1. + // + // The second Words are the arguments, which have index 1. + // In this example, there are two arguments, so when visiting the + // arguments individually, arg1 will have index 0 and arg2 will have index 1. + // + // TODO: It might be worth defining negative indexes to correspond + // to the fields "Cond", "Action", "Else", etc. + Index int + Element interface{} } @@ -40,15 +64,20 @@ func NewMkShWalker() *MkShWalker { // Path returns a representation of the path in the AST that is // currently visited. +// +// It is used for debugging only. +// +// See Test_MkShWalker_Walk, Callback.SimpleCommand for examples. func (w *MkShWalker) Path() string { var path []string for _, level := range w.Context { typeName := reflect.TypeOf(level.Element).Elem().Name() - abbreviated := strings.Replace(typeName, "MkSh", "", 1) + abbreviated := strings.TrimPrefix(typeName, "MkSh") if level.Index == -1 { + // TODO: This form should also be used if index == 0 and len == 1. path = append(path, abbreviated) } else { - path = append(path, fmt.Sprintf("%s[%d]", abbreviated, level.Index)) + path = append(path, sprintf("%s[%d]", abbreviated, level.Index)) } } return strings.Join(path, ".") @@ -59,6 +88,7 @@ func (w *MkShWalker) Path() string { func (w *MkShWalker) Walk(list *MkShList) { w.walkList(-1, list) + // If this fails, the calls to w.push and w.pop are unbalanced. G.Assertf(len(w.Context) == 0, "MkShWalker.Walk %v", w.Context) } @@ -136,7 +166,7 @@ func (w *MkShWalker) walkSimpleCommand(index int, command *MkShSimpleCommand) { if command.Name != nil { w.walkWord(-1, command.Name) } - w.walkWords(2, command.Args) + w.walkWords(1, command.Args) w.walkRedirects(-1, command.Redirections) w.pop() @@ -167,21 +197,21 @@ func (w *MkShWalker) walkCompoundCommand(index int, command *MkShCompoundCommand w.pop() } -func (w *MkShWalker) walkCase(caseClause *MkShCaseClause) { +func (w *MkShWalker) walkCase(caseClause *MkShCase) { w.push(-1, caseClause) if callback := w.Callback.Case; callback != nil { callback(caseClause) } - w.walkWord(0, caseClause.Word) + w.walkWord(-1, caseClause.Word) for i, caseItem := range caseClause.Cases { w.push(i, caseItem) if callback := w.Callback.CaseItem; callback != nil { callback(caseItem) } - w.walkWords(0, caseItem.Patterns) - w.walkList(1, caseItem.Action) + w.walkWords(-1, caseItem.Patterns) + w.walkList(-1, caseItem.Action) w.pop() } @@ -200,13 +230,14 @@ func (w *MkShWalker) walkFunctionDefinition(index int, funcdef *MkShFunctionDefi w.pop() } -func (w *MkShWalker) walkIf(ifClause *MkShIfClause) { +func (w *MkShWalker) walkIf(ifClause *MkShIf) { w.push(-1, ifClause) if callback := w.Callback.If; callback != nil { callback(ifClause) } + // TODO: Replace these indices with proper field names; see MkShWalkerPathElement.Index. for i, cond := range ifClause.Conds { w.walkList(2*i, cond) w.walkList(2*i+1, ifClause.Actions[i]) @@ -218,7 +249,7 @@ func (w *MkShWalker) walkIf(ifClause *MkShIfClause) { w.pop() } -func (w *MkShWalker) walkLoop(loop *MkShLoopClause) { +func (w *MkShWalker) walkLoop(loop *MkShLoop) { w.push(-1, loop) if callback := w.Callback.Loop; callback != nil { @@ -271,6 +302,9 @@ func (w *MkShWalker) walkRedirects(index int, redirects []*MkShRedirection) { } for i, redirect := range redirects { + // FIXME: The w.push/w.pop is missing here. + // How does the path look like? + // Are there ambiguities? if callback := w.Callback.Redirect; callback != nil { callback(redirect) } @@ -281,7 +315,7 @@ func (w *MkShWalker) walkRedirects(index int, redirects []*MkShRedirection) { w.pop() } -func (w *MkShWalker) walkFor(forClause *MkShForClause) { +func (w *MkShWalker) walkFor(forClause *MkShFor) { w.push(-1, forClause) if callback := w.Callback.For; callback != nil { diff --git a/pkgtools/pkglint/files/mkshwalker_test.go b/pkgtools/pkglint/files/mkshwalker_test.go index 30441feea8d..156f6917ce9 100644 --- a/pkgtools/pkglint/files/mkshwalker_test.go +++ b/pkgtools/pkglint/files/mkshwalker_test.go @@ -1,9 +1,6 @@ -package main +package pkglint -import ( - "fmt" - "gopkg.in/check.v1" -) +import "gopkg.in/check.v1" func (s *Suite) Test_MkShWalker_Walk(c *check.C) { list, err := parseShellProgram(dummyLine, ""+ @@ -22,8 +19,8 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { if format != "" && !contains(format, "%") { panic(format) } - detail := fmt.Sprintf(format, args...) - commands = append(commands, fmt.Sprintf("%16s %s", kind, detail)) + detail := sprintf(format, args...) + commands = append(commands, sprintf("%16s %s", kind, detail)) } walker := NewMkShWalker() @@ -37,20 +34,30 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { add("Path", "%s", walker.Path()) } callback.CompoundCommand = func(command *MkShCompoundCommand) { add("CompoundCommand", "") } - callback.Case = func(caseClause *MkShCaseClause) { add("Case", "with %d items", len(caseClause.Cases)) } + callback.Case = func(caseClause *MkShCase) { add("Case", "with %d items", len(caseClause.Cases)) } callback.CaseItem = func(caseItem *MkShCaseItem) { add("CaseItem", "with %d patterns", len(caseItem.Patterns)) } callback.FunctionDefinition = func(funcdef *MkShFunctionDefinition) { add("FunctionDef", "for %s", funcdef.Name) } - callback.If = func(ifClause *MkShIfClause) { add("If", "with %d then-branches", len(ifClause.Conds)) } - callback.Loop = func(loop *MkShLoopClause) { add("Loop", "") } + callback.If = func(ifClause *MkShIf) { add("If", "with %d then-branches", len(ifClause.Conds)) } + callback.Loop = func(loop *MkShLoop) { add("Loop", "") } callback.Words = func(words []*ShToken) { add("Words", "with %d words", len(words)) } callback.Word = func(word *ShToken) { add("Word", "%s", word.MkText) } callback.Redirects = func(redirects []*MkShRedirection) { add("Redirects", "with %d redirects", len(redirects)) } callback.Redirect = func(redirect *MkShRedirection) { add("Redirect", "%s", redirect.Op) } - callback.For = func(forClause *MkShForClause) { add("For", "variable %s", forClause.Varname) } + callback.For = func(forClause *MkShFor) { add("For", "variable %s", forClause.Varname) } callback.Varname = func(varname string) { add("Varname", "%s", varname) } walker.Walk(list) + // TODO: Provide a reduced AST that omits all "AndOr with 1 pipelines", etc. + // It should look like this: + // + // List with 5 andOrs (or generic Commands?) + // If with 1 then-branch(es) + // SimpleCommand condition + // SimpleCommand action + // Case with 1 item(s) + // ... + c.Check(commands, deepEquals, []string{ " List with 5 andOrs", " AndOr with 1 pipelines", @@ -63,14 +70,14 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " Pipeline with 1 commands", " Command ", " SimpleCommand condition", - " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.IfClause.List[0].AndOr[0].Pipeline[0].Command[0].SimpleCommand", + " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.If.List[0].AndOr[0].Pipeline[0].Command[0].SimpleCommand", " Word condition", " List with 1 andOrs", " AndOr with 1 pipelines", " Pipeline with 1 commands", " Command ", " SimpleCommand action", - " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.IfClause.List[1].AndOr[0].Pipeline[0].Command[0].SimpleCommand", + " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.If.List[1].AndOr[0].Pipeline[0].Command[0].SimpleCommand", " Word action", " List with 1 andOrs", " AndOr with 1 pipelines", @@ -87,9 +94,9 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " Pipeline with 1 commands", " Command ", " SimpleCommand case-item-action", - " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.IfClause." + - "List[2].AndOr[0].Pipeline[0].Command[0].CompoundCommand.CaseClause.CaseItem[0]." + - "List[1].AndOr[0].Pipeline[0].Command[0].SimpleCommand", + " Path List.AndOr[0].Pipeline[0].Command[0].CompoundCommand.If." + + "List[2].AndOr[0].Pipeline[0].Command[0].CompoundCommand.Case.CaseItem[0]." + + "List.AndOr[0].Pipeline[0].Command[0].SimpleCommand", " Word case-item-action", " AndOr with 1 pipelines", " Pipeline with 1 commands", @@ -120,7 +127,7 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " Pipeline with 1 commands", " Command ", " SimpleCommand [ \"$${lang}\" = \"wxstd.po\" ]", - " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.ForClause.List.AndOr[0].Pipeline[0].Command[0].SimpleCommand", + " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.For.List.AndOr[0].Pipeline[0].Command[0].SimpleCommand", " Word [", " Words with 4 words", " Word \"$${lang}\"", @@ -130,13 +137,13 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " Pipeline with 1 commands", " Command ", " SimpleCommand continue", - " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.ForClause.List.AndOr[0].Pipeline[1].Command[0].SimpleCommand", + " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.For.List.AndOr[0].Pipeline[1].Command[0].SimpleCommand", " Word continue", " AndOr with 1 pipelines", " Pipeline with 1 commands", " Command ", " SimpleCommand ${TOOLS_PATH.msgfmt} -c -o \"$${lang%.po}.mo\" \"$${lang}\"", - " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.ForClause.List.AndOr[1].Pipeline[0].Command[0].SimpleCommand", + " Path List.AndOr[3].Pipeline[0].Command[0].CompoundCommand.For.List.AndOr[1].Pipeline[0].Command[0].SimpleCommand", " Word ${TOOLS_PATH.msgfmt}", " Words with 4 words", " Word -c", @@ -153,7 +160,7 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " Pipeline with 1 commands", " Command ", " SimpleCommand :", - " Path List.AndOr[4].Pipeline[0].Command[0].CompoundCommand.LoopClause.List[0].AndOr[0].Pipeline[0].Command[0].SimpleCommand", + " Path List.AndOr[4].Pipeline[0].Command[0].CompoundCommand.Loop.List[0].AndOr[0].Pipeline[0].Command[0].SimpleCommand", " Word :", " List with 1 andOrs", " AndOr with 1 pipelines", @@ -166,7 +173,7 @@ func (s *Suite) Test_MkShWalker_Walk(c *check.C) { " Pipeline with 1 commands", " Command ", " SimpleCommand :", - " Path List.AndOr[4].Pipeline[0].Command[0].CompoundCommand.LoopClause." + + " Path List.AndOr[4].Pipeline[0].Command[0].CompoundCommand.Loop." + "List[1].AndOr[0].Pipeline[0].Command[0].FunctionDefinition.CompoundCommand." + "List.AndOr[0].Pipeline[0].Command[0].SimpleCommand", " Word :", diff --git a/pkgtools/pkglint/files/mktypes.go b/pkgtools/pkglint/files/mktypes.go index ddacc491a04..1a09ba801a5 100644 --- a/pkgtools/pkglint/files/mktypes.go +++ b/pkgtools/pkglint/files/mktypes.go @@ -1,17 +1,22 @@ -package main +package pkglint import ( "netbsd.org/pkglint/textproc" + "strings" "unicode" ) // MkToken represents a contiguous string from a Makefile. // It is either a literal string or a variable use. // -// Example (3 tokens): /usr/share/${PKGNAME}/data +// Example: /usr/share/${PKGNAME}/data consists of 3 tokens: +// 1. MkToken{Text: "/usr/share/"} +// 2. MkToken{Text: "${PKGNAME}", Varuse: &MkVarUse{varname: "PKGNAME"}} +// 3. MkToken{Text: "/data"} +// type MkToken struct { - Text string // Used for both literals and varuses. - Varuse *MkVarUse + Text string // Used for both literal text and variable uses + Varuse *MkVarUse // For literal text, it is nil } // MkVarUse represents a reference to a Make variable, with optional modifiers. @@ -28,6 +33,10 @@ type MkVarUse struct { modifiers []MkVarUseModifier // E.g. "Q", "S/from/to/" } +//func NewMkVarUse(varname string, modifiers ...MkVarUseModifier) *MkVarUse { +// return &MkVarUse{varname, modifiers} +//} + type MkVarUseModifier struct { Text string } @@ -113,15 +122,16 @@ func (m MkVarUseModifier) MatchMatch() (ok bool, positive bool, pattern string) func (m MkVarUseModifier) IsToLower() bool { return m.Text == "tl" } func (vu *MkVarUse) Mod() string { - mod := "" + var mod strings.Builder for _, modifier := range vu.modifiers { - mod += ":" + modifier.Text + mod.WriteString(":") + mod.WriteString(modifier.Text) } - return mod + return mod.String() } // IsExpression returns whether the varname is interpreted as a variable -// name (the usual case) or as a full expression (rare, only the modifiers +// name (the usual case) or as an expression (rare, only the modifiers // "?:" and "L" do this). func (vu *MkVarUse) IsExpression() bool { if len(vu.modifiers) == 0 { diff --git a/pkgtools/pkglint/files/mktypes_test.go b/pkgtools/pkglint/files/mktypes_test.go index 800b898a31e..25c680c9454 100644 --- a/pkgtools/pkglint/files/mktypes_test.go +++ b/pkgtools/pkglint/files/mktypes_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" @@ -18,8 +18,12 @@ func (s *Suite) Test_MkVarUse_Mod(c *check.C) { c.Check(varuse.Mod(), equals, ":Q") } +// AddCommand adds a command directly to a list of commands, +// creating all the intermediate nodes for the syntactic representation. +// As soon as that representation is replaced with a semantic representation, +// this method should no longer be necessary. func (list *MkShList) AddCommand(command *MkShCommand) *MkShList { - pipeline := NewMkShPipeline(false, command) + pipeline := NewMkShPipeline(false, []*MkShCommand{command}) andOr := NewMkShAndOr(pipeline) return list.AddAndOr(andOr) } @@ -47,3 +51,6 @@ func (s *Suite) Test_MkVarUseModifier_MatchSubst__backslash(c *check.C) { c.Check(to, equals, "\\:") c.Check(options, equals, "") } + +// TODO: Add test for :L in the middle of a MkVarUse. +// TODO: Add test for :L at the end of a MkVarUse. diff --git a/pkgtools/pkglint/files/options.go b/pkgtools/pkglint/files/options.go index fcc8f12dd32..d668bf48123 100755 --- a/pkgtools/pkglint/files/options.go +++ b/pkgtools/pkglint/files/options.go @@ -1,4 +1,4 @@ -package main +package pkglint func ChecklinesOptionsMk(mklines MkLines) { if trace.Tracing { @@ -15,8 +15,8 @@ func ChecklinesOptionsMk(mklines MkLines) { G.Explain( "The input variables in an options.mk file should always be", "mentioned in the same order: PKG_OPTIONS_VAR,", - "PKG_SUPPORTED_OPTIONS, PKG_SUGGESTED_OPTIONS. This way, the", - "options.mk files have the same structure and are easy to understand.") + "PKG_SUPPORTED_OPTIONS, PKG_SUGGESTED_OPTIONS.", + "This way, the options.mk files have the same structure and are easy to understand.") return } exp.Advance() @@ -91,8 +91,8 @@ loop: G.Explain( "For consistency among packages, the upper branch of this", ".if/.else statement should always handle the case where the", - "option is activated. A missing exclamation mark at this", - "point can easily be overlooked.") + "option is activated.", + "A missing exclamation mark at this point can easily be overlooked.") } } } @@ -100,17 +100,20 @@ loop: for _, option := range optionsInDeclarationOrder { declared := declaredOptions[option] handled := handledOptions[option] + if declared != nil && handled == nil { declared.Warnf("Option %q should be handled below in an .if block.", option) G.Explain( "If an option is not processed in this file, it may either be a", "typo, or the option does not have any effect.") } + if declared == nil && handled != nil { handled.Warnf("Option %q is handled but not added to PKG_SUPPORTED_OPTIONS.", option) G.Explain( "This block of code will never be run since PKG_OPTIONS cannot", - "contain this value. This is most probably a typo.") + "contain this value.", + "This is most probably a typo.") } } diff --git a/pkgtools/pkglint/files/options_test.go b/pkgtools/pkglint/files/options_test.go index 6f831613d05..d76b27e9dea 100755 --- a/pkgtools/pkglint/files/options_test.go +++ b/pkgtools/pkglint/files/options_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" @@ -43,6 +43,9 @@ func (s *Suite) Test_ChecklinesOptionsMk(c *check.C) { ".else", ".endif", "", + ".if empty(PKG_OPTIONS:Mnegative)", + ".endif", + "", ".if !empty(PKG_OPTIONS:Mncurses)", ".elif !empty(PKG_OPTIONS:Mslang)", ".endif", @@ -56,20 +59,24 @@ func (s *Suite) Test_ChecklinesOptionsMk(c *check.C) { t.CheckOutputLines( "WARN: ~/category/package/options.mk:6: l is used but not defined.", "WARN: ~/category/package/options.mk:18: Unknown option \"undeclared\".", - "NOTE: ~/category/package/options.mk:21: The positive branch of the .if/.else should be the one where the option is set.", - "WARN: ~/category/package/options.mk:6: Option \"mc-charset\" should be handled below in an .if block.", - "WARN: ~/category/package/options.mk:18: Option \"undeclared\" is handled but not added to PKG_SUPPORTED_OPTIONS.") + "NOTE: ~/category/package/options.mk:21: "+ + "The positive branch of the .if/.else should be the one where the option is set.", + // TODO: The diagnostics should appear in the correct order. + "WARN: ~/category/package/options.mk:6: "+ + "Option \"mc-charset\" should be handled below in an .if block.", + "WARN: ~/category/package/options.mk:18: "+ + "Option \"undeclared\" is handled but not added to PKG_SUPPORTED_OPTIONS.") } +// If there is no .include line after the declaration of the package-settable +// variables, the whole analysis stops. +// +// This case doesn't happen in practice and thus is not worth being handled in detail. func (s *Suite) Test_ChecklinesOptionsMk__unexpected_line(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wno-space") t.SetupVartypes() - t.SetupOption("mc-charset", "") - t.SetupOption("ncurses", "") - t.SetupOption("slang", "") - t.SetupOption("x11", "") t.CreateFileLines("mk/bsd.options.mk", MkRcsID) @@ -78,8 +85,6 @@ func (s *Suite) Test_ChecklinesOptionsMk__unexpected_line(c *check.C) { MkRcsID, "", "PKG_OPTIONS_VAR= PKG_OPTIONS.mc", - "PKG_SUPPORTED_OPTIONS= mc-charset x11 lang-${l}", - "PKG_SUGGESTED_OPTIONS= mc-charset", "", "pre-configure:", "\techo \"In the pre-configure stage.\"") @@ -87,7 +92,7 @@ func (s *Suite) Test_ChecklinesOptionsMk__unexpected_line(c *check.C) { ChecklinesOptionsMk(mklines) t.CheckOutputLines( - "WARN: ~/category/package/options.mk:7: Expected inclusion of \"../../mk/bsd.options.mk\".") + "WARN: ~/category/package/options.mk:5: Expected inclusion of \"../../mk/bsd.options.mk\".") } func (s *Suite) Test_ChecklinesOptionsMk__malformed_condition(c *check.C) { @@ -110,7 +115,7 @@ func (s *Suite) Test_ChecklinesOptionsMk__malformed_condition(c *check.C) { "PKG_SUPPORTED_OPTIONS= # none", "PKG_SUGGESTED_OPTIONS= # none", "", - "# Comment", + "# Comments and conditionals are allowed at this point.", ".if ${OPSYS} == NetBSD", ".endif", "", diff --git a/pkgtools/pkglint/files/package.go b/pkgtools/pkglint/files/package.go index 864fb02f772..2080784f762 100644 --- a/pkgtools/pkglint/files/package.go +++ b/pkgtools/pkglint/files/package.go @@ -1,16 +1,20 @@ -package main +package pkglint import ( - "fmt" "netbsd.org/pkglint/pkgver" "path" "strconv" "strings" ) +// TODO: What about package names that refer to other variables? const rePkgname = `^([\w\-.+]+)-(\d[.0-9A-Z_a-z]*)$` -// Package contains data for the pkgsrc package that is currently checked. +// Package is the pkgsrc package that is currently checked. +// +// Most of the information is loaded first, and after loading the actual checks take place. +// This is necessary because variables in Makefiles may be used before they are defined, +// and such dependencies often span multiple files that are included indirectly. type Package struct { dir string // The directory of the package, for resolving files Pkgpath string // e.g. "category/pkgdir" @@ -19,38 +23,45 @@ type Package struct { Patchdir string // PATCHDIR from the package Makefile DistinfoFile string // DISTINFO_FILE from the package Makefile EffectivePkgname string // PKGNAME or DISTNAME from the package Makefile, including nb13 - EffectivePkgbase string // The effective PKGNAME without the version + EffectivePkgbase string // EffectivePkgname without the version EffectivePkgversion string // The version part of the effective PKGNAME, excluding nb13 - EffectivePkgnameLine MkLine // The origin of the three effective_* values + EffectivePkgnameLine MkLine // The origin of the three Effective* values Plist PlistContent // Files and directories mentioned in the PLIST files - vars Scope - bl3 map[string]Line // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. - included map[string]Line // filename => line - seenMakefileCommon bool // Does the package have any .includes? - conditionalIncludes map[string]MkLine + vars Scope + bl3 map[string]MkLine // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. + included map[string]MkLine // filename => line + seenMakefileCommon bool // Does the package have any .includes? + + // Files from .include lines that are nested inside .if. + // They often depend on OPSYS or on the existence of files in the build environment. + conditionalIncludes map[string]MkLine + // Files from .include lines that are not nested. + // These are cross-checked with buildlink3.mk whether they are unconditional there, too. unconditionalIncludes map[string]MkLine - once Once - IgnoreMissingPatches bool // In distinfo, don't warn about patches that cannot be found. + + once Once + IgnoreMissingPatches bool // In distinfo, don't warn about patches that cannot be found. } func NewPackage(dir string) *Package { pkgpath := G.Pkgsrc.ToRel(dir) if strings.Count(pkgpath, "/") != 1 { - panic(fmt.Sprintf("Package directory %q must be two subdirectories below the pkgsrc root %q.", dir, G.Pkgsrc.File("."))) + G.Assertf(false, "Package directory %q must be two subdirectories below the pkgsrc root %q.", + dir, G.Pkgsrc.File(".")) } pkg := Package{ dir: dir, Pkgpath: pkgpath, Pkgdir: ".", - Filesdir: "files", - Patchdir: "patches", - DistinfoFile: "${PKGDIR}/distinfo", + Filesdir: "files", // TODO: Redundant, see the vars.Fallback below. + Patchdir: "patches", // TODO: Redundant, see the vars.Fallback below. + DistinfoFile: "${PKGDIR}/distinfo", // TODO: Redundant, see the vars.Fallback below. Plist: NewPlistContent(), vars: NewScope(), - bl3: make(map[string]Line), - included: make(map[string]Line), + bl3: make(map[string]MkLine), + included: make(map[string]MkLine), conditionalIncludes: make(map[string]MkLine), unconditionalIncludes: make(map[string]MkLine), } @@ -97,16 +108,22 @@ func (pkg *Package) checkPossibleDowngrade() { if change.Action == "Updated" { changeVersion := replaceAll(change.Version, `nb\d+$`, "") if pkgver.Compare(pkgversion, changeVersion) < 0 { - mkline.Warnf("The package is being downgraded from %s (see %s) to %s.", change.Version, mkline.Line.RefTo(change.Line), pkgversion) + mkline.Warnf("The package is being downgraded from %s (see %s) to %s.", + change.Version, mkline.Line.RefTo(change.Line), pkgversion) G.Explain( "The files in doc/CHANGES-*, in which all version changes are", "recorded, have a higher version number than what the package says.", "This is unusual, since packages are typically upgraded instead of", "downgraded.") + + // TODO: Check whether the current version is mentioned in doc/CHANGES. } } } +// checklinesBuildlink3Inclusion checks whether the package Makefile and +// the corresponding buildlink3.mk agree for all included buildlink3.mk +// files whether they are included conditionally or unconditionally. func (pkg *Package) checklinesBuildlink3Inclusion(mklines MkLines) { if trace.Tracing { defer trace.Call0()() @@ -148,6 +165,10 @@ func (pkg *Package) loadPackageMakefile() MkLines { return nil } + // TODO: Is this still necessary? This code is 20 years old and was introduced + // when pkglint loaded the package Makefile including all included files into + // a single string. Maybe it makes sense to print the file inclusion hierarchy + // to quickly see files that cannot be included because of unresolved variables. if G.Opts.DumpMakefile { G.out.WriteLine("Whole Makefile (with all included files) follows:") for _, line := range allLines.lines.Lines { @@ -155,6 +176,7 @@ func (pkg *Package) loadPackageMakefile() MkLines { } } + // See mk/tools/cmake.mk if pkg.vars.Defined("USE_CMAKE") { mainLines.Tools.def("cmake", "", false, AtRunTime) mainLines.Tools.def("cpack", "", false, AtRunTime) @@ -193,12 +215,13 @@ func (pkg *Package) loadPackageMakefile() MkLines { return mainLines } -func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines MkLines, includingFnameForUsedCheck string) (exists bool, result bool) { +// TODO: What is allLines used for, is it still necessary? Would it be better as a field in Package? +func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines MkLines, includingFileForUsedCheck string) (exists bool, result bool) { if trace.Tracing { defer trace.Call1(filename)() } - fileMklines := LoadMk(filename, NotEmpty) + fileMklines := LoadMk(filename, NotEmpty) // TODO: Document why omitting LogErrors is correct here. if fileMklines == nil { return false, false } @@ -215,43 +238,18 @@ func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines Mk allLines.mklines = append(allLines.mklines, mkline) allLines.lines.Lines = append(allLines.lines.Lines, mkline.Line) - var includedFile, incDir, incBase string - if mkline.IsInclude() { - includedFile = resolveVariableRefs(mkline.ResolveVarsInRelativePath(mkline.IncludedFile())) - if containsVarRef(includedFile) { - if !contains(filename, "/mk/") { - mkline.Notef("Skipping include file %q. This may result in false warnings.", includedFile) - } - includedFile = "" - } - incDir, incBase = path.Split(includedFile) - } - - if includedFile != "" { - if mkline.Basename != "buildlink3.mk" { - if m, bl3File := match1(includedFile, `^\.\./\.\./(.*)/buildlink3\.mk$`); m { - pkg.bl3[bl3File] = mkline.Line - if trace.Tracing { - trace.Step1("Buildlink3 file in package: %q", bl3File) - } - } - } - } + includedFile, incDir, incBase := pkg.findIncludedFile(mkline, filename) if includedFile != "" && pkg.included[includedFile] == nil { - pkg.included[includedFile] = mkline.Line + pkg.included[includedFile] = mkline + // TODO: "../../../.." also matches but shouldn't. if matches(includedFile, `^\.\./[^./][^/]*/[^/]+`) { mkline.Warnf("References to other packages should look like \"../../category/package\", not \"../package\".") mkline.ExplainRelativeDirs() } - if mkline.Basename == "Makefile" && !hasPrefix(incDir, "../../mk/") && incBase != "buildlink3.mk" && incBase != "builtin.mk" && incBase != "options.mk" { - if trace.Tracing { - trace.Step1("Including %q sets seenMakefileCommon.", includedFile) - } - pkg.seenMakefileCommon = true - } + pkg.collectUsedBy(mkline, incDir, incBase, includedFile) skip := contains(filename, "/mk/") || hasSuffix(includedFile, "/bsd.pkg.mk") || IsPrefs(includedFile) if !skip { @@ -310,12 +308,15 @@ func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines Mk atEnd := func(mkline MkLine) {} fileMklines.ForEachEnd(lineAction, atEnd) - if includingFnameForUsedCheck != "" { - fileMklines.CheckForUsedComment(G.Pkgsrc.ToRel(includingFnameForUsedCheck)) + if includingFileForUsedCheck != "" { + fileMklines.CheckForUsedComment(G.Pkgsrc.ToRel(includingFileForUsedCheck)) } // For every included buildlink3.mk, include the corresponding builtin.mk // automatically since the pkgsrc infrastructure does the same. + // + // Disabled for now since it increases the running time by about 20% + // and produces many new warnings, which must be evaluated first. if false && path.Base(filename) == "buildlink3.mk" { builtin := path.Join(path.Dir(filename), "builtin.mk") if fileExists(builtin) { @@ -326,6 +327,53 @@ func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines Mk return } +func (pkg *Package) collectUsedBy(mkline MkLine, incDir string, incBase string, includedFile string) { + switch { + case + mkline.Basename != "Makefile", + hasPrefix(incDir, "../../mk/"), + incBase == "buildlink3.mk", + incBase == "builtin.mk", + incBase == "options.mk": + return + } + + if trace.Tracing { + trace.Step1("Including %q sets seenMakefileCommon.", includedFile) + } + pkg.seenMakefileCommon = true +} + +func (pkg *Package) findIncludedFile(mkline MkLine, includingFilename string) (includedFile, incDir, incBase string) { + + if mkline.IsInclude() { + // TODO: resolveVariableRefs uses G.Pkg implicitly. It should be made explicit. + // TODO: Try to combine resolveVariableRefs and ResolveVarsInRelativePath. + includedFile = resolveVariableRefs(mkline.ResolveVarsInRelativePath(mkline.IncludedFile())) + if containsVarRef(includedFile) { + if trace.Tracing && !contains(includingFilename, "/mk/") { + trace.Stepf("%s:%s: Skipping include file %q. This may result in false warnings.", + mkline.Filename, mkline.Linenos(), includedFile) + } + includedFile = "" + } + incDir, incBase = path.Split(includedFile) + } + + if includedFile != "" { + if mkline.Basename != "buildlink3.mk" { + if m, bl3File := match1(includedFile, `^\.\./\.\./(.*)/buildlink3\.mk$`); m { + pkg.bl3[bl3File] = mkline + if trace.Tracing { + trace.Step1("Buildlink3 file in package: %q", bl3File) + } + } + } + } + + return +} + func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines) { if trace.Tracing { defer trace.Call1(filename)() @@ -337,10 +385,13 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines) { !vars.Defined("META_PACKAGE") && !fileExists(pkg.File(pkg.Pkgdir+"/PLIST")) && !fileExists(pkg.File(pkg.Pkgdir+"/PLIST.common")) { + // TODO: Move these technical details into the explanation, making space for an understandable warning. NewLineWhole(filename).Warnf("Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset.") } - if (vars.Defined("NO_CHECKSUM") || vars.Defined("META_PACKAGE")) && isEmptyDir(pkg.File(pkg.Patchdir)) { + if (vars.Defined("NO_CHECKSUM") || + vars.Defined("META_PACKAGE")) && isEmptyDir(pkg.File(pkg.Patchdir)) { + if distinfoFile := pkg.File(pkg.DistinfoFile); fileExists(distinfoFile) { NewLineWhole(distinfoFile).Warnf("This file should not exist if NO_CHECKSUM or META_PACKAGE is set.") } @@ -351,12 +402,17 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines) { } } - if perlLine, noconfLine := vars.FirstDefinition("REPLACE_PERL"), vars.FirstDefinition("NO_CONFIGURE"); perlLine != nil && noconfLine != nil { - perlLine.Warnf("REPLACE_PERL is ignored when NO_CONFIGURE is set (in %s).", perlLine.RefTo(noconfLine)) + // TODO: There are other REPLACE_* variables which are probably also affected by NO_CONFIGURE. + if noConfigureLine := vars.FirstDefinition("NO_CONFIGURE"); noConfigureLine != nil { + if replacePerlLine := vars.FirstDefinition("REPLACE_PERL"); replacePerlLine != nil { + replacePerlLine.Warnf("REPLACE_PERL is ignored when NO_CONFIGURE is set (in %s).", + replacePerlLine.RefTo(noConfigureLine)) + } } if !vars.Defined("LICENSE") && !vars.Defined("META_PACKAGE") && pkg.once.FirstTime("LICENSE") { NewLineWhole(filename).Errorf("Each package must define its LICENSE.") + // TODO: Explain why the LICENSE is necessary. } pkg.checkGnuConfigureUseLanguages() @@ -364,12 +420,14 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines) { pkg.checkPossibleDowngrade() if !vars.Defined("COMMENT") { - NewLineWhole(filename).Warnf("No COMMENT given.") + NewLineWhole(filename).Warnf("Each package should define a COMMENT.") } - if imake, x11 := vars.FirstDefinition("USE_IMAKE"), vars.FirstDefinition("USE_X11"); imake != nil && x11 != nil { - if !hasSuffix(x11.Filename, "/mk/x11.buildlink3.mk") { - imake.Notef("USE_IMAKE makes USE_X11 in %s superfluous.", imake.RefTo(x11)) + if imake := vars.FirstDefinition("USE_IMAKE"); imake != nil { + if x11 := vars.FirstDefinition("USE_X11"); x11 != nil { + if !hasSuffix(x11.Filename, "/mk/x11.buildlink3.mk") { + imake.Notef("USE_IMAKE makes USE_X11 in %s redundant.", imake.RefTo(x11)) + } } } @@ -382,20 +440,29 @@ func (pkg *Package) checkfilePackageMakefile(filename string, mklines MkLines) { func (pkg *Package) checkGnuConfigureUseLanguages() { vars := pkg.vars - if gnuLine, useLine := vars.FirstDefinition("GNU_CONFIGURE"), vars.FirstDefinition("USE_LANGUAGES"); gnuLine != nil && useLine != nil { - if matches(useLine.VarassignComment(), `(?-i)\b(?:c|empty|none)\b`) { - // Don't emit a warning, since the comment - // probably contains a statement that C is - // really not needed. + if gnuLine := vars.FirstDefinition("GNU_CONFIGURE"); gnuLine != nil { + if useLine := vars.FirstDefinition("USE_LANGUAGES"); useLine != nil { + + if matches(useLine.VarassignComment(), `(?-i)\b(?:c|empty|none)\b`) { + // Don't emit a warning since the comment probably contains a + // statement that C is really not needed. - } else if !matches(useLine.Value(), `(?:^|[\t ]+)(?:c|c99|objc)(?:[\t ]+|$)`) { - gnuLine.Warnf("GNU_CONFIGURE almost always needs a C compiler, but \"c\" is not added to USE_LANGUAGES in %s.", - gnuLine.RefTo(useLine)) + } else if !matches(useLine.Value(), `(?:^|[\t ]+)(?:c|c99|objc)(?:[\t ]+|$)`) { + gnuLine.Warnf( + "GNU_CONFIGURE almost always needs a C compiler, "+ + "but \"c\" is not added to USE_LANGUAGES in %s.", + gnuLine.RefTo(useLine)) + } } } } -func (pkg *Package) getNbpart() string { +// nbPart determines the smallest part of the package version number, +// typically "nb13" or an empty string. +// +// It is only used inside pkgsrc to mark changes that are +// independent from the upstream package. +func (pkg *Package) nbPart() string { pkgrevision, _ := pkg.vars.Value("PKGREVISION") if rev, err := strconv.Atoi(pkgrevision); err == nil { return "nb" + strconv.Itoa(rev) @@ -421,7 +488,7 @@ func (pkg *Package) determineEffectivePkgVars() { } if pkgname != "" && pkgname == distname && pkgnameLine.VarassignComment() == "" { - pkgnameLine.Notef("PKGNAME is ${DISTNAME} by default. You probably don't need to define PKGNAME.") + pkgnameLine.Notef("This assignment is probably redundant since PKGNAME is ${DISTNAME} by default.") } if pkgname == "" && distname != "" && !containsVarRef(distname) && !matches(distname, rePkgname) { @@ -430,20 +497,22 @@ func (pkg *Package) determineEffectivePkgVars() { if pkgname != "" && !containsVarRef(pkgname) { if m, m1, m2 := match2(pkgname, rePkgname); m { - pkg.EffectivePkgname = pkgname + pkg.getNbpart() + pkg.EffectivePkgname = pkgname + pkg.nbPart() pkg.EffectivePkgnameLine = pkgnameLine pkg.EffectivePkgbase = m1 pkg.EffectivePkgversion = m2 } } + if pkg.EffectivePkgnameLine == nil && distname != "" && !containsVarRef(distname) { if m, m1, m2 := match2(distname, rePkgname); m { - pkg.EffectivePkgname = distname + pkg.getNbpart() + pkg.EffectivePkgname = distname + pkg.nbPart() pkg.EffectivePkgnameLine = distnameLine pkg.EffectivePkgbase = m1 pkg.EffectivePkgversion = m2 } } + if pkg.EffectivePkgnameLine != nil { if trace.Tracing { trace.Stepf("Effective name=%q base=%q version=%q", @@ -455,6 +524,8 @@ func (pkg *Package) determineEffectivePkgVars() { func (pkg *Package) pkgnameFromDistname(pkgname, distname string) string { tokens := NewMkParser(nil, pkgname, false).MkTokens() + // TODO: Make this resolving of variable references available to all other variables as well. + result := "" for _, token := range tokens { if token.Varuse != nil && token.Varuse.varname == "DISTNAME" { @@ -478,30 +549,38 @@ func (pkg *Package) pkgnameFromDistname(pkgname, distname string) string { } func (pkg *Package) checkUpdate() { - if pkg.EffectivePkgbase != "" { - for _, sugg := range G.Pkgsrc.GetSuggestedPackageUpdates() { - if pkg.EffectivePkgbase != sugg.Pkgname { - continue - } + if pkg.EffectivePkgbase == "" { + return + } - suggver, comment := sugg.Version, sugg.Comment - if comment != "" { - comment = " (" + comment + ")" - } + for _, sugg := range G.Pkgsrc.GetSuggestedPackageUpdates() { + if pkg.EffectivePkgbase != sugg.Pkgname { + continue + } - pkgnameLine := pkg.EffectivePkgnameLine - cmp := pkgver.Compare(pkg.EffectivePkgversion, suggver) - switch { - case cmp < 0: - pkgnameLine.Warnf("This package should be updated to %s%s.", sugg.Version, comment) - G.Explain( - "The wishlist for package updates in doc/TODO mentions that a newer", - "version of this package is available.") - case cmp > 0: - pkgnameLine.Notef("This package is newer than the update request to %s%s.", suggver, comment) - default: - pkgnameLine.Notef("The update request to %s from doc/TODO%s has been done.", suggver, comment) - } + suggver, comment := sugg.Version, sugg.Comment + if comment != "" { + comment = " (" + comment + ")" + } + + pkgnameLine := pkg.EffectivePkgnameLine + cmp := pkgver.Compare(pkg.EffectivePkgversion, suggver) + switch { + + case cmp < 0: + pkgnameLine.Warnf("This package should be updated to %s%s.", + sugg.Version, comment) + G.Explain( + "The wishlist for package updates in doc/TODO mentions that a newer", + "version of this package is available.") + + case cmp > 0: + pkgnameLine.Notef("This package is newer than the update request to %s%s.", + suggver, comment) + + default: + pkgnameLine.Notef("The update request to %s from doc/TODO%s has been done.", + suggver, comment) } } } @@ -525,17 +604,22 @@ func (pkg *Package) CheckVarorder(mklines MkLines) { once many ) + type Variable struct { varname string repetition Repetition } + type Section struct { repetition Repetition vars []Variable } + variable := func(name string, repetition Repetition) Variable { return Variable{name, repetition} } section := func(repetition Repetition, vars ...Variable) Section { return Section{repetition, vars} } + // See doc/Makefile-example. + // See https://netbsd.org/docs/pkgsrc/pkgsrc.html#components.Makefile. var sections = []Section{ section(once, variable("GITHUB_PROJECT", optional), // either here or below MASTER_SITES @@ -583,11 +667,13 @@ func (pkg *Package) CheckVarorder(mklines MkLines) { section(optional, variable("BUILD_DEPENDS", many), variable("TOOL_DEPENDS", many), - variable("DEPENDS", many)), - } + variable("DEPENDS", many))} firstRelevant := -1 lastRelevant := -1 + + // TODO: understand and explain this code. + // It is much longer and much more complicated than it should be. skip := func() bool { relevantVars := make(map[string]bool) for _, section := range sections { @@ -705,6 +791,9 @@ func (pkg *Package) CheckVarorder(mklines MkLines) { canonical = canonical[:len(canonical)-1] } + // TODO: This leads to very long and complicated warnings. + // Those parts that are correct should not be mentioned, + // except if they are helpful for locating the mistakes. mkline := mklines.mklines[firstRelevant] mkline.Warnf("The canonical order of the variables is %s.", strings.Join(canonical, ", ")) G.Explain( @@ -715,6 +804,12 @@ func (pkg *Package) CheckVarorder(mklines MkLines) { seeGuide("Package components, Makefile", "components.Makefile")) } +// checkLocallyModified checks files that are about to be committed. +// Depending on whether the package has a MAINTAINER or an OWNER, +// the wording differs. +// +// Pkglint assumes that the local username is the same as the NetBSD +// username, which fits most scenarios. func (pkg *Package) checkLocallyModified(filename string) { if trace.Tracing { defer trace.Call(filename)() @@ -729,7 +824,7 @@ func (pkg *Package) checkLocallyModified(filename string) { return } - username := G.CurrentUsername + username := G.Username if trace.Tracing { trace.Stepf("user=%q owner=%q maintainer=%q", username, owner, maintainer) } @@ -738,18 +833,21 @@ func (pkg *Package) checkLocallyModified(filename string) { return } - if isLocallyModified(filename) { - if owner != "" { - NewLineWhole(filename).Warnf("Don't commit changes to this file without asking the OWNER, %s.", owner) - G.Explain( - seeGuide("Package components, Makefile", "components.Makefile")) - } - if maintainer != "" { - NewLineWhole(filename).Notef("Please only commit changes that %s would approve.", maintainer) - G.Explain( - "See the pkgsrc guide, section \"Package components\",", - "keyword \"maintainer\", for more information.") - } + if !isLocallyModified(filename) { + return + } + + if owner != "" { + NewLineWhole(filename).Warnf("Don't commit changes to this file without asking the OWNER, %s.", owner) + G.Explain( + seeGuide("Package components, Makefile", "components.Makefile")) + } + + if maintainer != "" { + NewLineWhole(filename).Notef("Please only commit changes that %s would approve.", maintainer) + G.Explain( + "See the pkgsrc guide, section \"Package components\",", + "keyword \"maintainer\", for more information.") } } @@ -766,13 +864,18 @@ func (pkg *Package) checkIncludeConditionally(mkline MkLine, indentation *Indent if indentation.IsConditional() { pkg.conditionalIncludes[includedFile] = mkline if other := pkg.unconditionalIncludes[includedFile]; other != nil { - mkline.Warnf("%q is included conditionally here (depending on %s) and unconditionally in %s.", + mkline.Warnf( + "%q is included conditionally here (depending on %s) "+ + "and unconditionally in %s.", cleanpath(includedFile), strings.Join(mkline.ConditionalVars(), ", "), mkline.RefTo(other)) } + } else { pkg.unconditionalIncludes[includedFile] = mkline if other := pkg.conditionalIncludes[includedFile]; other != nil { - mkline.Warnf("%q is included unconditionally here and conditionally in %s (depending on %s).", + mkline.Warnf( + "%q is included unconditionally here "+ + "and conditionally in %s (depending on %s).", cleanpath(includedFile), mkline.RefTo(other), strings.Join(other.ConditionalVars(), ", ")) } } diff --git a/pkgtools/pkglint/files/package_test.go b/pkgtools/pkglint/files/package_test.go index e1702ca3f93..306e0102ed0 100644 --- a/pkgtools/pkglint/files/package_test.go +++ b/pkgtools/pkglint/files/package_test.go @@ -1,6 +1,9 @@ -package main +package pkglint -import "gopkg.in/check.v1" +import ( + "gopkg.in/check.v1" + "strings" +) func (s *Suite) Test_Package_checklinesBuildlink3Inclusion__file_but_not_package(c *check.C) { t := s.Init(c) @@ -15,7 +18,8 @@ func (s *Suite) Test_Package_checklinesBuildlink3Inclusion__file_but_not_package G.Pkg.checklinesBuildlink3Inclusion(mklines) t.CheckOutputLines( - "WARN: category/package/buildlink3.mk:3: category/dependency/buildlink3.mk is included by this file but not by the package.") + "WARN: category/package/buildlink3.mk:3: " + + "category/dependency/buildlink3.mk is included by this file but not by the package.") } func (s *Suite) Test_Package_checklinesBuildlink3Inclusion__package_but_not_file(c *check.C) { @@ -23,16 +27,20 @@ func (s *Suite) Test_Package_checklinesBuildlink3Inclusion__package_but_not_file t.CreateFileLines("category/dependency/buildlink3.mk") G.Pkg = NewPackage(t.File("category/package")) - G.Pkg.bl3["../../category/dependency/buildlink3.mk"] = t.NewLine("filename", 1, "") + G.Pkg.bl3["../../category/dependency/buildlink3.mk"] = t.NewMkLine("filename", 1, "") mklines := t.NewMkLines("category/package/buildlink3.mk", MkRcsID) t.EnableTracingToLog() G.Pkg.checklinesBuildlink3Inclusion(mklines) + // This is only traced but not logged as a regular warning since + // several packages have build dependencies that are not needed + // for building other packages. These cannot be flagged as warnings. t.CheckOutputLines( "TRACE: + (*Package).checklinesBuildlink3Inclusion()", - "TRACE: 1 ../../category/dependency/buildlink3.mk/buildlink3.mk is included by the package but not by the buildlink3.mk file.", + "TRACE: 1 ../../category/dependency/buildlink3.mk/buildlink3.mk "+ + "is included by the package but not by the buildlink3.mk file.", "TRACE: - (*Package).checklinesBuildlink3Inclusion()") } @@ -42,20 +50,22 @@ func (s *Suite) Test_Package_pkgnameFromDistname(c *check.C) { pkg := NewPackage(t.File("category/package")) pkg.vars.Define("PKGNAME", t.NewMkLine("Makefile", 5, "PKGNAME=dummy")) - c.Check(pkg.pkgnameFromDistname("pkgname-1.0", "whatever"), equals, "pkgname-1.0") - c.Check(pkg.pkgnameFromDistname("${DISTNAME}", "distname-1.0"), equals, "distname-1.0") - c.Check(pkg.pkgnameFromDistname("${DISTNAME:S/dist/pkg/}", "distname-1.0"), equals, "pkgname-1.0") - c.Check(pkg.pkgnameFromDistname("${DISTNAME:S|a|b|g}", "panama-0.13"), equals, "pbnbmb-0.13") - c.Check(pkg.pkgnameFromDistname("${DISTNAME:S|^lib||}", "libncurses"), equals, "ncurses") - c.Check(pkg.pkgnameFromDistname("${DISTNAME:S|^lib||}", "mylib"), equals, "mylib") - c.Check(pkg.pkgnameFromDistname("${DISTNAME:tl:S/-/./g:S/he/-/1}", "SaxonHE9-5-0-1J"), equals, "saxon-9.5.0.1j") - c.Check(pkg.pkgnameFromDistname("${DISTNAME:C/beta/.0./}", "fspanel-0.8beta1"), equals, "${DISTNAME:C/beta/.0./}") - c.Check(pkg.pkgnameFromDistname("${DISTNAME:S/-0$/.0/1}", "aspell-af-0.50-0"), equals, "aspell-af-0.50.0") + test := func(pkgname, distname, expectedPkgname string) { + c.Check(pkg.pkgnameFromDistname(pkgname, distname), equals, expectedPkgname) + } - // FIXME: Should produce a parse error since the :S modifier is malformed; see Test_MkParser_MkTokens. - c.Check(pkg.pkgnameFromDistname("${DISTNAME:S,a,b,c,d}", "aspell-af-0.50-0"), equals, "bspell-af-0.50-0") + test("pkgname-1.0", "whatever", "pkgname-1.0") + test("${DISTNAME}", "distname-1.0", "distname-1.0") + test("${DISTNAME:S/dist/pkg/}", "distname-1.0", "pkgname-1.0") + test("${DISTNAME:S|a|b|g}", "panama-0.13", "pbnbmb-0.13") + test("${DISTNAME:S|^lib||}", "libncurses", "ncurses") + test("${DISTNAME:S|^lib||}", "mylib", "mylib") + test("${DISTNAME:tl:S/-/./g:S/he/-/1}", "SaxonHE9-5-0-1J", "saxon-9.5.0.1j") + test("${DISTNAME:C/beta/.0./}", "fspanel-0.8beta1", "${DISTNAME:C/beta/.0./}") + test("${DISTNAME:S/-0$/.0/1}", "aspell-af-0.50-0", "aspell-af-0.50.0") - t.CheckOutputEmpty() + // FIXME: Should produce a parse error since the :S modifier is malformed; see Test_MkParser_MkTokens. + test("${DISTNAME:S,a,b,c,d}", "aspell-af-0.50-0", "bspell-af-0.50-0") } func (s *Suite) Test_Package_CheckVarorder(c *check.C) { @@ -71,9 +81,11 @@ func (s *Suite) Test_Package_CheckVarorder(c *check.C) { "DISTNAME=9term", "CATEGORIES=x11")) + // TODO: Make this warning more specific to the actual situation. t.CheckOutputLines( "WARN: Makefile:3: The canonical order of the variables is " + - "GITHUB_PROJECT, DISTNAME, CATEGORIES, GITHUB_PROJECT, empty line, COMMENT, LICENSE.") + "GITHUB_PROJECT, DISTNAME, CATEGORIES, GITHUB_PROJECT, empty line, " + + "COMMENT, LICENSE.") pkg.CheckVarorder(t.NewMkLines("Makefile", MkRcsID, @@ -89,6 +101,7 @@ func (s *Suite) Test_Package_CheckVarorder(c *check.C) { } // Ensure that comments and empty lines do not lead to panics. +// This would be when accessing fields from the MkLine without checking the line type before. func (s *Suite) Test_Package_CheckVarorder__comments_do_not_crash(c *check.C) { t := s.Init(c) @@ -108,7 +121,8 @@ func (s *Suite) Test_Package_CheckVarorder__comments_do_not_crash(c *check.C) { t.CheckOutputLines( "WARN: Makefile:3: The canonical order of the variables is " + - "GITHUB_PROJECT, DISTNAME, CATEGORIES, GITHUB_PROJECT, empty line, COMMENT, LICENSE.") + "GITHUB_PROJECT, DISTNAME, CATEGORIES, GITHUB_PROJECT, empty line, " + + "COMMENT, LICENSE.") } func (s *Suite) Test_Package_CheckVarorder__comments_are_ignored(c *check.C) { @@ -150,12 +164,35 @@ func (s *Suite) Test_Package_CheckVarorder__skip_if_there_are_directives(c *chec ".endif", "LICENSE=\tgnu-gpl-v2")) - // No warning about the missing COMMENT since the directive + // No warning about the missing COMMENT since the .if directive // causes the whole check to be skipped. t.CheckOutputEmpty() } -func (s *Suite) Test_Package_CheckVarorder__GitHub(c *check.C) { +// TODO: Add more tests like skip_if_there_are_directives for other line types. + +func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_top(c *check.C) { + t := s.Init(c) + + t.SetupCommandLine("-Worder") + pkg := NewPackage(t.File("x11/9term")) + + pkg.CheckVarorder(t.NewMkLines("Makefile", + MkRcsID, + "", + "GITHUB_PROJECT=\t\tautocutsel", + "DISTNAME=\t\tautocutsel-0.10.0", + "CATEGORIES=\t\tx11", + "MASTER_SITES=\t\t${MASTER_SITE_GITHUB:=sigmike/}", + "GITHUB_TAG=\t\t${PKGVERSION_NOREV}", + "", + "COMMENT=\tComment", + "LICENSE=\tgnu-gpl-v2")) + + t.CheckOutputEmpty() +} + +func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_bottom(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Worder") @@ -226,8 +263,6 @@ func (s *Suite) Test_Package_CheckVarorder__MASTER_SITES(c *check.C) { t.CheckOutputEmpty() } -// The diagnostics must be helpful. -// In the case of wip/ioping, they were ambiguous and wrong. func (s *Suite) Test_Package_CheckVarorder__diagnostics(c *check.C) { t := s.Init(c) @@ -256,7 +291,8 @@ func (s *Suite) Test_Package_CheckVarorder__diagnostics(c *check.C) { t.CheckOutputLines( "WARN: Makefile:3: The canonical order of the variables is " + - "GITHUB_PROJECT, DISTNAME, PKGNAME, CATEGORIES, MASTER_SITES, GITHUB_PROJECT, DIST_SUBDIR, empty line, " + + "GITHUB_PROJECT, DISTNAME, PKGNAME, CATEGORIES, " + + "MASTER_SITES, GITHUB_PROJECT, DIST_SUBDIR, empty line, " + "MAINTAINER, HOMEPAGE, COMMENT, LICENSE.") // After moving the variables according to the warning: @@ -280,20 +316,21 @@ func (s *Suite) Test_Package_CheckVarorder__diagnostics(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_Package_getNbpart(c *check.C) { +func (s *Suite) Test_Package_nbPart(c *check.C) { t := s.Init(c) pkg := NewPackage(t.File("category/pkgbase")) pkg.vars.Define("PKGREVISION", t.NewMkLine("Makefile", 1, "PKGREVISION=14")) - c.Check(pkg.getNbpart(), equals, "nb14") + c.Check(pkg.nbPart(), equals, "nb14") pkg.vars = NewScope() pkg.vars.Define("PKGREVISION", t.NewMkLine("Makefile", 1, "PKGREVISION=asdf")) - c.Check(pkg.getNbpart(), equals, "") + c.Check(pkg.nbPart(), equals, "") } +// PKGNAME is stronger than DISTNAME. func (s *Suite) Test_Package_determineEffectivePkgVars__precedence(c *check.C) { t := s.Init(c) @@ -325,7 +362,7 @@ func (s *Suite) Test_Package_determineEffectivePkgVars__same(c *check.C) { t.CheckOutputLines( "NOTE: ~/category/package/Makefile:20: " + - "PKGNAME is ${DISTNAME} by default. You probably don't need to define PKGNAME.") + "This assignment is probably redundant since PKGNAME is ${DISTNAME} by default.") } func (s *Suite) Test_Package_determineEffectivePkgVars__invalid_DISTNAME(c *check.C) { @@ -390,6 +427,8 @@ func (s *Suite) Test_Package_loadPackageMakefile__dump(c *check.C) { "", "COMMENT=\tComment", "LICENSE=\t2-clause-bsd") + // TODO: There is no .include line at the end of the Makefile. + // This should always be checked though. G.checkdirPackage(t.File("category/package")) @@ -442,7 +481,7 @@ func (s *Suite) Test_Package__varuse_at_load_time(c *check.C) { "", ".include \"../../mk/bsd.prefs.mk\"", // - // Now all tools from USE_TOOLS are defined with their variables. + // At this point, all tools from USE_TOOLS are defined with their variables. // ${FALSE} works, but a plain "false" might call the wrong tool. // That's because the tool wrappers are not set up yet. This // happens between the post-depends and pre-fetch stages. Even @@ -477,6 +516,7 @@ func (s *Suite) Test_Package__varuse_at_load_time(c *check.C) { t.CheckOutputLines( "WARN: ~/category/pkgbase/Makefile:14: To use the tool ${FALSE} at load time, bsd.prefs.mk has to be included before.", + // TODO: "before including bsd.prefs.mk in line ###". "WARN: ~/category/pkgbase/Makefile:15: To use the tool ${NICE} at load time, it has to be added to USE_TOOLS before including bsd.prefs.mk.", "WARN: ~/category/pkgbase/Makefile:16: To use the tool ${TRUE} at load time, bsd.prefs.mk has to be included before.", "WARN: ~/category/pkgbase/Makefile:25: To use the tool ${NICE} at load time, it has to be added to USE_TOOLS before including bsd.prefs.mk.") @@ -535,44 +575,33 @@ func (s *Suite) Test_Package_checkIncludeConditionally__conditional_and_uncondit t := s.Init(c) t.SetupVartypes() - t.CreateFileLines("devel/zlib/buildlink3.mk", "") - t.CreateFileLines("licenses/gnu-gpl-v2", "") - t.CreateFileLines("mk/bsd.pkg.mk", "") - t.CreateFileLines("sysutils/coreutils/buildlink3.mk", "") - - t.Chdir("category/package") - t.CreateFileLines("Makefile", - MkRcsID, - "", - "COMMENT=\tDescription", - "LICENSE=\tgnu-gpl-v2", + t.SetupOption("zlib", "") + t.SetupPackage("category/package", ".include \"../../devel/zlib/buildlink3.mk\"", ".if ${OPSYS} == \"Linux\"", ".include \"../../sysutils/coreutils/buildlink3.mk\"", - ".endif", - ".include \"../../mk/bsd.pkg.mk\"") - t.CreateFileLines("options.mk", + ".endif") + t.CreateFileLines("devel/zlib/buildlink3.mk", "") + t.CreateFileLines("sysutils/coreutils/buildlink3.mk", "") + + t.CreateFileLines("category/package/options.mk", MkRcsID, "", ".if !empty(PKG_OPTIONS:Mzlib)", ". include \"../../devel/zlib/buildlink3.mk\"", ".endif", ".include \"../../sysutils/coreutils/buildlink3.mk\"") - t.CreateFileLines("PLIST", - PlistRcsID, - "bin/program") - t.CreateFileLines("distinfo", - RcsID) + t.Chdir("category/package") G.checkdirPackage(".") t.CheckOutputLines( - "WARN: Makefile:3: The canonical order of the variables is CATEGORIES, empty line, COMMENT, LICENSE.", - "WARN: options.mk:3: Unknown option \"zlib\".", "WARN: options.mk:4: \"../../devel/zlib/buildlink3.mk\" is "+ - "included conditionally here (depending on PKG_OPTIONS) and unconditionally in Makefile:5.", + "included conditionally here (depending on PKG_OPTIONS) "+ + "and unconditionally in Makefile:20.", "WARN: options.mk:6: \"../../sysutils/coreutils/buildlink3.mk\" is "+ - "included unconditionally here and conditionally in Makefile:7 (depending on OPSYS).", + "included unconditionally here "+ + "and conditionally in Makefile:22 (depending on OPSYS).", "WARN: options.mk:3: Expected definition of PKG_OPTIONS_VAR.") } @@ -581,18 +610,13 @@ func (s *Suite) Test_Package__include_without_exists(c *check.C) { t := s.Init(c) t.SetupVartypes() - t.CreateFileLines("mk/bsd.pkg.mk") - t.CreateFileLines("category/package/Makefile", - MkRcsID, - "", - ".include \"options.mk\"", - "", - ".include \"../../mk/bsd.pkg.mk\"") + t.SetupPackage("category/package", + ".include \"options.mk\"") G.checkdirPackage(t.File("category/package")) t.CheckOutputLines( - "ERROR: ~/category/package/Makefile:3: Cannot read \"options.mk\".") + "ERROR: ~/category/package/Makefile:20: Cannot read \"options.mk\".") } // See https://github.com/rillig/pkglint/issues/1 @@ -600,25 +624,16 @@ func (s *Suite) Test_Package__include_after_exists(c *check.C) { t := s.Init(c) t.SetupVartypes() - t.CreateFileLines("mk/bsd.pkg.mk") - t.Chdir("category/package") - t.CreateFileLines("Makefile", - MkRcsID, - "", + t.SetupPackage("category/package", ".if exists(options.mk)", ". include \"options.mk\"", - ".endif", - "", - ".include \"../../mk/bsd.pkg.mk\"") + ".endif") - G.checkdirPackage(".") + G.checkdirPackage(t.File("category/package")) + // FIXME: This error message should not appear at all because of the .if exists before. t.CheckOutputLines( - "WARN: Makefile: Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset.", - "WARN: distinfo: File not found. Please run \""+confMake+" makesum\" or define NO_CHECKSUM=yes in the package Makefile.", - "ERROR: Makefile: Each package must define its LICENSE.", - "WARN: Makefile: No COMMENT given.", - "ERROR: Makefile:4: Relative path \"options.mk\" does not exist.") + "ERROR: ~/category/package/Makefile:21: Relative path \"options.mk\" does not exist.") } // See https://github.com/rillig/pkglint/issues/1 @@ -626,20 +641,15 @@ func (s *Suite) Test_Package_readMakefile__include_other_after_exists(c *check.C t := s.Init(c) t.SetupVartypes() - t.CreateFileLines("mk/bsd.pkg.mk") - t.CreateFileLines("category/package/Makefile", - MkRcsID, - "", + t.SetupPackage("category/package", ".if exists(options.mk)", ". include \"another.mk\"", - ".endif", - "", - ".include \"../../mk/bsd.pkg.mk\"") + ".endif") G.checkdirPackage(t.File("category/package")) t.CheckOutputLines( - "ERROR: ~/category/package/Makefile:4: Cannot read \"another.mk\".") + "ERROR: ~/category/package/Makefile:21: Cannot read \"another.mk\".") } // See https://mail-index.netbsd.org/tech-pkg/2018/07/22/msg020092.html @@ -681,7 +691,12 @@ func (s *Suite) Test_Package__redundant_master_sites(c *check.C) { func (s *Suite) Test_Package_checkUpdate(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetupPackage("category/pkg1", + "PKGNAME= package1-1.0") + t.SetupPackage("category/pkg2", + "PKGNAME= package2-1.0") + t.SetupPackage("category/pkg3", + "PKGNAME= package3-5.0") t.CreateFileLines("doc/TODO", "Suggested package updates", "", @@ -691,30 +706,6 @@ func (s *Suite) Test_Package_checkUpdate(c *check.C) { "\t"+"o package1-1.0", "\t"+"o package2-2.0 [nice new features]", "\t"+"o package3-3.0 [security update]") - t.CreateFileLines("licenses/gnu-gpl-v2", - "The licenses for most software are designed to take away ...") - - t.CreateFileLines("category/pkg1/Makefile", - MkRcsID, - "", - "PKGNAME= package1-1.0", - "GENERATE_PLIST+= echo \"bin/program\";", - "NO_CHECKSUM= yes", - "LICENSE= gnu-gpl-v2") - t.CreateFileLines("category/pkg2/Makefile", - MkRcsID, - "", - "PKGNAME= package2-1.0", - "GENERATE_PLIST+= echo \"bin/program\";", - "NO_CHECKSUM= yes", - "LICENSE= gnu-gpl-v2") - t.CreateFileLines("category/pkg3/Makefile", - MkRcsID, - "", - "PKGNAME= package3-5.0", - "GENERATE_PLIST+= echo \"bin/program\";", - "NO_CHECKSUM= yes", - "LICENSE= gnu-gpl-v2") t.Chdir(".") G.Main("pkglint", "-Wall,no-space,no-order", "category/pkg1", "category/pkg2", "category/pkg3") @@ -723,16 +714,10 @@ func (s *Suite) Test_Package_checkUpdate(c *check.C) { "WARN: category/pkg1/../../doc/TODO:3: Invalid line format \"\".", "WARN: category/pkg1/../../doc/TODO:4: Invalid line format \"\\tO wrong bullet\".", "WARN: category/pkg1/../../doc/TODO:5: Invalid package name \"package-without-version\".", - "WARN: category/pkg1/Makefile: No COMMENT given.", - "NOTE: category/pkg1/Makefile:3: The update request to 1.0 from doc/TODO has been done.", - "WARN: category/pkg1/Makefile:4: Please use \"${ECHO}\" instead of \"echo\".", - "WARN: category/pkg2/Makefile: No COMMENT given.", - "WARN: category/pkg2/Makefile:3: This package should be updated to 2.0 ([nice new features]).", - "WARN: category/pkg2/Makefile:4: Please use \"${ECHO}\" instead of \"echo\".", - "WARN: category/pkg3/Makefile: No COMMENT given.", - "NOTE: category/pkg3/Makefile:3: This package is newer than the update request to 3.0 ([security update]).", - "WARN: category/pkg3/Makefile:4: Please use \"${ECHO}\" instead of \"echo\".", - "0 errors and 10 warnings found.", + "NOTE: category/pkg1/Makefile:20: The update request to 1.0 from doc/TODO has been done.", + "WARN: category/pkg2/Makefile:20: This package should be updated to 2.0 ([nice new features]).", + "NOTE: category/pkg3/Makefile:20: This package is newer than the update request to 3.0 ([security update]).", + "0 errors and 4 warnings found.", "(Run \"pkglint -e\" to show explanations.)") } @@ -746,7 +731,7 @@ func (s *Suite) Test_NewPackage(c *check.C) { c.Check( func() { NewPackage("category") }, check.PanicMatches, - `Package directory "category" must be two subdirectories below the pkgsrc root ".*".`) + `Pkglint internal error: Package directory "category" must be two subdirectories below the pkgsrc root ".*".`) } // Before 2018-09-09, the .CURDIR variable did not have a fallback value. @@ -780,7 +765,7 @@ func (s *Suite) Test__distinfo_from_other_package(c *check.C) { t.CheckOutputLines( "WARN: x11/gst-x11/Makefile: Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset.", "ERROR: x11/gst-x11/Makefile: Each package must define its LICENSE.", - "WARN: x11/gst-x11/Makefile: No COMMENT given.", + "WARN: x11/gst-x11/Makefile: Each package should define a COMMENT.", "WARN: x11/gst-x11/../../multimedia/gst-base/distinfo:3: Patch file \"patch-aa\" does not exist in directory \"../../x11/gst-x11/patches\".") } @@ -798,6 +783,8 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__GNU_CONFIGURE(c *check.C) "WARN: ~/category/package/Makefile:20: GNU_CONFIGURE almost always needs a C compiler, but \"c\" is not added to USE_LANGUAGES in line 21.") } +// Packages that define GNU_CONFIGURE should also set at least USE_LANGUAGES=c. +// Except if they know what they are doing, as documented in the comment "none, really". func (s *Suite) Test_Package_checkfilePackageMakefile__GNU_CONFIGURE_ok(c *check.C) { t := s.Init(c) @@ -848,7 +835,7 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__USE_IMAKE_and_USE_X11(c * G.CheckDirent(pkg) t.CheckOutputLines( - "NOTE: ~/category/package/Makefile:21: USE_IMAKE makes USE_X11 in line 20 superfluous.") + "NOTE: ~/category/package/Makefile:21: USE_IMAKE makes USE_X11 in line 20 redundant.") } func (s *Suite) Test_Package_readMakefile__skipping(c *check.C) { @@ -858,12 +845,28 @@ func (s *Suite) Test_Package_readMakefile__skipping(c *check.C) { pkg := t.SetupPackage("category/package", ".include \"${MYSQL_PKGSRCDIR:S/-client$/-server/}/buildlink3.mk\"") + t.EnableTracingToLog() G.CheckDirent(pkg) + t.EnableSilentTracing() - t.CheckOutputLines( - "NOTE: ~/category/package/Makefile:20: " + + // Since 2018-12-16 there is no warning or note anymore for the + // buildlink3.mk file being skipped since it didn't help the average + // pkglint user. + + // The information is still available in the trace log though. + + output := t.Output() + var relevant []string + for _, line := range strings.Split(output, "\n") { + if contains(line, "Skipping") { + relevant = append(relevant, line) + } + } + + c.Check(relevant, deepEquals, []string{ + "TRACE: 1 2 3 4 ~/category/package/Makefile:20: " + "Skipping include file \"${MYSQL_PKGSRCDIR:S/-client$/-server/}/buildlink3.mk\". " + - "This may result in false warnings.") + "This may result in false warnings."}) } func (s *Suite) Test_Package_readMakefile__not_found(c *check.C) { @@ -892,21 +895,25 @@ func (s *Suite) Test_Package_readMakefile__relative(c *check.C) { // FIXME: One of the below warnings is redundant. t.CheckOutputLines( - "WARN: ~/category/package/Makefile:20: References to other packages should look like \"../../category/package\", not \"../package\".", + "WARN: ~/category/package/Makefile:20: "+ + "References to other packages should look "+ + "like \"../../category/package\", not \"../package\".", "WARN: ~/category/package/Makefile:20: Invalid relative path \"../package/extra.mk\".") } func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { t := s.Init(c) + // no-order since SetupPackage doesn't place OWNER correctly. t.SetupCommandLine("-Wall,no-order") - G.CurrentUsername = "example-user" + G.Username = "example-user" t.CreateFileLines("category/package/CVS/Entries", "/Makefile//modified//") - // Since MAINTAINER= pkgsrc-users@NetBSD.org, everyone may commit changes. + // In packages without specific MAINTAINER, everyone may commit changes. - pkg := t.SetupPackage("category/package") + pkg := t.SetupPackage("category/package", + "MAINTAINER=\tpkgsrc-users@NetBSD.org") G.CheckDirent(pkg) @@ -914,19 +921,8 @@ func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { // A package with a MAINTAINER may be edited with care. - t.CreateFileLines("category/package/Makefile", - MkRcsID, - "", - "DISTNAME=\tdistname-1.0", - "CATEGORIES=\tcategory", - "MASTER_SITES=\t# none", - "", - "MAINTAINER=\tmaintainer@example.org", // Different from default value - "HOMEPAGE=\t# none", - "COMMENT=\tDummy package", - "LICENSE=\t2-clause-bsd", - "", - ".include \"../../mk/bsd.pkg.mk\"") + t.SetupPackage("category/package", + "MAINTAINER=\tmaintainer@example.org") G.CheckDirent(pkg) @@ -937,6 +933,7 @@ func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { // A package with an OWNER may NOT be edited by others. pkg = t.SetupPackage("category/package", + "#MAINTAINER=\t# undefined", "OWNER=\towner@example.org") G.CheckDirent(pkg) @@ -945,9 +942,23 @@ func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { "WARN: ~/category/package/Makefile: " + "Don't commit changes to this file without asking the OWNER, owner@example.org.") + // In a package with both OWNER and MAINTAINER, OWNER wins. + + pkg = t.SetupPackage("category/package", + "MAINTAINER=\tmaintainer@example.org", + "OWNER=\towner@example.org") + + G.CheckDirent(pkg) + + t.CheckOutputLines( + "WARN: ~/category/package/Makefile: "+ + "Don't commit changes to this file without asking the OWNER, owner@example.org.", + "NOTE: ~/category/package/Makefile: "+ + "Please only commit changes that maintainer@example.org would approve.") + // ... unless you are the owner, of course. - G.CurrentUsername = "owner" + G.Username = "owner" G.CheckDirent(pkg) diff --git a/pkgtools/pkglint/files/parser.go b/pkgtools/pkglint/files/parser.go index 723be2e5508..ab9e6f9ea65 100644 --- a/pkgtools/pkglint/files/parser.go +++ b/pkgtools/pkglint/files/parser.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "netbsd.org/pkglint/textproc" @@ -97,17 +97,21 @@ func (p *Parser) Dependency() *DependencyPattern { lexer.Reset(mark2) } } + if dp.LowerOp != "" || dp.UpperOp != "" { return &dp } + if lexer.SkipByte('-') && lexer.Rest() != "" { dp.Wildcard = lexer.Rest() lexer.Skip(len(lexer.Rest())) return &dp } + if hasPrefix(dp.Pkgbase, "${") && hasSuffix(dp.Pkgbase, "}") { return &dp } + if hasSuffix(dp.Pkgbase, "-*") { dp.Pkgbase = strings.TrimSuffix(dp.Pkgbase, "-*") dp.Wildcard = "*" diff --git a/pkgtools/pkglint/files/parser_test.go b/pkgtools/pkglint/files/parser_test.go index 90f4e8f7098..e5754a9bde9 100644 --- a/pkgtools/pkglint/files/parser_test.go +++ b/pkgtools/pkglint/files/parser_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" @@ -43,22 +43,55 @@ func (s *Suite) Test_Parser_Dependency(c *check.C) { testRest(pattern, expected, "") } - test("fltk>=1.1.5rc1<1.3", DependencyPattern{"fltk", ">=", "1.1.5rc1", "<", "1.3", ""}) - test("libwcalc-1.0*", DependencyPattern{"libwcalc", "", "", "", "", "1.0*"}) - test("${PHP_PKG_PREFIX}-pdo-5.*", DependencyPattern{"${PHP_PKG_PREFIX}-pdo", "", "", "", "", "5.*"}) - test("${PYPKGPREFIX}-metakit-[0-9]*", DependencyPattern{"${PYPKGPREFIX}-metakit", "", "", "", "", "[0-9]*"}) - test("boost-build-1.59.*", DependencyPattern{"boost-build", "", "", "", "", "1.59.*"}) - test("${_EMACS_REQD}", DependencyPattern{"${_EMACS_REQD}", "", "", "", "", ""}) - test("{gcc46,gcc46-libs}>=4.6.0", DependencyPattern{"{gcc46,gcc46-libs}", ">=", "4.6.0", "", "", ""}) - test("perl5-*", DependencyPattern{"perl5", "", "", "", "", "*"}) - test("verilog{,-current}-[0-9]*", DependencyPattern{"verilog{,-current}", "", "", "", "", "[0-9]*"}) - test("mpg123{,-esound,-nas}>=0.59.18", DependencyPattern{"mpg123{,-esound,-nas}", ">=", "0.59.18", "", "", ""}) - test("mysql*-{client,server}-[0-9]*", DependencyPattern{"mysql*-{client,server}", "", "", "", "", "[0-9]*"}) - test("postgresql8[0-35-9]-${module}-[0-9]*", DependencyPattern{"postgresql8[0-35-9]-${module}", "", "", "", "", "[0-9]*"}) - test("ncurses-${NC_VERS}{,nb*}", DependencyPattern{"ncurses", "", "", "", "", "${NC_VERS}{,nb*}"}) - test("xulrunner10>=${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", DependencyPattern{"xulrunner10", ">=", "${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", "", "", ""}) - testRest("gnome-control-center>=2.20.1{,nb*}", DependencyPattern{"gnome-control-center", ">=", "2.20.1", "", "", ""}, "{,nb*}") + test("fltk>=1.1.5rc1<1.3", + DependencyPattern{"fltk", ">=", "1.1.5rc1", "<", "1.3", ""}) + + test("libwcalc-1.0*", + DependencyPattern{"libwcalc", "", "", "", "", "1.0*"}) + + test("${PHP_PKG_PREFIX}-pdo-5.*", + DependencyPattern{"${PHP_PKG_PREFIX}-pdo", "", "", "", "", "5.*"}) + + test("${PYPKGPREFIX}-metakit-[0-9]*", + DependencyPattern{"${PYPKGPREFIX}-metakit", "", "", "", "", "[0-9]*"}) + + test("boost-build-1.59.*", + DependencyPattern{"boost-build", "", "", "", "", "1.59.*"}) + + test("${_EMACS_REQD}", + DependencyPattern{"${_EMACS_REQD}", "", "", "", "", ""}) + + test("{gcc46,gcc46-libs}>=4.6.0", + DependencyPattern{"{gcc46,gcc46-libs}", ">=", "4.6.0", "", "", ""}) + + test("perl5-*", + DependencyPattern{"perl5", "", "", "", "", "*"}) + + test("verilog{,-current}-[0-9]*", + DependencyPattern{"verilog{,-current}", "", "", "", "", "[0-9]*"}) + + test("mpg123{,-esound,-nas}>=0.59.18", + DependencyPattern{"mpg123{,-esound,-nas}", ">=", "0.59.18", "", "", ""}) + + test("mysql*-{client,server}-[0-9]*", + DependencyPattern{"mysql*-{client,server}", "", "", "", "", "[0-9]*"}) + + test("postgresql8[0-35-9]-${module}-[0-9]*", + DependencyPattern{"postgresql8[0-35-9]-${module}", "", "", "", "", "[0-9]*"}) + + test("ncurses-${NC_VERS}{,nb*}", + DependencyPattern{"ncurses", "", "", "", "", "${NC_VERS}{,nb*}"}) + + test("xulrunner10>=${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", + DependencyPattern{"xulrunner10", ">=", "${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", "", "", ""}) + + testRest("gnome-control-center>=2.20.1{,nb*}", + DependencyPattern{"gnome-control-center", ">=", "2.20.1", "", "", ""}, "{,nb*}") + testNil(">=2.20.1{,nb*}") + testNil("pkgbase<=") + + // TODO: support this edge case someday. // "{ssh{,6}-[0-9]*,openssh-[0-9]*}" is not representable using the current data structure } diff --git a/pkgtools/pkglint/files/patches.go b/pkgtools/pkglint/files/patches.go index 0863e9f6a12..f14718cd598 100644 --- a/pkgtools/pkglint/files/patches.go +++ b/pkgtools/pkglint/files/patches.go @@ -1,4 +1,4 @@ -package main +package pkglint // Checks for patch files. @@ -103,7 +103,7 @@ func (ck *PatchChecker) Check() { } } -// See http://www.gnu.org/software/diffutils/manual/html_node/Detailed-Unified.html +// See https://www.gnu.org/software/diffutils/manual/html_node/Detailed-Unified.html func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { if trace.Tracing { defer trace.Call0()() @@ -129,22 +129,32 @@ func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { for !ck.exp.EOF() && (linesToDel > 0 || linesToAdd > 0 || hasPrefix(ck.exp.CurrentLine().Text, "\\")) { line := ck.exp.CurrentLine() ck.exp.Advance() + text := line.Text switch { + case text == "": + // There should be a space here, but that was a trailing space and + // has been trimmed down somewhere on its way. Doesn't matter, + // all the patch programs can handle this situation. linesToDel-- linesToAdd-- + case hasPrefix(text, " "), hasPrefix(text, "\t"): linesToDel-- linesToAdd-- ck.checklineContext(text[1:], patchedFileType) + case hasPrefix(text, "-"): linesToDel-- + case hasPrefix(text, "+"): linesToAdd-- ck.checklineAdded(text[1:], patchedFileType) + case hasPrefix(text, "\\"): // \ No newline at end of file (or a translation of that message) + default: line.Errorf("Invalid line in unified patch hunk: %s", text) return @@ -161,18 +171,20 @@ func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { linesToDel, linesToAdd) } } + if !hasHunks { ck.exp.CurrentLine().Errorf("No patch hunks for %q.", patchedFile) } + if !ck.exp.EOF() { line := ck.exp.CurrentLine() if !ck.isEmptyLine(line.Text) && !matches(line.Text, rePatchUniFileDel) { line.Warnf("Empty line or end of file expected.") G.Explain( - "This line is not part of the patch anymore, although it may", - "look so. To make this situation clear, there should be an", - "empty line before this line. If the line doesn't contain", - "useful information, it should be removed.") + "This line is not part of the patch anymore, although it may look so.", + "To make this situation clear, there should be an", + "empty line before this line.", + "If the line doesn't contain useful information, it should be removed.") } } } @@ -185,10 +197,10 @@ func (ck *PatchChecker) checkBeginDiff(line Line, patchedFiles int) { if !ck.seenDocumentation && patchedFiles == 0 { line.Errorf("Each patch must be documented.") G.Explain( - "Pkgsrc tries to have as few patches as possible. Therefore, each", - "patch must document why it is necessary. Typical reasons are", - "portability or security. A typical documented patch looks like", - "this:", + "Pkgsrc tries to have as few patches as possible.", + "Therefore, each patch must document why it is necessary.", + "Typical reasons are portability or security.", + "A typical documented patch looks like this:", "", "\t$"+"NetBSD$", "", @@ -199,8 +211,8 @@ func (ck *PatchChecker) checkBeginDiff(line Line, patchedFiles int) { "corresponding CVE identifier.", "", "Each patch should be sent to the upstream maintainers of the", - "package, so that they can include it in future versions. After", - "submitting a patch upstream, the corresponding bug report should", + "package, so that they can include it in future versions.", + "After submitting a patch upstream, the corresponding bug report should", "be mentioned in this file, to prevent duplicate work.") } if G.Opts.WarnSpace && !ck.previousLineEmpty { @@ -259,6 +271,8 @@ func (ck *PatchChecker) checktextUniHunkCr() { line := ck.exp.PreviousLine() if hasSuffix(line.Text, "\r") { + // This code has been introduced around 2006. + // As of 2018, this might be fixed by now. fix := line.Autofix() fix.Errorf("The hunk header must not end with a CR character.") fix.Explain( @@ -281,6 +295,9 @@ func (ck *PatchChecker) checktextRcsid(text string) { } } +// isEmptyLine tests whether a line provides essentially no interesting content. +// The focus here is on human-generated content that is intended for other human readers. +// Therefore text that is typical for patch generators is considered empty as well. func (ck *PatchChecker) isEmptyLine(text string) bool { return text == "" || hasPrefix(text, "index ") || @@ -291,6 +308,9 @@ func (ck *PatchChecker) isEmptyLine(text string) bool { type FileType uint8 +// TODO: Is this type really useful? It is mainly used for warning about absolute filenames, +// and that check is questionable in itself. + const ( ftSource FileType = iota ftShell diff --git a/pkgtools/pkglint/files/patches_test.go b/pkgtools/pkglint/files/patches_test.go index ab34ec0cc1e..0ac69793c1d 100644 --- a/pkgtools/pkglint/files/patches_test.go +++ b/pkgtools/pkglint/files/patches_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" @@ -9,8 +9,11 @@ func (s *Suite) Test_ChecklinesPatch__with_comment(c *check.C) { lines := t.NewLines("patch-WithComment", RcsID, "", - "Text", - "Text", + "This part describes:", + "* the purpose of the patch,", + "* to which operating systems it applies", + "* either why it is specific to pkgsrc", + "* or where it has been reported upstream", "", "--- file.orig", "+++ file", @@ -272,6 +275,10 @@ func (s *Suite) Test_ChecklinesPatch__two_patched_files(c *check.C) { lines := t.NewLines("patch-aa", RcsID, "", + "A single patch file can apply to more than one file at a time.", + "It shouldn't though, to keep the relation between patch files", + "and patched files simple.", + "", "--- oldfile", "+++ newfile", "@@ -1 +1 @@", @@ -286,7 +293,6 @@ func (s *Suite) Test_ChecklinesPatch__two_patched_files(c *check.C) { ChecklinesPatch(lines) t.CheckOutputLines( - "ERROR: patch-aa:3: Each patch must be documented.", "WARN: patch-aa: Contains patches for 2 files, should be only one.") } @@ -480,7 +486,8 @@ func (s *Suite) Test_ChecklinesPatch__context_lines_with_tab_instead_of_space(c t.CheckOutputEmpty() } -// Must not panic. +// Before 2018-01-28, pkglint had panicked when checking an empty +// patch file, as a slice index was out of bounds. func (s *Suite) Test_ChecklinesPatch__autofix_empty_patch(c *check.C) { t := s.Init(c) @@ -493,7 +500,8 @@ func (s *Suite) Test_ChecklinesPatch__autofix_empty_patch(c *check.C) { t.CheckOutputEmpty() } -// Must not panic. +// Before 2018-01-28, pkglint had panicked when checking an empty +// patch file, as a slice index was out of bounds. func (s *Suite) Test_ChecklinesPatch__autofix_long_empty_patch(c *check.C) { t := s.Init(c) @@ -507,7 +515,7 @@ func (s *Suite) Test_ChecklinesPatch__autofix_long_empty_patch(c *check.C) { t.CheckOutputEmpty() } -func (s *Suite) Test_ChecklinesPatch__crlf(c *check.C) { +func (s *Suite) Test_ChecklinesPatch__crlf_autofix(c *check.C) { t := s.Init(c) t.SetupCommandLine("-Wall", "--autofix") @@ -524,6 +532,9 @@ func (s *Suite) Test_ChecklinesPatch__crlf(c *check.C) { ChecklinesPatch(lines) + // To relieve the pkgsrc package maintainers from this boring work, + // the pkgsrc infrastructure could fix these issues before actually + // applying the patches. t.CheckOutputLines( "AUTOFIX: ~/patch-aa:7: Replacing \"\\r\\n\" with \"\\n\".") } @@ -591,11 +602,6 @@ func (s *Suite) Test_ChecklinesPatch__invalid_line_in_hunk(c *check.C) { ChecklinesPatch(lines) - // The first context line should start with a single space character, - // but that would mean trailing whitespace, so it may be left out. - // The last context line is omitted completely because it would also - // have trailing whitespace, and if that were removed, would be a - // trailing empty line. t.CheckOutputLines( "ERROR: ~/patch-aa:10: Invalid line in unified patch hunk: <<<<<<<<") } diff --git a/pkgtools/pkglint/files/pkglint.go b/pkgtools/pkglint/files/pkglint.go index 08f19abe7d6..a62d9da7163 100644 --- a/pkgtools/pkglint/files/pkglint.go +++ b/pkgtools/pkglint/files/pkglint.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "fmt" @@ -26,10 +26,10 @@ type Pkglint struct { Mk MkLines // The Makefile (or fragment) that is currently checked, or nil. Todo []string // The files or directories that still need to be checked. - Wip bool // Is the currently checked item from pkgsrc-wip? - Infrastructure bool // Is the currently checked item from the pkgsrc infrastructure? + Wip bool // Is the currently checked file or package from pkgsrc-wip? + Infrastructure bool // Is the currently checked file from the pkgsrc infrastructure? Testing bool // Is pkglint in self-testing mode (only during development)? - CurrentUsername string // For checking against OWNER and MAINTAINER + Username string // For checking against OWNER and MAINTAINER CvsEntriesDir string // Cached to avoid I/O CvsEntriesLines Lines @@ -47,6 +47,11 @@ func NewPkglint() Pkglint { } type CmdOpts struct { + // TODO: Are these Check* options really necessary? + // + // They had been introduced in order to make pkglint more flexible, + // but without any actual need. + CheckAlternatives, CheckBuildlink3, CheckDescr, @@ -61,6 +66,13 @@ type CmdOpts struct { CheckPatches, CheckPlist bool + // TODO: Are these Warn* options really all necessary? + // + // Some of them may have been unreliable in the past when they were new. + // Instead of these fine-grained options, there is already --only, which + // could be contrasted by a future --ignore option, in order to suppress + // individual checks. + WarnAbsname, WarnDirectcmd, WarnExtra, @@ -87,7 +99,7 @@ type CmdOpts struct { type Hash struct { hash string - line Line + line Line // TODO: Maybe a Location object would already be enough. } type pkglintFatal struct{} @@ -97,19 +109,18 @@ type pkglintFatal struct{} var ( G = NewPkglint() trace tracePkg.Tracer - exit = os.Exit // Indirect access, to allow main() to be tested. ) -func main() { +func Main() int { G.out = NewSeparatorWriter(os.Stdout) G.err = NewSeparatorWriter(os.Stderr) trace.Out = os.Stdout - exitcode := G.Main(os.Args...) + exitCode := G.Main(os.Args...) if G.Opts.Profiling { G = Pkglint{} // Free all memory. - runtime.GC() // Detect possible memory leaks. + runtime.GC() // For detecting possible memory leaks; see qa-pkglint. } - exit(exitcode) + return exitCode } // Main runs the main program with the given arguments. @@ -120,19 +131,19 @@ func main() { // back to false. // // It also discards the -Wall option that is used by default in other tests. -func (pkglint *Pkglint) Main(argv ...string) (exitcode int) { +func (pkglint *Pkglint) Main(argv ...string) (exitCode int) { defer func() { if r := recover(); r != nil { if _, ok := r.(pkglintFatal); ok { - exitcode = 1 + exitCode = 1 } else { panic(r) } } }() - if exitcode := pkglint.ParseCommandLine(argv); exitcode != nil { - return *exitcode + if exitcode := pkglint.ParseCommandLine(argv); exitcode != -1 { + return exitcode } if pkglint.Opts.Profiling { @@ -154,7 +165,7 @@ func (pkglint *Pkglint) Main(argv ...string) (exitcode int) { pkglint.histo.PrintStats(pkglint.out.out, "loghisto", -1) pkglint.res.PrintStats(pkglint.out.out) pkglint.loaded.PrintStats(pkglint.out.out, "loaded", 10) - pkglint.out.WriteLine(fmt.Sprintf("fileCache: %d hits, %d misses", pkglint.fileCache.hits, pkglint.fileCache.misses)) + pkglint.out.WriteLine(sprintf("fileCache: %d hits, %d misses", pkglint.fileCache.hits, pkglint.fileCache.misses)) }() } @@ -184,7 +195,7 @@ func (pkglint *Pkglint) Main(argv ...string) (exitcode int) { currentUser, err := user.Current() if err == nil { // On Windows, this is `Computername\Username`. - pkglint.CurrentUsername = replaceAll(currentUser.Username, `^.*\\`, "") + pkglint.Username = replaceAll(currentUser.Username, `^.*\\`, "") } for len(pkglint.Todo) > 0 { @@ -202,7 +213,7 @@ func (pkglint *Pkglint) Main(argv ...string) (exitcode int) { return 0 } -func (pkglint *Pkglint) ParseCommandLine(args []string) *int { +func (pkglint *Pkglint) ParseCommandLine(args []string) int { gopts := &pkglint.Opts lopts := &pkglint.Logger.Opts opts := getopt.NewOptions() @@ -253,28 +264,31 @@ func (pkglint *Pkglint) ParseCommandLine(args []string) *int { remainingArgs, err := opts.Parse(args) if err != nil { - _, _ = fmt.Fprintf(pkglint.err.out, "%s\n\n", err) - opts.Help(pkglint.err.out, "pkglint [options] dir...") - exitcode := 1 - return &exitcode + errOut := pkglint.err.out + _, _ = fmt.Fprintln(errOut, err) + _, _ = fmt.Fprintln(errOut, "") + opts.Help(errOut, "pkglint [options] dir...") + return 1 } gopts.args = remainingArgs if gopts.ShowHelp { opts.Help(pkglint.out.out, "pkglint [options] dir...") - exitcode := 0 - return &exitcode + return 0 } if pkglint.Opts.ShowVersion { _, _ = fmt.Fprintf(pkglint.out.out, "%s\n", confVersion) - exitcode := 0 - return &exitcode + return 0 } - return nil + return -1 } +// CheckDirent checks a directory or a single file. +// +// During tests, it assumes that Pkgsrc.LoadInfrastructure has been called. +// It is the most high-level method for testing pkglint. func (pkglint *Pkglint) CheckDirent(filename string) { if trace.Tracing { defer trace.Call1(filename)() @@ -352,6 +366,7 @@ func (pkglint *Pkglint) checkdirPackage(dir string) { havePatches := false // Determine the used variables and PLIST directories before checking any of the Makefile fragments. + // TODO: Why is this code necessary? What effect does it have? for _, filename := range files { basename := path.Base(filename) if (hasPrefix(basename, "Makefile.") || hasSuffix(filename, ".mk")) && @@ -395,6 +410,7 @@ func (pkglint *Pkglint) checkdirPackage(dir string) { if pkg.Pkgdir == "." && pkglint.Opts.CheckDistinfo && pkglint.Opts.CheckPatches { if havePatches && !haveDistinfo { + // TODO: Add Line.RefTo to make the context clear. NewLineWhole(pkg.File(pkg.DistinfoFile)).Warnf("File not found. Please run %q.", bmake("makepatchsum")) } } @@ -407,7 +423,7 @@ func (pkglint *Pkglint) checkdirPackage(dir string) { // For runtime errors, use dummyLine.Fatalf. func (pkglint *Pkglint) Assertf(cond bool, format string, args ...interface{}) { if !cond { - panic("Pkglint internal error: " + fmt.Sprintf(format, args...)) + panic("Pkglint internal error: " + sprintf(format, args...)) } } @@ -493,9 +509,8 @@ func ChecklinesDescr(lines Lines) { line.Warnf("File too long (should be no more than %d lines).", maxLines) G.Explain( - "The DESCR file should fit on a traditional terminal of 80x25", - "characters. It is also intended to give a _brief_ summary about", - "the package's contents.") + "The DESCR file should fit on a traditional terminal of 80x25 characters.", + "It is also intended to give a _brief_ summary about the package's contents.") } SaveAutofixChanges(lines) @@ -733,9 +748,10 @@ func (pkglint *Pkglint) checkExecutable(filename string, st os.FileInfo) { fix := line.Autofix() fix.Warnf("Should not be executable.") fix.Explain( - "No package file should ever be executable. Even the INSTALL and", - "DEINSTALL scripts are usually not usable in the form they have in", - "the package, as the pathnames get adjusted during installation.", + "No package file should ever be executable.", + "Even the INSTALL and DEINSTALL scripts are usually not usable", + "in the form they have in the package,", + "as the pathnames get adjusted during installation.", "So there is no need to have any file executable.") fix.Custom(func(showAutofix, autofix bool) { fix.Describef(0, "Clearing executable bits") diff --git a/pkgtools/pkglint/files/pkglint_test.go b/pkgtools/pkglint/files/pkglint_test.go index 852f007a23b..768900bd58f 100644 --- a/pkgtools/pkglint/files/pkglint_test.go +++ b/pkgtools/pkglint/files/pkglint_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "io/ioutil" @@ -44,8 +44,8 @@ func (s *Suite) Test_Pkglint_Main__only(c *check.C) { exitcode := G.ParseCommandLine([]string{"pkglint", "-Wall", "-o", ":Q", "--version"}) - if c.Check(exitcode, check.NotNil) { - c.Check(*exitcode, equals, 0) + if exitcode != -1 { + c.Check(exitcode, equals, 0) } c.Check(G.Opts.LogOnly, deepEquals, []string{":Q"}) t.CheckOutputLines( @@ -337,7 +337,7 @@ func (s *Suite) Test_Pkglint_CheckDirent__manual_patch(c *check.C) { "WARN: ~/category/package/Makefile: Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset.", "WARN: ~/category/package/distinfo: File not found. Please run \""+confMake+" makesum\" or define NO_CHECKSUM=yes in the package Makefile.", "ERROR: ~/category/package/Makefile: Each package must define its LICENSE.", - "WARN: ~/category/package/Makefile: No COMMENT given.") + "WARN: ~/category/package/Makefile: Each package should define a COMMENT.") } func (s *Suite) Test_Pkglint_CheckDirent(c *check.C) { @@ -384,9 +384,9 @@ func (s *Suite) Test_resolveVariableRefs__circular_reference(c *check.C) { func (s *Suite) Test_resolveVariableRefs__multilevel(c *check.C) { t := s.Init(c) - mkline1 := t.NewMkLine("filename", 10, "_=${SECOND}") - mkline2 := t.NewMkLine("filename", 11, "_=${THIRD}") - mkline3 := t.NewMkLine("filename", 12, "_=got it") + mkline1 := t.NewMkLine("filename", 10, "FIRST=\t${SECOND}") + mkline2 := t.NewMkLine("filename", 11, "SECOND=\t${THIRD}") + mkline3 := t.NewMkLine("filename", 12, "THIRD=\tgot it") G.Pkg = NewPackage(t.File("category/pkgbase")) defineVar(mkline1, "FIRST") defineVar(mkline2, "SECOND") @@ -880,6 +880,9 @@ func (s *Suite) Test_Pkglint_checkMode__skipped(c *check.C) { "ERROR: device: Only files and directories are allowed in pkgsrc.") } +// A package that is very incomplete may produce lots of warnings. +// This case is unrealistic since most packages are either generated by url2pkg +// or copied from an existing working package. func (s *Suite) Test_Pkglint_checkdirPackage(c *check.C) { t := s.Init(c) @@ -893,7 +896,7 @@ func (s *Suite) Test_Pkglint_checkdirPackage(c *check.C) { "WARN: Makefile: Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset.", "WARN: distinfo: File not found. Please run \""+confMake+" makesum\" or define NO_CHECKSUM=yes in the package Makefile.", "ERROR: Makefile: Each package must define its LICENSE.", - "WARN: Makefile: No COMMENT given.") + "WARN: Makefile: Each package should define a COMMENT.") } func (s *Suite) Test_Pkglint_checkdirPackage__PKGDIR(c *check.C) { @@ -923,8 +926,8 @@ func (s *Suite) Test_Pkglint_checkdirPackage__PKGDIR(c *check.C) { "LICENSE=\t2-clause-bsd", "PKGDIR=\t\t../../other/package") - // DISTINFO_FILE is resolved relative to PKGDIR, the other places - // are resolved relative to the package base directory. + // DISTINFO_FILE is resolved relative to PKGDIR, + // the other locations are resolved relative to the package base directory. G.checkdirPackage(".") t.CheckOutputLines( @@ -942,8 +945,11 @@ func (s *Suite) Test_Pkglint_checkdirPackage__patch_without_distinfo(c *check.C) // FIXME: One of the below warnings is redundant. t.CheckOutputLines( - "WARN: ~/category/package/distinfo: File not found. Please run \""+confMake+" makesum\" or define NO_CHECKSUM=yes in the package Makefile.", - "WARN: ~/category/package/distinfo: File not found. Please run \""+confMake+" makepatchsum\".") + "WARN: ~/category/package/distinfo: File not found. "+ + "Please run \""+confMake+" makesum\" "+ + "or define NO_CHECKSUM=yes in the package Makefile.", + "WARN: ~/category/package/distinfo: File not found. "+ + "Please run \""+confMake+" makepatchsum\".") } func (s *Suite) Test_Pkglint_checkdirPackage__meta_package_without_license(c *check.C) { @@ -958,14 +964,15 @@ func (s *Suite) Test_Pkglint_checkdirPackage__meta_package_without_license(c *ch G.checkdirPackage(".") + // No error about missing LICENSE since meta-packages don't need a license. + // They are so simple that there is no reason to have any license. t.CheckOutputLines( - "WARN: Makefile: No COMMENT given.") // No error about missing LICENSE. + "WARN: Makefile: Each package should define a COMMENT.") } func (s *Suite) Test_Pkglint_checkdirPackage__filename_with_variable(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-order") pkg := t.SetupPackage("category/package", ".include \"../../mk/bsd.prefs.mk\"", "", @@ -981,6 +988,7 @@ func (s *Suite) Test_Pkglint_checkdirPackage__filename_with_variable(c *check.C) // because the variable \"rv\" comes from a .for loop. // // TODO: iterate over variables in simple .for loops like the above. + // TODO: when implementing the above, take care of deeply nested loops (42.zip). G.CheckDirent(pkg) t.CheckOutputEmpty() @@ -1043,7 +1051,7 @@ func (s *Suite) Test_Pkglint_checkExecutable__already_committed(c *check.C) { // See the "Too late" comment in Pkglint.checkExecutable. t.CheckOutputEmpty() } -func (s *Suite) Test_main(c *check.C) { +func (s *Suite) Test_Main(c *check.C) { t := s.Init(c) out, err := os.Create(t.CreateFileLines("out")) @@ -1058,21 +1066,17 @@ func (s *Suite) Test_main(c *check.C) { args := os.Args stdout := os.Stdout stderr := os.Stderr - prevExit := exit defer func() { os.Stderr = stderr os.Stdout = stdout os.Args = args - exit = prevExit }() os.Args = commandLine os.Stdout = out os.Stderr = out - exit = func(code int) { - c.Check(code, equals, 0) - } - main() + exitCode := Main() + c.Check(exitCode, equals, 0) } runMain(out, "pkglint", ".") diff --git a/pkgtools/pkglint/files/pkgsrc.go b/pkgtools/pkglint/files/pkgsrc.go index 19143394cb0..1617242072c 100644 --- a/pkgtools/pkglint/files/pkgsrc.go +++ b/pkgtools/pkglint/files/pkgsrc.go @@ -1,8 +1,9 @@ -package main +package pkglint import ( "io/ioutil" "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/textproc" "os" "path/filepath" "sort" @@ -488,8 +489,8 @@ func (src *Pkgsrc) loadDocChangesFromFile(filename string) []*Change { "the changes entry.") } } - } else if text := line.Text; len(text) >= 2 && text[0] == '\t' && 'A' <= text[1] && text[1] <= 'Z' { - line.Warnf("Unknown doc/CHANGES line: %s", text) + } else if lex := textproc.NewLexer(line.Text); lex.SkipByte('\t') && lex.TestByteSet(textproc.Upper) { + line.Warnf("Unknown doc/CHANGES line: %s", line.Text) G.Explain("See mk/misc/developer.mk for the rules.") } } diff --git a/pkgtools/pkglint/files/pkgsrc_test.go b/pkgtools/pkglint/files/pkgsrc_test.go index 55106a8d4f5..2ff92dab83f 100644 --- a/pkgtools/pkglint/files/pkgsrc_test.go +++ b/pkgtools/pkglint/files/pkgsrc_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" @@ -256,8 +256,7 @@ func (s *Suite) Test_Pkgsrc__deprecated(c *check.C) { t.CheckOutputLines( "WARN: Makefile:2: Definition of USE_PERL5 is deprecated. Use USE_TOOLS+=perl or USE_TOOLS+=perl:run instead.", "WARN: Makefile:3: Definition of SUBST_POSTCMD.class is deprecated. Has been removed, as it seemed unused.", - "WARN: Makefile:4: Use of \"PKG_JVM\" is deprecated. Use PKG_DEFAULT_JVM instead.", - "WARN: Makefile:4: BUILDLINK_CPPFLAGS.${PKG_JVM} may not be used in any file; it is a write-only variable.") + "WARN: Makefile:4: Use of \"PKG_JVM\" is deprecated. Use PKG_DEFAULT_JVM instead.") } func (s *Suite) Test_Pkgsrc_ListVersions__no_basedir(c *check.C) { diff --git a/pkgtools/pkglint/files/pkgver/vercmp.go b/pkgtools/pkglint/files/pkgver/vercmp.go index 5b6cd8edcd0..2cc7a05be0f 100644 --- a/pkgtools/pkglint/files/pkgver/vercmp.go +++ b/pkgtools/pkglint/files/pkgver/vercmp.go @@ -3,6 +3,8 @@ package pkgver // See pkgtools/pkg_install/files/lib/dewey.c import ( + "netbsd.org/pkglint/textproc" + "strconv" "strings" ) @@ -42,50 +44,34 @@ type version struct { func newVersion(vstr string) *version { v := new(version) - rest := strings.ToLower(vstr) - for rest != "" { + lex := textproc.NewLexer(strings.ToLower(vstr)) + for !lex.EOF() { + switch { - case isdigit(rest[0]): - n := 0 - i := 0 - for i < len(rest) && isdigit(rest[i]) { - n = 10*n + int(rest[i]-'0') - i++ - } - rest = rest[i:] + case lex.TestByteSet(textproc.Digit): + num := lex.NextBytesSet(textproc.Digit) + n, _ := strconv.Atoi(num) v.Add(n) - case rest[0] == '_' || rest[0] == '.': + case lex.SkipByte('_') || lex.SkipByte('.'): v.Add(0) - rest = rest[1:] - case strings.HasPrefix(rest, "alpha"): + case lex.SkipString("alpha"): v.Add(-3) - rest = rest[5:] - case strings.HasPrefix(rest, "beta"): + case lex.SkipString("beta"): v.Add(-2) - rest = rest[4:] - case strings.HasPrefix(rest, "pre"): + case lex.SkipString("pre"): v.Add(-1) - rest = rest[3:] - case strings.HasPrefix(rest, "rc"): + case lex.SkipString("rc"): v.Add(-1) - rest = rest[2:] - case strings.HasPrefix(rest, "pl"): + case lex.SkipString("pl"): v.Add(0) - rest = rest[2:] - case strings.HasPrefix(rest, "nb"): - i := 2 - n := 0 - for i < len(rest) && isdigit(rest[i]) { - n = 10*n + int(rest[i]-'0') - i++ - } - v.nb = n - rest = rest[i:] - case rest[0]-'a' <= 'z'-'a': - v.Add(int(rest[0] - 'a' + 1)) - rest = rest[1:] + case lex.SkipString("nb"): + num := lex.NextBytesSet(textproc.Digit) + v.nb, _ = strconv.Atoi(num) + case lex.TestByteSet(textproc.Lower): + v.Add(int(lex.Rest()[0] - 'a' + 1)) + lex.Skip(1) default: - rest = rest[1:] + lex.Skip(1) } } return v diff --git a/pkgtools/pkglint/files/plist.go b/pkgtools/pkglint/files/plist.go index 21e32ff5b06..1dda4c7203c 100644 --- a/pkgtools/pkglint/files/plist.go +++ b/pkgtools/pkglint/files/plist.go @@ -1,6 +1,7 @@ -package main +package pkglint import ( + "netbsd.org/pkglint/textproc" "path" "sort" "strings" @@ -20,9 +21,8 @@ func ChecklinesPlist(lines Lines) { "and that the author didn't run \"bmake print-PLIST\" after installing", "the files.", "", - "Another reason, common for Perl packages, is that the final PLIST is", - "automatically generated. Since the source PLIST is not used at all,", - "you can remove it.", + "Another reason, common for Perl packages, is that the final PLIST is automatically generated.", + "Since the source PLIST is not used at all, it can be removed.", "", "Meta packages also don't need a PLIST file.") } @@ -90,15 +90,14 @@ func (ck *PlistChecker) NewLines(lines Lines) []*PlistLine { return plines } +var plistLineStart = textproc.NewByteSet("$0-9A-Za-z") + func (ck *PlistChecker) collectFilesAndDirs(plines []*PlistLine) { for _, pline := range plines { if text := pline.text; len(text) > 0 { first := text[0] switch { - case 'a' <= first && first <= 'z', - first == '$', - 'A' <= first && first <= 'Z', - '0' <= first && first <= '9': + case plistLineStart.Contains(first): if prev := ck.allFiles[text]; prev == nil || pline.condition < prev.condition { ck.allFiles[text] = pline } @@ -237,9 +236,9 @@ func (ck *PlistChecker) checkpathBin(pline *PlistLine, dirname, basename string) pline.Warnf("The bin/ directory should not have subdirectories.") G.Explain( "The programs in bin/ are collected there to be executable by the", - "user without having to type an absolute path. This advantage does", - "not apply to programs in subdirectories of bin/. These programs", - "should rather be placed in libexec/PKGBASE.") + "user without having to type an absolute path.", + "This advantage does not apply to programs in subdirectories of bin/.", + "These programs should rather be placed in libexec/PKGBASE.") return } } @@ -325,9 +324,9 @@ func (ck *PlistChecker) checkpathMan(pline *PlistLine) { fix.Notef("The .gz extension is unnecessary for manual pages.") fix.Explain( "Whether the manual pages are installed in compressed form or not is", - "configured by the pkgsrc user. Compression and decompression takes", - "place automatically, no matter if the .gz extension is mentioned in", - "the PLIST or not.") + "configured by the pkgsrc user.", + "Compression and decompression takes place automatically,", + "no matter if the .gz extension is mentioned in the PLIST or not.") fix.ReplaceRegex(`\.gz\n`, "\n", 1) fix.Apply() } @@ -446,8 +445,8 @@ func (pline *PlistLine) warnImakeMannewsuffix() { pline.Warnf("IMAKE_MANNEWSUFFIX is not meant to appear in PLISTs.") G.Explain( "This is the result of a print-PLIST call that has not been edited", - "manually by the package maintainer. Please replace the", - "IMAKE_MANNEWSUFFIX with:", + "manually by the package maintainer.", + "Please replace the IMAKE_MANNEWSUFFIX with:", "", "\tIMAKE_MAN_SUFFIX for programs,", "\tIMAKE_LIBMAN_SUFFIX for library functions,", diff --git a/pkgtools/pkglint/files/plist_test.go b/pkgtools/pkglint/files/plist_test.go index 209811ec004..8e8c5950a8d 100644 --- a/pkgtools/pkglint/files/plist_test.go +++ b/pkgtools/pkglint/files/plist_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/shell.go b/pkgtools/pkglint/files/shell.go index 8f747188c0f..a83295704ff 100644 --- a/pkgtools/pkglint/files/shell.go +++ b/pkgtools/pkglint/files/shell.go @@ -1,4 +1,4 @@ -package main +package pkglint // Parsing and checking shell commands embedded in Makefiles @@ -87,9 +87,13 @@ outer: case atom.Type == shtSubshell: line.Warnf("Invoking subshells via $(...) is not portable enough.") G.Explain( - "The Solaris /bin/sh does not know this way to execute a command in a", - "subshell. Please use backticks (`...`) as a replacement.") - return // To avoid internal pkglint parse errors + "The Solaris /bin/sh does not know this way to execute a command in a subshell.", + "Please use backticks (`...`) as a replacement.") + + // Early return to avoid further parse errors. + // As of December 2018, it might be worth continuing again since the + // shell parser has improved in 2018. + return case atom.Type == shtText: break @@ -104,7 +108,7 @@ outer: } if trimHspace(tok.Rest()) != "" { - line.Warnf("Pkglint parse error in ShellLine.CheckWord at %q (quoting=%s), rest: %s", token, quoting, tok.Rest()) + line.Warnf("Internal pkglint error in ShellLine.CheckWord at %q (quoting=%s), rest: %s", token, quoting, tok.Rest()) } } @@ -118,17 +122,16 @@ func (shline *ShellLine) checkShVarUse(atom *ShAtom, checkQuoting bool) { } else if G.Opts.WarnQuoting && checkQuoting && shline.variableNeedsQuoting(shVarname) { line.Warnf("Unquoted shell variable %q.", shVarname) G.Explain( - "When a shell variable contains whitespace, it is expanded (split", - "into multiple words) when it is written as $variable in a shell", - "script. If that is not intended, you should add quotation marks", - "around it, like \"$variable\". Then, the variable will always expand", - "to a single word, preserving all whitespace and other special", - "characters.", + "When a shell variable contains whitespace, it is expanded (split into multiple words)", + "when it is written as $variable in a shell script.", + "If that is not intended, it should be surrounded by quotation marks, like \"$variable\".", + "This way it always expands to a single word, preserving all whitespace and other special characters.", "", "Example:", "\tfname=\"Curriculum vitae.doc\"", "\tcp $filename /tmp", "\t# tries to copy the two files \"Curriculum\" and \"Vitae.doc\"", + "", "\tcp \"$filename\" /tmp", "\t# copies one file, as intended") } @@ -165,8 +168,8 @@ func (shline *ShellLine) checkVaruseToken(atoms *[]*ShAtom, quoting ShQuoting) b case quoting == shqDquot && varuse.IsQ(): shline.mkline.Warnf("Please don't use the :Q operator in double quotes.") G.Explain( - "Either remove the :Q or the double quotes. In most cases, it is", - "more appropriate to remove the double quotes.") + "Either remove the :Q or the double quotes.", + "In most cases, it is more appropriate to remove the double quotes.") } if varname != "@" { @@ -269,10 +272,10 @@ func (shline *ShellLine) CheckShellCommandLine(shelltext string) { sprintf("Run %q for more information.", makeHelp("subst"))) if contains(shelltext, "#") { G.Explain( - "When migrating to the SUBST framework, pay attention to \"#\"", - "characters. In shell commands, make(1) does not interpret them as", - "comment character, but in variable assignments it does. Therefore,", - "instead of the shell command", + "When migrating to the SUBST framework, pay attention to \"#\" characters.", + "In shell commands, make(1) does not interpret them as", + "comment character, but in variable assignments it does.", + "Therefore, instead of the shell command", "", "\tsed -e 's,#define foo,,'", "", @@ -282,10 +285,6 @@ func (shline *ShellLine) CheckShellCommandLine(shelltext string) { } } - if m, cmd := match1(shelltext, `^@*-(.*(?:MKDIR|INSTALL.*-d|INSTALL_.*_DIR).*)`); m { - line.Notef("You don't need to use \"-\" before %q.", cmd) - } - lexer := textproc.NewLexer(shelltext) lexer.NextHspace() hiddenAndSuppress := lexer.NextBytesFunc(func(b byte) bool { return b == '-' || b == '@' }) @@ -384,12 +383,12 @@ func (shline *ShellLine) checkHiddenAndSuppress(hiddenAndSuppress, rest string) shline.mkline.Warnf("The shell command %q should not be hidden.", cmd) G.Explain( "Hidden shell commands do not appear on the terminal or in the log", - "file when they are executed. When they fail, the error message", + "file when they are executed.", + "When they fail, the error message", "cannot be assigned to the command, which is very difficult to debug.", "", - "It is better to insert ${RUN} at the beginning of the whole command", - "line. This will hide the command by default but shows it when", - "PKG_DEBUG_LEVEL is set.") + "It is better to insert ${RUN} at the beginning of the whole command line", + "This will hide the command by default but shows it when PKG_DEBUG_LEVEL is set.") } } } @@ -397,8 +396,8 @@ func (shline *ShellLine) checkHiddenAndSuppress(hiddenAndSuppress, rest string) if contains(hiddenAndSuppress, "-") { shline.mkline.Warnf("Using a leading \"-\" to suppress errors is deprecated.") G.Explain( - "If you really want to ignore any errors from this command, append", - "\"|| ${TRUE}\" to the command.") + "If you really want to ignore any errors from this command, append \"|| ${TRUE}\" to the command.", + "This is more visible than a single hyphen, and it should be.") } } @@ -457,9 +456,10 @@ func (scc *SimpleCommandChecker) checkCommandStart() { if G.Opts.WarnExtra && !(G.Mk != nil && G.Mk.indentation.DependsOn("OPSYS")) { scc.shline.mkline.Warnf("Unknown shell command %q.", shellword) G.Explain( - "If you want your package to be portable to all platforms that pkgsrc", - "supports, you should only use shell commands that are covered by the", - "tools framework.") + "To make the package portable to all platforms that pkgsrc supports,", + "it should only use shell commands that are covered by the tools framework.", + "", + "To run custom shell commands, prefix them with \"./\" or with \"${PREFIX}/\".") } } } @@ -567,9 +567,10 @@ func (scc *SimpleCommandChecker) handleComment() bool { G.Explain( "When you split a shell command into multiple lines that are", "continued with a backslash, they will nevertheless be converted to", - "a single line before the shell sees them. That means that even if", - "it _looks_ like that the comment only spans one line in the", - "Makefile, in fact it spans until the end of the whole shell command.", + "a single line before the shell sees them.", + "That means that even if it _looks_ like that the comment only spans", + "one line in the Makefile, in fact it spans until the end of the whole", + "shell command.", "", "To insert a comment into shell code, you can write it like this:", "", @@ -627,13 +628,14 @@ func (scc *SimpleCommandChecker) checkAutoMkdirs() { scc.shline.mkline.Notef("You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= %s\" instead of %q.", dirname, cmdname) G.Explain( "Many packages include a list of all needed directories in their", - "PLIST file. In such a case, you can just set AUTO_MKDIRS=yes and", - "be done. The pkgsrc infrastructure will then create all directories", - "in advance.", + "PLIST file.", + "In such a case, you can just set AUTO_MKDIRS=yes and be done.", + "The pkgsrc infrastructure will then create all directories in advance.", "", - "To create directories that are not mentioned in the PLIST file, it", - "is easier to just list them in INSTALLATION_DIRS than to execute the", - "commands explicitly. That way, you don't have to think about which", + "To create directories that are not mentioned in the PLIST file,", + "it is easier to just list them in INSTALLATION_DIRS than to execute the", + "commands explicitly.", + "That way, you don't have to think about which", "of the many INSTALL_*_DIR variables is appropriate, since", "INSTALLATION_DIRS takes care of that.") } else { @@ -641,9 +643,10 @@ func (scc *SimpleCommandChecker) checkAutoMkdirs() { G.Explain( "To create directories during installation, it is easier to just", "list them in INSTALLATION_DIRS than to execute the commands", - "explicitly. That way, you don't have to think about which", - "of the many INSTALL_*_DIR variables is appropriate, since", - "INSTALLATION_DIRS takes care of that.") + "explicitly.", + "That way, you don't have to think about which", + "of the many INSTALL_*_DIR variables is appropriate,", + "since INSTALLATION_DIRS takes care of that.") } } } @@ -734,14 +737,14 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { } walker := NewMkShWalker() - walker.Callback.If = func(ifClause *MkShIfClause) { + walker.Callback.If = func(ifClause *MkShIf) { for _, cond := range ifClause.Conds { if simple := getSimple(cond); simple != nil { checkConditionalCd(simple) } } } - walker.Callback.Loop = func(loop *MkShLoopClause) { + walker.Callback.Loop = func(loop *MkShLoop) { if simple := getSimple(loop.Cond); simple != nil { checkConditionalCd(simple) } @@ -751,7 +754,8 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { spc.shline.mkline.Warnf("The Solaris /bin/sh does not support negation of shell commands.") G.Explain( "The GNU Autoconf manual has many more details of what shell", - "features to avoid for portable programs. It can be read at:", + "features to avoid for portable programs.", + "It can be read at:", "https://www.gnu.org/software/autoconf/manual/autoconf.html#Limitations-of-Builtins") } } @@ -795,8 +799,8 @@ func (spc *ShellProgramChecker) checkPipeExitcode(line Line, pipeline *MkShPipel "on the left side of the \"|\" fails, this failure is ignored.", "", "If you need to detect the failure of the left-hand-side command, use", - "temporary files to save the output of the command. A good place to", - "create those files is in ${WRKDIR}.") + "temporary files to save the output of the command.", + "A good place to create those files is in ${WRKDIR}.") } } } @@ -891,9 +895,10 @@ func (spc *ShellProgramChecker) checkSetE(list *MkShList, index int, andor *MkSh line.Warnf("Please switch to \"set -e\" mode before using a semicolon (after %q) to separate commands.", NewStrCommand(command.Simple).String()) G.Explain( - "Normally, when a shell command fails (returns non-zero), the", - "remaining commands are still executed. For example, the following", - "commands would remove all files from the HOME directory:", + "Normally, when a shell command fails (returns non-zero),", + "the remaining commands are still executed.", + "For example, the following commands would remove", + "all files from the HOME directory:", "", "\tcd \"$HOME\"; cd /nonexistent; rm -rf *", "", @@ -935,14 +940,15 @@ func (shline *ShellLine) checkInstallCommand(shellcmd string) { line.Warnf("The shell command %q should not be used in the install phase.", shellcmd) G.Explain( "In the install phase, the only thing that should be done is to", - "install the prepared files to their final location. The file's", - "contents should not be changed anymore.") + "install the prepared files to their final location.", + "The file's contents should not be changed anymore.") case "cp", "${CP}": line.Warnf("${CP} should not be used to install files.") G.Explain( "The ${CP} command is highly platform dependent and cannot overwrite", - "read-only files. Please use ${PAX} instead.", + "read-only files.", + "Please use ${PAX} instead.", "", "For example, instead of", "\t${CP} -R ${WRKSRC}/* ${PREFIX}/foodir", diff --git a/pkgtools/pkglint/files/shell.y b/pkgtools/pkglint/files/shell.y index 74c4c8f21dc..3c4f6cc92b5 100644 --- a/pkgtools/pkglint/files/shell.y +++ b/pkgtools/pkglint/files/shell.y @@ -1,5 +1,5 @@ %{ -package main +package pkglint %} %token <Word> tkWORD @@ -25,11 +25,11 @@ package main Separator MkShSeparator Simple *MkShSimpleCommand FuncDef *MkShFunctionDefinition - For *MkShForClause - If *MkShIfClause - Case *MkShCaseClause + For *MkShFor + If *MkShIf + Case *MkShCase CaseItem *MkShCaseItem - Loop *MkShLoopClause + Loop *MkShLoop Words []*ShToken Word *ShToken Redirections []*MkShRedirection @@ -86,7 +86,7 @@ pipeline : tkEXCLAM pipe_sequence { } pipe_sequence : command { - $$ = NewMkShPipeline(false, $1) + $$ = NewMkShPipeline(false, []*MkShCommand{$1}) } pipe_sequence : pipe_sequence tkPIPE linebreak command { $$.Add($4) @@ -156,13 +156,13 @@ for_clause : tkFOR tkWORD linebreak do_group { &ShAtom{shtText, "\"", shqDquot, nil}, &ShAtom{shtShVarUse, "$$@", shqDquot, "@"}, &ShAtom{shtText, "\"", shqPlain, nil}) - $$ = &MkShForClause{$2.MkText, []*ShToken{args}, $4} + $$ = &MkShFor{$2.MkText, []*ShToken{args}, $4} } for_clause : tkFOR tkWORD linebreak tkIN sequential_sep do_group { - $$ = &MkShForClause{$2.MkText, nil, $6} + $$ = &MkShFor{$2.MkText, nil, $6} } for_clause : tkFOR tkWORD linebreak tkIN wordlist sequential_sep do_group { - $$ = &MkShForClause{$2.MkText, $5, $7} + $$ = &MkShFor{$2.MkText, $5, $7} } wordlist : tkWORD { @@ -181,11 +181,11 @@ case_clause : tkCASE tkWORD linebreak tkIN linebreak case_list_ns tkESAC { $$.Word = $2 } case_clause : tkCASE tkWORD linebreak tkIN linebreak tkESAC { - $$ = &MkShCaseClause{$2, nil} + $$ = &MkShCase{$2, nil} } case_list_ns : case_item_ns { - $$ = &MkShCaseClause{nil, nil} + $$ = &MkShCase{nil, nil} $$.Cases = append($$.Cases, $1) } case_list_ns : case_list case_item_ns { @@ -193,7 +193,7 @@ case_list_ns : case_list case_item_ns { } case_list : case_item { - $$ = &MkShCaseClause{nil, nil} + $$ = &MkShCase{nil, nil} $$.Cases = append($$.Cases, $1) } case_list : case_list case_item { @@ -237,12 +237,12 @@ if_clause : tkIF compound_list tkTHEN compound_list else_part tkFI { $$.Prepend($2, $4) } if_clause : tkIF compound_list tkTHEN compound_list tkFI { - $$ = &MkShIfClause{} + $$ = &MkShIf{} $$.Prepend($2, $4) } else_part : tkELIF compound_list tkTHEN compound_list { - $$ = &MkShIfClause{} + $$ = &MkShIf{} $$.Prepend($2, $4) } else_part : tkELIF compound_list tkTHEN compound_list else_part { @@ -250,14 +250,14 @@ else_part : tkELIF compound_list tkTHEN compound_list else_part { $$.Prepend($2, $4) } else_part : tkELSE compound_list { - $$ = &MkShIfClause{nil, nil, $2} + $$ = &MkShIf{nil, nil, $2} } while_clause : tkWHILE compound_list do_group { - $$ = &MkShLoopClause{$2, $3, false} + $$ = &MkShLoop{$2, $3, false} } until_clause : tkUNTIL compound_list do_group { - $$ = &MkShLoopClause{$2, $3, true} + $$ = &MkShLoop{$2, $3, true} } function_definition : tkWORD tkLPAREN tkRPAREN linebreak compound_command { /* Apply rule 9 */ diff --git a/pkgtools/pkglint/files/shell_test.go b/pkgtools/pkglint/files/shell_test.go index 22388bbcfdb..9cf8173cd74 100644 --- a/pkgtools/pkglint/files/shell_test.go +++ b/pkgtools/pkglint/files/shell_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" @@ -14,7 +14,7 @@ func (s *Suite) Test_splitIntoShellTokens__line_continuation(c *check.C) { c.Check(rest, equals, "\\") t.CheckOutputLines( - "WARN: Pkglint parse error in ShTokenizer.ShAtom at \"\\\\\" (quoting=plain).") + "WARN: Internal pkglint error in ShTokenizer.ShAtom at \"\\\\\" (quoting=plain).") } func (s *Suite) Test_splitIntoShellTokens__dollar_slash(c *check.C) { @@ -276,7 +276,6 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { checkShellCommandLine("-${MKDIR} deeply/nested/subdir") t.CheckOutputLines( - "NOTE: filename:1: You don't need to use \"-\" before \"${MKDIR} deeply/nested/subdir\".", "WARN: filename:1: Using a leading \"-\" to suppress errors is deprecated.") G.Pkg = NewPackage(t.File("category/pkgbase")) @@ -558,8 +557,8 @@ func (s *Suite) Test_ShellLine_CheckWord__squot_dollar(c *check.C) { // FIXME: Should be parsed correctly. Make passes the dollar through (probably), // and the shell parser should complain about the unfinished string literal. t.CheckOutputLines( - "WARN: filename:1: Pkglint parse error in ShTokenizer.ShAtom at \"$\" (quoting=s).", - "WARN: filename:1: Pkglint parse error in ShellLine.CheckWord at \"'$\" (quoting=s), rest: $") + "WARN: filename:1: Internal pkglint error in ShTokenizer.ShAtom at \"$\" (quoting=s).", + "WARN: filename:1: Internal pkglint error in ShellLine.CheckWord at \"'$\" (quoting=s), rest: $") } func (s *Suite) Test_ShellLine_CheckWord__dquot_dollar(c *check.C) { @@ -572,8 +571,8 @@ func (s *Suite) Test_ShellLine_CheckWord__dquot_dollar(c *check.C) { // FIXME: Should be parsed correctly. Make passes the dollar through (probably), // and the shell parser should complain about the unfinished string literal. t.CheckOutputLines( - "WARN: filename:1: Pkglint parse error in ShTokenizer.ShAtom at \"$\" (quoting=d).", - "WARN: filename:1: Pkglint parse error in ShellLine.CheckWord at \"\\\"$\" (quoting=d), rest: $") + "WARN: filename:1: Internal pkglint error in ShTokenizer.ShAtom at \"$\" (quoting=d).", + "WARN: filename:1: Internal pkglint error in ShellLine.CheckWord at \"\\\"$\" (quoting=d), rest: $") } func (s *Suite) Test_ShellLine_CheckWord__dollar_subshell(c *check.C) { @@ -729,10 +728,10 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__shell_variables(c *check.C shline.CheckShellCommandLine(text) t.CheckOutputLines( - "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Makefile variable or $$f if you mean a shell variable.", - "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Makefile variable or $$f if you mean a shell variable.", - "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Makefile variable or $$f if you mean a shell variable.", - "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Makefile variable or $$f if you mean a shell variable.", + "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Make variable or $$f if you mean a shell variable.", + "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Make variable or $$f if you mean a shell variable.", + "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Make variable or $$f if you mean a shell variable.", + "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Make variable or $$f if you mean a shell variable.", "NOTE: Makefile:3: Please use the SUBST framework instead of ${SED} and ${MV}.", "WARN: Makefile:3: f is used but not defined.", "WARN: Makefile:3: f is used but not defined.", @@ -1057,11 +1056,11 @@ func (s *Suite) Test_ShellLine_CheckShellCommand__subshell(c *check.C) { // FIXME: "(" is not a shell command, it's an operator. t.CheckOutputLines( "WARN: Makefile:4: The shell command \"(\" should not be hidden.", - "WARN: Makefile:5: Pkglint parse error in ShTokenizer.ShAtom at \"$$(echo 1024))\" (quoting=S).", + "WARN: Makefile:5: Internal pkglint error in ShTokenizer.ShAtom at \"$$(echo 1024))\" (quoting=S).", "WARN: Makefile:5: Invoking subshells via $(...) is not portable enough.", - "WARN: Makefile:6: Pkglint parse error in ShTokenizer.ShAtom at \"$$(echo 1024)))\" (quoting=S).", + "WARN: Makefile:6: Internal pkglint error in ShTokenizer.ShAtom at \"$$(echo 1024)))\" (quoting=S).", "WARN: Makefile:6: The shell command \"(\" should not be hidden.", - "WARN: Makefile:6: Pkglint parse error in ShTokenizer.ShAtom at \"$$(echo 1024)))\" (quoting=S).", + "WARN: Makefile:6: Internal pkglint error in ShTokenizer.ShAtom at \"$$(echo 1024)))\" (quoting=S).", "WARN: Makefile:6: Invoking subshells via $(...) is not portable enough.") } @@ -1230,7 +1229,7 @@ func (s *Suite) Test_ShellProgramChecker_checkConditionalCd(c *check.C) { // FIXME: Fix the parse error. t.CheckOutputLines( "ERROR: Makefile:3: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", - "WARN: Pkglint parse error in ShTokenizer.ShAtom at \"$$\" (quoting=plain).", + "WARN: Internal pkglint error in ShTokenizer.ShAtom at \"$$\" (quoting=plain).", "WARN: Makefile:4: The exitcode of \"ls\" at the left of the | operator is ignored.") } diff --git a/pkgtools/pkglint/files/shtokenizer.go b/pkgtools/pkglint/files/shtokenizer.go index 7cb7ca40082..7d6422e7111 100644 --- a/pkgtools/pkglint/files/shtokenizer.go +++ b/pkgtools/pkglint/files/shtokenizer.go @@ -1,4 +1,6 @@ -package main +package pkglint + +import "netbsd.org/pkglint/textproc" type ShTokenizer struct { parser *Parser @@ -61,7 +63,7 @@ func (p *ShTokenizer) ShAtom(quoting ShQuoting) *ShAtom { if hasPrefix(lexer.Rest(), "${") { p.parser.Line.Warnf("Unclosed Make variable starting at %q.", shorten(lexer.Rest(), 20)) } else { - p.parser.Line.Warnf("Pkglint parse error in ShTokenizer.ShAtom at %q (quoting=%s).", lexer.Rest(), quoting) + p.parser.Line.Warnf("Internal pkglint error in ShTokenizer.ShAtom at %q (quoting=%s).", lexer.Rest(), quoting) } } return atom @@ -315,7 +317,7 @@ func (p *ShTokenizer) shVarUse(q ShQuoting) *ShAtom { return nil } - if lexer.PeekByte() >= '0' && lexer.PeekByte() <= '9' { + if lexer.TestByteSet(textproc.Digit) { lexer.Skip(1) text := lexer.Since(beforeDollar) return &ShAtom{shtShVarUse, text, q, text[2:]} diff --git a/pkgtools/pkglint/files/shtokenizer_test.go b/pkgtools/pkglint/files/shtokenizer_test.go index 83121d55af0..2460345b2b2 100644 --- a/pkgtools/pkglint/files/shtokenizer_test.go +++ b/pkgtools/pkglint/files/shtokenizer_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" @@ -588,30 +588,30 @@ func (s *Suite) Test_ShTokenizer__examples_from_fuzzing(c *check.C) { // Just good that these redundant error messages don't occur every day. t.CheckOutputLines( - "WARN: fuzzing.mk:4: Pkglint parse error in ShTokenizer.ShAtom at \"`\" (quoting=bd).", + "WARN: fuzzing.mk:4: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=bd).", "WARN: fuzzing.mk:4: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", - "WARN: fuzzing.mk:5: Pkglint parse error in ShTokenizer.ShAtom at \"$`\" (quoting=bs).", + "WARN: fuzzing.mk:5: Internal pkglint error in ShTokenizer.ShAtom at \"$`\" (quoting=bs).", "WARN: fuzzing.mk:5: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", - "WARN: fuzzing.mk:5: Pkglint parse error in MkLine.Tokenize at \"$`\".", + "WARN: fuzzing.mk:5: Internal pkglint error in MkLine.Tokenize at \"$`\".", "WARN: fuzzing.mk:6: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", - "WARN: fuzzing.mk:7: Pkglint parse error in ShTokenizer.ShAtom at \"$|\" (quoting=db).", + "WARN: fuzzing.mk:7: Internal pkglint error in ShTokenizer.ShAtom at \"$|\" (quoting=db).", "WARN: fuzzing.mk:7: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", - "WARN: fuzzing.mk:7: Pkglint parse error in MkLine.Tokenize at \"$|\".", + "WARN: fuzzing.mk:7: Internal pkglint error in MkLine.Tokenize at \"$|\".", - "WARN: fuzzing.mk:8: Pkglint parse error in ShTokenizer.ShAtom at \"`\" (quoting=dbd).", + "WARN: fuzzing.mk:8: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=dbd).", "WARN: fuzzing.mk:8: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", "WARN: fuzzing.mk:9: Invoking subshells via $(...) is not portable enough.", - "WARN: fuzzing.mk:10: Pkglint parse error in ShTokenizer.ShAtom at \"`\" (quoting=S).", + "WARN: fuzzing.mk:10: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=S).", "WARN: fuzzing.mk:10: Invoking subshells via $(...) is not portable enough.", - "WARN: fuzzing.mk:11: Pkglint parse error in ShTokenizer.ShAtom at \"$)\" (quoting=Ss).", + "WARN: fuzzing.mk:11: Internal pkglint error in ShTokenizer.ShAtom at \"$)\" (quoting=Ss).", "WARN: fuzzing.mk:11: Invoking subshells via $(...) is not portable enough.", - "WARN: fuzzing.mk:11: Pkglint parse error in MkLine.Tokenize at \"$)\".", + "WARN: fuzzing.mk:11: Internal pkglint error in MkLine.Tokenize at \"$)\".", "WARN: fuzzing.mk:12: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}") } diff --git a/pkgtools/pkglint/files/shtypes.go b/pkgtools/pkglint/files/shtypes.go index dd1ddfdded9..59066d2381e 100644 --- a/pkgtools/pkglint/files/shtypes.go +++ b/pkgtools/pkglint/files/shtypes.go @@ -1,8 +1,4 @@ -package main - -import ( - "fmt" -) +package pkglint //go:generate goyacc -o shellyacc.go -v shellyacc.log -p shyy shell.y @@ -50,13 +46,13 @@ type ShAtom struct { func (atom *ShAtom) String() string { if atom.Type == shtText && atom.Quoting == shqPlain && atom.data == nil { - return fmt.Sprintf("%q", atom.MkText) + return sprintf("%q", atom.MkText) } if atom.Type == shtVaruse { varuse := atom.VarUse() - return fmt.Sprintf("varuse(%q)", varuse.varname+varuse.Mod()) + return sprintf("varuse(%q)", varuse.varname+varuse.Mod()) } - return fmt.Sprintf("ShAtom(%v, %q, %s)", atom.Type, atom.MkText, atom.Quoting) + return sprintf("ShAtom(%v, %q, %s)", atom.Type, atom.MkText, atom.Quoting) } // VarUse returns a read access to a Makefile variable, or nil for plain shell tokens. @@ -133,5 +129,5 @@ func NewShToken(mkText string, atoms ...*ShAtom) *ShToken { } func (token *ShToken) String() string { - return fmt.Sprintf("ShToken(%v)", token.Atoms) + return sprintf("ShToken(%v)", token.Atoms) } diff --git a/pkgtools/pkglint/files/shtypes_test.go b/pkgtools/pkglint/files/shtypes_test.go index a9f1c754401..ca890e0ec37 100644 --- a/pkgtools/pkglint/files/shtypes_test.go +++ b/pkgtools/pkglint/files/shtypes_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/substcontext.go b/pkgtools/pkglint/files/substcontext.go index 12f9262af58..007d9b8e679 100644 --- a/pkgtools/pkglint/files/substcontext.go +++ b/pkgtools/pkglint/files/substcontext.go @@ -1,4 +1,4 @@ -package main +package pkglint import "netbsd.org/pkglint/textproc" diff --git a/pkgtools/pkglint/files/substcontext_test.go b/pkgtools/pkglint/files/substcontext_test.go index c0eaaf99a72..a93379164c5 100644 --- a/pkgtools/pkglint/files/substcontext_test.go +++ b/pkgtools/pkglint/files/substcontext_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "fmt" diff --git a/pkgtools/pkglint/files/testnames_test.go b/pkgtools/pkglint/files/testnames_test.go index 7a9d2f36c99..7bb217a5b8c 100644 --- a/pkgtools/pkglint/files/testnames_test.go +++ b/pkgtools/pkglint/files/testnames_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" @@ -15,7 +15,6 @@ func (s *Suite) Test__test_names(c *check.C) { ck.AllowPrefix("ShellParser", "mkshparser.go") ck.AllowCamelCaseDescriptions( "comparing_YesNo_variable_to_string", - "GitHub", "enumFrom", "enumFromDirs", "dquotBacktDquot", diff --git a/pkgtools/pkglint/files/textproc/lexer.go b/pkgtools/pkglint/files/textproc/lexer.go index 2b42bd0771e..3881b5d5c05 100644 --- a/pkgtools/pkglint/files/textproc/lexer.go +++ b/pkgtools/pkglint/files/textproc/lexer.go @@ -9,6 +9,13 @@ import ( // Lexer provides a flexible way of splitting a string into several parts // by repeatedly chopping off a prefix that matches a string, a function // or a set of byte values. +// +// The Next* methods chop off and return the matched portion. +// +// The Skip* methods chop off the matched portion and return whether something matched. +// +// PeekByte and TestByteSet look at the next byte without chopping it off. +// They are typically used in switch statements, which don't allow variable declarations. type Lexer struct { rest string } @@ -45,6 +52,13 @@ func (l *Lexer) PeekByte() int { return -1 } +// TestByteSet returns whether the remaining string starts with a byte +// from the given set. +func (l *Lexer) TestByteSet(set *ByteSet) bool { + rest := l.rest + return 0 < len(rest) && set.Contains(rest[0]) +} + // Skip skips the next n bytes. func (l *Lexer) Skip(n int) bool { l.rest = l.rest[n:] @@ -251,6 +265,8 @@ var ( Alnum = NewByteSet("A-Za-z0-9") // Alphanumerical, without underscore AlnumU = NewByteSet("A-Za-z0-9_") // Alphanumerical, including underscore Digit = NewByteSet("0-9") // The digits zero to nine + Upper = NewByteSet("A-Z") // The uppercase letters from A to Z + Lower = NewByteSet("a-z") // The lowercase letters from a to z Space = NewByteSet("\t\n ") // Tab, newline, space Hspace = NewByteSet("\t ") // Tab, space XPrint = NewByteSet("\n\t -~") // Printable ASCII, plus tab and newline diff --git a/pkgtools/pkglint/files/textproc/lexer_test.go b/pkgtools/pkglint/files/textproc/lexer_test.go index 1fda0105311..61ddbfa3fca 100644 --- a/pkgtools/pkglint/files/textproc/lexer_test.go +++ b/pkgtools/pkglint/files/textproc/lexer_test.go @@ -58,6 +58,19 @@ func (s *Suite) Test_Lexer_PeekByte(c *check.C) { c.Check(lexer.PeekByte(), equals, -1) } +func (s *Suite) Test_Lexer_TestByteSet(c *check.C) { + lexer := NewLexer("text") + + c.Check(lexer.TestByteSet(Upper), equals, false) + c.Check(lexer.TestByteSet(Lower), equals, true) + c.Check(lexer.TestByteSet(NewByteSet("t")), equals, true) + + c.Check(lexer.NextString("text"), equals, "text") + + c.Check(lexer.TestByteSet(Upper), equals, false) + c.Check(lexer.TestByteSet(Lower), equals, false) +} + func (s *Suite) Test_Lexer_Skip(c *check.C) { lexer := NewLexer("example text") diff --git a/pkgtools/pkglint/files/tools.go b/pkgtools/pkglint/files/tools.go index 13aba20ccc4..dea1c798adf 100644 --- a/pkgtools/pkglint/files/tools.go +++ b/pkgtools/pkglint/files/tools.go @@ -1,7 +1,6 @@ -package main +package pkglint import ( - "fmt" "sort" "strings" ) @@ -23,7 +22,7 @@ type Tool struct { } func (tool *Tool) String() string { - return fmt.Sprintf("%s:%s:%s:%s", + return sprintf("%s:%s:%s:%s", tool.Name, tool.Varname, ifelseStr(tool.MustUseVarForm, "var", ""), tool.Validity) } diff --git a/pkgtools/pkglint/files/tools_test.go b/pkgtools/pkglint/files/tools_test.go index a3be44a2cc9..74e86d068dd 100644 --- a/pkgtools/pkglint/files/tools_test.go +++ b/pkgtools/pkglint/files/tools_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/toplevel.go b/pkgtools/pkglint/files/toplevel.go index bb7d4f24c94..5feaeb87580 100644 --- a/pkgtools/pkglint/files/toplevel.go +++ b/pkgtools/pkglint/files/toplevel.go @@ -1,4 +1,4 @@ -package main +package pkglint type Toplevel struct { dir string diff --git a/pkgtools/pkglint/files/toplevel_test.go b/pkgtools/pkglint/files/toplevel_test.go index 904c15f5c9a..6a52568269d 100644 --- a/pkgtools/pkglint/files/toplevel_test.go +++ b/pkgtools/pkglint/files/toplevel_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/util.go b/pkgtools/pkglint/files/util.go index caf12deebd9..aed027c5d63 100644 --- a/pkgtools/pkglint/files/util.go +++ b/pkgtools/pkglint/files/util.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "fmt" @@ -116,7 +116,7 @@ func mustMatch(s string, re regex.Pattern) []string { if m := G.res.Match(s, re); m != nil { return m } - panic(fmt.Sprintf("mustMatch %q %q", s, re)) + panic(sprintf("mustMatch %q %q", s, re)) } func isEmptyDir(filename string) bool { diff --git a/pkgtools/pkglint/files/util_test.go b/pkgtools/pkglint/files/util_test.go index 43cc28db8db..fc2cc3d0027 100644 --- a/pkgtools/pkglint/files/util_test.go +++ b/pkgtools/pkglint/files/util_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/var.go b/pkgtools/pkglint/files/var.go new file mode 100644 index 00000000000..9050bb62597 --- /dev/null +++ b/pkgtools/pkglint/files/var.go @@ -0,0 +1,24 @@ +package pkglint + +// Var describes a variable in a Makefile snippet. +// +// TODO: Remove this type in June 2019 if it is still a stub. +type Var struct { + Name string + Type *Vartype +} + +func NewVar(name string) *Var { return &Var{name, nil} } + +// Constant returns whether the variable is only ever assigned a single value, +// without being dependent on any other variable. +// +// Multiple assignments (such as VAR=1, VAR+=2, VAR+=3) are considered constant +// as well, as long as the variable is not used in-between these assignments. +// That is, no .include or .if may appear there, and none of the ::= modifiers may +// be involved. +// +// Simple .for loops that append to the variable are ok though. +func (v *Var) Constant() bool { return false } + +func (v *Var) ConstantValue() string { return "" } diff --git a/pkgtools/pkglint/files/var_test.go b/pkgtools/pkglint/files/var_test.go new file mode 100644 index 00000000000..eb37ff58616 --- /dev/null +++ b/pkgtools/pkglint/files/var_test.go @@ -0,0 +1,19 @@ +package pkglint + +import "gopkg.in/check.v1" + +func (s *Suite) Test_Var_Constant(c *check.C) { + v := NewVar("VARNAME") + + // FIXME: Replace this test with an actual use case. + + c.Check(v.Constant(), equals, false) +} + +func (s *Suite) Test_Var_ConstantValue(c *check.C) { + v := NewVar("VARNAME") + + // FIXME: Replace this test with an actual use case. + + c.Check(v.ConstantValue(), equals, "") +} diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go index 7746c66ed44..1bf114f6d06 100644 --- a/pkgtools/pkglint/files/vardefs.go +++ b/pkgtools/pkglint/files/vardefs.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "netbsd.org/pkglint/regex" @@ -77,7 +77,7 @@ func (src *Pkgsrc) InitVartypes() { } bl3list := func(varname string, kindOfList KindOfList, checker *BasicType) { - acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk: append") + acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk: append; *: use") } cmdline := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk:; *: use-loadtime, use") @@ -518,6 +518,7 @@ func (src *Pkgsrc) InitVartypes() { pkglist("BROKEN_EXCEPT_ON_PLATFORM", lkSpace, BtMachinePlatformPattern) pkglist("BROKEN_ON_PLATFORM", lkSpace, BtMachinePlatformPattern) sys("BSD_MAKE_ENV", lkShell, BtShellWord) + // TODO: Align the permissions of the various BUILDLINK_*.* variables with each other. acl("BUILDLINK_ABI_DEPENDS.*", lkSpace, BtDependency, "buildlink3.mk, builtin.mk: append, use-loadtime; *: append") acl("BUILDLINK_API_DEPENDS.*", lkSpace, BtDependency, "buildlink3.mk, builtin.mk: append, use-loadtime; *: append") acl("BUILDLINK_AUTO_DIRS.*", lkNone, BtYesNo, "buildlink3.mk: append") @@ -1215,11 +1216,11 @@ func enum(values string) *BasicType { valueMap[value] = true } name := "enum: " + values + " " // See IsEnum - basicType := &BasicType{name, nil} + basicType := BasicType{name, nil} basicType.checker = func(check *VartypeCheck) { - check.Enum(valueMap, basicType) + check.Enum(valueMap, &basicType) } - return basicType + return &basicType } func parseACLEntries(varname string, aclEntries string) []ACLEntry { diff --git a/pkgtools/pkglint/files/vardefs_test.go b/pkgtools/pkglint/files/vardefs_test.go index 9cecb499bc0..647a5fb2322 100644 --- a/pkgtools/pkglint/files/vardefs_test.go +++ b/pkgtools/pkglint/files/vardefs_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import "gopkg.in/check.v1" diff --git a/pkgtools/pkglint/files/vartype.go b/pkgtools/pkglint/files/vartype.go index 8c8f1579a6d..1ef9f9fc6f6 100644 --- a/pkgtools/pkglint/files/vartype.go +++ b/pkgtools/pkglint/files/vartype.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "path" @@ -37,10 +37,10 @@ const ( aclpUseLoadtime // OTHER := ${VAR}, OTHER != ${VAR} aclpUse // OTHER = ${VAR} aclpUnknown - aclpAll = aclpAppend | aclpSetDefault | aclpSet | aclpUseLoadtime | aclpUse - aclpAllRuntime = aclpAppend | aclpSetDefault | aclpSet | aclpUse aclpAllWrite = aclpSet | aclpSetDefault | aclpAppend aclpAllRead = aclpUseLoadtime | aclpUse + aclpAll = aclpAllWrite | aclpAllRead + aclpAllRuntime = aclpAll &^ aclpUseLoadtime ) func (perms ACLPermissions) Contains(subset ACLPermissions) bool { diff --git a/pkgtools/pkglint/files/vartype_test.go b/pkgtools/pkglint/files/vartype_test.go index 3142ba187f8..e1b485553ad 100644 --- a/pkgtools/pkglint/files/vartype_test.go +++ b/pkgtools/pkglint/files/vartype_test.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "gopkg.in/check.v1" @@ -44,6 +44,18 @@ func (s *Suite) Test_ACLPermissions_String(c *check.C) { c.Check(aclpUnknown.String(), equals, "unknown") } +func (s *Suite) Test_ACLPermissions_HumanString(c *check.C) { + + c.Check(ACLPermissions(0).HumanString(), + equals, "") // Doesn't happen in practice + + c.Check(aclpAll.HumanString(), + equals, "set, given a default value, appended to, used at load time, used") + + c.Check(aclpUnknown.HumanString(), + equals, "") // Doesn't happen in practice +} + func (s *Suite) Test_Vartype_IsConsideredList(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go index 621eb31ac59..50302598efd 100644 --- a/pkgtools/pkglint/files/vartypecheck.go +++ b/pkgtools/pkglint/files/vartypecheck.go @@ -1,4 +1,4 @@ -package main +package pkglint import ( "path" @@ -222,9 +222,9 @@ func (cv *VartypeCheck) Comment() { if m, isA := match1(value, ` (is a|is an) `); m { cv.Warnf("COMMENT should not contain %q.", isA) G.Explain( - "The words \"package is a\" are redundant. Since every package comment", - "could start with them, it is better to remove this redundancy in all", - "cases.") + "The words \"package is a\" are redundant.", + "Since every package comment could start with them,", + "it is better to remove this redundancy in all cases.") } if G.Pkg != nil && G.Pkg.EffectivePkgbase != "" { pkgbase := G.Pkg.EffectivePkgbase @@ -285,8 +285,9 @@ func (cv *VartypeCheck) Dependency() { G.Explain( "The \"{,nb*}\" extension is only necessary for dependencies of the", "form \"pkgbase-1.2\", since the pattern \"pkgbase-1.2\" doesn't match", - "the version \"pkgbase-1.2nb5\". For dependency patterns using the", - "comparison operators, this is not necessary.") + "the version \"pkgbase-1.2nb5\".", + "For dependency patterns using the comparison operators,", + "this is not necessary.") } else if deppat == nil || !parser.EOF() { cv.Warnf("Invalid dependency pattern %q.", value) @@ -305,9 +306,9 @@ func (cv *VartypeCheck) Dependency() { if inside != "0-9" { cv.Warnf("Only [0-9]* is allowed in the numeric part of a dependency.") G.Explain( - "The pattern -[0-9] means any version. All other version patterns", - "should be expressed using the comparison operators like < or >= or", - "even >=2<3.", + "The pattern -[0-9] means any version.", + "All other version patterns should be expressed using", + "the comparison operators like < or >= or even >=2<3.", "", "Patterns like -[0-7] will only match the first digit of the version", "number and will not do the correct thing when the package reaches", @@ -333,8 +334,9 @@ func (cv *VartypeCheck) Dependency() { cv.Warnf("Please use \"%[1]s-[0-9]*\" instead of \"%[1]s-*\".", deppat.Pkgbase) G.Explain( "If you use a * alone, the package specification may match other", - "packages that have the same prefix but a longer name. For example,", - "foo-* matches foo-1.2 but also foo-client-1.2 and foo-server-1.2.") + "packages that have the same prefix but a longer name.", + "For example, foo-* matches foo-1.2 but also", + "foo-client-1.2 and foo-server-1.2.") } withoutCharClasses := replaceAll(wildcard, `\[[\d-]+\]`, "") @@ -676,9 +678,9 @@ func (cv *VartypeCheck) Message() { cv.Warnf("%s should not be quoted.", varname) G.Explain( "The quoting is only needed for variables which are interpreted as", - "multiple words (or, generally speaking, a list of something). A", - "single text message does not belong to this class, since it is only", - "printed as a whole.") + "multiple words (or, generally speaking, a list of something).", + "A single text message does not belong to this class,", + "since it is only printed as a whole.") } } @@ -698,8 +700,8 @@ func (cv *VartypeCheck) Option() { if _, found := G.Pkgsrc.PkgOptions[optname]; !found { // There's a difference between empty and absent here. cv.Warnf("Unknown option %q.", optname) G.Explain( - "This option is not documented in the mk/defaults/options.description", - "file. Please think of a brief but precise description and either", + "This option is not documented in the mk/defaults/options.description file.", + "Please think of a brief but precise description and either", "update that file yourself or suggest a description for this option", "on the tech-pkg@NetBSD.org mailing list.") } @@ -832,8 +834,8 @@ func (cv *VartypeCheck) PkgRevision() { "Usually, different packages using the same Makefile.common have", "different dependencies and will be bumped at different times (e.g.", "for shlib major bumps) and thus the PKGREVISIONs must be in the", - "separate Makefiles. There is no practical way of having this", - "information in a commonly used Makefile.") + "separate Makefiles.", + "There is no practical way of having this information in a commonly used Makefile.") } } @@ -896,8 +898,8 @@ func (cv *VartypeCheck) PythonDependency() { cv.Warnf("Invalid Python dependency %q.", cv.Value) G.Explain( "Python dependencies must be an identifier for a package, as", - "specified in lang/python/versioned_dependencies.mk. This", - "identifier may be followed by :build for a build-time only", + "specified in lang/python/versioned_dependencies.mk.", + "This identifier may be followed by :build for a build-time only", "dependency, or by :link for a run-time only dependency.") } } @@ -920,7 +922,8 @@ func (cv *VartypeCheck) Restricted() { cv.Warnf("The only valid value for %s is ${RESTRICTED}.", cv.Varname) G.Explain( "These variables are used to control which files may be mirrored on", - "FTP servers or CD-ROM collections. They are not intended to mark", + "FTP servers or CD-ROM collections.", + "They are not intended to mark", "packages whose only MASTER_SITES are on ftp.NetBSD.org.") } } @@ -1169,10 +1172,9 @@ func (cv *VartypeCheck) Yes() { if !matches(cv.Value, `^(?:YES|yes)(?:[\t ]+#.*)?$`) { cv.Warnf("%s should be set to YES or yes.", cv.Varname) G.Explain( - "This variable means \"yes\" if it is defined, and \"no\" if it is", - "undefined. Even when it has the value \"no\", this means \"yes\".", - "Therefore when it is defined, its value should correspond to its", - "meaning.") + "This variable means \"yes\" if it is defined, and \"no\" if it is undefined.", + "Even when it has the value \"no\", this means \"yes\".", + "Therefore when it is defined, its value should correspond to its meaning.") } } } @@ -1194,8 +1196,9 @@ func (cv *VartypeCheck) YesNo() { cv.Warnf("%s should be matched against %q or %q, not compared with %q.", cv.Varname, yes1, no1, cv.Value) G.Explain( "The yes/no value can be written in either upper or lower case, and", - "both forms are actually used. As long as this is the case, when", - "checking the variable value, both must be accepted.") + "both forms are actually used.", + "As long as this is the case, when checking the variable value,", + "both must be accepted.") } else if !matches(cv.Value, `^(?:YES|yes|NO|no)(?:[\t ]+#.*)?$`) { cv.Warnf("%s should be set to YES, yes, NO, or no.", cv.Varname) } diff --git a/pkgtools/pkglint/files/vartypecheck_test.go b/pkgtools/pkglint/files/vartypecheck_test.go index bd733ae0c3d..02cb9d1984a 100644 --- a/pkgtools/pkglint/files/vartypecheck_test.go +++ b/pkgtools/pkglint/files/vartypecheck_test.go @@ -1,8 +1,6 @@ -package main +package pkglint import ( - "fmt" - "gopkg.in/check.v1" ) @@ -16,7 +14,7 @@ func (s *Suite) Test_VartypeCheck_AwkCommand(c *check.C) { "{print $$0}") vt.Output( - "WARN: filename:1: $0 is ambiguous. Use ${0} if you mean a Makefile variable or $$0 if you mean a shell variable.") + "WARN: filename:1: $0 is ambiguous. Use ${0} if you mean a Make variable or $$0 if you mean a shell variable.") } func (s *Suite) Test_VartypeCheck_BasicRegularExpression(c *check.C) { @@ -28,7 +26,7 @@ func (s *Suite) Test_VartypeCheck_BasicRegularExpression(c *check.C) { ".*\\.pl$$") vt.Output( - "WARN: filename:1: Pkglint parse error in MkLine.Tokenize at \"$\".") + "WARN: filename:1: Internal pkglint error in MkLine.Tokenize at \"$\".") } func (s *Suite) Test_VartypeCheck_BuildlinkDepmethod(c *check.C) { @@ -1241,7 +1239,7 @@ func (vt *VartypeCheckTester) Values(values ...string) { text = varname + opStr + value } case op == opUseMatch: - text = fmt.Sprintf(".if ${%s:M%s} == \"\"", varname, value) + text = sprintf(".if ${%s:M%s} == \"\"", varname, value) default: panic("Invalid operator: " + opStr) } |