diff options
55 files changed, 3217 insertions, 2598 deletions
diff --git a/pkgtools/pkglint/Makefile b/pkgtools/pkglint/Makefile index 80fc4895d0f..5098bb89818 100644 --- a/pkgtools/pkglint/Makefile +++ b/pkgtools/pkglint/Makefile @@ -1,6 +1,6 @@ -# $NetBSD: Makefile,v 1.508 2017/01/11 23:20:13 tron Exp $ +# $NetBSD: Makefile,v 1.509 2017/01/17 22:37:27 rillig Exp $ -PKGNAME= pkglint-5.4.15 +PKGNAME= pkglint-5.4.16 DISTFILES= # none CATEGORIES= pkgtools diff --git a/pkgtools/pkglint/files/buildlink3.go b/pkgtools/pkglint/files/buildlink3.go index 1105c696053..dfa8cd821f9 100644 --- a/pkgtools/pkglint/files/buildlink3.go +++ b/pkgtools/pkglint/files/buildlink3.go @@ -2,12 +2,13 @@ package main import ( "netbsd.org/pkglint/pkgver" + "netbsd.org/pkglint/trace" "strings" ) func ChecklinesBuildlink3Mk(mklines *MkLines) { - if G.opts.Debug { - defer tracecall1(mklines.lines[0].Fname)() + if trace.Tracing { + defer trace.Call1(mklines.lines[0].Filename())() } mklines.Check() @@ -17,7 +18,7 @@ func ChecklinesBuildlink3Mk(mklines *MkLines) { for exp.AdvanceIfPrefix("#") { line := exp.PreviousLine() // See pkgtools/createbuildlink/files/createbuildlink - if hasPrefix(line.Text, "# XXX This file was created automatically") { + if hasPrefix(line.Text(), "# XXX This file was created automatically") { line.Errorf("This comment indicates unfinished work (url2pkg).") } } @@ -31,7 +32,7 @@ func ChecklinesBuildlink3Mk(mklines *MkLines) { } pkgbaseLine, pkgbase := exp.CurrentLine(), "" - var abiLine, apiLine *Line + var abiLine, apiLine Line var abi, api *DependencyPattern // First paragraph: Introduction of the package identifier @@ -42,7 +43,7 @@ func ChecklinesBuildlink3Mk(mklines *MkLines) { pkgbase = exp.m[1] if containsVarRef(pkgbase) { warned := false - for _, pair := range []struct{ varuse, simple string }{ + for _, pair := range [...]struct{ varuse, simple string }{ {"${PYPKGPREFIX}", "py"}, {"${RUBY_BASE}", "ruby"}, {"${RUBY_PKGPREFIX}", "ruby"}, @@ -133,16 +134,16 @@ func ChecklinesBuildlink3Mk(mklines *MkLines) { } doCheck = true } - if doCheck && abi != nil && api != nil && abi.pkgbase != api.pkgbase && !hasPrefix(api.pkgbase, "{") { + if doCheck && abi != nil && api != nil && abi.Pkgbase != api.Pkgbase && !hasPrefix(api.Pkgbase, "{") { abiLine.Warnf("Package name mismatch between ABI %q and API %q (from %s).", - abi.pkgbase, api.pkgbase, apiLine.ReferenceFrom(abiLine)) + abi.Pkgbase, api.Pkgbase, apiLine.ReferenceFrom(abiLine)) } if doCheck { - if abi != nil && abi.lower != "" && !containsVarRef(abi.lower) { - if api != nil && api.lower != "" && !containsVarRef(api.lower) { - if pkgver.Compare(abi.lower, api.lower) < 0 { + if abi != nil && abi.Lower != "" && !containsVarRef(abi.Lower) { + if api != nil && api.Lower != "" && !containsVarRef(api.Lower) { + if pkgver.Compare(abi.Lower, api.Lower) < 0 { abiLine.Warnf("ABI version %q should be at least API version %q (see %s).", - abi.lower, api.lower, apiLine.ReferenceFrom(abiLine)) + abi.Lower, api.Lower, apiLine.ReferenceFrom(abiLine)) } } } @@ -175,8 +176,8 @@ func ChecklinesBuildlink3Mk(mklines *MkLines) { } } else { - if G.opts.Debug { - traceStep1("Unchecked line %s in third paragraph.", exp.CurrentLine().linenos()) + if trace.Tracing { + trace.Step1("Unchecked line %s in third paragraph.", exp.CurrentLine().Linenos()) } exp.Advance() } diff --git a/pkgtools/pkglint/files/category.go b/pkgtools/pkglint/files/category.go index 6f2fa691238..4e874ea2a6f 100644 --- a/pkgtools/pkglint/files/category.go +++ b/pkgtools/pkglint/files/category.go @@ -1,12 +1,13 @@ package main import ( + "netbsd.org/pkglint/trace" "sort" ) func CheckdirCategory() { - if G.opts.Debug { - defer tracecall1(G.CurrentDir)() + if trace.Tracing { + defer trace.Call1(G.CurrentDir)() } lines := LoadNonemptyLines(G.CurrentDir+"/Makefile", true) @@ -23,7 +24,7 @@ func CheckdirCategory() { exp.ExpectEmptyLine() if exp.AdvanceIfMatches(`^COMMENT=\t*(.*)`) { - mklines.mklines[exp.index-1].CheckValidCharactersInValue(`[- '(),/0-9A-Za-z]`) + MkLineChecker{mklines.mklines[exp.index-1]}.CheckValidCharactersInValue(`[- '(),/0-9A-Za-z]`) } else { exp.CurrentLine().Errorf("COMMENT= line expected.") } @@ -31,7 +32,7 @@ func CheckdirCategory() { type subdir struct { name string - line *Line + line Line active bool } @@ -46,7 +47,7 @@ func CheckdirCategory() { prevSubdir := "" for !exp.EOF() { line := exp.CurrentLine() - text := line.Text + text := line.Text() if m, commentFlag, indentation, name, comment := match4(text, `^(#?)SUBDIR\+=(\s*)(\S+)\s*(?:#\s*(.*?)\s*|)$`); m { commentedOut := commentFlag == "#" @@ -71,7 +72,7 @@ func CheckdirCategory() { exp.Advance() } else { - if line.Text != "" { + if line.Text() != "" { line.Errorf("SUBDIR+= line or empty line expected.") } break @@ -95,7 +96,7 @@ func CheckdirCategory() { var subdirs []string - var line *Line + var line Line mActive := false for !(mAtend && fAtend) { diff --git a/pkgtools/pkglint/files/check_test.go b/pkgtools/pkglint/files/check_test.go index 291f76bc054..f6471cb1535 100644 --- a/pkgtools/pkglint/files/check_test.go +++ b/pkgtools/pkglint/files/check_test.go @@ -11,6 +11,8 @@ import ( "testing" check "gopkg.in/check.v1" + "netbsd.org/pkglint/textproc" + "netbsd.org/pkglint/trace" ) var equals = check.Equals @@ -79,8 +81,8 @@ func (s *Suite) NewRawLines(args ...interface{}) []*RawLine { return rawlines[:j] } -func (s *Suite) NewLines(fname string, texts ...string) []*Line { - result := make([]*Line, len(texts)) +func (s *Suite) NewLines(fname string, texts ...string) []Line { + result := make([]Line, len(texts)) for i, text := range texts { textnl := text + "\n" result[i] = NewLine(fname, i+1, text, s.NewRawLines(i+1, textnl)) @@ -93,15 +95,15 @@ func (s *Suite) NewMkLines(fname string, lines ...string) *MkLines { } func (s *Suite) BeginDebugToStdout() { - G.debugOut = os.Stdout G.logOut = os.Stdout - G.opts.Debug = true + trace.Out = os.Stdout + trace.Tracing = true } func (s *Suite) EndDebugToStdout() { - G.debugOut = &s.stdout G.logOut = &s.stdout - G.opts.Debug = false + trace.Out = &s.stdout + trace.Tracing = false } func (s *Suite) UseCommandLine(args ...string) { @@ -185,7 +187,8 @@ func (s *Suite) ExpectFatalError(action func()) { func (s *Suite) SetUpTest(c *check.C) { G = GlobalVars{Testing: true} - G.logOut, G.logErr, G.debugOut = &s.stdout, &s.stderr, &s.stdout + textproc.Testing = true + G.logOut, G.logErr, trace.Out = &s.stdout, &s.stderr, &s.stdout s.checkC = c s.UseCommandLine( /* no arguments */ ) s.checkC = nil @@ -194,6 +197,7 @@ func (s *Suite) SetUpTest(c *check.C) { func (s *Suite) TearDownTest(c *check.C) { G = GlobalVars{} + textproc.Testing = false if out := s.Output(); out != "" { fmt.Fprintf(os.Stderr, "Unchecked output in %q; check with: c.Check(s.Output(), equals, %q)", c.TestName(), out) } diff --git a/pkgtools/pkglint/files/distinfo.go b/pkgtools/pkglint/files/distinfo.go index 2cfb4b38c62..90c445601bc 100644 --- a/pkgtools/pkglint/files/distinfo.go +++ b/pkgtools/pkglint/files/distinfo.go @@ -5,15 +5,16 @@ import ( "crypto/sha1" "fmt" "io/ioutil" + "netbsd.org/pkglint/trace" "strings" ) -func ChecklinesDistinfo(lines []*Line) { - if G.opts.Debug { - defer tracecall1(lines[0].Fname)() +func ChecklinesDistinfo(lines []Line) { + if trace.Tracing { + defer trace.Call1(lines[0].Filename())() } - fname := lines[0].Fname + fname := lines[0].Filename() patchesDir := "patches" patchesDirSet := false if G.Pkg != nil && contains(fname, "lang/php") { @@ -26,8 +27,8 @@ func ChecklinesDistinfo(lines []*Line) { if G.Pkg != nil && !patchesDirSet && dirExists(G.CurrentDir+"/"+G.Pkg.Patchdir) { patchesDir = G.Pkg.Patchdir } - if G.opts.Debug { - traceStep1("patchesDir=%q", patchesDir) + if trace.Tracing { + trace.Step1("patchesDir=%q", patchesDir) } ck := &distinfoLinesChecker{ @@ -45,15 +46,15 @@ type distinfoLinesChecker struct { distinfoIsCommitted bool patches map[string]bool // "patch-aa" => true - currentFirstLine *Line + currentFirstLine Line currentFilename string isPatch bool algorithms []string } -func (ck *distinfoLinesChecker) checkLines(lines []*Line) { - lines[0].CheckRcsid(``, "") - if 1 < len(lines) && lines[1].Text != "" { +func (ck *distinfoLinesChecker) checkLines(lines []Line) { + LineChecker{lines[0]}.CheckRcsid(``, "") + if 1 < len(lines) && lines[1].Text() != "" { lines[1].Notef("Empty line expected.") } @@ -61,7 +62,7 @@ func (ck *distinfoLinesChecker) checkLines(lines []*Line) { if i < 2 { continue } - m, alg, filename, hash := match3(line.Text, `^(\w+) \((\w[^)]*)\) = (.*)(?: bytes)?$`) + m, alg, filename, hash := match3(line.Text(), `^(\w+) \((\w[^)]*)\) = (.*)(?: bytes)?$`) if !m { line.Errorf("Invalid line.") continue @@ -78,7 +79,7 @@ func (ck *distinfoLinesChecker) checkLines(lines []*Line) { ck.onFilenameChange(NewLineEOF(ck.distinfoFilename), "") } -func (ck *distinfoLinesChecker) onFilenameChange(line *Line, nextFname string) { +func (ck *distinfoLinesChecker) onFilenameChange(line Line, nextFname string) { currentFname := ck.currentFilename if currentFname != "" { algorithms := strings.Join(ck.algorithms, ", ") @@ -105,7 +106,7 @@ func (ck *distinfoLinesChecker) onFilenameChange(line *Line, nextFname string) { ck.algorithms = nil } -func (ck *distinfoLinesChecker) checkPatchSha1(line *Line, patchFname, distinfoSha1Hex string) { +func (ck *distinfoLinesChecker) checkPatchSha1(line Line, patchFname, distinfoSha1Hex string) { patchBytes, err := ioutil.ReadFile(G.CurrentDir + "/" + patchFname) if err != nil { line.Errorf("%s does not exist.", patchFname) @@ -130,8 +131,8 @@ func (ck *distinfoLinesChecker) checkPatchSha1(line *Line, patchFname, distinfoS func (ck *distinfoLinesChecker) checkUnrecordedPatches() { files, err := ioutil.ReadDir(G.CurrentDir + "/" + ck.patchdir) if err != nil { - if G.opts.Debug { - traceStep("Cannot read patchesDir %q: %s", ck.patchdir, err) + if trace.Tracing { + trace.Stepf("Cannot read patchesDir %q: %s", ck.patchdir, err) } return } @@ -145,7 +146,7 @@ func (ck *distinfoLinesChecker) checkUnrecordedPatches() { } // Inter-package check for differing distfile checksums. -func (ck *distinfoLinesChecker) checkGlobalMismatch(line *Line, fname, alg, hash string) { +func (ck *distinfoLinesChecker) checkGlobalMismatch(line Line, fname, alg, hash string) { if G.Hash != nil && !hasPrefix(fname, "patch-") { // Intentionally checking the filename instead of ck.isPatch key := alg + ":" + fname otherHash := G.Hash[key] @@ -160,7 +161,7 @@ func (ck *distinfoLinesChecker) checkGlobalMismatch(line *Line, fname, alg, hash } } -func (ck *distinfoLinesChecker) checkUncommittedPatch(line *Line, patchName, sha1Hash string) { +func (ck *distinfoLinesChecker) checkUncommittedPatch(line Line, patchName, sha1Hash string) { if ck.isPatch { patchFname := ck.patchdir + "/" + patchName if ck.distinfoIsCommitted && !isCommitted(G.CurrentDir+"/"+patchFname) { diff --git a/pkgtools/pkglint/files/expecter.go b/pkgtools/pkglint/files/expecter.go index 5e188d8a077..e0f141cba7a 100644 --- a/pkgtools/pkglint/files/expecter.go +++ b/pkgtools/pkglint/files/expecter.go @@ -1,25 +1,30 @@ package main +import ( + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/trace" +) + // Expecter records the state when checking a list of lines from top to bottom. type Expecter struct { - lines []*Line + lines []Line index int m []string } -func NewExpecter(lines []*Line) *Expecter { +func NewExpecter(lines []Line) *Expecter { return &Expecter{lines, 0, nil} } -func (exp *Expecter) CurrentLine() *Line { +func (exp *Expecter) CurrentLine() Line { if exp.index < len(exp.lines) { return exp.lines[exp.index] } - return NewLineEOF(exp.lines[0].Fname) + return NewLineEOF(exp.lines[0].Filename()) } -func (exp *Expecter) PreviousLine() *Line { +func (exp *Expecter) PreviousLine() Line { return exp.lines[exp.index-1] } @@ -37,13 +42,13 @@ func (exp *Expecter) StepBack() { exp.index-- } -func (exp *Expecter) AdvanceIfMatches(re RegexPattern) bool { - if G.opts.Debug { - defer tracecall(exp.CurrentLine().Text, re)() +func (exp *Expecter) AdvanceIfMatches(re regex.RegexPattern) bool { + if trace.Tracing { + defer trace.Call(exp.CurrentLine().Text(), re)() } if !exp.EOF() { - if m := match(exp.lines[exp.index].Text, re); m != nil { + if m := regex.Match(exp.lines[exp.index].Text(), re); m != nil { exp.index++ exp.m = m return true @@ -53,19 +58,19 @@ func (exp *Expecter) AdvanceIfMatches(re RegexPattern) bool { } func (exp *Expecter) AdvanceIfPrefix(prefix string) bool { - if G.opts.Debug { - defer tracecall2(exp.CurrentLine().Text, prefix)() + if trace.Tracing { + defer trace.Call2(exp.CurrentLine().Text(), prefix)() } - return !exp.EOF() && hasPrefix(exp.lines[exp.index].Text, prefix) && exp.Advance() + return !exp.EOF() && hasPrefix(exp.lines[exp.index].Text(), prefix) && exp.Advance() } func (exp *Expecter) AdvanceIfEquals(text string) bool { - if G.opts.Debug { - defer tracecall2(exp.CurrentLine().Text, text)() + if trace.Tracing { + defer trace.Call2(exp.CurrentLine().Text(), text)() } - return !exp.EOF() && exp.lines[exp.index].Text == text && exp.Advance() + return !exp.EOF() && exp.lines[exp.index].Text() == text && exp.Advance() } func (exp *Expecter) ExpectEmptyLine() bool { @@ -82,7 +87,7 @@ func (exp *Expecter) ExpectEmptyLine() bool { } func (exp *Expecter) ExpectText(text string) bool { - if !exp.EOF() && exp.lines[exp.index].Text == text { + if !exp.EOF() && exp.lines[exp.index].Text() == text { exp.index++ exp.m = nil return true diff --git a/pkgtools/pkglint/files/files.go b/pkgtools/pkglint/files/files.go index cfdf4339b33..9faada41728 100644 --- a/pkgtools/pkglint/files/files.go +++ b/pkgtools/pkglint/files/files.go @@ -6,7 +6,7 @@ import ( "strings" ) -func LoadNonemptyLines(fname string, joinBackslashLines bool) []*Line { +func LoadNonemptyLines(fname string, joinBackslashLines bool) []Line { lines, err := readLines(fname, joinBackslashLines) if err != nil { NewLineWhole(fname).Errorf("Cannot be read.") @@ -19,7 +19,7 @@ func LoadNonemptyLines(fname string, joinBackslashLines bool) []*Line { return lines } -func LoadExistingLines(fname string, joinBackslashLines bool) []*Line { +func LoadExistingLines(fname string, joinBackslashLines bool) []Line { lines, err := readLines(fname, joinBackslashLines) if err != nil { NewLineWhole(fname).Fatalf("Cannot be read.") @@ -30,7 +30,7 @@ func LoadExistingLines(fname string, joinBackslashLines bool) []*Line { return lines } -func getLogicalLine(fname string, rawLines []*RawLine, pindex *int) *Line { +func getLogicalLine(fname string, rawLines []*RawLine, pindex *int) Line { { // Handle the common case efficiently index := *pindex rawLine := rawLines[index] @@ -101,7 +101,7 @@ func splitRawLine(textnl string) (leadingWhitespace, text, trailingWhitespace, c return } -func readLines(fname string, joinBackslashLines bool) ([]*Line, error) { +func readLines(fname string, joinBackslashLines bool) ([]Line, error) { rawText, err := ioutil.ReadFile(fname) if err != nil { return nil, err @@ -110,7 +110,7 @@ func readLines(fname string, joinBackslashLines bool) ([]*Line, error) { return convertToLogicalLines(fname, string(rawText), joinBackslashLines), nil } -func convertToLogicalLines(fname string, rawText string, joinBackslashLines bool) []*Line { +func convertToLogicalLines(fname string, rawText string, joinBackslashLines bool) []Line { var rawLines []*RawLine for lineno, rawLine := range strings.SplitAfter(rawText, "\n") { if rawLine != "" { @@ -118,7 +118,7 @@ func convertToLogicalLines(fname string, rawText string, joinBackslashLines bool } } - var loglines []*Line + var loglines []Line if joinBackslashLines { for lineno := 0; lineno < len(rawLines); { loglines = append(loglines, getLogicalLine(fname, rawLines, &lineno)) @@ -138,10 +138,10 @@ func convertToLogicalLines(fname string, rawText string, joinBackslashLines bool return loglines } -func SaveAutofixChanges(lines []*Line) (autofixed bool) { +func SaveAutofixChanges(lines []Line) (autofixed bool) { if !G.opts.Autofix { for _, line := range lines { - if line.changed { + if line.IsChanged() { G.autofixAvailable = true } } @@ -151,10 +151,10 @@ func SaveAutofixChanges(lines []*Line) (autofixed bool) { changes := make(map[string][]string) changed := make(map[string]bool) for _, line := range lines { - if line.changed { - changed[line.Fname] = true + if line.IsChanged() { + changed[line.Filename()] = true } - changes[line.Fname] = append(changes[line.Fname], line.modifiedLines()...) + changes[line.Filename()] = append(changes[line.Filename()], line.(*LineImpl).modifiedLines()...) } for fname := range changed { diff --git a/pkgtools/pkglint/files/files_test.go b/pkgtools/pkglint/files/files_test.go index 556fb50fe40..35e19b1cd42 100644 --- a/pkgtools/pkglint/files/files_test.go +++ b/pkgtools/pkglint/files/files_test.go @@ -70,7 +70,7 @@ func (s *Suite) Test_show_autofix(c *check.C) { } SaveAutofixChanges(lines) - c.Check(lines[1].raw[0].textnl, equals, "XXXXX\n") + c.Check(lines[1].(*LineImpl).raw[0].textnl, equals, "XXXXX\n") c.Check(s.LoadTmpFile("Makefile"), equals, "line1\nline2\nline3\n") c.Check(s.Output(), equals, ""+ "WARN: ~/Makefile:2: Something's wrong here.\n"+ diff --git a/pkgtools/pkglint/files/getopt/getopt_test.go b/pkgtools/pkglint/files/getopt/getopt_test.go index 338fd3974ec..509d158d414 100644 --- a/pkgtools/pkglint/files/getopt/getopt_test.go +++ b/pkgtools/pkglint/files/getopt/getopt_test.go @@ -5,8 +5,7 @@ import ( "testing" ) -type Suite struct { -} +type Suite struct{} var _ = check.Suite(new(Suite)) diff --git a/pkgtools/pkglint/files/globaldata.go b/pkgtools/pkglint/files/globaldata.go index f82a3c935ab..f9cd34e4618 100644 --- a/pkgtools/pkglint/files/globaldata.go +++ b/pkgtools/pkglint/files/globaldata.go @@ -2,6 +2,8 @@ package main import ( "io/ioutil" + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/trace" "path" "sort" "strings" @@ -26,7 +28,7 @@ type GlobalData struct { // Change is a change entry from the `doc/CHANGES-*` files. type Change struct { - Line *Line + Line Line Action string Pkgpath string Version string @@ -36,7 +38,7 @@ type Change struct { // SuggestedUpdate is from the `doc/TODO` file. type SuggestedUpdate struct { - Line *Line + Line Line Pkgname string Version string Comment string @@ -64,7 +66,7 @@ func (gd *GlobalData) Initialize() { gd.loadDeprecatedVars() } -func (gd *GlobalData) Latest(category string, re RegexPattern, repl string) string { +func (gd *GlobalData) Latest(category string, re regex.RegexPattern, repl string) string { key := category + "/" + string(re) + " => " + repl if latest, found := gd.latest[key]; found { return latest @@ -88,7 +90,7 @@ func (gd *GlobalData) Latest(category string, re RegexPattern, repl string) stri latest := "" for _, fileInfo := range all { if matches(fileInfo.Name(), re) { - latest = regcomp(re).ReplaceAllString(fileInfo.Name(), repl) + latest = regex.Compile(re).ReplaceAllString(fileInfo.Name(), repl) } } if latest == "" { @@ -106,7 +108,7 @@ func (gd *GlobalData) loadDistSites() { name2url := make(map[string]string) url2name := make(map[string]string) for _, line := range lines { - if m, varname, _, _, _, urls, _, _ := MatchVarassign(line.Text); m { + if m, varname, _, _, _, urls, _, _ := MatchVarassign(line.Text()); m { if hasPrefix(varname, "MASTER_SITE_") && varname != "MASTER_SITE_BACKUP" { for _, url := range splitOnSpace(urls) { if matches(url, `^(?:http://|https://|ftp://)`) { @@ -123,8 +125,8 @@ func (gd *GlobalData) loadDistSites() { // Explicitly allowed, although not defined in mk/fetch/sites.mk. name2url["MASTER_SITE_LOCAL"] = "ftp://ftp.NetBSD.org/pub/pkgsrc/distfiles/LOCAL_PORTS/" - if G.opts.Debug { - traceStep("Loaded %d MASTER_SITE_* URLs.", len(url2name)) + if trace.Tracing { + trace.Stepf("Loaded %d MASTER_SITE_* URLs.", len(url2name)) } gd.MasterSiteURLToVar = url2name gd.MasterSiteVarToURL = name2url @@ -136,7 +138,7 @@ func (gd *GlobalData) loadPkgOptions() { gd.PkgOptions = make(map[string]string) for _, line := range lines { - if m, optname, optdescr := match2(line.Text, `^([-0-9a-z_+]+)(?:\s+(.*))?$`); m { + if m, optname, optdescr := match2(line.Text(), `^([-0-9a-z_+]+)(?:\s+(.*))?$`); m { gd.PkgOptions[optname] = optdescr } else { line.Fatalf("Unknown line format.") @@ -150,7 +152,7 @@ func (gd *GlobalData) loadTools() { fname := G.globalData.Pkgsrcdir + "/mk/tools/bsd.tools.mk" lines := LoadExistingLines(fname, true) for _, line := range lines { - if m, _, _, includefile := MatchMkInclude(line.Text); m { + if m, _, _, includefile := MatchMkInclude(line.Text()); m { if !contains(includefile, "/") { toolFiles = append(toolFiles, includefile) } @@ -178,23 +180,23 @@ func (gd *GlobalData) loadTools() { } } - for _, basename := range []string{"bsd.prefs.mk", "bsd.pkg.mk"} { + for _, basename := range [...]string{"bsd.prefs.mk", "bsd.pkg.mk"} { fname := G.globalData.Pkgsrcdir + "/mk/" + basename condDepth := 0 lines := LoadExistingLines(fname, true) for _, line := range lines { - text := line.Text + text := line.Text() if m, varname, _, _, _, value, _, _ := MatchVarassign(text); m { if varname == "USE_TOOLS" { - if G.opts.Debug { - traceStep("[condDepth=%d] %s", condDepth, value) + if trace.Tracing { + trace.Stepf("[condDepth=%d] %s", condDepth, value) } if condDepth == 0 || condDepth == 1 && basename == "bsd.prefs.mk" { for _, toolname := range splitOnSpace(value) { if !containsVarRef(toolname) { - for _, tool := range []*Tool{reg.Register(toolname), reg.Register("TOOLS_" + toolname)} { + for _, tool := range [...]*Tool{reg.Register(toolname), reg.Register("TOOLS_" + toolname)} { tool.Predefined = true if basename == "bsd.prefs.mk" { tool.UsableAtLoadtime = true @@ -221,11 +223,11 @@ func (gd *GlobalData) loadTools() { } } - if G.opts.Debug { + if trace.Tracing { reg.Trace() } - if G.opts.Debug { - traceStep("systemBuildDefs: %v", systemBuildDefs) + if trace.Tracing { + trace.Stepf("systemBuildDefs: %v", systemBuildDefs) } // Some user-defined variables do not influence the binary @@ -251,11 +253,11 @@ func loadSuggestedUpdates(fname string) []SuggestedUpdate { return parselinesSuggestedUpdates(lines) } -func parselinesSuggestedUpdates(lines []*Line) []SuggestedUpdate { +func parselinesSuggestedUpdates(lines []Line) []SuggestedUpdate { var updates []SuggestedUpdate state := 0 for _, line := range lines { - text := line.Text + text := line.Text() if state == 0 && text == "Suggested package updates" { state = 1 @@ -292,9 +294,9 @@ func (gd *GlobalData) loadSuggestedUpdates() { func (gd *GlobalData) loadDocChangesFromFile(fname string) []*Change { lines := LoadExistingLines(fname, false) - parseChange := func(line *Line) *Change { - text := line.Text - if !hasPrefix(line.Text, "\t") { + parseChange := func(line Line) *Change { + text := line.Text() + if !hasPrefix(text, "\t") { return nil } @@ -327,8 +329,8 @@ func (gd *GlobalData) loadDocChangesFromFile(fname string) []*Change { for _, line := range lines { if change := parseChange(line); change != nil { changes = append(changes, change) - } else if len(line.Text) >= 2 && line.Text[0] == '\t' && 'A' <= line.Text[1] && line.Text[1] <= 'Z' { - line.Warnf("Unknown doc/CHANGES line: %q", line.Text) + } else if text := line.Text(); len(text) >= 2 && text[0] == '\t' && 'A' <= text[1] && text[1] <= 'Z' { + line.Warnf("Unknown doc/CHANGES line: %q", text) Explain("See mk/misc/developer.mk for the rules.") } } @@ -586,8 +588,8 @@ func (tr *ToolRegistry) RegisterTool(tool *Tool) { } func (tr *ToolRegistry) Trace() { - if G.opts.Debug { - defer tracecall0()() + if trace.Tracing { + defer trace.Call0()() } var keys []string @@ -597,12 +599,12 @@ func (tr *ToolRegistry) Trace() { sort.Strings(keys) for _, toolname := range keys { - traceStep("tool %+v", tr.byName[toolname]) + trace.Stepf("tool %+v", tr.byName[toolname]) } } -func (tr *ToolRegistry) ParseToolLine(line *Line) { - if m, varname, _, _, _, value, _, _ := MatchVarassign(line.Text); m { +func (tr *ToolRegistry) ParseToolLine(line Line) { + if m, varname, _, _, _, value, _, _ := MatchVarassign(line.Text()); m { if varname == "TOOLS_CREATE" && (value == "[" || matches(value, `^?[-\w.]+$`)) { tr.Register(value) diff --git a/pkgtools/pkglint/files/globaldata_test.go b/pkgtools/pkglint/files/globaldata_test.go index 438ace98068..700878da1f1 100644 --- a/pkgtools/pkglint/files/globaldata_test.go +++ b/pkgtools/pkglint/files/globaldata_test.go @@ -2,6 +2,7 @@ package main import ( check "gopkg.in/check.v1" + "netbsd.org/pkglint/trace" ) func (s *Suite) Test_GlobalData_InitVartypes(c *check.C) { @@ -55,7 +56,7 @@ func (s *Suite) Test_GlobalData_loadTools(c *check.C) { G.globalData.loadTools() - G.opts.Debug = true + trace.Tracing = true G.globalData.Tools.Trace() c.Check(s.Output(), equals, ""+ @@ -102,7 +103,7 @@ func (s *Suite) Test_GlobalData_deprecated(c *check.C) { G.globalData.loadDeprecatedVars() line := NewLine("Makefile", 5, "USE_PERL5=\tyes", nil) - NewMkLine(line).checkVarassign() + MkLineChecker{NewMkLine(line)}.checkVarassign() c.Check(s.Output(), equals, "WARN: Makefile:5: Definition of USE_PERL5 is deprecated. Use USE_TOOLS+=perl or USE_TOOLS+=perl:run instead.\n") } diff --git a/pkgtools/pkglint/files/globalvars.go b/pkgtools/pkglint/files/globalvars.go index 2cf78ae367e..40b1842b13d 100644 --- a/pkgtools/pkglint/files/globalvars.go +++ b/pkgtools/pkglint/files/globalvars.go @@ -2,7 +2,7 @@ package main import ( "io" - "regexp" + "netbsd.org/pkglint/histogram" ) type GlobalVars struct { @@ -19,7 +19,7 @@ type GlobalVars struct { Testing bool // Is pkglint in self-testing mode (only during development)? CurrentUsername string // For checking against OWNER and MAINTAINER CvsEntriesDir string // Cached to avoid I/O - CvsEntriesLines []*Line + CvsEntriesLines []Line Hash map[string]*Hash // Maps "alg:fname" => hash (inter-package check). UsedLicenses map[string]bool // Maps "license name" => true (inter-package check). @@ -30,16 +30,10 @@ type GlobalVars struct { explanationsAvailable bool explanationsGiven map[string]bool autofixAvailable bool - traceDepth int logOut io.Writer logErr io.Writer - debugOut io.Writer - res map[RegexPattern]*regexp.Regexp // Compiled regular expressions - rematch *Histogram - renomatch *Histogram - retime *Histogram // Total time taken by matching a regular expression - loghisto *Histogram + loghisto *histogram.Histogram } type CmdOpts struct { @@ -78,7 +72,6 @@ type CmdOpts struct { Profiling, Quiet, Recursive, - Debug, PrintAutofix, PrintSource, PrintVersion bool @@ -88,7 +81,7 @@ type CmdOpts struct { type Hash struct { hash string - line *Line + line Line } var G GlobalVars diff --git a/pkgtools/pkglint/files/histogram/histogram.go b/pkgtools/pkglint/files/histogram/histogram.go new file mode 100644 index 00000000000..a915c382daf --- /dev/null +++ b/pkgtools/pkglint/files/histogram/histogram.go @@ -0,0 +1,55 @@ +package histogram + +import ( + "fmt" + "io" + "sort" +) + +type Histogram struct { + histo map[string]int +} + +func New() *Histogram { + return &Histogram{make(map[string]int)} +} + +func (h *Histogram) Add(s string, n int) { + h.histo[s] += n +} + +func (h *Histogram) PrintStats(caption string, out io.Writer, limit int) { + entries := make([]entry, len(h.histo)) + + i := 0 + for s, count := range h.histo { + entries[i] = entry{s, count} + i++ + } + + sort.Sort(byCountDesc(entries)) + + for i, entry := range entries { + fmt.Fprintf(out, "%s %6d %s\n", caption, entry.count, entry.s) + if limit > 0 && i >= limit { + break + } + } +} + +type entry struct { + s string + count int +} + +type byCountDesc []entry + +func (a byCountDesc) Len() int { + return len(a) +} +func (a byCountDesc) Swap(i, j int) { + a[i], a[j] = a[j], a[i] +} +func (a byCountDesc) Less(i, j int) bool { + return a[j].count < a[i].count || (a[i].count == a[j].count && a[i].s < a[j].s) +} diff --git a/pkgtools/pkglint/files/license.y b/pkgtools/pkglint/files/license.y deleted file mode 100755 index 802fc8a3713..00000000000 --- a/pkgtools/pkglint/files/license.y +++ /dev/null @@ -1,35 +0,0 @@ -%{ -package main -%} - -%token <Node> ltNAME -%token ltAND ltOR ltOPEN ltCLOSE - -%union { - Node *LicenseCondition -} - -%type <Node> start list condition - -%% - -start : list { - liyylex.(*licenseLexer).result = $$ -} - -list : condition { - $$ = $1 -} -list : list ltAND condition { - $$.And = append($$.And, $3) -} -list : list ltOR condition { - $$.Or = append($$.Or, $3) -} - -condition : ltNAME { - $$ = $1 -} -condition : ltOPEN list ltCLOSE { - $$ = &LicenseCondition{Main: $2} -} diff --git a/pkgtools/pkglint/files/licenses.go b/pkgtools/pkglint/files/licenses.go index 5051560ad99..dfa5e66700c 100644 --- a/pkgtools/pkglint/files/licenses.go +++ b/pkgtools/pkglint/files/licenses.go @@ -2,77 +2,9 @@ package main import ( "io/ioutil" + "netbsd.org/pkglint/licenses" ) -//go:generate go tool yacc -p liyy -o licenseyacc.go -v licenseyacc.log license.y - -// LicenseCondition describes a complex license condition. -// It has either `Name` or `Main` set. -type LicenseCondition struct { - Name string - Main *LicenseCondition - And []*LicenseCondition - Or []*LicenseCondition -} - -func (lc *LicenseCondition) Walk(callback func(*LicenseCondition)) { - callback(lc) - if lc.Main != nil { - lc.Main.Walk(callback) - } - for _, and := range lc.And { - and.Walk(callback) - } - for _, or := range lc.Or { - or.Walk(callback) - } -} - -type licenseLexer struct { - repl *PrefixReplacer - result *LicenseCondition - error string -} - -func (lexer *licenseLexer) Lex(llval *liyySymType) int { - repl := lexer.repl - repl.AdvanceHspace() - switch { - case repl.rest == "": - return 0 - case repl.AdvanceStr("("): - return ltOPEN - case repl.AdvanceStr(")"): - return ltCLOSE - case repl.AdvanceRegexp(`^[\w-.]+`): - word := repl.m[0] - switch word { - case "AND": - return ltAND - case "OR": - return ltOR - default: - llval.Node = &LicenseCondition{Name: word} - return ltNAME - } - } - return -1 -} - -func (lexer *licenseLexer) Error(s string) { - lexer.error = s -} - -func parseLicenses(licenses string) *LicenseCondition { - expanded := resolveVariableRefs(licenses) // For ${PERL5_LICENSE} - lexer := &licenseLexer{repl: NewPrefixReplacer(expanded)} - result := liyyNewParser().Parse(lexer) - if result == 0 { - return lexer.result - } - return nil -} - func checkToplevelUnusedLicenses() { if G.UsedLicenses == nil { return @@ -96,13 +28,14 @@ type LicenseChecker struct { } func (lc *LicenseChecker) Check(value string, op MkOperator) { - licenses := parseLicenses(ifelseStr(op == opAssignAppend, "append-placeholder ", "") + value) + expanded := resolveVariableRefs(value) // For ${PERL5_LICENSE} + licenses := licenses.Parse(ifelseStr(op == opAssignAppend, "append-placeholder ", "") + expanded) if licenses == nil { if op == opAssign { - lc.MkLine.Line.Errorf("Parse error for license condition %q.", value) + lc.MkLine.Errorf("Parse error for license condition %q.", value) } else { - lc.MkLine.Line.Errorf("Parse error for appended license condition %q.", value) + lc.MkLine.Errorf("Parse error for appended license condition %q.", value) } return } @@ -110,12 +43,7 @@ func (lc *LicenseChecker) Check(value string, op MkOperator) { licenses.Walk(lc.checkNode) } -func (lc *LicenseChecker) checkNode(cond *LicenseCondition) { - license := cond.Name - if license == "" || license == "append-placeholder" { - return - } - +func (lc *LicenseChecker) checkLicenseName(license string) { var licenseFile string if G.Pkg != nil { if licenseFileValue, ok := G.Pkg.varValue("LICENSE_FILE"); ok { @@ -146,9 +74,16 @@ func (lc *LicenseChecker) checkNode(cond *LicenseCondition) { "and define LICENSE to that file name. See the pkgsrc guide,", "keyword LICENSE, for more information.") } +} + +func (lc *LicenseChecker) checkNode(cond *licenses.Condition) { + if license := cond.Name; license != "" && license != "append-placeholder" { + lc.checkLicenseName(license) + return + } - if len(cond.And) > 0 && len(cond.Or) > 0 { - lc.MkLine.Line.Errorf("AND and OR operators in license conditions can only be combined using parentheses.") + if cond.And && cond.Or { + lc.MkLine.Errorf("AND and OR operators in license conditions can only be combined using parentheses.") Explain( "Examples for valid license conditions are:", "", diff --git a/pkgtools/pkglint/files/licenses/licenses.go b/pkgtools/pkglint/files/licenses/licenses.go new file mode 100644 index 00000000000..8305b2ea2e7 --- /dev/null +++ b/pkgtools/pkglint/files/licenses/licenses.go @@ -0,0 +1,96 @@ +package licenses + +import "netbsd.org/pkglint/textproc" + +// Condition describes a complex license condition. +// It has either `Name` or `Paren` or `Children` set. +// In the `Children` case, `And` and `Or` specify the operators used. +// Malformed license conditions can have both `And` and `Or` set. +type Condition struct { + Name string `json:",omitempty"` + Paren *Condition `json:",omitempty"` + And bool `json:",omitempty"` + Or bool `json:",omitempty"` + Children []*Condition `json:",omitempty"` +} + +func Parse(licenses string) *Condition { + lexer := &licenseLexer{repl: textproc.NewPrefixReplacer(licenses)} + result := liyyNewParser().Parse(lexer) + if result == 0 { + return lexer.result + } + return nil +} + +func (cond *Condition) String() string { + if cond.Name != "" { + return cond.Name + } + if cond.Paren != nil { + return "(" + cond.Paren.String() + ")" + } + s := "" + separator := [...]string{"", " AND ", " OR ", " MIXED "}[b2i(cond.And)+2*b2i(cond.Or)] + for i, child := range cond.Children { + if i != 0 { + s += separator + } + s += child.String() + } + return s +} + +func (cond *Condition) Walk(callback func(*Condition)) { + if cond.Paren != nil { + cond.Paren.Walk(callback) + } + for _, child := range cond.Children { + child.Walk(callback) + } + callback(cond) +} + +//go:generate go tool yacc -p liyy -o licensesyacc.go -v licensesyacc.log licenses.y + +type licenseLexer struct { + repl *textproc.PrefixReplacer + result *Condition + error string +} + +func (lexer *licenseLexer) Lex(llval *liyySymType) int { + repl := lexer.repl + repl.AdvanceHspace() + switch { + case repl.EOF(): + return 0 + case repl.AdvanceStr("("): + return ltOPEN + case repl.AdvanceStr(")"): + return ltCLOSE + case repl.AdvanceRegexp(`^[\w-.]+`): + word := repl.Group(0) + switch word { + case "AND": + return ltAND + case "OR": + return ltOR + default: + llval.Node = &Condition{Name: word} + return ltNAME + } + } + return -1 +} + +func (lexer *licenseLexer) Error(s string) { + lexer.error = s +} + +func b2i(x bool) int { + if x { + return 1 + } + return 0 +} diff --git a/pkgtools/pkglint/files/licenses/licenses.y b/pkgtools/pkglint/files/licenses/licenses.y new file mode 100644 index 00000000000..d3e1c82b43a --- /dev/null +++ b/pkgtools/pkglint/files/licenses/licenses.y @@ -0,0 +1,37 @@ +%{ +package licenses +%} + +%token <Node> ltNAME +%token ltAND ltOR ltOPEN ltCLOSE + +%union { + Node *Condition +} + +%type <Node> start list node + +%% + +start : list { + liyylex.(*licenseLexer).result = $$ +} + +list : node { + $$ = &Condition{Children: []*Condition{$1}} +} +list : list ltAND node { + $$.Children = append($$.Children, $3) + $$.And = true +} +list : list ltOR node { + $$.Children = append($$.Children, $3) + $$.Or = true +} + +node : ltNAME { + $$ = $1 +} +node : ltOPEN list ltCLOSE { + $$ = &Condition{Paren: $2} +} diff --git a/pkgtools/pkglint/files/licenses/licenses_test.go b/pkgtools/pkglint/files/licenses/licenses_test.go new file mode 100644 index 00000000000..871a44aba04 --- /dev/null +++ b/pkgtools/pkglint/files/licenses/licenses_test.go @@ -0,0 +1,106 @@ +package licenses + +import ( + "encoding/json" + "gopkg.in/check.v1" + "strings" + "testing" +) + +type Suite struct{} + +func (s *Suite) Test_Parse(c *check.C) { + checkParse := func(cond string, expected string) { + c.Check(toJSON(Parse(cond)), check.Equals, expected) + } + + c.Check(Parse("gnu-gpl-v2"), check.DeepEquals, NewSingleton(NewName("gnu-gpl-v2"))) + + checkParse("gnu-gpl-v2", "{Children:[{Name:gnu-gpl-v2}]}") + checkParse("a AND b", "{And:true,Children:[{Name:a},{Name:b}]}") + checkParse("a OR b", "{Or:true,Children:[{Name:a},{Name:b}]}") + + checkParse("a OR (b AND c)", "{Or:true,Children:[{Name:a},{Paren:{And:true,Children:[{Name:b},{Name:c}]}}]}") + checkParse("(a OR b) AND c", "{And:true,Children:[{Paren:{Or:true,Children:[{Name:a},{Name:b}]}},{Name:c}]}") + + checkParse("a AND b AND c AND d", "{And:true,Children:[{Name:a},{Name:b},{Name:c},{Name:d}]}") + c.Check( + Parse("a AND b AND c AND d"), + check.DeepEquals, + NewAnd(NewName("a"), NewName("b"), NewName("c"), NewName("d"))) + + checkParse("a OR b OR c OR d", "{Or:true,Children:[{Name:a},{Name:b},{Name:c},{Name:d}]}") + c.Check( + Parse("a OR b OR c OR d"), + check.DeepEquals, + NewOr(NewName("a"), NewName("b"), NewName("c"), NewName("d"))) + + checkParse("(a OR b) AND (c AND d)", "{And:true,Children:[{Paren:{Or:true,Children:[{Name:a},{Name:b}]}},{Paren:{And:true,Children:[{Name:c},{Name:d}]}}]}") + c.Check( + (Parse("(a OR b) AND (c AND d)")), + check.DeepEquals, + NewAnd( + NewParen(NewOr(NewName("a"), NewName("b"))), + NewParen(NewAnd(NewName("c"), NewName("d"))))) + + checkParse("a AND b OR c AND d", "{And:true,Or:true,Children:[{Name:a},{Name:b},{Name:c},{Name:d}]}") + checkParse("((a AND (b AND c)))", "{Children:[{Paren:{Children:[{Paren:{And:true,Children:[{Name:a},{Paren:{And:true,Children:[{Name:b},{Name:c}]}}]}}]}}]}") + + c.Check(Parse("a AND b OR c AND d").String(), check.Equals, "a MIXED b MIXED c MIXED d") + + c.Check(Parse("AND artistic"), check.IsNil) +} + +func (s *Suite) Test_Condition_String(c *check.C) { + c.Check( + NewName("a").String(), + check.Equals, + "a") + + c.Check( + NewAnd(NewName("a"), NewName("b")).String(), + check.Equals, + "a AND b") + + c.Check( + NewOr(NewName("a"), NewName("b")).String(), + check.Equals, + "a OR b") + + c.Check( + NewAnd( + NewParen(NewOr(NewName("a"), NewName("b"))), + NewParen(NewOr(NewName("c"), NewName("d")))).String(), + check.Equals, + "(a OR b) AND (c OR d)") + + mixed := NewAnd(NewName("a"), NewName("b"), NewName("c")) + mixed.Or = true + c.Check(mixed.String(), check.Equals, "a MIXED b MIXED c") +} + +func NewName(name string) *Condition { + return &Condition{Name: name} +} +func NewParen(child *Condition) *Condition { + return &Condition{Paren: child} +} +func NewSingleton(child *Condition) *Condition { + return &Condition{Children: []*Condition{child}} +} +func NewAnd(parts ...*Condition) *Condition { + return &Condition{Children: parts, And: true} +} +func NewOr(parts ...*Condition) *Condition { + return &Condition{Children: parts, Or: true} +} + +func toJSON(cond *Condition) string { + json, _ := json.Marshal(cond) + return strings.Replace(string(json), "\"", "", -1) +} + +func Test(t *testing.T) { + check.Suite(new(Suite)) + check.TestingT(t) +} diff --git a/pkgtools/pkglint/files/licenses_test.go b/pkgtools/pkglint/files/licenses_test.go index ce1c2808322..4f76137e3b7 100644 --- a/pkgtools/pkglint/files/licenses_test.go +++ b/pkgtools/pkglint/files/licenses_test.go @@ -4,11 +4,6 @@ import ( check "gopkg.in/check.v1" ) -func (s *Suite) Test_parseLicenses(c *check.C) { - c.Check(parseLicenses("gnu-gpl-v2"), check.DeepEquals, &LicenseCondition{Name: "gnu-gpl-v2"}) - c.Check(parseLicenses("AND artistic"), check.IsNil) -} - func (s *Suite) Test_checklineLicense(c *check.C) { s.Init(c) s.CreateTmpFile("licenses/gnu-gpl-v2", "Most software \u2026") diff --git a/pkgtools/pkglint/files/line.go b/pkgtools/pkglint/files/line.go index 29ace63f733..121d61244b8 100644 --- a/pkgtools/pkglint/files/line.go +++ b/pkgtools/pkglint/files/line.go @@ -16,6 +16,7 @@ package main import ( "fmt" "io" + "netbsd.org/pkglint/regex" "path" "strconv" "strings" @@ -31,11 +32,33 @@ func (rline *RawLine) String() string { return strconv.Itoa(rline.Lineno) + ":" + rline.textnl } -type Line struct { - Fname string +type Line interface { + fmt.Stringer + + Filename() string + Linenos() string + Text() string + IsMultiline() bool + IsChanged() bool + + Fatalf(fmt string, args ...interface{}) + Errorf(fmt string, args ...interface{}) + Warnf(fmt string, args ...interface{}) + Notef(fmt string, args ...interface{}) + ReferenceFrom(Line) string + + AutofixReplace(from, to string) bool + AutofixReplaceRegexp(from regex.RegexPattern, to string) bool + AutofixInsertBefore(text string) bool + AutofixDelete() bool + AutofixMark(reason string) +} + +type LineImpl struct { + fname string firstLine int32 // Zero means not applicable, -1 means EOF lastLine int32 // Usually the same as firstLine, may differ in Makefiles - Text string + text string raw []*RawLine changed bool before []string @@ -43,26 +66,38 @@ type Line struct { autofixMessage string } -func NewLine(fname string, lineno int, text string, rawLines []*RawLine) *Line { +func NewLine(fname string, lineno int, text string, rawLines []*RawLine) Line { return NewLineMulti(fname, lineno, lineno, text, rawLines) } // NewLineMulti is for logical Makefile lines that end with backslash. -func NewLineMulti(fname string, firstLine, lastLine int, text string, rawLines []*RawLine) *Line { - return &Line{fname, int32(firstLine), int32(lastLine), text, rawLines, false, nil, nil, ""} +func NewLineMulti(fname string, firstLine, lastLine int, text string, rawLines []*RawLine) Line { + return &LineImpl{fname, int32(firstLine), int32(lastLine), text, rawLines, false, nil, nil, ""} } // NewLineEOF creates a dummy line for logging, with the "line number" EOF. -func NewLineEOF(fname string) *Line { +func NewLineEOF(fname string) Line { return NewLineMulti(fname, -1, 0, "", nil) } // NewLineWhole creates a dummy line for logging messages that affect a file as a whole. -func NewLineWhole(fname string) *Line { +func NewLineWhole(fname string) Line { return NewLine(fname, 0, "", nil) } -func (line *Line) modifiedLines() []string { +func (line *LineImpl) Filename() string { + return line.fname +} + +func (line *LineImpl) Text() string { + return line.text +} + +func (line *LineImpl) IsChanged() bool { + return line.changed +} + +func (line *LineImpl) modifiedLines() []string { var result []string result = append(result, line.before...) for _, raw := range line.raw { @@ -72,7 +107,7 @@ func (line *Line) modifiedLines() []string { return result } -func (line *Line) linenos() string { +func (line *LineImpl) Linenos() string { switch { case line.firstLine == -1: return "EOF" @@ -85,18 +120,18 @@ func (line *Line) linenos() string { } } -func (line *Line) ReferenceFrom(other *Line) string { - if line.Fname != other.Fname { - return cleanpath(relpath(path.Dir(other.Fname), line.Fname)) + ":" + line.linenos() +func (line *LineImpl) ReferenceFrom(other Line) string { + if line.fname != other.Filename() { + return cleanpath(relpath(path.Dir(other.Filename()), line.fname)) + ":" + line.Linenos() } - return "line " + line.linenos() + return "line " + line.Linenos() } -func (line *Line) IsMultiline() bool { +func (line *LineImpl) IsMultiline() bool { return line.firstLine > 0 && line.firstLine != line.lastLine } -func (line *Line) printSource(out io.Writer) { +func (line *LineImpl) printSource(out io.Writer) { if G.opts.PrintSource { io.WriteString(out, "\n") for _, before := range line.before { @@ -120,55 +155,55 @@ func (line *Line) printSource(out io.Writer) { } } -func (line *Line) Fatalf(format string, args ...interface{}) { +func (line *LineImpl) Fatalf(format string, args ...interface{}) { line.printSource(G.logErr) - logs(llFatal, line.Fname, line.linenos(), format, fmt.Sprintf(format, args...)) + logs(llFatal, line.fname, line.Linenos(), format, fmt.Sprintf(format, args...)) } -func (line *Line) Errorf(format string, args ...interface{}) { +func (line *LineImpl) Errorf(format string, args ...interface{}) { line.printSource(G.logOut) - logs(llError, line.Fname, line.linenos(), format, fmt.Sprintf(format, args...)) + logs(llError, line.fname, line.Linenos(), format, fmt.Sprintf(format, args...)) line.logAutofix() } -func (line *Line) Warnf(format string, args ...interface{}) { +func (line *LineImpl) Warnf(format string, args ...interface{}) { line.printSource(G.logOut) - logs(llWarn, line.Fname, line.linenos(), format, fmt.Sprintf(format, args...)) + logs(llWarn, line.fname, line.Linenos(), format, fmt.Sprintf(format, args...)) line.logAutofix() } -func (line *Line) Notef(format string, args ...interface{}) { +func (line *LineImpl) Notef(format string, args ...interface{}) { line.printSource(G.logOut) - logs(llNote, line.Fname, line.linenos(), format, fmt.Sprintf(format, args...)) + logs(llNote, line.fname, line.Linenos(), format, fmt.Sprintf(format, args...)) line.logAutofix() } -func (line *Line) String() string { - return line.Fname + ":" + line.linenos() + ": " + line.Text +func (line *LineImpl) String() string { + return line.fname + ":" + line.Linenos() + ": " + line.text } -func (line *Line) logAutofix() { +func (line *LineImpl) logAutofix() { if line.autofixMessage != "" { - logs(llAutofix, line.Fname, line.linenos(), "%s", line.autofixMessage) + logs(llAutofix, line.fname, line.Linenos(), "%s", line.autofixMessage) line.autofixMessage = "" } } -func (line *Line) AutofixInsertBefore(text string) bool { +func (line *LineImpl) AutofixInsertBefore(text string) bool { if G.opts.PrintAutofix || G.opts.Autofix { line.before = append(line.before, text+"\n") } return line.RememberAutofix("Inserting a line %q before this line.", text) } -func (line *Line) AutofixInsertAfter(text string) bool { +func (line *LineImpl) AutofixInsertAfter(text string) bool { if G.opts.PrintAutofix || G.opts.Autofix { line.after = append(line.after, text+"\n") } return line.RememberAutofix("Inserting a line %q after this line.", text) } -func (line *Line) AutofixDelete() bool { +func (line *LineImpl) AutofixDelete() bool { if G.opts.PrintAutofix || G.opts.Autofix { for _, rawLine := range line.raw { rawLine.textnl = "" @@ -177,7 +212,7 @@ func (line *Line) AutofixDelete() bool { return line.RememberAutofix("Deleting this line.") } -func (line *Line) AutofixReplace(from, to string) bool { +func (line *LineImpl) AutofixReplace(from, to string) bool { for _, rawLine := range line.raw { if rawLine.Lineno != 0 { if replaced := strings.Replace(rawLine.textnl, from, to, 1); replaced != rawLine.textnl { @@ -191,10 +226,10 @@ func (line *Line) AutofixReplace(from, to string) bool { return false } -func (line *Line) AutofixReplaceRegexp(from RegexPattern, to string) bool { +func (line *LineImpl) AutofixReplaceRegexp(from regex.RegexPattern, to string) bool { for _, rawLine := range line.raw { if rawLine.Lineno != 0 { - if replaced := regcomp(from).ReplaceAllString(rawLine.textnl, to); replaced != rawLine.textnl { + if replaced := regex.Compile(from).ReplaceAllString(rawLine.textnl, to); replaced != rawLine.textnl { if G.opts.PrintAutofix || G.opts.Autofix { rawLine.textnl = replaced } @@ -205,13 +240,13 @@ func (line *Line) AutofixReplaceRegexp(from RegexPattern, to string) bool { return false } -func (line *Line) RememberAutofix(format string, args ...interface{}) (hasBeenFixed bool) { +func (line *LineImpl) RememberAutofix(format string, args ...interface{}) (hasBeenFixed bool) { if line.firstLine < 1 { return false } line.changed = true if G.opts.Autofix { - logs(llAutofix, line.Fname, line.linenos(), format, fmt.Sprintf(format, args...)) + logs(llAutofix, line.fname, line.Linenos(), format, fmt.Sprintf(format, args...)) return true } if G.opts.PrintAutofix { @@ -220,72 +255,8 @@ func (line *Line) RememberAutofix(format string, args ...interface{}) (hasBeenFi return false } -func (line *Line) CheckAbsolutePathname(text string) { - if G.opts.Debug { - defer tracecall1(text)() - } - - // In the GNU coding standards, DESTDIR is defined as a (usually - // empty) prefix that can be used to install files to a different - // location from what they have been built for. Therefore - // everything following it is considered an absolute pathname. - // - // Another context where absolute pathnames usually appear is in - // assignments like "bindir=/bin". - if m, path := match1(text, `(?:^|\$[{(]DESTDIR[)}]|[\w_]+\s*=\s*)(/(?:[^"'\s]|"[^"*]"|'[^']*')*)`); m { - if matches(path, `^/\w`) { - checkwordAbsolutePathname(line, path) - } - } -} - -func (line *Line) CheckLength(maxlength int) { - if len(line.Text) > maxlength { - line.Warnf("Line too long (should be no more than %d characters).", maxlength) - Explain( - "Back in the old time, terminals with 80x25 characters were common.", - "And this is still the default size of many terminal emulators.", - "Moderately short lines also make reading easier.") - } -} - -func (line *Line) CheckValidCharacters(reChar RegexPattern) { - rest := regcomp(reChar).ReplaceAllString(line.Text, "") - if rest != "" { - uni := "" - for _, c := range rest { - uni += fmt.Sprintf(" %U", c) - } - line.Warnf("Line contains invalid characters (%s).", uni[1:]) - } -} - -func (line *Line) CheckTrailingWhitespace() { - if hasSuffix(line.Text, " ") || hasSuffix(line.Text, "\t") { - if !line.AutofixReplaceRegexp(`\s+\n$`, "\n") { - line.Notef("Trailing white-space.") - Explain( - "When a line ends with some white-space, that space is in most cases", - "irrelevant and can be removed.") - } - } -} - -func (line *Line) CheckRcsid(prefixRe RegexPattern, suggestedPrefix string) bool { - if G.opts.Debug { - defer tracecall(prefixRe, suggestedPrefix)() - } - - if matches(line.Text, `^`+prefixRe+`\$`+`NetBSD(?::[^\$]+)?\$$`) { - return true - } - - if !line.AutofixInsertBefore(suggestedPrefix + "$" + "NetBSD$") { - line.Errorf("Expected %q.", suggestedPrefix+"$"+"NetBSD$") - Explain( - "Several files in pkgsrc must contain the CVS Id, so that their", - "current version can be traced back later from a binary package.", - "This is to ensure reproducible builds, for example for finding bugs.") - } - return false +func (line *LineImpl) AutofixMark(reason string) { + line.RememberAutofix(reason) + line.logAutofix() + line.changed = true } diff --git a/pkgtools/pkglint/files/line_test.go b/pkgtools/pkglint/files/line_test.go index 8ec338757c3..1913f541d74 100644 --- a/pkgtools/pkglint/files/line_test.go +++ b/pkgtools/pkglint/files/line_test.go @@ -8,7 +8,7 @@ func (s *Suite) Test_Line_modifications(c *check.C) { s.Init(c) s.UseCommandLine("--show-autofix") - line := NewLine("fname", 1, "dummy", s.NewRawLines(1, "original\n")) + line := NewLine("fname", 1, "dummy", s.NewRawLines(1, "original\n")).(*LineImpl) c.Check(line.changed, equals, false) c.Check(line.raw, check.DeepEquals, s.NewRawLines(1, "original\n")) @@ -54,15 +54,6 @@ func (s *Suite) Test_Line_modifications(c *check.C) { "after\n"}) } -func (s *Suite) Test_Line_CheckAbsolutePathname(c *check.C) { - line := NewLine("Makefile", 1, "# dummy", nil) - - line.CheckAbsolutePathname("bindir=/bin") - line.CheckAbsolutePathname("bindir=/../lib") - - c.Check(s.Output(), equals, "WARN: Makefile:1: Found absolute pathname: /bin\n") -} - func (s *Suite) Test_Line_show_autofix_AutofixReplace(c *check.C) { s.Init(c) s.UseCommandLine("--show-autofix", "--source") @@ -117,29 +108,3 @@ func (s *Suite) Test_Line_show_autofix_AutofixDelete(c *check.C) { "WARN: Makefile:30: Dummy\n"+ "AUTOFIX: Makefile:30: Deleting this line.\n") } - -func (s *Suite) Test_Line_CheckTrailingWhitespace(c *check.C) { - line := NewLine("Makefile", 32, "The line must go on ", nil) - - line.CheckTrailingWhitespace() - - c.Check(s.Output(), equals, "NOTE: Makefile:32: Trailing white-space.\n") -} - -func (s *Suite) Test_Line_CheckRcsid(c *check.C) { - lines := s.NewLines("fname", - "$"+"NetBSD: dummy $", - "$"+"NetBSD$", - "$"+"Id: dummy $", - "$"+"Id$", - "$"+"FreeBSD$") - - for _, line := range lines { - line.CheckRcsid(``, "") - } - - c.Check(s.Output(), equals, ""+ - "ERROR: fname:3: Expected \"$"+"NetBSD$\".\n"+ - "ERROR: fname:4: Expected \"$"+"NetBSD$\".\n"+ - "ERROR: fname:5: Expected \"$"+"NetBSD$\".\n") -} diff --git a/pkgtools/pkglint/files/linechecker.go b/pkgtools/pkglint/files/linechecker.go new file mode 100644 index 00000000000..6408cd84001 --- /dev/null +++ b/pkgtools/pkglint/files/linechecker.go @@ -0,0 +1,81 @@ +package main + +import ( + "fmt" + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/trace" +) + +type LineChecker struct { + Line Line +} + +func (ck LineChecker) CheckAbsolutePathname(text string) { + if trace.Tracing { + defer trace.Call1(text)() + } + + // In the GNU coding standards, DESTDIR is defined as a (usually + // empty) prefix that can be used to install files to a different + // location from what they have been built for. Therefore + // everything following it is considered an absolute pathname. + // + // Another context where absolute pathnames usually appear is in + // assignments like "bindir=/bin". + if m, path := match1(text, `(?:^|\$[{(]DESTDIR[)}]|[\w_]+\s*=\s*)(/(?:[^"'\s]|"[^"*]"|'[^']*')*)`); m { + if matches(path, `^/\w`) { + checkwordAbsolutePathname(ck.Line, path) + } + } +} + +func (ck LineChecker) CheckLength(maxlength int) { + if len(ck.Line.Text()) > maxlength { + ck.Line.Warnf("Line too long (should be no more than %d characters).", maxlength) + Explain( + "Back in the old time, terminals with 80x25 characters were common.", + "And this is still the default size of many terminal emulators.", + "Moderately short lines also make reading easier.") + } +} + +func (ck LineChecker) CheckValidCharacters(reChar regex.RegexPattern) { + rest := regex.Compile(reChar).ReplaceAllString(ck.Line.Text(), "") + if rest != "" { + uni := "" + for _, c := range rest { + uni += fmt.Sprintf(" %U", c) + } + ck.Line.Warnf("Line contains invalid characters (%s).", uni[1:]) + } +} + +func (ck LineChecker) CheckTrailingWhitespace() { + if hasSuffix(ck.Line.Text(), " ") || hasSuffix(ck.Line.Text(), "\t") { + if !ck.Line.AutofixReplaceRegexp(`\s+\n$`, "\n") { + ck.Line.Notef("Trailing white-space.") + Explain( + "When a line ends with some white-space, that space is in most cases", + "irrelevant and can be removed.") + } + } +} + +func (ck LineChecker) CheckRcsid(prefixRe regex.RegexPattern, suggestedPrefix string) bool { + if trace.Tracing { + defer trace.Call(prefixRe, suggestedPrefix)() + } + + if matches(ck.Line.Text(), `^`+prefixRe+`\$`+`NetBSD(?::[^\$]+)?\$$`) { + return true + } + + if !ck.Line.AutofixInsertBefore(suggestedPrefix + "$" + "NetBSD$") { + ck.Line.Errorf("Expected %q.", suggestedPrefix+"$"+"NetBSD$") + Explain( + "Several files in pkgsrc must contain the CVS Id, so that their", + "current version can be traced back later from a binary package.", + "This is to ensure reproducible builds, for example for finding bugs.") + } + return false +} diff --git a/pkgtools/pkglint/files/linechecker_test.go b/pkgtools/pkglint/files/linechecker_test.go new file mode 100644 index 00000000000..824acb89e8b --- /dev/null +++ b/pkgtools/pkglint/files/linechecker_test.go @@ -0,0 +1,38 @@ +package main + +import "gopkg.in/check.v1" + +func (s *Suite) Test_LineChecker_CheckAbsolutePathname(c *check.C) { + ck := LineChecker{NewLine("Makefile", 1, "# dummy", nil)} + + ck.CheckAbsolutePathname("bindir=/bin") + ck.CheckAbsolutePathname("bindir=/../lib") + + c.Check(s.Output(), equals, "WARN: Makefile:1: Found absolute pathname: /bin\n") +} + +func (s *Suite) Test_LineChecker_CheckTrailingWhitespace(c *check.C) { + ck := LineChecker{NewLine("Makefile", 32, "The line must go on ", nil)} + + ck.CheckTrailingWhitespace() + + c.Check(s.Output(), equals, "NOTE: Makefile:32: Trailing white-space.\n") +} + +func (s *Suite) Test_LineChecker_CheckRcsid(c *check.C) { + lines := s.NewLines("fname", + "$"+"NetBSD: dummy $", + "$"+"NetBSD$", + "$"+"Id: dummy $", + "$"+"Id$", + "$"+"FreeBSD$") + + for _, line := range lines { + LineChecker{line}.CheckRcsid(``, "") + } + + c.Check(s.Output(), equals, ""+ + "ERROR: fname:3: Expected \"$"+"NetBSD$\".\n"+ + "ERROR: fname:4: Expected \"$"+"NetBSD$\".\n"+ + "ERROR: fname:5: Expected \"$"+"NetBSD$\".\n") +} diff --git a/pkgtools/pkglint/files/logging.go b/pkgtools/pkglint/files/logging.go index c5361ed46b6..81ea2af3b9d 100644 --- a/pkgtools/pkglint/files/logging.go +++ b/pkgtools/pkglint/files/logging.go @@ -99,10 +99,13 @@ func Explain(explanation ...string) { io.WriteString(G.logOut, "\t"+explanationLine+"\n") } io.WriteString(G.logOut, "\n") - } else if G.Testing { + } + + if G.Testing { for _, s := range explanation { if l := tabLength(s); l > 68 && contains(s, " ") { - print(fmt.Sprintf("Long explanation line (%d): %s\n", l, s)) + lastSpace := strings.LastIndexByte(s[:68], ' ') + print(fmt.Sprintf("Long explanation line: %s\nBreak after: %s\n", s, s[:lastSpace])) } if m, before := match1(s, `(.+)\. [^ ]`); m { if !matches(before, `\d$|e\.g`) { diff --git a/pkgtools/pkglint/files/mkline.go b/pkgtools/pkglint/files/mkline.go index a0e0d84a612..f4572e62417 100644 --- a/pkgtools/pkglint/files/mkline.go +++ b/pkgtools/pkglint/files/mkline.go @@ -4,14 +4,13 @@ package main import ( "fmt" - "os" - "path" - "strconv" + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/trace" "strings" ) type MkLine struct { - *Line + Line data interface{} // One of the following mkLine* types } type mkLineAssign struct { @@ -45,10 +44,10 @@ type mkLineDependency struct { sources string } -func NewMkLine(line *Line) (mkline *MkLine) { +func NewMkLine(line Line) (mkline *MkLine) { mkline = &MkLine{Line: line} - text := line.Text + text := line.Text() if hasPrefix(text, " ") { mkline.Warnf("Makefile lines should not start with space characters.") @@ -143,7 +142,7 @@ func NewMkLine(line *Line) (mkline *MkLine) { } func (mkline *MkLine) String() string { - return fmt.Sprintf("%s:%s", mkline.Line.Fname, mkline.Line.linenos()) + return fmt.Sprintf("%s:%s", mkline.Line.Filename(), mkline.Line.Linenos()) } func (mkline *MkLine) IsVarassign() bool { _, ok := mkline.data.(mkLineAssign); return ok } func (mkline *MkLine) IsShellcmd() bool { _, ok := mkline.data.(mkLineShell); return ok } @@ -181,216 +180,9 @@ func (mkline *MkLine) Includefile() string { return mkline.data.(mkLineInclude). func (mkline *MkLine) Targets() string { return mkline.data.(mkLineDependency).targets } func (mkline *MkLine) Sources() string { return mkline.data.(mkLineDependency).sources } -func (mkline *MkLine) Check() { - mkline.Line.CheckTrailingWhitespace() - mkline.Line.CheckValidCharacters(`[\t -~]`) - - switch { - case mkline.IsVarassign(): - mkline.checkVarassign() - - case mkline.IsShellcmd(): - shellcmd := mkline.Shellcmd() - mkline.checkText(shellcmd) - NewShellLine(mkline).CheckShellCommandLine(shellcmd) - - case mkline.IsComment(): - if hasPrefix(mkline.Line.Text, "# url2pkg-marker") { - mkline.Line.Errorf("This comment indicates unfinished work (url2pkg).") - } - - case mkline.IsInclude(): - mkline.checkInclude() - } -} - -func (mkline *MkLine) checkInclude() { - if G.opts.Debug { - defer tracecall0()() - } - - if mkline.Indent() != "" { - mkline.checkDirectiveIndentation(G.Mk.indentation.Depth()) - } - - includefile := mkline.Includefile() - mustExist := mkline.MustExist() - if G.opts.Debug { - traceStep2("includingFile=%s includefile=%s", mkline.Fname, includefile) - } - mkline.CheckRelativePath(includefile, mustExist) - - switch { - case hasSuffix(includefile, "/Makefile"): - mkline.Line.Errorf("Other Makefiles must not be included directly.") - Explain( - "If you want to include portions of another Makefile, extract", - "the common parts and put them into a Makefile.common. After", - "that, both this one and the other package should include the", - "Makefile.common.") - - case includefile == "../../mk/bsd.prefs.mk": - if path.Base(mkline.Line.Fname) == "buildlink3.mk" { - mkline.Notef("For efficiency reasons, please include bsd.fast.prefs.mk instead of bsd.prefs.mk.") - } - if G.Pkg != nil { - G.Pkg.setSeenBsdPrefsMk() - } - - case includefile == "../../mk/bsd.fast.prefs.mk", includefile == "../../mk/buildlink3/bsd.builtin.mk": - if G.Pkg != nil { - G.Pkg.setSeenBsdPrefsMk() - } - - case hasSuffix(includefile, "/x11-links/buildlink3.mk"): - mkline.Errorf("%s must not be included directly. Include \"../../mk/x11.buildlink3.mk\" instead.", includefile) - - case hasSuffix(includefile, "/jpeg/buildlink3.mk"): - mkline.Errorf("%s must not be included directly. Include \"../../mk/jpeg.buildlink3.mk\" instead.", includefile) - - case hasSuffix(includefile, "/intltool/buildlink3.mk"): - mkline.Warnf("Please write \"USE_TOOLS+= intltool\" instead of this line.") - - case hasSuffix(includefile, "/builtin.mk"): - mkline.Line.Errorf("%s must not be included directly. Include \"%s/buildlink3.mk\" instead.", includefile, path.Dir(includefile)) - } -} - -func (mkline *MkLine) checkCond(forVars map[string]bool) { - directive := mkline.Directive() - args := mkline.Args() - indentation := &G.Mk.indentation - - switch directive { - case "endif", "endfor": - if indentation.Len() > 1 { - indentation.Pop() - } else { - mkline.Errorf("Unmatched .%s.", directive) - } - } - - expectedDepth := indentation.Depth() - if directive == "elif" || directive == "else" { - expectedDepth = indentation.depth[len(indentation.depth)-2] - } - mkline.checkDirectiveIndentation(expectedDepth) - - if directive == "if" && matches(args, `^!defined\([\w]+_MK\)$`) { - indentation.Push(indentation.Depth()) - } else if matches(directive, `^(?:if|ifdef|ifndef|for)$`) { - indentation.Push(indentation.Depth() + 2) - } - - needsArgument := matches(directive, `^(?:if|ifdef|ifndef|elif|for|undef)$`) - if needsArgument != (args != "") { - if needsArgument { - mkline.Errorf("\".%s\" requires arguments.", directive) - } else { - mkline.Errorf("\".%s\" does not take arguments.", directive) - if directive == "else" { - mkline.Notef("If you meant \"else if\", use \".elif\".") - } - } - - } else if directive == "if" || directive == "elif" { - mkline.CheckCond() - - } else if directive == "ifdef" || directive == "ifndef" { - mkline.Line.Warnf("The \".%s\" directive is deprecated. Please use \".if %sdefined(%s)\" instead.", - directive, ifelseStr(directive == "ifdef", "", "!"), args) - - } else if directive == "for" { - if m, vars, values := match2(args, `^(\S+(?:\s*\S+)*?)\s+in\s+(.*)$`); m { - for _, forvar := range splitOnSpace(vars) { - if !G.Infrastructure && hasPrefix(forvar, "_") { - mkline.Warnf("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", forvar) - } - - if matches(forvar, `^[_a-z][_a-z0-9]*$`) { - // Fine. - } else if matches(forvar, `[A-Z]`) { - mkline.Warnf(".for variable names should not contain uppercase letters.") - } else { - mkline.Errorf("Invalid variable name %q.", forvar) - } - - forVars[forvar] = true - } - - // Check if any of the value's types is not guessed. - guessed := true - for _, value := range splitOnSpace(values) { - if m, vname := match1(value, `^\$\{(.*)\}`); m { - vartype := mkline.getVariableType(vname) - if vartype != nil && !vartype.guessed { - guessed = false - } - } - } - - forLoopType := &Vartype{lkSpace, BtUnknown, []AclEntry{{"*", aclpAllRead}}, guessed} - forLoopContext := &VarUseContext{forLoopType, vucTimeParse, vucQuotFor, false} - for _, forLoopVar := range mkline.extractUsedVariables(values) { - mkline.CheckVaruse(&MkVarUse{forLoopVar, nil}, forLoopContext) - } - } - - } else if directive == "undef" && args != "" { - for _, uvar := range splitOnSpace(args) { - if forVars[uvar] { - mkline.Notef("Using \".undef\" after a \".for\" loop is unnecessary.") - } - } - } -} - -func (mkline *MkLine) checkDirectiveIndentation(expectedDepth int) { - if G.Mk == nil { - return - } - indent := mkline.Indent() - if expected := strings.Repeat(" ", expectedDepth); indent != expected { - if G.opts.WarnSpace && !mkline.Line.AutofixReplace("."+indent, "."+expected) { - mkline.Line.Notef("This directive should be indented by %d spaces.", expectedDepth) - } - } -} - -func (mkline *MkLine) checkDependencyRule(allowedTargets map[string]bool) { - targets := splitOnSpace(mkline.Targets()) - sources := splitOnSpace(mkline.Sources()) - - for _, source := range sources { - if source == ".PHONY" { - for _, target := range targets { - allowedTargets[target] = true - } - } - } - - for _, target := range targets { - if target == ".PHONY" { - for _, dep := range sources { - allowedTargets[dep] = true - } - - } else if target == ".ORDER" { - // TODO: Check for spelling mistakes. - - } else if !allowedTargets[target] { - mkline.Warnf("Unusual target %q.", target) - Explain( - "If you want to define your own targets, you can \"declare\"", - "them by inserting a \".PHONY: my-target\" line before this line. This", - "will tell make(1) to not interpret this target's name as a filename.") - } - } -} - func (mkline *MkLine) Tokenize(s string) []*MkToken { - if G.opts.Debug { - defer tracecall(mkline, s)() + if trace.Tracing { + defer trace.Call(mkline, s)() } p := NewMkParser(mkline.Line, s, true) @@ -401,705 +193,11 @@ func (mkline *MkLine) Tokenize(s string) []*MkToken { return tokens } -func (mkline *MkLine) checkVarassignDefPermissions() { - if !G.opts.WarnPerm { - return - } - if G.opts.Debug { - defer tracecall()() - } - - varname := mkline.Varname() - op := mkline.Op() - vartype := mkline.getVariableType(varname) - if vartype == nil { - if G.opts.Debug { - traceStep1("No type definition found for %q.", varname) - } - return - } - - perms := vartype.EffectivePermissions(mkline.Line.Fname) - var needed AclPermissions - switch op { - case opAssign, opAssignShell, opAssignEval: - needed = aclpSet - case opAssignDefault: - needed = aclpSetDefault - case opAssignAppend: - needed = aclpAppend - } - - switch { - case perms.Contains(needed): - break - case perms == aclpUnknown: - if G.opts.Debug { - traceStep1("Unknown permissions for %q.", varname) - } - default: - alternativeActions := perms & aclpAllWrite - alternativeFiles := vartype.AllowedFiles(needed) - switch { - case alternativeActions != 0 && alternativeFiles != "": - mkline.Line.Warnf("The variable %s may not be %s (only %s) in this file; it would be ok in %s.", - varname, needed.HumanString(), alternativeActions.HumanString(), alternativeFiles) - case alternativeFiles != "": - mkline.Line.Warnf("The variable %s may not be %s in this file; it would be ok in %s.", - varname, needed.HumanString(), alternativeFiles) - case alternativeActions != 0: - mkline.Line.Warnf("The variable %s may not be %s (only %s) in this file.", - varname, needed.HumanString(), alternativeActions.HumanString()) - default: - mkline.Line.Warnf("The variable %s may not be %s by any package.", - varname, needed.HumanString()) - } - Explain( - "The allowed actions for a variable are determined based on the file", - "name in which the variable is used or defined. The exact rules are", - "hard-coded into pkglint. If they seem to be incorrect, please ask", - "on the tech-pkg@NetBSD.org mailing list.") - } -} - -func (mkline *MkLine) CheckVaruse(varuse *MkVarUse, vuc *VarUseContext) { - if G.opts.Debug { - defer tracecall(mkline, varuse, vuc)() - } - - if varuse.IsExpression() { - return - } - - varname := varuse.varname - vartype := mkline.getVariableType(varname) - if G.opts.WarnExtra && - (vartype == nil || vartype.guessed) && - !varIsUsed(varname) && - !(G.Mk != nil && G.Mk.forVars[varname]) && - !containsVarRef(varname) { - mkline.Warnf("%s is used but not defined. Spelling mistake?", varname) - } - - mkline.CheckVarusePermissions(varname, vartype, vuc) - - if varname == "LOCALBASE" && !G.Infrastructure { - mkline.WarnVaruseLocalbase() - } - - needsQuoting := mkline.variableNeedsQuoting(varname, vartype, vuc) - - if vuc.quoting == vucQuotFor { - mkline.checkVaruseFor(varname, vartype, needsQuoting) - } - - if G.opts.WarnQuoting && vuc.quoting != vucQuotUnknown && needsQuoting != nqDontKnow { - mkline.CheckVaruseShellword(varname, vartype, vuc, varuse.Mod(), needsQuoting) - } - - if G.globalData.UserDefinedVars[varname] != nil && !G.globalData.SystemBuildDefs[varname] && !G.Mk.buildDefs[varname] { - mkline.Warnf("The user-defined variable %s is used but not added to BUILD_DEFS.", varname) - Explain( - "When a pkgsrc package is built, many things can be configured by the", - "pkgsrc user in the mk.conf file. All these configurations should be", - "recorded in the binary package, so the package can be reliably", - "rebuilt. The BUILD_DEFS variable contains a list of all these", - "user-settable variables, so please add your variable to it, too.") - } -} - -func (mkline *MkLine) CheckVarusePermissions(varname string, vartype *Vartype, vuc *VarUseContext) { - if !G.opts.WarnPerm { - return - } - if G.opts.Debug { - defer tracecall(varname, vuc)() - } - - // This is the type of the variable that is being used. Not to - // be confused with vuc.vartype, which is the type of the - // context in which the variable is used (often a ShellCommand - // or, in an assignment, the type of the left hand side variable). - if vartype == nil { - if G.opts.Debug { - traceStep1("No type definition found for %q.", varname) - } - return - } - - perms := vartype.EffectivePermissions(mkline.Line.Fname) - - isLoadTime := false // Will the variable be used at load time? - - // Might the variable be used indirectly at load time, for example - // by assigning it to another variable which then gets evaluated? - isIndirect := false - - switch { - case vuc.vartype != nil && vuc.vartype.guessed: - // Don't warn about unknown variables. - - case vuc.time == vucTimeParse && !perms.Contains(aclpUseLoadtime): - isLoadTime = true - - case vuc.vartype != nil && vuc.vartype.Union().Contains(aclpUseLoadtime) && !perms.Contains(aclpUseLoadtime): - isLoadTime = true - isIndirect = true - } - - done := false - tool := G.globalData.Tools.byVarname[varname] - - if isLoadTime && tool != nil { - done = tool.Predefined && (G.Mk == nil || G.Mk.SeenBsdPrefsMk || G.Pkg == nil || G.Pkg.SeenBsdPrefsMk) - - if !done && G.Pkg != nil && !G.Pkg.SeenBsdPrefsMk && G.Mk != nil && !G.Mk.SeenBsdPrefsMk { - mkline.Warnf("To use the tool %q at load time, bsd.prefs.mk has to be included before.", varname) - done = true - } - - if !done && G.Pkg != nil { - usable, defined := G.Pkg.loadTimeTools[tool.Name] - if usable { - done = true - } - if defined && !usable { - mkline.Warnf("To use the tool %q at load time, it has to be added to USE_TOOLS before including bsd.prefs.mk.", varname) - done = true - } - } - } - - if !done && isLoadTime && !isIndirect { - mkline.Warnf("%s should not be evaluated at load time.", varname) - Explain( - "Many variables, especially lists of something, get their values", - "incrementally. Therefore it is generally unsafe to rely on their", - "value until it is clear that it will never change again. This", - "point is reached when the whole package Makefile is loaded and", - "execution of the shell commands starts; in some cases earlier.", - "", - "Additionally, when using the \":=\" operator, each $$ is replaced", - "with a single $, so variables that have references to shell", - "variables or regular expressions are modified in a subtle way.") - done = true - } - - if !done && isLoadTime && isIndirect { - mkline.Warnf("%s should not be evaluated indirectly at load time.", varname) - Explain( - "The variable on the left-hand side may be evaluated at load time,", - "but the variable on the right-hand side may not. Because of the", - "assignment in this line, the variable might be used indirectly", - "at load time, before it is guaranteed to be properly initialized.") - done = true - } - - if !perms.Contains(aclpUseLoadtime) && !perms.Contains(aclpUse) { - needed := aclpUse - if isLoadTime { - needed = aclpUseLoadtime - } - alternativeFiles := vartype.AllowedFiles(needed) - if alternativeFiles != "" { - mkline.Warnf("%s may not be used in this file; it would be ok in %s.", - varname, alternativeFiles) - } else { - mkline.Warnf("%s may not be used in any file; it is a write-only variable.", varname) - } - Explain( - "The allowed actions for a variable are determined based on the file", - "name in which the variable is used or defined. The exact rules are", - "hard-coded into pkglint. If they seem to be incorrect, please ask", - "on the tech-pkg@NetBSD.org mailing list.") - } -} - -func (mkline *MkLine) WarnVaruseLocalbase() { - mkline.Warnf("The LOCALBASE variable should not be used by packages.") - Explain( - // from jlam via private mail. - "Currently, LOCALBASE is typically used in these cases:", - "", - "(1) To locate a file or directory from another package.", - "(2) To refer to own files after installation.", - "", - "Example for (1):", - "", - " STRLIST= ${LOCALBASE}/bin/strlist", - " do-build:", - " cd ${WRKSRC} && ${STRLIST} *.str", - "", - "This should better be:", - "", - " EVAL_PREFIX= STRLIST_PREFIX=strlist", - " STRLIST= ${STRLIST_PREFIX}/bin/strlist", - " do-build:", - " cd ${WRKSRC} && ${STRLIST} *.str", - "", - "Example for (2):", - "", - " CONFIGURE_ENV+= --with-datafiles=${LOCALBASE}/share/pkgbase", - "", - "This should better be:", - "", - " CONFIGURE_ENV+= --with-datafiles=${PREFIX}/share/pkgbase") -} - -func (mkline *MkLine) checkVaruseFor(varname string, vartype *Vartype, needsQuoting NeedsQuoting) { - if G.opts.Debug { - defer tracecall(varname, vartype, needsQuoting)() - } - - if false && // Too many false positives - vartype != nil && - vartype.kindOfList != lkSpace && - needsQuoting != nqDoesntMatter { - mkline.Warnf("The variable %s should not be used in .for loops.", varname) - Explain( - "The .for loop splits its argument at sequences of white-space, as", - "opposed to all other places in make(1), which act like the shell.", - "Therefore only variables that are split at whitespace or don't", - "contain any special characters should be used here.") - } -} - -func (mkline *MkLine) CheckVaruseShellword(varname string, vartype *Vartype, vuc *VarUseContext, mod string, needsQuoting NeedsQuoting) { - if G.opts.Debug { - defer tracecall(varname, vartype, vuc, mod, needsQuoting)() - } - - // In GNU configure scripts, a few variables need to be - // passed through the :M* operator before they reach the - // configure scripts. - // - // When doing checks outside a package, the :M* operator is needed for safety. - needMstar := matches(varname, `^(?:.*_)?(?:CFLAGS||CPPFLAGS|CXXFLAGS|FFLAGS|LDFLAGS|LIBS)$`) && - (G.Pkg == nil || G.Pkg.vardef["GNU_CONFIGURE"] != nil) - - strippedMod := mod - if m, stripped := match1(mod, `(.*?)(?::M\*)?(?::Q)?$`); m { - strippedMod = stripped - } - - if mod == ":M*:Q" && !needMstar { - mkline.Line.Notef("The :M* modifier is not needed here.") - - } else if needsQuoting == nqYes { - correctMod := strippedMod + ifelseStr(needMstar, ":M*:Q", ":Q") - if correctMod == mod+":Q" && vuc.IsWordPart && !vartype.IsShell() { - mkline.Line.Warnf("The list variable %s should not be embedded in a word.", varname) - Explain( - "When a list variable has multiple elements, this expression expands", - "to something unexpected:", - "", - "Example: ${MASTER_SITE_SOURCEFORGE}directory/ expands to", - "", - "\thttps://mirror1.sf.net/ https://mirror2.sf.net/directory/", - "", - "The first URL is missing the directory. To fix this, write", - "\t${MASTER_SITE_SOURCEFORGE:=directory/}.", - "", - "Example: -l${LIBS} expands to", - "", - "\t-llib1 lib2", - "", - "The second library is missing the -l. To fix this, write", - "${LIBS:@lib@-l${lib}@}.") - - } else if mod != correctMod { - if vuc.quoting == vucQuotPlain { - if !mkline.Line.AutofixReplace("${"+varname+mod+"}", "${"+varname+correctMod+"}") { - mkline.Line.Warnf("Please use ${%s%s} instead of ${%s%s}.", varname, correctMod, varname, mod) - } - } else { - mkline.Line.Warnf("Please use ${%s%s} instead of ${%s%s} and make sure"+ - " the variable appears outside of any quoting characters.", varname, correctMod, varname, mod) - } - Explain( - "See the pkgsrc guide, section \"quoting guideline\", for details.") - - } else if vuc.quoting != vucQuotPlain { - mkline.Line.Warnf("Please move ${%s%s} outside of any quoting characters.", varname, mod) - Explain( - "The :Q modifier only works reliably when it is used outside of any", - "quoting characters.", - "", - "Examples:", - "Instead of CFLAGS=\"${CFLAGS:Q}\",", - " write CFLAGS=${CFLAGS:Q}.", - "Instead of 's,@CFLAGS@,${CFLAGS:Q},',", - " write 's,@CFLAGS@,'${CFLAGS:Q}','.") - } - } - - if hasSuffix(mod, ":Q") && (needsQuoting == nqNo || needsQuoting == nqDoesntMatter) { - bad := "${" + varname + mod + "}" - good := "${" + varname + strings.TrimSuffix(mod, ":Q") + "}" - needExplain := false - if needsQuoting == nqNo && !mkline.Line.AutofixReplace(bad, good) { - needExplain = true - mkline.Warnf("The :Q operator should not be used for ${%s} here.", varname) - } - if needsQuoting == nqDoesntMatter && !mkline.Line.AutofixReplace(bad, good) { - needExplain = true - mkline.Line.Notef("The :Q operator isn't necessary for ${%s} here.", varname) - } - if needExplain { - Explain( - "Many variables in pkgsrc do not need the :Q operator, since they", - "are not expected to contain white-space or other special characters.", - "Examples for these \"safe\" variables are:", - "", - "\t* filenames", - "\t* directory names", - "\t* user and group names", - "\t* tool names and tool paths", - "\t* variable names", - "\t* PKGNAME") - } - } -} - -func (mkline *MkLine) checkVarassignPythonVersions(varname, value string) { - if G.opts.Debug { - defer tracecall2(varname, value)() - } - - strversions := splitOnSpace(value) - intversions := make([]int, len(strversions)) - for i, strversion := range strversions { - iver, err := strconv.Atoi(strversion) - if err != nil || !(iver > 0) { - mkline.Errorf("All values for %s must be positive integers.", varname) - return - } - intversions[i] = iver - } - - for i, ver := range intversions { - if i > 0 && ver >= intversions[i-1] { - mkline.Warnf("The values for %s should be in decreasing order.", varname) - Explain( - "If they aren't, it may be possible that needless versions of", - "packages are installed.") - } - } -} - -func (mkline *MkLine) checkVarassign() { - varname := mkline.Varname() - op := mkline.Op() - value := mkline.Value() - comment := mkline.VarassignComment() - varcanon := varnameCanon(varname) - - if G.opts.Debug { - defer tracecall(varname, op, value)() - } - - defineVar(mkline, varname) - mkline.checkVarassignDefPermissions() - mkline.checkVarassignBsdPrefs() - - mkline.checkText(value) - mkline.CheckVartype(varname, op, value, comment) - - // If the variable is not used and is untyped, it may be a spelling mistake. - if op == opAssignEval && varname == strings.ToLower(varname) { - if G.opts.Debug { - traceStep1("%s might be unused unless it is an argument to a procedure file.", varname) - } - - } else if !varIsUsed(varname) { - if vartypes := G.globalData.vartypes; vartypes[varname] != nil || vartypes[varcanon] != nil { - // Ok - } else if deprecated := G.globalData.Deprecated; deprecated[varname] != "" || deprecated[varcanon] != "" { - // Ok - } else { - mkline.Warnf("%s is defined but not used. Spelling mistake?", varname) - } - } - - mkline.checkVarassignSpecific() - - if varname == "EVAL_PREFIX" { - if m, evalVarname := match1(value, `^([\w_]+)=`); m { - - // The variables mentioned in EVAL_PREFIX will later be - // defined by find-prefix.mk. Therefore, they are marked - // as known in the current file. - G.Mk.vardef[evalVarname] = mkline - } - } - - if varname == "USE_TOOLS" { - for _, fullToolname := range splitOnSpace(value) { - toolname := strings.Split(fullToolname, ":")[0] - if G.Pkg != nil { - if !G.Pkg.SeenBsdPrefsMk { - G.Pkg.loadTimeTools[toolname] = true - if G.opts.Debug { - traceStep1("loadTimeTool %q", toolname) - } - } else if !G.Pkg.loadTimeTools[toolname] { - G.Pkg.loadTimeTools[toolname] = false - if G.opts.Debug { - traceStep1("too late for loadTimeTool %q", toolname) - } - } - } - } - } - - if fix := G.globalData.Deprecated[varname]; fix != "" { - mkline.Warnf("Definition of %s is deprecated. %s", varname, fix) - } else if fix := G.globalData.Deprecated[varcanon]; fix != "" { - mkline.Warnf("Definition of %s is deprecated. %s", varname, fix) - } - - mkline.checkVarassignPlistComment(varname, value) - mkline.checkVarassignVaruse() -} - -func (mkline *MkLine) checkVarassignVaruse() { - if G.opts.Debug { - defer tracecall()() - } - - op := mkline.Op() - - time := vucTimeRun - if op == opAssignEval || op == opAssignShell { - time = vucTimeParse - } - - vartype := mkline.getVariableType(mkline.Varname()) - if op == opAssignShell { - vartype = shellcommandsContextType - } - - if vartype != nil && vartype.IsShell() { - mkline.checkVarassignVaruseShell(vartype, time) - } else { - mkline.checkVarassignVaruseMk(vartype, time) - } -} - -func (mkline *MkLine) checkVarassignVaruseMk(vartype *Vartype, time vucTime) { - if G.opts.Debug { - defer tracecall(vartype, time)() - } - tokens := NewMkParser(mkline.Line, mkline.Value(), false).MkTokens() - for i, token := range tokens { - if token.Varuse != nil { - spaceLeft := i-1 < 0 || matches(tokens[i-1].Text, `\s$`) - spaceRight := i+1 >= len(tokens) || matches(tokens[i+1].Text, `^\s`) - isWordPart := !(spaceLeft && spaceRight) - vuc := &VarUseContext{vartype, time, vucQuotPlain, isWordPart} - mkline.CheckVaruse(token.Varuse, vuc) - } - } -} - -func (mkline *MkLine) checkVarassignVaruseShell(vartype *Vartype, time vucTime) { - if G.opts.Debug { - defer tracecall(vartype, time)() - } - - isWordPart := func(tokens []*ShAtom, i int) bool { - if i-1 >= 0 && tokens[i-1].Type.IsWord() { - return true - } - if i+1 < len(tokens) && tokens[i+1].Type.IsWord() { - return true - } - return false - } - - atoms := NewShTokenizer(mkline.Line, mkline.Value(), false).ShAtoms() - for i, atom := range atoms { - if atom.Type == shtVaruse { - isWordPart := isWordPart(atoms, i) - vuc := &VarUseContext{vartype, time, atom.Quoting.ToVarUseContext(), isWordPart} - mkline.CheckVaruse(atom.Data.(*MkVarUse), vuc) - } - } -} - -func (mkline *MkLine) checkVarassignSpecific() { - varname := mkline.Varname() - value := mkline.Value() - - if contains(value, "/etc/rc.d") { - mkline.Warnf("Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") - } - - if hasPrefix(varname, "_") && !G.Infrastructure { - mkline.Warnf("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", varname) - } - - if varname == "PERL5_PACKLIST" && G.Pkg != nil { - if m, p5pkgname := match1(G.Pkg.EffectivePkgbase, `^p5-(.*)`); m { - guess := "auto/" + strings.Replace(p5pkgname, "-", "/", -1) + "/.packlist" - - ucvalue, ucguess := strings.ToUpper(value), strings.ToUpper(guess) - if ucvalue != ucguess && ucvalue != "${PERL5_SITEARCH}/"+ucguess { - mkline.Warnf("Unusual value for PERL5_PACKLIST -- %q expected.", guess) - } - } - } - - if varname == "CONFIGURE_ARGS" && contains(value, "=${PREFIX}/share/kde") { - mkline.Notef("Please .include \"../../meta-pkgs/kde3/kde3.mk\" instead of this line.") - Explain( - "That file does many things automatically and consistently that this", - "package also does. When using kde3.mk, you can probably also leave", - "out some explicit dependencies.") - } - - if varname == "PYTHON_VERSIONS_ACCEPTED" { - mkline.checkVarassignPythonVersions(varname, value) - } - - if mkline.VarassignComment() == "# defined" && !hasSuffix(varname, "_MK") && !hasSuffix(varname, "_COMMON") { - mkline.Notef("Please use \"# empty\", \"# none\" or \"yes\" instead of \"# defined\".") - Explain( - "The value #defined says something about the state of the variable,", - "but not what that _means_. In some cases a variable that is defined", - "means \"yes\", in other cases it is an empty list (which is also", - "only the state of the variable), whose meaning could be described", - "with \"none\". It is this meaning that should be described.") - } - - if m, revvarname := match1(value, `\$\{(PKGNAME|PKGVERSION)[:\}]`); m { - if varname == "DIST_SUBDIR" || varname == "WRKSRC" { - mkline.Line.Warnf("%s should not be used in %s, as it includes the PKGREVISION. Please use %s_NOREV instead.", revvarname, varname, revvarname) - } - } - - if hasPrefix(varname, "SITES_") { - mkline.Warnf("SITES_* is deprecated. Please use SITES.* instead.") - } - - if varname == "PKG_SKIP_REASON" && G.Mk.indentation.DependsOn("OPSYS") { - mkline.Notef("Consider defining NOT_FOR_PLATFORM instead of setting PKG_SKIP_REASON depending on ${OPSYS}.") - } -} - -func (mkline *MkLine) checkVarassignBsdPrefs() { - if G.opts.WarnExtra && mkline.Op() == opAssignDefault && G.Pkg != nil && !G.Pkg.SeenBsdPrefsMk { - switch mkline.Varcanon() { - case "BUILDLINK_PKGSRCDIR.*", "BUILDLINK_DEPMETHOD.*", "BUILDLINK_ABI_DEPENDS.*": - return - } - - mkline.Warnf("Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".") - Explain( - "The ?= operator is used to provide a default value to a variable.", - "In pkgsrc, many variables can be set by the pkgsrc user in the", - "mk.conf file. This file must be included explicitly. If a ?=", - "operator appears before mk.conf has been included, it will not care", - "about the user's preferences, which can result in unexpected", - "behavior.", - "", - "The easiest way to include the mk.conf file is by including the", - "bsd.prefs.mk file, which will take care of everything.") - } -} - -func (mkline *MkLine) checkVarassignPlistComment(varname, value string) { - if false && // This is currently neither correct nor helpful - contains(value, "@comment") && !matches(value, `="@comment "`) { - mkline.Warnf("Please don't use @comment in %s.", varname) - Explain( - "If you are defining a PLIST conditional here, use one of the", - "following patterns instead:", - "", - "1. The direct way, without intermediate variable", - "", - "\tPLIST_SUBST+=\tMY_VAR=\"@comment \"", - "", - "2. The indirect way, with a separate variable", - "", - "\tPLIST_VARS+=\tMY_VAR", - "\t.if ...", - "\tMY_VAR?=\tyes", - "\t.endif") - } - - // Mark the variable as PLIST condition. This is later used in checkfile_PLIST. - if G.Pkg != nil { - if m, plistVarname := match1(value, `(.+)=.*@comment.*`); m { - G.Pkg.plistSubstCond[plistVarname] = true - } - } -} - -func (mkline *MkLine) CheckVartype(varname string, op MkOperator, value, comment string) { - if G.opts.Debug { - defer tracecall(varname, op, value, comment)() - } - - if !G.opts.WarnTypes { - return - } - - vartype := mkline.getVariableType(varname) - - if op == opAssignAppend { - if vartype != nil && !vartype.MayBeAppendedTo() { - mkline.Warnf("The \"+=\" operator should only be used with lists.") - } - } - - switch { - case vartype == nil: - if G.opts.Debug { - traceStep1("Unchecked variable assignment for %s.", varname) - } - - case op == opAssignShell: - if G.opts.Debug { - traceStep1("Unchecked use of !=: %q", value) - } - - case vartype.kindOfList == lkNone: - mkline.CheckVartypePrimitive(varname, vartype.basicType, op, value, comment, vartype.guessed) - - case value == "": - break - - case vartype.kindOfList == lkSpace: - for _, word := range splitOnSpace(value) { - mkline.CheckVartypePrimitive(varname, vartype.basicType, op, word, comment, vartype.guessed) - } - - case vartype.kindOfList == lkShell: - words, _ := splitIntoMkWords(mkline.Line, value) - for _, word := range words { - mkline.CheckVartypePrimitive(varname, vartype.basicType, op, word, comment, vartype.guessed) - } - } -} - -// For some variables (like `BuildlinkDepth`), `op` influences the valid values. -// The `comment` parameter comes from a variable assignment, when a part of the line is commented out. -func (mkline *MkLine) CheckVartypePrimitive(varname string, checker *BasicType, op MkOperator, value, comment string, guessed bool) { - if G.opts.Debug { - defer tracecall(varname, checker.name, op, value, comment, guessed)() - } - - valueNoVar := mkline.withoutMakeVariables(value) - ctx := &VartypeCheck{mkline, mkline.Line, varname, op, value, valueNoVar, comment, guessed} - checker.checker(ctx) -} - func (mkline *MkLine) withoutMakeVariables(value string) string { valueNovar := value for { var m []string - m, valueNovar = replaceFirst(valueNovar, `\$\{[^{}]*\}`, "") + m, valueNovar = regex.ReplaceFirst(valueNovar, `\$\{[^{}]*\}`, "") if m == nil { return valueNovar } @@ -1138,123 +236,12 @@ func (mkline *MkLine) resolveVarsInRelativePath(relpath string, adjustDepth bool } } - if G.opts.Debug { - traceStep2("resolveVarsInRelativePath: %q => %q", relpath, tmp) + if trace.Tracing { + trace.Step2("resolveVarsInRelativePath: %q => %q", relpath, tmp) } return tmp } -func (mkline *MkLine) checkText(text string) { - if G.opts.Debug { - defer tracecall1(text)() - } - - if contains(text, "${WRKSRC}/..") { - mkline.Warnf("Building the package should take place entirely inside ${WRKSRC}, not \"${WRKSRC}/..\".") - Explain( - "WRKSRC should be defined so that there is no need to do anything", - "outside of this directory.", - "", - "Example:", - "", - "\tWRKSRC=\t${WRKDIR}", - "\tCONFIGURE_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src", - "\tBUILD_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src ${WRKSRC}/cmd", - "", - "See the pkgsrc guide, section \"Directories used during the build", - "process\" for more information.") - } - - // Note: A simple -R is not detected, as the rate of false positives is too high. - if m, flag := match1(text, `\b(-Wl,--rpath,|-Wl,-rpath-link,|-Wl,-rpath,|-Wl,-R)\b`); m { - mkline.Warnf("Please use ${COMPILER_RPATH_FLAG} instead of %q.", flag) - } - - rest := text - for { - m, r := replaceFirst(rest, `(?:^|[^$])\$\{([-A-Z0-9a-z_]+)(\.[\-0-9A-Z_a-z]+)?(?::[^\}]+)?\}`, "") - if m == nil { - break - } - rest = r - - varbase, varext := m[1], m[2] - varname := varbase + varext - varcanon := varnameCanon(varname) - instead := G.globalData.Deprecated[varname] - if instead == "" { - instead = G.globalData.Deprecated[varcanon] - } - if instead != "" { - mkline.Warnf("Use of %q is deprecated. %s", varname, instead) - } - } -} - -func (mkline *MkLine) CheckCond() { - if G.opts.Debug { - defer tracecall1(mkline.Args())() - } - - p := NewMkParser(mkline.Line, mkline.Args(), false) - cond := p.MkCond() - if !p.EOF() { - mkline.Warnf("Invalid conditional %q.", mkline.Args()) - return - } - - cond.Visit("empty", func(node *Tree) { - varuse := node.args[0].(MkVarUse) - varname := varuse.varname - if matches(varname, `^\$.*:[MN]`) { - mkline.Warnf("The empty() function takes a variable name as parameter, not a variable expression.") - Explain( - "Instead of empty(${VARNAME:Mpattern}), you should write either", - "of the following:", - "", - "\tempty(VARNAME:Mpattern)", - "\t${VARNAME:Mpattern} == \"\"", - "", - "Instead of !empty(${VARNAME:Mpattern}), you should write either", - "of the following:", - "", - "\t!empty(VARNAME:Mpattern)", - "\t${VARNAME:Mpattern}") - } - for _, modifier := range varuse.modifiers { - if modifier[0] == 'M' || modifier[0] == 'N' { - mkline.CheckVartype(varname, opUseMatch, modifier[1:], "") - } - } - }) - - cond.Visit("compareVarStr", func(node *Tree) { - varuse := node.args[0].(MkVarUse) - varname := varuse.varname - varmods := varuse.modifiers - value := node.args[2].(string) - if len(varmods) == 0 { - mkline.checkCompareVarStr(varname, node.args[1].(string), value) - } else if len(varmods) == 1 && matches(varmods[0], `^[MN]`) && value != "" { - mkline.CheckVartype(varname, opUseMatch, value, "") - } - }) - - mkline.rememberUsedVariables(cond) -} - -func (mkline *MkLine) checkCompareVarStr(varname, op, value string) { - mkline.CheckVartype(varname, opUseCompare, value, "") - - if varname == "PKGSRC_COMPILER" { - mkline.Line.Warnf("Use ${PKGSRC_COMPILER:%s%s} instead of the %s operator.", ifelseStr(op == "==", "M", "N"), value, op) - Explain( - "The PKGSRC_COMPILER can be a list of chained compilers, e.g. \"ccache", - "distcc clang\". Therefore, comparing it using == or != leads to", - "wrong results in these cases.") - } -} - func (mkline *MkLine) rememberUsedVariables(cond *Tree) { if G.Mk == nil { return @@ -1281,17 +268,6 @@ func (mkline *MkLine) rememberUsedVariables(cond *Tree) { cond.Visit("compareVarVar", arg2varuse) } -func (mkline *MkLine) CheckValidCharactersInValue(reValid RegexPattern) { - rest := regcomp(reValid).ReplaceAllString(mkline.Value(), "") - if rest != "" { - uni := "" - for _, c := range rest { - uni += fmt.Sprintf(" %U", c) - } - mkline.Warnf("%s contains invalid characters (%s).", mkline.Varname(), uni[1:]) - } -} - func (mkline *MkLine) explainRelativeDirs() { Explain( "Directories in the form \"../../category/package\" make it easier to", @@ -1299,61 +275,6 @@ func (mkline *MkLine) explainRelativeDirs() { "main pkgsrc repository.") } -func (mkline *MkLine) CheckRelativePkgdir(pkgdir string) { - if G.opts.Debug { - defer tracecall1(pkgdir)() - } - - mkline.CheckRelativePath(pkgdir, true) - pkgdir = mkline.resolveVarsInRelativePath(pkgdir, false) - - if m, otherpkgpath := match1(pkgdir, `^(?:\./)?\.\./\.\./([^/]+/[^/]+)$`); m { - if !fileExists(G.globalData.Pkgsrcdir + "/" + otherpkgpath + "/Makefile") { - mkline.Errorf("There is no package in %q.", otherpkgpath) - } - - } else if !containsVarRef(pkgdir) { - mkline.Warnf("%q is not a valid relative package directory.", pkgdir) - Explain( - "A relative pathname always starts with \"../../\", followed", - "by a category, a slash and a the directory name of the package.", - "For example, \"../../misc/screen\" is a valid relative pathname.") - } -} - -func (mkline *MkLine) CheckRelativePath(path string, mustExist bool) { - if G.opts.Debug { - defer tracecall(path, mustExist)() - } - - if !G.Wip && contains(path, "/wip/") { - mkline.Line.Errorf("A main pkgsrc package must not depend on a pkgsrc-wip package.") - } - - resolvedPath := mkline.resolveVarsInRelativePath(path, true) - if containsVarRef(resolvedPath) { - return - } - - abs := resolvedPath - if !hasPrefix(abs, "/") { - abs = G.CurrentDir + "/" + abs - } - if _, err := os.Stat(abs); err != nil { - if mustExist { - mkline.Errorf("%q does not exist.", resolvedPath) - } - return - } - - if hasPrefix(path, "../") && - !matches(path, `^\.\./\.\./[^/]+/[^/]`) && - !(G.CurPkgsrcdir == ".." && hasPrefix(path, "../mk/")) && // For category Makefiles. - !hasPrefix(path, "../../mk/") { - mkline.Warnf("Invalid relative path %q.", path) - } -} - func matchMkCond(text string) (m bool, indent, directive, args string) { i, n := 0, len(text) if i < n && text[i] == '.' { @@ -1414,8 +335,8 @@ func (nq NeedsQuoting) String() string { } func (mkline *MkLine) variableNeedsQuoting(varname string, vartype *Vartype, vuc *VarUseContext) (needsQuoting NeedsQuoting) { - if G.opts.Debug { - defer tracecall(varname, vartype, vuc, "=>", &needsQuoting)() + if trace.Tracing { + defer trace.Call(varname, vartype, vuc, "=>", &needsQuoting)() } if vartype == nil || vuc.vartype == nil { @@ -1441,8 +362,8 @@ func (mkline *MkLine) variableNeedsQuoting(varname string, vartype *Vartype, vuc // Determine whether the context expects a list of shell words or not. wantList := vuc.vartype.IsConsideredList() haveList := vartype.IsConsideredList() - if G.opts.Debug { - traceStep("wantList=%v, haveList=%v", wantList, haveList) + if trace.Tracing { + trace.Stepf("wantList=%v, haveList=%v", wantList, haveList) } // A shell word may appear as part of a shell word, for example COMPILER_RPATH_FLAG. @@ -1514,8 +435,8 @@ func (mkline *MkLine) variableNeedsQuoting(varname string, vartype *Vartype, vuc return nqYes } - if G.opts.Debug { - traceStep1("Don't know whether :Q is needed for %q", varname) + if trace.Tracing { + trace.Step1("Don't know whether :Q is needed for %q", varname) } return nqDontKnow } @@ -1523,8 +444,8 @@ func (mkline *MkLine) variableNeedsQuoting(varname string, vartype *Vartype, vuc // Returns the type of the variable (maybe guessed based on the variable name), // or nil if the type cannot even be guessed. func (mkline *MkLine) getVariableType(varname string) *Vartype { - if G.opts.Debug { - defer tracecall1(varname)() + if trace.Tracing { + defer trace.Call1(varname)() } if vartype := G.globalData.vartypes[varname]; vartype != nil { @@ -1536,8 +457,8 @@ func (mkline *MkLine) getVariableType(varname string) *Vartype { if tool := G.globalData.Tools.byVarname[varname]; tool != nil { perms := aclpUse - if G.opts.Debug { - traceStep("Use of tool %+v", tool) + if trace.Tracing { + trace.Stepf("Use of tool %+v", tool) } if tool.UsableAtLoadtime { if G.Pkg == nil || G.Pkg.SeenBsdPrefsMk || G.Pkg.loadTimeTools[tool.Name] { @@ -1590,11 +511,11 @@ func (mkline *MkLine) getVariableType(varname string) *Vartype { gtype = &Vartype{lkNone, BtYes, allowAll, true} } - if G.opts.Debug { + if trace.Tracing { if gtype != nil { - traceStep2("The guessed type of %q is %q.", varname, gtype.String()) + trace.Step2("The guessed type of %q is %q.", varname, gtype.String()) } else { - traceStep1("No type definition found for %q.", varname) + trace.Step1("No type definition found for %q.", varname) } } return gtype @@ -1602,7 +523,7 @@ func (mkline *MkLine) getVariableType(varname string) *Vartype { // TODO: merge with determineUsedVariables func (mkline *MkLine) extractUsedVariables(text string) []string { - re := regcomp(`^(?:[^\$]+|\$[\$*<>?@]|\$\{([.0-9A-Z_a-z]+)(?::(?:[^\${}]|\$[^{])+)?\})`) + re := regex.Compile(`^(?:[^\$]+|\$[\$*<>?@]|\$\{([.0-9A-Z_a-z]+)(?::(?:[^\${}]|\$[^{])+)?\})`) rest := text var result []string for { @@ -1617,14 +538,14 @@ func (mkline *MkLine) extractUsedVariables(text string) []string { } } - if G.opts.Debug && rest != "" { - traceStep1("extractUsedVariables: rest=%q", rest) + if trace.Tracing && rest != "" { + trace.Step1("extractUsedVariables: rest=%q", rest) } return result } func (mkline *MkLine) determineUsedVariables() (varnames []string) { - rest := mkline.Line.Text + rest := mkline.Line.Text() if strings.HasPrefix(rest, "#") { return @@ -1653,7 +574,7 @@ func (mkline *MkLine) determineUsedVariables() (varnames []string) { } rest = rest[min:] - m := regcomp(`(?:\$\{|\$\(|defined\(|empty\()([*+\-.0-9A-Z_a-z]+)[:})]`).FindStringSubmatchIndex(rest) + m := regex.Compile(`(?:\$\{|\$\(|defined\(|empty\()([*+\-.0-9A-Z_a-z]+)[:})]`).FindStringSubmatchIndex(rest) if m == nil { return } @@ -1732,6 +653,10 @@ func (vuc *VarUseContext) String() string { return fmt.Sprintf("(%s time:%s quoting:%s wordpart:%v)", typename, vuc.time, vuc.quoting, vuc.IsWordPart) } +// Indentation remembers the stack of preprocessing directives and their +// indentation. By convention, each directive is indented by 2 spaces. +// An excepting are multiple-inclusion guards, they don't increase the +// indentation. type Indentation struct { depth []int // Number of space characters; always a multiple of 2 conditionVars [][]string // Variables on which the current path depends diff --git a/pkgtools/pkglint/files/mkline_test.go b/pkgtools/pkglint/files/mkline_test.go index a2a66eee717..84f81042c17 100644 --- a/pkgtools/pkglint/files/mkline_test.go +++ b/pkgtools/pkglint/files/mkline_test.go @@ -4,35 +4,6 @@ import ( check "gopkg.in/check.v1" ) -func (s *Suite) Test_MkLine_CheckVartype_simple_type(c *check.C) { - s.Init(c) - s.UseCommandLine("-Wtypes") - G.globalData.InitVartypes() - mkline := NewMkLine(NewLine("fname", 1, "COMMENT=\tA nice package", nil)) - - vartype1 := G.globalData.vartypes["COMMENT"] - c.Assert(vartype1, check.NotNil) - c.Check(vartype1.guessed, equals, false) - - vartype := mkline.getVariableType("COMMENT") - - c.Assert(vartype, check.NotNil) - c.Check(vartype.basicType.name, equals, "Comment") - c.Check(vartype.guessed, equals, false) - c.Check(vartype.kindOfList, equals, lkNone) - - mkline.CheckVartype("COMMENT", opAssign, "A nice package", "") - - c.Check(s.Stdout(), equals, "WARN: fname:1: COMMENT should not begin with \"A\".\n") -} - -func (s *Suite) Test_MkLine_CheckVartype(c *check.C) { - G.globalData.InitVartypes() - mkline := NewMkLine(NewLine("fname", 1, "DISTNAME=gcc-${GCC_VERSION}", nil)) - - mkline.CheckVartype("DISTNAME", opAssign, "gcc-${GCC_VERSION}", "") -} - func (s *Suite) Test_VaralignBlock_Check_autofix(c *check.C) { s.Init(c) s.UseCommandLine("-Wspace", "-f") @@ -57,20 +28,20 @@ func (s *Suite) Test_VaralignBlock_Check_autofix(c *check.C) { } varalign.Finish() - c.Check(lines[0].changed, equals, true) - c.Check(lines[0].raw[0].String(), equals, "1:VAR=\tvalue\n") - c.Check(lines[2].changed, equals, true) - c.Check(lines[2].raw[0].String(), equals, "3:VAR=\tvalue\n") - c.Check(lines[4].changed, equals, true) - c.Check(lines[4].raw[0].String(), equals, "5:VAR=\tvalue\n") - c.Check(lines[6].changed, equals, true) - c.Check(lines[6].raw[0].String(), equals, "7:VAR=\tvalue\n") - c.Check(lines[8].changed, equals, true) - c.Check(lines[8].raw[0].String(), equals, "9:VAR=\tvalue\n") - c.Check(lines[10].changed, equals, true) - c.Check(lines[10].raw[0].String(), equals, "11:VAR=\tvalue\n") - c.Check(lines[12].changed, equals, false) - c.Check(lines[12].raw[0].String(), equals, "13:VAR=\tvalue\n") + c.Check(lines[0].IsChanged(), equals, true) + c.Check(lines[0].(*LineImpl).raw[0].String(), equals, "1:VAR=\tvalue\n") + c.Check(lines[2].IsChanged(), equals, true) + c.Check(lines[2].(*LineImpl).raw[0].String(), equals, "3:VAR=\tvalue\n") + c.Check(lines[4].IsChanged(), equals, true) + c.Check(lines[4].(*LineImpl).raw[0].String(), equals, "5:VAR=\tvalue\n") + c.Check(lines[6].IsChanged(), equals, true) + c.Check(lines[6].(*LineImpl).raw[0].String(), equals, "7:VAR=\tvalue\n") + c.Check(lines[8].IsChanged(), equals, true) + c.Check(lines[8].(*LineImpl).raw[0].String(), equals, "9:VAR=\tvalue\n") + c.Check(lines[10].IsChanged(), equals, true) + c.Check(lines[10].(*LineImpl).raw[0].String(), equals, "11:VAR=\tvalue\n") + c.Check(lines[12].IsChanged(), equals, false) + c.Check(lines[12].(*LineImpl).raw[0].String(), equals, "13:VAR=\tvalue\n") c.Check(s.Output(), equals, ""+ "NOTE: file.mk:1: This variable value should be aligned with tabs, not spaces, to column 9.\n"+ "AUTOFIX: file.mk:1: Replacing \"VAR= \" with \"VAR=\\t\".\n"+ @@ -86,7 +57,7 @@ func (s *Suite) Test_VaralignBlock_Check_autofix(c *check.C) { "AUTOFIX: file.mk:11: Replacing \"VAR= \\t\" with \"VAR=\\t\".\n") } -func (s *Suite) Test_VaralignBlock_Check_reduce_indentation(c *check.C) { +func (s *Suite) Test_VaralignBlock_Check__reduce_indentation(c *check.C) { s.Init(c) s.UseCommandLine("-Wspace") mklines := s.NewMkLines("file.mk", @@ -245,64 +216,6 @@ func (s *Suite) Test_NewMkLine__autofix_space_after_varname(c *check.C) { "pkgbase := pkglint\n") } -// Pkglint once interpreted all lists as consisting of shell tokens, -// splitting this URL at the ampersands. -func (s *Suite) Test_MkLine_checkVarassign_URL_with_shell_special_characters(c *check.C) { - G.Pkg = NewPackage("graphics/gimp-fix-ca") - G.globalData.InitVartypes() - mkline := NewMkLine(NewLine("fname", 10, "MASTER_SITES=http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=", nil)) - - mkline.checkVarassign() - - c.Check(s.Output(), equals, "") -} - -func (s *Suite) Test_MkLine_Check_conditions(c *check.C) { - s.Init(c) - s.UseCommandLine("-Wtypes") - G.globalData.InitVartypes() - - NewMkLine(NewLine("fname", 1, ".if !empty(PKGSRC_COMPILER:Mmycc)", nil)).CheckCond() - - c.Check(s.Stdout(), equals, "WARN: fname:1: The pattern \"mycc\" cannot match any of "+ - "{ ccache ccc clang distcc f2c gcc hp icc ido "+ - "gcc mipspro mipspro-ucode pcc sunpro xlc } for PKGSRC_COMPILER.\n") - - NewMkLine(NewLine("fname", 1, ".elif ${A} != ${B}", nil)).CheckCond() - - c.Check(s.Stdout(), equals, "") - - NewMkLine(NewLine("fname", 1, ".if ${HOMEPAGE} == \"mailto:someone@example.org\"", nil)).CheckCond() - - c.Check(s.Output(), equals, "WARN: fname:1: \"mailto:someone@example.org\" is not a valid URL.\n") - - NewMkLine(NewLine("fname", 1, ".if !empty(PKGSRC_RUN_TEST:M[Y][eE][sS])", nil)).CheckCond() - - c.Check(s.Output(), equals, "WARN: fname:1: PKGSRC_RUN_TEST should be matched against \"[yY][eE][sS]\" or \"[nN][oO]\", not \"[Y][eE][sS]\".\n") - - NewMkLine(NewLine("fname", 1, ".if !empty(IS_BUILTIN.Xfixes:M[yY][eE][sS])", nil)).CheckCond() - - c.Check(s.Output(), equals, "") - - NewMkLine(NewLine("fname", 1, ".if !empty(${IS_BUILTIN.Xfixes:M[yY][eE][sS]})", nil)).CheckCond() - - c.Check(s.Output(), equals, "WARN: fname:1: The empty() function takes a variable name as parameter, not a variable expression.\n") - - NewMkLine(NewLine("fname", 1, ".if ${EMUL_PLATFORM} == \"linux-x386\"", nil)).CheckCond() - - c.Check(s.Output(), equals, "WARN: fname:1: \"x386\" is not valid for the hardware architecture part of EMUL_PLATFORM. Use one of { aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } instead.\n") - - NewMkLine(NewLine("fname", 1, ".if ${EMUL_PLATFORM:Mlinux-x386}", nil)).CheckCond() - - c.Check(s.Output(), equals, "WARN: fname:1: The pattern \"x386\" cannot match any of { aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } for the hardware architecture part of EMUL_PLATFORM.\n") - - NewMkLine(NewLine("fname", 98, ".if ${MACHINE_PLATFORM:MUnknownOS-*-*} || ${MACHINE_ARCH:Mx86}", nil)).CheckCond() - - c.Check(s.Output(), equals, ""+ - "WARN: fname:98: The pattern \"UnknownOS\" cannot match any of { AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare } for the operating system part of MACHINE_PLATFORM.\n"+ - "WARN: fname:98: The pattern \"x86\" cannot match any of { aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } for MACHINE_ARCH.\n") -} - func (s *Suite) Test_MkLine_getVariableType_varparam(c *check.C) { mkline := NewMkLine(NewLine("fname", 1, "# dummy", nil)) G.globalData.InitVartypes() @@ -327,18 +240,6 @@ func (s *Suite) Test_VarUseContext_String(c *check.C) { c.Check(vuc.String(), equals, "(PkgName time:unknown quoting:backt wordpart:false)") } -func (s *Suite) Test_MkLine_checkVarassign(c *check.C) { - G.globalData.InitVartypes() - - G.Mk = s.NewMkLines("Makefile", - mkrcsid, - "ac_cv_libpari_libs+=\t-L${BUILDLINK_PREFIX.pari}/lib") // From math/clisp-pari/Makefile, rev. 1.8 - - G.Mk.mklines[1].checkVarassign() - - c.Check(s.Output(), equals, "WARN: Makefile:2: ac_cv_libpari_libs is defined but not used. Spelling mistake?\n") -} - // In variable assignments, a plain '#' introduces a line comment, unless // it is escaped by a backslash. In shell commands, on the other hand, it // is interpreted literally. @@ -370,61 +271,6 @@ func (s *Suite) Test_NewMkLine_leading_space(c *check.C) { c.Check(s.Output(), equals, "WARN: rubyversion.mk:427: Makefile lines should not start with space characters.\n") } -func (s *Suite) Test_MkLine_checkVarassignDefPermissions(c *check.C) { - s.Init(c) - s.UseCommandLine("-Wall") - G.globalData.InitVartypes() - mkline := NewMkLine(NewLine("options.mk", 2, "PKG_DEVELOPER?=\tyes", nil)) - - mkline.checkVarassignDefPermissions() - - c.Check(s.Output(), equals, "WARN: options.mk:2: The variable PKG_DEVELOPER may not be given a default value by any package.\n") -} - -func (s *Suite) Test_MkLine_CheckVarusePermissions(c *check.C) { - s.Init(c) - s.UseCommandLine("-Wall") - G.globalData.InitVartypes() - mklines := s.NewMkLines("options.mk", - mkrcsid, - "COMMENT=\t${GAMES_USER}", - "COMMENT:=\t${PKGBASE}", - "PYPKGPREFIX=${PKGBASE}") - G.globalData.UserDefinedVars = map[string]*MkLine{ - "GAMES_USER": mklines.mklines[0], - } - - mklines.Check() - - c.Check(s.Output(), equals, ""+ - "WARN: options.mk:2: The user-defined variable GAMES_USER is used but not added to BUILD_DEFS.\n"+ - "WARN: options.mk:3: PKGBASE should not be evaluated at load time.\n"+ - "WARN: options.mk:4: The variable PYPKGPREFIX may not be set in this file; it would be ok in pyversion.mk.\n"+ - "WARN: options.mk:4: PKGBASE should not be evaluated indirectly at load time.\n"+ - "NOTE: options.mk:4: This variable value should be aligned to column 17.\n") -} - -func (s *Suite) Test_MkLine_CheckVarusePermissions__load_time(c *check.C) { - s.Init(c) - s.UseCommandLine("-Wall") - G.globalData.InitVartypes() - mklines := s.NewMkLines("options.mk", - mkrcsid, - "WRKSRC:=${.CURDIR}") - - mklines.Check() - - c.Check(s.Output(), equals, "") // Don't warn that ".CURDIR should not be evaluated at load time." -} - -func (s *Suite) Test_MkLine_WarnVaruseLocalbase(c *check.C) { - mkline := NewMkLine(NewLine("options.mk", 56, "PKGNAME=${LOCALBASE}", nil)) - - mkline.WarnVaruseLocalbase() - - c.Check(s.Output(), equals, "WARN: options.mk:56: The LOCALBASE variable should not be used by packages.\n") -} - func (s *Suite) Test_MkLines_Check__extra(c *check.C) { s.Init(c) s.UseCommandLine("-Wextra") @@ -453,28 +299,6 @@ func (s *Suite) Test_MkLines_Check__extra(c *check.C) { "NOTE: options.mk:10: You can use \"../build\" instead of \"${WRKSRC}/../build\".\n") } -func (s *Suite) Test_MkLine_CheckRelativePkgdir(c *check.C) { - mkline := NewMkLine(NewLine("Makefile", 46, "# dummy", nil)) - - mkline.CheckRelativePkgdir("../pkgbase") - - c.Check(s.Output(), equals, ""+ - "ERROR: Makefile:46: \"../pkgbase\" does not exist.\n"+ - "WARN: Makefile:46: \"../pkgbase\" is not a valid relative package directory.\n") -} - -// PR pkg/46570, item 2 -func (s *Suite) Test_MkLine__unclosed_varuse(c *check.C) { - mkline := NewMkLine(NewLine("Makefile", 93, "EGDIRS=${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d", nil)) - - mkline.checkVarassign() - - c.Check(s.Output(), equals, ""+ - "WARN: Makefile:93: Pkglint parse error in MkLine.Tokenize at \"${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".\n"+ - "WARN: Makefile:93: Pkglint parse error in ShTokenizer.ShAtom at \"${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\" (quoting=plain)\n"+ - "WARN: Makefile:93: EGDIRS is defined but not used. Spelling mistake?\n") -} - func (s *Suite) Test_MkLine_variableNeedsQuoting__unknown_rhs(c *check.C) { mkline := NewMkLine(NewLine("fname", 1, "PKGNAME := ${UNKNOWN}", nil)) G.globalData.InitVartypes() @@ -497,7 +321,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__append_URL_to_list_of_URLs(c * c.Check(nq, equals, nqNo) - mkline.checkVarassign() + MkLineChecker{mkline}.checkVarassign() c.Check(s.Output(), equals, "") // Up to pkglint 5.3.6, it warned about a missing :Q here, which was wrong. } @@ -510,7 +334,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__append_list_to_list(c *check.C s.RegisterMasterSite("MASTER_SITE_SOURCEFORGE", "http://downloads.sourceforge.net/sourceforge/") mkline := NewMkLine(NewLine("Makefile", 96, "MASTER_SITES=\t${MASTER_SITE_SOURCEFORGE:=squirrel-sql/}", nil)) - mkline.checkVarassign() + MkLineChecker{mkline}.checkVarassign() c.Check(s.Output(), equals, "") } @@ -521,7 +345,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__eval_shell(c *check.C) { G.globalData.InitVartypes() mkline := NewMkLine(NewLine("builtin.mk", 3, "USE_BUILTIN.Xfixes!=\t${PKG_ADMIN} pmatch 'pkg-[0-9]*' ${BUILTIN_PKG.Xfixes:Q}", nil)) - mkline.checkVarassign() + MkLineChecker{mkline}.checkVarassign() c.Check(s.Output(), equals, ""+ "WARN: builtin.mk:3: PKG_ADMIN should not be evaluated at load time.\n"+ @@ -534,7 +358,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__command_in_single_quotes(c *ch G.globalData.InitVartypes() mkline := NewMkLine(NewLine("Makefile", 3, "SUBST_SED.hpath=\t-e 's|^\\(INSTALL[\t:]*=\\).*|\\1${INSTALL}|'", nil)) - mkline.checkVarassign() + MkLineChecker{mkline}.checkVarassign() c.Check(s.Output(), equals, "WARN: Makefile:3: Please use ${INSTALL:Q} instead of ${INSTALL} and make sure the variable appears outside of any quoting characters.\n") } @@ -551,7 +375,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__command_in_command(c *check.C) "GENERATE_PLIST= cd ${DESTDIR}${PREFIX}; ${FIND} * \\( -type f -or -type l \\) | ${SORT};") G.Mk.determineDefinedVariables() - G.Mk.mklines[1].Check() + MkLineChecker{G.Mk.mklines[1]}.Check() c.Check(s.Output(), equals, ""+ "WARN: Makefile:2: The exitcode of the left-hand-side command of the pipe operator is ignored.\n") @@ -565,7 +389,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__word_as_part_of_word(c *check. mkrcsid, "EGDIR=\t${EGDIR}/${MACHINE_GNU_PLATFORM}") - G.Mk.mklines[1].Check() + MkLineChecker{G.Mk.mklines[1]}.Check() c.Check(s.Output(), equals, "") } @@ -587,8 +411,8 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__command_as_command_argument(c "\t${RUN} cd ${WRKSRC} && ( ${ECHO} ${PERL5:Q} ; ${ECHO} ) | ${BASH} ./install", "\t${RUN} cd ${WRKSRC} && ( ${ECHO} ${PERL5} ; ${ECHO} ) | ${BASH} ./install") - G.Mk.mklines[1].Check() - G.Mk.mklines[2].Check() + MkLineChecker{G.Mk.mklines[1]}.Check() + MkLineChecker{G.Mk.mklines[2]}.Check() c.Check(s.Output(), equals, ""+ "WARN: Makefile:2: The exitcode of the left-hand-side command of the pipe operator is ignored.\n"+ @@ -604,7 +428,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__URL_as_part_of_word_in_list(c mkrcsid, "MASTER_SITES=${HOMEPAGE}archive/") - G.Mk.mklines[1].Check() + MkLineChecker{G.Mk.mklines[1]}.Check() c.Check(s.Output(), equals, "") // Don't suggest to use ${HOMEPAGE:Q}. } @@ -625,8 +449,8 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__command_in_subshell(c *check.C "\t id=$$(${AWK} '{print}' < ${WRKSRC}/idfile) && echo \"$$id\"", "\t id=`${AWK} '{print}' < ${WRKSRC}/idfile` && echo \"$$id\"") - G.Mk.mklines[1].Check() - G.Mk.mklines[2].Check() + MkLineChecker{G.Mk.mklines[1]}.Check() + MkLineChecker{G.Mk.mklines[2]}.Check() c.Check(s.Output(), equals, "WARN: xpi.mk:2: Invoking subshells via $(...) is not portable enough.\n") // Don't suggest to use ${AWK:Q}. } @@ -643,8 +467,8 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__LDFLAGS_in_single_quotes(c *ch "SUBST_SED.link=-e 's|(LIBTOOL_LINK).*(LIBS)|& ${LDFLAGS:M*:Q}|g'", "SUBST_SED.link=-e 's|(LIBTOOL_LINK).*(LIBS)|& '${LDFLAGS:M*:Q}'|g'") - G.Mk.mklines[1].Check() - G.Mk.mklines[2].Check() + MkLineChecker{G.Mk.mklines[1]}.Check() + MkLineChecker{G.Mk.mklines[2]}.Check() c.Check(s.Output(), equals, "WARN: x11/mlterm/Makefile:2: Please move ${LDFLAGS:M*:Q} outside of any quoting characters.\n") } @@ -680,7 +504,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__tool_in_quotes_in_subshell_in_ mkrcsid, "CONFIGURE_ARGS+=\t-tklibs \"`${SH} -c '${ECHO} $$TK_LD_FLAGS'`\"") - G.Mk.mklines[1].Check() + MkLineChecker{G.Mk.mklines[1]}.Check() c.Check(s.Output(), equals, "") // Don't suggest ${ECHO:Q} here. } @@ -692,25 +516,25 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting__LDADD_in_BUILDLINK_TRANSFORM(c G.Mk = s.NewMkLines("x11/qt5-qtbase/Makefile.common", "BUILDLINK_TRANSFORM+=opt:-ldl:${BUILDLINK_LDADD.dl:M*}") - G.Mk.mklines[0].Check() + MkLineChecker{G.Mk.mklines[0]}.Check() // Note: The :M* modifier is not necessary, since this is not a GNU Configure package. c.Check(s.Output(), equals, "WARN: x11/qt5-qtbase/Makefile.common:1: Please use ${BUILDLINK_LDADD.dl:Q} instead of ${BUILDLINK_LDADD.dl:M*}.\n") } -func (s *Suite) Test_MkLine_variableNeedsQuoting_command_in_message(c *check.C) { +func (s *Suite) Test_MkLine_variableNeedsQuoting__command_in_message(c *check.C) { s.Init(c) s.UseCommandLine("-Wall") G.globalData.InitVartypes() G.Mk = s.NewMkLines("benchmarks/iozone/Makefile", "SUBST_MESSAGE.crlf=\tStripping EOL CR in ${REPLACE_PERL}") - G.Mk.mklines[0].Check() + MkLineChecker{G.Mk.mklines[0]}.Check() c.Check(s.Output(), equals, "") // Don't suggest ${REPLACE_PERL:Q}. } -func (s *Suite) Test_MkLine_variableNeedsQuoting_guessed_list_variable_in_quotes(c *check.C) { +func (s *Suite) Test_MkLine_variableNeedsQuoting__guessed_list_variable_in_quotes(c *check.C) { s.Init(c) s.UseCommandLine("-Wall") G.globalData.InitVartypes() @@ -724,7 +548,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting_guessed_list_variable_in_quotes c.Check(s.Output(), equals, "WARN: audio/jack-rack/Makefile:3: The list variable LADSPA_PLUGIN_PATH should not be embedded in a word.\n") } -func (s *Suite) Test_MkLine_variableNeedsQuoting_list_in_list(c *check.C) { +func (s *Suite) Test_MkLine_variableNeedsQuoting__list_in_list(c *check.C) { s.Init(c) s.UseCommandLine("-Wall") G.globalData.InitVartypes() @@ -737,7 +561,7 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting_list_in_list(c *check.C) { c.Check(s.Output(), equals, "") // Don't warn about missing :Q modifiers. } -func (s *Suite) Test_MkLine_variableNeedsQuoting_PKGNAME_and_URL_list_in_URL_list(c *check.C) { +func (s *Suite) Test_MkLine_variableNeedsQuoting__PKGNAME_and_URL_list_in_URL_list(c *check.C) { s.Init(c) s.UseCommandLine("-Wall") s.RegisterMasterSite("MASTER_SITE_GNOME", "http://ftp.gnome.org/") @@ -746,12 +570,12 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting_PKGNAME_and_URL_list_in_URL_lis mkrcsid, "MASTER_SITES=\tftp://ftp.gtk.org/${PKGNAME}/ ${MASTER_SITE_GNOME:=subdir/}") - G.Mk.mklines[1].checkVarassignVaruse() + MkLineChecker{G.Mk.mklines[1]}.checkVarassignVaruse() c.Check(s.Output(), equals, "") // Don't warn about missing :Q modifiers. } -func (s *Suite) Test_MkLine_variableNeedsQuoting_tool_in_CONFIGURE_ENV(c *check.C) { +func (s *Suite) Test_MkLine_variableNeedsQuoting__tool_in_CONFIGURE_ENV(c *check.C) { s.Init(c) s.UseCommandLine("-Wall") G.globalData.InitVartypes() @@ -763,54 +587,13 @@ func (s *Suite) Test_MkLine_variableNeedsQuoting_tool_in_CONFIGURE_ENV(c *check. "", "CONFIGURE_ENV+=\tSYS_TAR_COMMAND_PATH=${TOOLS_TAR:Q}") - mklines.mklines[2].checkVarassignVaruse() + MkLineChecker{mklines.mklines[2]}.checkVarassignVaruse() // The TOOLS_* variables only contain the path to the tool, // without any additional arguments that might be necessary. c.Check(s.Output(), equals, "NOTE: Makefile:3: The :Q operator isn't necessary for ${TOOLS_TAR} here.\n") } -func (s *Suite) Test_MkLine_Varuse_Modifier_L(c *check.C) { - s.Init(c) - s.UseCommandLine("-Wall") - G.globalData.InitVartypes() - G.Mk = s.NewMkLines("x11/xkeyboard-config/Makefile", - "FILES_SUBST+=XKBCOMP_SYMLINK=${${XKBBASE}/xkbcomp:L:Q}") - - G.Mk.mklines[0].Check() - - c.Check(s.Output(), equals, "") // Don't warn that ${XKBBASE}/xkbcomp is used but not defined. -} - -func (s *Suite) Test_MkLine_CheckCond_comparison_with_shell_command(c *check.C) { - s.Init(c) - s.UseCommandLine("-Wall") - G.globalData.InitVartypes() - G.Mk = s.NewMkLines("security/openssl/Makefile", - mkrcsid, - ".if ${PKGSRC_COMPILER} == \"gcc\" && ${CC} == \"cc\"", - ".endif") - - G.Mk.Check() - - // Don't warn about unknown shell command "cc". - c.Check(s.Output(), equals, "WARN: security/openssl/Makefile:2: Use ${PKGSRC_COMPILER:Mgcc} instead of the == operator.\n") -} - -func (s *Suite) Test_MkLine_CheckCond_comparing_PKGSRC_COMPILER_with_eqeq(c *check.C) { - s.Init(c) - s.UseCommandLine("-Wall") - G.globalData.InitVartypes() - G.Mk = s.NewMkLines("audio/pulseaudio/Makefile", - mkrcsid, - ".if ${OPSYS} == \"Darwin\" && ${PKGSRC_COMPILER} == \"clang\"", - ".endif") - - G.Mk.Check() - - c.Check(s.Output(), equals, "WARN: audio/pulseaudio/Makefile:2: Use ${PKGSRC_COMPILER:Mclang} instead of the == operator.\n") -} - func (s *Suite) Test_MkLine_Pkgmandir(c *check.C) { s.Init(c) s.UseCommandLine("-Wall") @@ -825,26 +608,7 @@ func (s *Suite) Test_MkLine_Pkgmandir(c *check.C) { c.Check(s.Output(), equals, "WARN: chat/ircII/Makefile:2: Please use ${PKGMANDIR} instead of \"man\".\n") } -func (s *Suite) Test_MkLine_Check_CFLAGS_with_backticks(c *check.C) { - s.Init(c) - s.UseCommandLine("-Wall") - G.globalData.InitVartypes() - G.Mk = s.NewMkLines("chat/pidgin-icb/Makefile", - mkrcsid, - "CFLAGS+=\t`pkg-config pidgin --cflags`") - mkline := G.Mk.mklines[1] - - words, rest := splitIntoMkWords(mkline.Line, mkline.Value()) - - c.Check(words, deepEquals, []string{"`pkg-config pidgin --cflags`"}) - c.Check(rest, equals, "") - - G.Mk.mklines[1].CheckVartype("CFLAGS", opAssignAppend, "`pkg-config pidgin --cflags`", "") - - c.Check(s.Output(), equals, "") // No warning about "`pkg-config" being an unknown CFlag. -} - -func (s *Suite) Test_MkLine_Check_VERSION_as_wordpart_in_MASTER_SITES(c *check.C) { +func (s *Suite) Test_MkLines_Check__VERSION_as_wordpart_in_MASTER_SITES(c *check.C) { s.Init(c) s.UseCommandLine("-Wall") G.globalData.InitVartypes() @@ -858,7 +622,7 @@ func (s *Suite) Test_MkLine_Check_VERSION_as_wordpart_in_MASTER_SITES(c *check.C "The list variable MASTER_SITE_SOURCEFORGE should not be embedded in a word.\n") } -func (s *Suite) Test_MkLine_Check_shell_command_as_wordpart_in_ENV_list(c *check.C) { +func (s *Suite) Test_MkLines_Check__shell_command_as_wordpart_in_ENV_list(c *check.C) { s.Init(c) s.UseCommandLine("-Wall") G.globalData.InitVartypes() @@ -896,22 +660,6 @@ func (s *Suite) Test_MkLine_getVariableType(c *check.C) { c.Check(mkline.getVariableType("SOMEDIR").guessed, equals, true) } -// See PR 46570, Ctrl+F "4. Shell quoting". -// Pkglint is correct, since the shell sees this definition for -// CPPFLAGS as three words, not one word. -func (s *Suite) Test_MkLine_CheckVartype_CFLAGS(c *check.C) { - G.globalData.InitVartypes() - mklines := s.NewMkLines("Makefile", - mkrcsid, - "CPPFLAGS.SunOS+=\t-DPIPECOMMAND=\\\"/usr/sbin/sendmail -bs %s\\\"") - - mklines.Check() - - c.Check(s.Output(), equals, ""+ - "WARN: Makefile:2: Unknown compiler flag \"-bs\".\n"+ - "WARN: Makefile:2: Compiler flag \"%s\\\\\\\"\" should start with a hyphen.\n") -} - // PR 51696, security/py-pbkdf2/Makefile, r1.2 func (s *Suite) Test_MkLine__comment_in_comment(c *check.C) { G.globalData.InitVartypes() diff --git a/pkgtools/pkglint/files/mklinechecker.go b/pkgtools/pkglint/files/mklinechecker.go new file mode 100644 index 00000000000..e9f89cb7bd4 --- /dev/null +++ b/pkgtools/pkglint/files/mklinechecker.go @@ -0,0 +1,1118 @@ +package main + +import ( + "fmt" + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/trace" + "os" + "path" + "strconv" + "strings" +) + +type MkLineChecker struct { + MkLine *MkLine +} + +func (ck MkLineChecker) Check() { + mkline := ck.MkLine + + LineChecker{mkline.Line}.CheckTrailingWhitespace() + LineChecker{mkline.Line}.CheckValidCharacters(`[\t -~]`) + + switch { + case mkline.IsVarassign(): + ck.checkVarassign() + + case mkline.IsShellcmd(): + shellcmd := mkline.Shellcmd() + ck.checkText(shellcmd) + NewShellLine(mkline).CheckShellCommandLine(shellcmd) + + case mkline.IsComment(): + if hasPrefix(mkline.Line.Text(), "# url2pkg-marker") { + mkline.Line.Errorf("This comment indicates unfinished work (url2pkg).") + } + + case mkline.IsInclude(): + ck.checkInclude() + } +} + +func (ck MkLineChecker) checkInclude() { + if trace.Tracing { + defer trace.Call0()() + } + + mkline := ck.MkLine + if mkline.Indent() != "" { + ck.checkDirectiveIndentation(G.Mk.indentation.Depth()) + } + + includefile := mkline.Includefile() + mustExist := mkline.MustExist() + if trace.Tracing { + trace.Step2("includingFile=%s includefile=%s", mkline.Filename(), includefile) + } + ck.CheckRelativePath(includefile, mustExist) + + switch { + case hasSuffix(includefile, "/Makefile"): + mkline.Line.Errorf("Other Makefiles must not be included directly.") + Explain( + "If you want to include portions of another Makefile, extract", + "the common parts and put them into a Makefile.common. After", + "that, both this one and the other package should include the", + "Makefile.common.") + + case includefile == "../../mk/bsd.prefs.mk": + if path.Base(mkline.Line.Filename()) == "buildlink3.mk" { + mkline.Notef("For efficiency reasons, please include bsd.fast.prefs.mk instead of bsd.prefs.mk.") + } + if G.Pkg != nil { + G.Pkg.setSeenBsdPrefsMk() + } + + case includefile == "../../mk/bsd.fast.prefs.mk", includefile == "../../mk/buildlink3/bsd.builtin.mk": + if G.Pkg != nil { + G.Pkg.setSeenBsdPrefsMk() + } + + case hasSuffix(includefile, "/x11-links/buildlink3.mk"): + mkline.Errorf("%s must not be included directly. Include \"../../mk/x11.buildlink3.mk\" instead.", includefile) + + case hasSuffix(includefile, "/jpeg/buildlink3.mk"): + mkline.Errorf("%s must not be included directly. Include \"../../mk/jpeg.buildlink3.mk\" instead.", includefile) + + case hasSuffix(includefile, "/intltool/buildlink3.mk"): + mkline.Warnf("Please write \"USE_TOOLS+= intltool\" instead of this line.") + + case hasSuffix(includefile, "/builtin.mk"): + mkline.Line.Errorf("%s must not be included directly. Include \"%s/buildlink3.mk\" instead.", includefile, path.Dir(includefile)) + } +} + +func (ck MkLineChecker) checkCond(forVars map[string]bool) { + mkline := ck.MkLine + + directive := mkline.Directive() + args := mkline.Args() + indentation := &G.Mk.indentation + + switch directive { + case "endif", "endfor": + if indentation.Len() > 1 { + indentation.Pop() + } else { + mkline.Errorf("Unmatched .%s.", directive) + } + } + + expectedDepth := indentation.Depth() + if directive == "elif" || directive == "else" { + expectedDepth = indentation.depth[len(indentation.depth)-2] + } + ck.checkDirectiveIndentation(expectedDepth) + + if directive == "if" && matches(args, `^!defined\([\w]+_MK\)$`) { + indentation.Push(indentation.Depth()) + } else if matches(directive, `^(?:if|ifdef|ifndef|for)$`) { + indentation.Push(indentation.Depth() + 2) + } + + needsArgument := matches(directive, `^(?:if|ifdef|ifndef|elif|for|undef)$`) + if needsArgument != (args != "") { + if needsArgument { + mkline.Errorf("\".%s\" requires arguments.", directive) + } else { + mkline.Errorf("\".%s\" does not take arguments.", directive) + if directive == "else" { + mkline.Notef("If you meant \"else if\", use \".elif\".") + } + } + + } else if directive == "if" || directive == "elif" { + ck.CheckCond() + + } else if directive == "ifdef" || directive == "ifndef" { + mkline.Line.Warnf("The \".%s\" directive is deprecated. Please use \".if %sdefined(%s)\" instead.", + directive, ifelseStr(directive == "ifdef", "", "!"), args) + + } else if directive == "for" { + if m, vars, values := match2(args, `^(\S+(?:\s*\S+)*?)\s+in\s+(.*)$`); m { + for _, forvar := range splitOnSpace(vars) { + if !G.Infrastructure && hasPrefix(forvar, "_") { + mkline.Warnf("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", forvar) + } + + if matches(forvar, `^[_a-z][_a-z0-9]*$`) { + // Fine. + } else if matches(forvar, `[A-Z]`) { + mkline.Warnf(".for variable names should not contain uppercase letters.") + } else { + mkline.Errorf("Invalid variable name %q.", forvar) + } + + forVars[forvar] = true + } + + // Check if any of the value's types is not guessed. + guessed := true + for _, value := range splitOnSpace(values) { + if m, vname := match1(value, `^\$\{(.*)\}`); m { + vartype := mkline.getVariableType(vname) + if vartype != nil && !vartype.guessed { + guessed = false + } + } + } + + forLoopType := &Vartype{lkSpace, BtUnknown, []AclEntry{{"*", aclpAllRead}}, guessed} + forLoopContext := &VarUseContext{forLoopType, vucTimeParse, vucQuotFor, false} + for _, forLoopVar := range mkline.extractUsedVariables(values) { + ck.CheckVaruse(&MkVarUse{forLoopVar, nil}, forLoopContext) + } + } + + } else if directive == "undef" && args != "" { + for _, uvar := range splitOnSpace(args) { + if forVars[uvar] { + mkline.Notef("Using \".undef\" after a \".for\" loop is unnecessary.") + } + } + } +} + +func (ck MkLineChecker) checkDirectiveIndentation(expectedDepth int) { + if G.Mk == nil { + return + } + mkline := ck.MkLine + indent := mkline.Indent() + if expected := strings.Repeat(" ", expectedDepth); indent != expected { + if G.opts.WarnSpace && !mkline.AutofixReplace("."+indent, "."+expected) { + mkline.Notef("This directive should be indented by %d spaces.", expectedDepth) + } + } +} + +func (ck MkLineChecker) checkDependencyRule(allowedTargets map[string]bool) { + mkline := ck.MkLine + targets := splitOnSpace(mkline.Targets()) + sources := splitOnSpace(mkline.Sources()) + + for _, source := range sources { + if source == ".PHONY" { + for _, target := range targets { + allowedTargets[target] = true + } + } + } + + for _, target := range targets { + if target == ".PHONY" { + for _, dep := range sources { + allowedTargets[dep] = true + } + + } else if target == ".ORDER" { + // TODO: Check for spelling mistakes. + + } else if !allowedTargets[target] { + mkline.Warnf("Unusual target %q.", target) + Explain( + "If you want to define your own targets, you can \"declare\"", + "them by inserting a \".PHONY: my-target\" line before this line. This", + "will tell make(1) to not interpret this target's name as a filename.") + } + } +} + +func (ck MkLineChecker) checkVarassignDefPermissions() { + if !G.opts.WarnPerm { + return + } + if trace.Tracing { + defer trace.Call()() + } + + mkline := ck.MkLine + varname := mkline.Varname() + op := mkline.Op() + vartype := mkline.getVariableType(varname) + if vartype == nil { + if trace.Tracing { + trace.Step1("No type definition found for %q.", varname) + } + return + } + + perms := vartype.EffectivePermissions(mkline.Line.Filename()) + var needed AclPermissions + switch op { + case opAssign, opAssignShell, opAssignEval: + needed = aclpSet + case opAssignDefault: + needed = aclpSetDefault + case opAssignAppend: + needed = aclpAppend + } + + switch { + case perms.Contains(needed): + break + case perms == aclpUnknown: + if trace.Tracing { + trace.Step1("Unknown permissions for %q.", varname) + } + default: + alternativeActions := perms & aclpAllWrite + alternativeFiles := vartype.AllowedFiles(needed) + switch { + case alternativeActions != 0 && alternativeFiles != "": + mkline.Line.Warnf("The variable %s may not be %s (only %s) in this file; it would be ok in %s.", + varname, needed.HumanString(), alternativeActions.HumanString(), alternativeFiles) + case alternativeFiles != "": + mkline.Line.Warnf("The variable %s may not be %s in this file; it would be ok in %s.", + varname, needed.HumanString(), alternativeFiles) + case alternativeActions != 0: + mkline.Line.Warnf("The variable %s may not be %s (only %s) in this file.", + varname, needed.HumanString(), alternativeActions.HumanString()) + default: + mkline.Line.Warnf("The variable %s may not be %s by any package.", + varname, needed.HumanString()) + } + Explain( + "The allowed actions for a variable are determined based on the file", + "name in which the variable is used or defined. The exact rules are", + "hard-coded into pkglint. If they seem to be incorrect, please ask", + "on the tech-pkg@NetBSD.org mailing list.") + } +} + +func (ck MkLineChecker) CheckVaruse(varuse *MkVarUse, vuc *VarUseContext) { + mkline := ck.MkLine + if trace.Tracing { + defer trace.Call(mkline, varuse, vuc)() + } + + if varuse.IsExpression() { + return + } + + varname := varuse.varname + vartype := mkline.getVariableType(varname) + if G.opts.WarnExtra && + (vartype == nil || vartype.guessed) && + !varIsUsed(varname) && + !(G.Mk != nil && G.Mk.forVars[varname]) && + !containsVarRef(varname) { + mkline.Warnf("%s is used but not defined. Spelling mistake?", varname) + } + + ck.CheckVarusePermissions(varname, vartype, vuc) + + if varname == "LOCALBASE" && !G.Infrastructure { + ck.WarnVaruseLocalbase() + } + + needsQuoting := mkline.variableNeedsQuoting(varname, vartype, vuc) + + if vuc.quoting == vucQuotFor { + ck.checkVaruseFor(varname, vartype, needsQuoting) + } + + if G.opts.WarnQuoting && vuc.quoting != vucQuotUnknown && needsQuoting != nqDontKnow { + ck.CheckVaruseShellword(varname, vartype, vuc, varuse.Mod(), needsQuoting) + } + + if G.globalData.UserDefinedVars[varname] != nil && !G.globalData.SystemBuildDefs[varname] && !G.Mk.buildDefs[varname] { + mkline.Warnf("The user-defined variable %s is used but not added to BUILD_DEFS.", varname) + Explain( + "When a pkgsrc package is built, many things can be configured by the", + "pkgsrc user in the mk.conf file. All these configurations should be", + "recorded in the binary package, so the package can be reliably", + "rebuilt. The BUILD_DEFS variable contains a list of all these", + "user-settable variables, so please add your variable to it, too.") + } +} + +func (ck MkLineChecker) CheckVarusePermissions(varname string, vartype *Vartype, vuc *VarUseContext) { + if !G.opts.WarnPerm { + return + } + if trace.Tracing { + defer trace.Call(varname, vuc)() + } + + // This is the type of the variable that is being used. Not to + // be confused with vuc.vartype, which is the type of the + // context in which the variable is used (often a ShellCommand + // or, in an assignment, the type of the left hand side variable). + if vartype == nil { + if trace.Tracing { + trace.Step1("No type definition found for %q.", varname) + } + return + } + + mkline := ck.MkLine + perms := vartype.EffectivePermissions(mkline.Filename()) + + isLoadTime := false // Will the variable be used at load time? + + // Might the variable be used indirectly at load time, for example + // by assigning it to another variable which then gets evaluated? + isIndirect := false + + switch { + case vuc.vartype != nil && vuc.vartype.guessed: + // Don't warn about unknown variables. + + case vuc.time == vucTimeParse && !perms.Contains(aclpUseLoadtime): + isLoadTime = true + + case vuc.vartype != nil && vuc.vartype.Union().Contains(aclpUseLoadtime) && !perms.Contains(aclpUseLoadtime): + isLoadTime = true + isIndirect = true + } + + done := false + tool := G.globalData.Tools.byVarname[varname] + + if isLoadTime && tool != nil { + done = tool.Predefined && (G.Mk == nil || G.Mk.SeenBsdPrefsMk || G.Pkg == nil || G.Pkg.SeenBsdPrefsMk) + + if !done && G.Pkg != nil && !G.Pkg.SeenBsdPrefsMk && G.Mk != nil && !G.Mk.SeenBsdPrefsMk { + mkline.Warnf("To use the tool %q at load time, bsd.prefs.mk has to be included before.", varname) + done = true + } + + if !done && G.Pkg != nil { + usable, defined := G.Pkg.loadTimeTools[tool.Name] + if usable { + done = true + } + if defined && !usable { + mkline.Warnf("To use the tool %q at load time, it has to be added to USE_TOOLS before including bsd.prefs.mk.", varname) + done = true + } + } + } + + if !done && isLoadTime && !isIndirect { + mkline.Warnf("%s should not be evaluated at load time.", varname) + Explain( + "Many variables, especially lists of something, get their values", + "incrementally. Therefore it is generally unsafe to rely on their", + "value until it is clear that it will never change again. This", + "point is reached when the whole package Makefile is loaded and", + "execution of the shell commands starts; in some cases earlier.", + "", + "Additionally, when using the \":=\" operator, each $$ is replaced", + "with a single $, so variables that have references to shell", + "variables or regular expressions are modified in a subtle way.") + done = true + } + + if !done && isLoadTime && isIndirect { + mkline.Warnf("%s should not be evaluated indirectly at load time.", varname) + Explain( + "The variable on the left-hand side may be evaluated at load time,", + "but the variable on the right-hand side may not. Because of the", + "assignment in this line, the variable might be used indirectly", + "at load time, before it is guaranteed to be properly initialized.") + done = true + } + + if !perms.Contains(aclpUseLoadtime) && !perms.Contains(aclpUse) { + needed := aclpUse + if isLoadTime { + needed = aclpUseLoadtime + } + alternativeFiles := vartype.AllowedFiles(needed) + if alternativeFiles != "" { + mkline.Warnf("%s may not be used in this file; it would be ok in %s.", + varname, alternativeFiles) + } else { + mkline.Warnf("%s may not be used in any file; it is a write-only variable.", varname) + } + Explain( + "The allowed actions for a variable are determined based on the file", + "name in which the variable is used or defined. The exact rules are", + "hard-coded into pkglint. If they seem to be incorrect, please ask", + "on the tech-pkg@NetBSD.org mailing list.") + } +} + +func (ck MkLineChecker) WarnVaruseLocalbase() { + ck.MkLine.Warnf("The LOCALBASE variable should not be used by packages.") + Explain( + // from jlam via private mail. + "Currently, LOCALBASE is typically used in these cases:", + "", + "(1) To locate a file or directory from another package.", + "(2) To refer to own files after installation.", + "", + "Example for (1):", + "", + " STRLIST= ${LOCALBASE}/bin/strlist", + " do-build:", + " cd ${WRKSRC} && ${STRLIST} *.str", + "", + "This should better be:", + "", + " EVAL_PREFIX= STRLIST_PREFIX=strlist", + " STRLIST= ${STRLIST_PREFIX}/bin/strlist", + " do-build:", + " cd ${WRKSRC} && ${STRLIST} *.str", + "", + "Example for (2):", + "", + " CONFIGURE_ENV+= --with-datafiles=${LOCALBASE}/share/pkgbase", + "", + "This should better be:", + "", + " CONFIGURE_ENV+= --with-datafiles=${PREFIX}/share/pkgbase") +} + +func (ck MkLineChecker) checkVaruseFor(varname string, vartype *Vartype, needsQuoting NeedsQuoting) { + if trace.Tracing { + defer trace.Call(varname, vartype, needsQuoting)() + } + + if false && // Too many false positives + vartype != nil && + vartype.kindOfList != lkSpace && + needsQuoting != nqDoesntMatter { + ck.MkLine.Warnf("The variable %s should not be used in .for loops.", varname) + Explain( + "The .for loop splits its argument at sequences of white-space, as", + "opposed to all other places in make(1), which act like the shell.", + "Therefore only variables that are split at whitespace or don't", + "contain any special characters should be used here.") + } +} + +func (ck MkLineChecker) CheckVaruseShellword(varname string, vartype *Vartype, vuc *VarUseContext, mod string, needsQuoting NeedsQuoting) { + if trace.Tracing { + defer trace.Call(varname, vartype, vuc, mod, needsQuoting)() + } + + // In GNU configure scripts, a few variables need to be + // passed through the :M* operator before they reach the + // configure scripts. + // + // When doing checks outside a package, the :M* operator is needed for safety. + needMstar := matches(varname, `^(?:.*_)?(?:CFLAGS||CPPFLAGS|CXXFLAGS|FFLAGS|LDFLAGS|LIBS)$`) && + (G.Pkg == nil || G.Pkg.vardef["GNU_CONFIGURE"] != nil) + + strippedMod := mod + if m, stripped := match1(mod, `(.*?)(?::M\*)?(?::Q)?$`); m { + strippedMod = stripped + } + + mkline := ck.MkLine + if mod == ":M*:Q" && !needMstar { + mkline.Notef("The :M* modifier is not needed here.") + + } else if needsQuoting == nqYes { + correctMod := strippedMod + ifelseStr(needMstar, ":M*:Q", ":Q") + if correctMod == mod+":Q" && vuc.IsWordPart && !vartype.IsShell() { + mkline.Warnf("The list variable %s should not be embedded in a word.", varname) + Explain( + "When a list variable has multiple elements, this expression expands", + "to something unexpected:", + "", + "Example: ${MASTER_SITE_SOURCEFORGE}directory/ expands to", + "", + "\thttps://mirror1.sf.net/ https://mirror2.sf.net/directory/", + "", + "The first URL is missing the directory. To fix this, write", + "\t${MASTER_SITE_SOURCEFORGE:=directory/}.", + "", + "Example: -l${LIBS} expands to", + "", + "\t-llib1 lib2", + "", + "The second library is missing the -l. To fix this, write", + "${LIBS:@lib@-l${lib}@}.") + + } else if mod != correctMod { + if vuc.quoting == vucQuotPlain { + if !mkline.AutofixReplace("${"+varname+mod+"}", "${"+varname+correctMod+"}") { + mkline.Warnf("Please use ${%s%s} instead of ${%s%s}.", varname, correctMod, varname, mod) + } + } else { + mkline.Warnf("Please use ${%s%s} instead of ${%s%s} and make sure"+ + " the variable appears outside of any quoting characters.", varname, correctMod, varname, mod) + } + Explain( + "See the pkgsrc guide, section \"quoting guideline\", for details.") + + } else if vuc.quoting != vucQuotPlain { + mkline.Warnf("Please move ${%s%s} outside of any quoting characters.", varname, mod) + Explain( + "The :Q modifier only works reliably when it is used outside of any", + "quoting characters.", + "", + "Examples:", + "Instead of CFLAGS=\"${CFLAGS:Q}\",", + " write CFLAGS=${CFLAGS:Q}.", + "Instead of 's,@CFLAGS@,${CFLAGS:Q},',", + " write 's,@CFLAGS@,'${CFLAGS:Q}','.") + } + } + + if hasSuffix(mod, ":Q") && (needsQuoting == nqNo || needsQuoting == nqDoesntMatter) { + bad := "${" + varname + mod + "}" + good := "${" + varname + strings.TrimSuffix(mod, ":Q") + "}" + needExplain := false + if needsQuoting == nqNo && !mkline.AutofixReplace(bad, good) { + needExplain = true + mkline.Warnf("The :Q operator should not be used for ${%s} here.", varname) + } + if needsQuoting == nqDoesntMatter && !mkline.AutofixReplace(bad, good) { + needExplain = true + mkline.Notef("The :Q operator isn't necessary for ${%s} here.", varname) + } + if needExplain { + Explain( + "Many variables in pkgsrc do not need the :Q operator, since they", + "are not expected to contain white-space or other special characters.", + "Examples for these \"safe\" variables are:", + "", + "\t* filenames", + "\t* directory names", + "\t* user and group names", + "\t* tool names and tool paths", + "\t* variable names", + "\t* PKGNAME") + } + } +} + +func (ck MkLineChecker) checkVarassignPythonVersions(varname, value string) { + if trace.Tracing { + defer trace.Call2(varname, value)() + } + + mkline := ck.MkLine + strversions := splitOnSpace(value) + intversions := make([]int, len(strversions)) + for i, strversion := range strversions { + iver, err := strconv.Atoi(strversion) + if err != nil || !(iver > 0) { + mkline.Errorf("All values for %s must be positive integers.", varname) + return + } + intversions[i] = iver + } + + for i, ver := range intversions { + if i > 0 && ver >= intversions[i-1] { + mkline.Warnf("The values for %s should be in decreasing order.", varname) + Explain( + "If they aren't, it may be possible that needless versions of", + "packages are installed.") + } + } +} + +func (ck MkLineChecker) checkVarassign() { + mkline := ck.MkLine + varname := mkline.Varname() + op := mkline.Op() + value := mkline.Value() + comment := mkline.VarassignComment() + varcanon := varnameCanon(varname) + + if trace.Tracing { + defer trace.Call(varname, op, value)() + } + + defineVar(mkline, varname) + ck.checkVarassignDefPermissions() + ck.checkVarassignBsdPrefs() + + ck.checkText(value) + ck.CheckVartype(varname, op, value, comment) + + // If the variable is not used and is untyped, it may be a spelling mistake. + if op == opAssignEval && varname == strings.ToLower(varname) { + if trace.Tracing { + trace.Step1("%s might be unused unless it is an argument to a procedure file.", varname) + } + + } else if !varIsUsed(varname) { + if vartypes := G.globalData.vartypes; vartypes[varname] != nil || vartypes[varcanon] != nil { + // Ok + } else if deprecated := G.globalData.Deprecated; deprecated[varname] != "" || deprecated[varcanon] != "" { + // Ok + } else { + mkline.Warnf("%s is defined but not used. Spelling mistake?", varname) + } + } + + ck.checkVarassignSpecific() + + if varname == "EVAL_PREFIX" { + if m, evalVarname := match1(value, `^([\w_]+)=`); m { + + // The variables mentioned in EVAL_PREFIX will later be + // defined by find-prefix.mk. Therefore, they are marked + // as known in the current file. + G.Mk.vardef[evalVarname] = mkline + } + } + + if varname == "USE_TOOLS" { + for _, fullToolname := range splitOnSpace(value) { + toolname := strings.Split(fullToolname, ":")[0] + if G.Pkg != nil { + if !G.Pkg.SeenBsdPrefsMk { + G.Pkg.loadTimeTools[toolname] = true + if trace.Tracing { + trace.Step1("loadTimeTool %q", toolname) + } + } else if !G.Pkg.loadTimeTools[toolname] { + G.Pkg.loadTimeTools[toolname] = false + if trace.Tracing { + trace.Step1("too late for loadTimeTool %q", toolname) + } + } + } + } + } + + if fix := G.globalData.Deprecated[varname]; fix != "" { + mkline.Warnf("Definition of %s is deprecated. %s", varname, fix) + } else if fix := G.globalData.Deprecated[varcanon]; fix != "" { + mkline.Warnf("Definition of %s is deprecated. %s", varname, fix) + } + + ck.checkVarassignPlistComment(varname, value) + ck.checkVarassignVaruse() +} + +func (ck MkLineChecker) checkVarassignVaruse() { + if trace.Tracing { + defer trace.Call()() + } + + mkline := ck.MkLine + op := mkline.Op() + + time := vucTimeRun + if op == opAssignEval || op == opAssignShell { + time = vucTimeParse + } + + vartype := mkline.getVariableType(mkline.Varname()) + if op == opAssignShell { + vartype = shellcommandsContextType + } + + if vartype != nil && vartype.IsShell() { + ck.checkVarassignVaruseShell(vartype, time) + } else { + ck.checkVarassignVaruseMk(vartype, time) + } +} + +func (ck MkLineChecker) checkVarassignVaruseMk(vartype *Vartype, time vucTime) { + if trace.Tracing { + defer trace.Call(vartype, time)() + } + mkline := ck.MkLine + tokens := NewMkParser(mkline.Line, mkline.Value(), false).MkTokens() + for i, token := range tokens { + if token.Varuse != nil { + spaceLeft := i-1 < 0 || matches(tokens[i-1].Text, `\s$`) + spaceRight := i+1 >= len(tokens) || matches(tokens[i+1].Text, `^\s`) + isWordPart := !(spaceLeft && spaceRight) + vuc := &VarUseContext{vartype, time, vucQuotPlain, isWordPart} + ck.CheckVaruse(token.Varuse, vuc) + } + } +} + +func (ck MkLineChecker) checkVarassignVaruseShell(vartype *Vartype, time vucTime) { + if trace.Tracing { + defer trace.Call(vartype, time)() + } + + isWordPart := func(tokens []*ShAtom, i int) bool { + if i-1 >= 0 && tokens[i-1].Type.IsWord() { + return true + } + if i+1 < len(tokens) && tokens[i+1].Type.IsWord() { + return true + } + return false + } + + mkline := ck.MkLine + atoms := NewShTokenizer(mkline.Line, mkline.Value(), false).ShAtoms() + for i, atom := range atoms { + if atom.Type == shtVaruse { + isWordPart := isWordPart(atoms, i) + vuc := &VarUseContext{vartype, time, atom.Quoting.ToVarUseContext(), isWordPart} + ck.CheckVaruse(atom.Data.(*MkVarUse), vuc) + } + } +} + +func (ck MkLineChecker) checkVarassignSpecific() { + mkline := ck.MkLine + varname := mkline.Varname() + value := mkline.Value() + + if contains(value, "/etc/rc.d") { + mkline.Warnf("Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") + } + + if hasPrefix(varname, "_") && !G.Infrastructure { + mkline.Warnf("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", varname) + } + + if varname == "PERL5_PACKLIST" && G.Pkg != nil { + if m, p5pkgname := match1(G.Pkg.EffectivePkgbase, `^p5-(.*)`); m { + guess := "auto/" + strings.Replace(p5pkgname, "-", "/", -1) + "/.packlist" + + ucvalue, ucguess := strings.ToUpper(value), strings.ToUpper(guess) + if ucvalue != ucguess && ucvalue != "${PERL5_SITEARCH}/"+ucguess { + mkline.Warnf("Unusual value for PERL5_PACKLIST -- %q expected.", guess) + } + } + } + + if varname == "CONFIGURE_ARGS" && contains(value, "=${PREFIX}/share/kde") { + mkline.Notef("Please .include \"../../meta-pkgs/kde3/kde3.mk\" instead of this line.") + Explain( + "That file does many things automatically and consistently that this", + "package also does. When using kde3.mk, you can probably also leave", + "out some explicit dependencies.") + } + + if varname == "PYTHON_VERSIONS_ACCEPTED" { + ck.checkVarassignPythonVersions(varname, value) + } + + if mkline.VarassignComment() == "# defined" && !hasSuffix(varname, "_MK") && !hasSuffix(varname, "_COMMON") { + mkline.Notef("Please use \"# empty\", \"# none\" or \"yes\" instead of \"# defined\".") + Explain( + "The value #defined says something about the state of the variable,", + "but not what that _means_. In some cases a variable that is defined", + "means \"yes\", in other cases it is an empty list (which is also", + "only the state of the variable), whose meaning could be described", + "with \"none\". It is this meaning that should be described.") + } + + if m, revvarname := match1(value, `\$\{(PKGNAME|PKGVERSION)[:\}]`); m { + if varname == "DIST_SUBDIR" || varname == "WRKSRC" { + mkline.Line.Warnf("%s should not be used in %s, as it includes the PKGREVISION. Please use %s_NOREV instead.", revvarname, varname, revvarname) + } + } + + if hasPrefix(varname, "SITES_") { + mkline.Warnf("SITES_* is deprecated. Please use SITES.* instead.") + } + + if varname == "PKG_SKIP_REASON" && G.Mk.indentation.DependsOn("OPSYS") { + mkline.Notef("Consider defining NOT_FOR_PLATFORM instead of setting PKG_SKIP_REASON depending on ${OPSYS}.") + } +} + +func (ck MkLineChecker) checkVarassignBsdPrefs() { + mkline := ck.MkLine + if G.opts.WarnExtra && mkline.Op() == opAssignDefault && G.Pkg != nil && !G.Pkg.SeenBsdPrefsMk { + switch mkline.Varcanon() { + case "BUILDLINK_PKGSRCDIR.*", "BUILDLINK_DEPMETHOD.*", "BUILDLINK_ABI_DEPENDS.*": + return + } + + mkline.Warnf("Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".") + Explain( + "The ?= operator is used to provide a default value to a variable.", + "In pkgsrc, many variables can be set by the pkgsrc user in the", + "mk.conf file. This file must be included explicitly. If a ?=", + "operator appears before mk.conf has been included, it will not care", + "about the user's preferences, which can result in unexpected", + "behavior.", + "", + "The easiest way to include the mk.conf file is by including the", + "bsd.prefs.mk file, which will take care of everything.") + } +} + +func (ck MkLineChecker) checkVarassignPlistComment(varname, value string) { + if false && // This is currently neither correct nor helpful + contains(value, "@comment") && !matches(value, `="@comment "`) { + ck.MkLine.Warnf("Please don't use @comment in %s.", varname) + Explain( + "If you are defining a PLIST conditional here, use one of the", + "following patterns instead:", + "", + "1. The direct way, without intermediate variable", + "", + "\tPLIST_SUBST+=\tMY_VAR=\"@comment \"", + "", + "2. The indirect way, with a separate variable", + "", + "\tPLIST_VARS+=\tMY_VAR", + "\t.if ...", + "\tMY_VAR?=\tyes", + "\t.endif") + } + + // Mark the variable as PLIST condition. This is later used in checkfile_PLIST. + if G.Pkg != nil { + if m, plistVarname := match1(value, `(.+)=.*@comment.*`); m { + G.Pkg.plistSubstCond[plistVarname] = true + } + } +} + +func (ck MkLineChecker) CheckVartype(varname string, op MkOperator, value, comment string) { + if trace.Tracing { + defer trace.Call(varname, op, value, comment)() + } + + if !G.opts.WarnTypes { + return + } + + mkline := ck.MkLine + vartype := mkline.getVariableType(varname) + + if op == opAssignAppend { + if vartype != nil && !vartype.MayBeAppendedTo() { + mkline.Warnf("The \"+=\" operator should only be used with lists.") + } + } + + switch { + case vartype == nil: + if trace.Tracing { + trace.Step1("Unchecked variable assignment for %s.", varname) + } + + case op == opAssignShell: + if trace.Tracing { + trace.Step1("Unchecked use of !=: %q", value) + } + + case vartype.kindOfList == lkNone: + ck.CheckVartypePrimitive(varname, vartype.basicType, op, value, comment, vartype.guessed) + + case value == "": + break + + case vartype.kindOfList == lkSpace: + for _, word := range splitOnSpace(value) { + ck.CheckVartypePrimitive(varname, vartype.basicType, op, word, comment, vartype.guessed) + } + + case vartype.kindOfList == lkShell: + words, _ := splitIntoMkWords(mkline.Line, value) + for _, word := range words { + ck.CheckVartypePrimitive(varname, vartype.basicType, op, word, comment, vartype.guessed) + } + } +} + +// For some variables (like `BuildlinkDepth`), `op` influences the valid values. +// The `comment` parameter comes from a variable assignment, when a part of the line is commented out. +func (ck MkLineChecker) CheckVartypePrimitive(varname string, checker *BasicType, op MkOperator, value, comment string, guessed bool) { + if trace.Tracing { + defer trace.Call(varname, checker.name, op, value, comment, guessed)() + } + + mkline := ck.MkLine + valueNoVar := mkline.withoutMakeVariables(value) + ctx := &VartypeCheck{mkline, mkline.Line, varname, op, value, valueNoVar, comment, guessed} + checker.checker(ctx) +} + +func (ck MkLineChecker) checkText(text string) { + if trace.Tracing { + defer trace.Call1(text)() + } + + mkline := ck.MkLine + if contains(text, "${WRKSRC}/..") { + mkline.Warnf("Building the package should take place entirely inside ${WRKSRC}, not \"${WRKSRC}/..\".") + Explain( + "WRKSRC should be defined so that there is no need to do anything", + "outside of this directory.", + "", + "Example:", + "", + "\tWRKSRC=\t${WRKDIR}", + "\tCONFIGURE_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src", + "\tBUILD_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src ${WRKSRC}/cmd", + "", + "See the pkgsrc guide, section \"Directories used during the build", + "process\" for more information.") + } + + // Note: A simple -R is not detected, as the rate of false positives is too high. + if m, flag := match1(text, `\b(-Wl,--rpath,|-Wl,-rpath-link,|-Wl,-rpath,|-Wl,-R)\b`); m { + mkline.Warnf("Please use ${COMPILER_RPATH_FLAG} instead of %q.", flag) + } + + rest := text + for { + m, r := regex.ReplaceFirst(rest, `(?:^|[^$])\$\{([-A-Z0-9a-z_]+)(\.[\-0-9A-Z_a-z]+)?(?::[^\}]+)?\}`, "") + if m == nil { + break + } + rest = r + + varbase, varext := m[1], m[2] + varname := varbase + varext + varcanon := varnameCanon(varname) + instead := G.globalData.Deprecated[varname] + if instead == "" { + instead = G.globalData.Deprecated[varcanon] + } + if instead != "" { + mkline.Warnf("Use of %q is deprecated. %s", varname, instead) + } + } +} + +func (ck MkLineChecker) CheckCond() { + mkline := ck.MkLine + if trace.Tracing { + defer trace.Call1(mkline.Args())() + } + + p := NewMkParser(mkline.Line, mkline.Args(), false) + cond := p.MkCond() + if !p.EOF() { + mkline.Warnf("Invalid conditional %q.", mkline.Args()) + return + } + + cond.Visit("empty", func(node *Tree) { + varuse := node.args[0].(MkVarUse) + varname := varuse.varname + if matches(varname, `^\$.*:[MN]`) { + mkline.Warnf("The empty() function takes a variable name as parameter, not a variable expression.") + Explain( + "Instead of empty(${VARNAME:Mpattern}), you should write either", + "of the following:", + "", + "\tempty(VARNAME:Mpattern)", + "\t${VARNAME:Mpattern} == \"\"", + "", + "Instead of !empty(${VARNAME:Mpattern}), you should write either", + "of the following:", + "", + "\t!empty(VARNAME:Mpattern)", + "\t${VARNAME:Mpattern}") + } + for _, modifier := range varuse.modifiers { + if modifier[0] == 'M' || modifier[0] == 'N' { + ck.CheckVartype(varname, opUseMatch, modifier[1:], "") + } + } + }) + + cond.Visit("compareVarStr", func(node *Tree) { + varuse := node.args[0].(MkVarUse) + varname := varuse.varname + varmods := varuse.modifiers + value := node.args[2].(string) + if len(varmods) == 0 { + ck.checkCompareVarStr(varname, node.args[1].(string), value) + } else if len(varmods) == 1 && matches(varmods[0], `^[MN]`) && value != "" { + ck.CheckVartype(varname, opUseMatch, value, "") + } + }) + + mkline.rememberUsedVariables(cond) +} + +func (ck MkLineChecker) checkCompareVarStr(varname, op, value string) { + ck.CheckVartype(varname, opUseCompare, value, "") + + if varname == "PKGSRC_COMPILER" { + ck.MkLine.Warnf("Use ${PKGSRC_COMPILER:%s%s} instead of the %s operator.", ifelseStr(op == "==", "M", "N"), value, op) + Explain( + "The PKGSRC_COMPILER can be a list of chained compilers, e.g. \"ccache", + "distcc clang\". Therefore, comparing it using == or != leads to", + "wrong results in these cases.") + } +} + +func (ck MkLineChecker) CheckValidCharactersInValue(reValid regex.RegexPattern) { + mkline := ck.MkLine + rest := regex.Compile(reValid).ReplaceAllString(mkline.Value(), "") + if rest != "" { + uni := "" + for _, c := range rest { + uni += fmt.Sprintf(" %U", c) + } + mkline.Warnf("%s contains invalid characters (%s).", mkline.Varname(), uni[1:]) + } +} + +func (ck MkLineChecker) CheckRelativePkgdir(pkgdir string) { + if trace.Tracing { + defer trace.Call1(pkgdir)() + } + + mkline := ck.MkLine + ck.CheckRelativePath(pkgdir, true) + pkgdir = mkline.resolveVarsInRelativePath(pkgdir, false) + + if m, otherpkgpath := match1(pkgdir, `^(?:\./)?\.\./\.\./([^/]+/[^/]+)$`); m { + if !fileExists(G.globalData.Pkgsrcdir + "/" + otherpkgpath + "/Makefile") { + mkline.Errorf("There is no package in %q.", otherpkgpath) + } + + } else if !containsVarRef(pkgdir) { + mkline.Warnf("%q is not a valid relative package directory.", pkgdir) + Explain( + "A relative pathname always starts with \"../../\", followed", + "by a category, a slash and a the directory name of the package.", + "For example, \"../../misc/screen\" is a valid relative pathname.") + } +} + +func (ck MkLineChecker) CheckRelativePath(path string, mustExist bool) { + if trace.Tracing { + defer trace.Call(path, mustExist)() + } + + mkline := ck.MkLine + if !G.Wip && contains(path, "/wip/") { + mkline.Errorf("A main pkgsrc package must not depend on a pkgsrc-wip package.") + } + + resolvedPath := mkline.resolveVarsInRelativePath(path, true) + if containsVarRef(resolvedPath) { + return + } + + abs := resolvedPath + if !hasPrefix(abs, "/") { + abs = G.CurrentDir + "/" + abs + } + if _, err := os.Stat(abs); err != nil { + if mustExist { + mkline.Errorf("%q does not exist.", resolvedPath) + } + return + } + + if hasPrefix(path, "../") && + !matches(path, `^\.\./\.\./[^/]+/[^/]`) && + !(G.CurPkgsrcdir == ".." && hasPrefix(path, "../mk/")) && // For category Makefiles. + !hasPrefix(path, "../../mk/") { + mkline.Warnf("Invalid relative path %q.", path) + } +} diff --git a/pkgtools/pkglint/files/mklinechecker_test.go b/pkgtools/pkglint/files/mklinechecker_test.go new file mode 100644 index 00000000000..f008a4cf1ef --- /dev/null +++ b/pkgtools/pkglint/files/mklinechecker_test.go @@ -0,0 +1,255 @@ +package main + +import "gopkg.in/check.v1" + +func (s *Suite) Test_MkLineChecker_CheckVartype__simple_type(c *check.C) { + s.Init(c) + s.UseCommandLine("-Wtypes") + G.globalData.InitVartypes() + mkline := NewMkLine(NewLine("fname", 1, "COMMENT=\tA nice package", nil)) + + vartype1 := G.globalData.vartypes["COMMENT"] + c.Assert(vartype1, check.NotNil) + c.Check(vartype1.guessed, equals, false) + + vartype := mkline.getVariableType("COMMENT") + + c.Assert(vartype, check.NotNil) + c.Check(vartype.basicType.name, equals, "Comment") + c.Check(vartype.guessed, equals, false) + c.Check(vartype.kindOfList, equals, lkNone) + + MkLineChecker{mkline}.CheckVartype("COMMENT", opAssign, "A nice package", "") + + c.Check(s.Stdout(), equals, "WARN: fname:1: COMMENT should not begin with \"A\".\n") +} + +func (s *Suite) Test_MkLineChecker_CheckVartype(c *check.C) { + G.globalData.InitVartypes() + mkline := NewMkLine(NewLine("fname", 1, "DISTNAME=gcc-${GCC_VERSION}", nil)) + + MkLineChecker{mkline}.CheckVartype("DISTNAME", opAssign, "gcc-${GCC_VERSION}", "") +} + +// Pkglint once interpreted all lists as consisting of shell tokens, +// splitting this URL at the ampersands. +func (s *Suite) Test_MkLineChecker_checkVarassign__URL_with_shell_special_characters(c *check.C) { + G.Pkg = NewPackage("graphics/gimp-fix-ca") + G.globalData.InitVartypes() + mkline := NewMkLine(NewLine("fname", 10, "MASTER_SITES=http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=", nil)) + + MkLineChecker{mkline}.checkVarassign() + + c.Check(s.Output(), equals, "") +} + +func (s *Suite) Test_MkLineChecker_Check__conditions(c *check.C) { + s.Init(c) + s.UseCommandLine("-Wtypes") + G.globalData.InitVartypes() + + MkLineChecker{NewMkLine(NewLine("fname", 1, ".if !empty(PKGSRC_COMPILER:Mmycc)", nil))}.CheckCond() + + c.Check(s.Stdout(), equals, "WARN: fname:1: The pattern \"mycc\" cannot match any of "+ + "{ ccache ccc clang distcc f2c gcc hp icc ido "+ + "mipspro mipspro-ucode pcc sunpro xlc } for PKGSRC_COMPILER.\n") + + MkLineChecker{NewMkLine(NewLine("fname", 1, ".elif ${A} != ${B}", nil))}.CheckCond() + + c.Check(s.Stdout(), equals, "") + + MkLineChecker{NewMkLine(NewLine("fname", 1, ".if ${HOMEPAGE} == \"mailto:someone@example.org\"", nil))}.CheckCond() + + c.Check(s.Output(), equals, "WARN: fname:1: \"mailto:someone@example.org\" is not a valid URL.\n") + + MkLineChecker{NewMkLine(NewLine("fname", 1, ".if !empty(PKGSRC_RUN_TEST:M[Y][eE][sS])", nil))}.CheckCond() + + c.Check(s.Output(), equals, "WARN: fname:1: PKGSRC_RUN_TEST should be matched against \"[yY][eE][sS]\" or \"[nN][oO]\", not \"[Y][eE][sS]\".\n") + + MkLineChecker{NewMkLine(NewLine("fname", 1, ".if !empty(IS_BUILTIN.Xfixes:M[yY][eE][sS])", nil))}.CheckCond() + + c.Check(s.Output(), equals, "") + + MkLineChecker{NewMkLine(NewLine("fname", 1, ".if !empty(${IS_BUILTIN.Xfixes:M[yY][eE][sS]})", nil))}.CheckCond() + + c.Check(s.Output(), equals, "WARN: fname:1: The empty() function takes a variable name as parameter, not a variable expression.\n") + + MkLineChecker{NewMkLine(NewLine("fname", 1, ".if ${EMUL_PLATFORM} == \"linux-x386\"", nil))}.CheckCond() + + c.Check(s.Output(), equals, "WARN: fname:1: \"x386\" is not valid for the hardware architecture part of EMUL_PLATFORM. Use one of { aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } instead.\n") + + MkLineChecker{NewMkLine(NewLine("fname", 1, ".if ${EMUL_PLATFORM:Mlinux-x386}", nil))}.CheckCond() + + c.Check(s.Output(), equals, "WARN: fname:1: The pattern \"x386\" cannot match any of { aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } for the hardware architecture part of EMUL_PLATFORM.\n") + + MkLineChecker{NewMkLine(NewLine("fname", 98, ".if ${MACHINE_PLATFORM:MUnknownOS-*-*} || ${MACHINE_ARCH:Mx86}", nil))}.CheckCond() + + c.Check(s.Output(), equals, ""+ + "WARN: fname:98: The pattern \"UnknownOS\" cannot match any of { AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare } for the operating system part of MACHINE_PLATFORM.\n"+ + "WARN: fname:98: The pattern \"x86\" cannot match any of { aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } for MACHINE_ARCH.\n") +} + +func (s *Suite) Test_MkLineChecker_checkVarassign(c *check.C) { + G.globalData.InitVartypes() + + G.Mk = s.NewMkLines("Makefile", + mkrcsid, + "ac_cv_libpari_libs+=\t-L${BUILDLINK_PREFIX.pari}/lib") // From math/clisp-pari/Makefile, rev. 1.8 + + MkLineChecker{G.Mk.mklines[1]}.checkVarassign() + + c.Check(s.Output(), equals, "WARN: Makefile:2: ac_cv_libpari_libs is defined but not used. Spelling mistake?\n") +} + +func (s *Suite) Test_MkLineChecker_checkVarassignDefPermissions(c *check.C) { + s.Init(c) + s.UseCommandLine("-Wall") + G.globalData.InitVartypes() + mkline := NewMkLine(NewLine("options.mk", 2, "PKG_DEVELOPER?=\tyes", nil)) + + MkLineChecker{mkline}.checkVarassignDefPermissions() + + c.Check(s.Output(), equals, "WARN: options.mk:2: The variable PKG_DEVELOPER may not be given a default value by any package.\n") +} + +func (s *Suite) Test_MkLineChecker_CheckVarusePermissions(c *check.C) { + s.Init(c) + s.UseCommandLine("-Wall") + G.globalData.InitVartypes() + mklines := s.NewMkLines("options.mk", + mkrcsid, + "COMMENT=\t${GAMES_USER}", + "COMMENT:=\t${PKGBASE}", + "PYPKGPREFIX=${PKGBASE}") + G.globalData.UserDefinedVars = map[string]*MkLine{ + "GAMES_USER": mklines.mklines[0], + } + + mklines.Check() + + c.Check(s.Output(), equals, ""+ + "WARN: options.mk:2: The user-defined variable GAMES_USER is used but not added to BUILD_DEFS.\n"+ + "WARN: options.mk:3: PKGBASE should not be evaluated at load time.\n"+ + "WARN: options.mk:4: The variable PYPKGPREFIX may not be set in this file; it would be ok in pyversion.mk.\n"+ + "WARN: options.mk:4: PKGBASE should not be evaluated indirectly at load time.\n"+ + "NOTE: options.mk:4: This variable value should be aligned to column 17.\n") +} + +func (s *Suite) Test_MkLineChecker_CheckVarusePermissions__load_time(c *check.C) { + s.Init(c) + s.UseCommandLine("-Wall") + G.globalData.InitVartypes() + mklines := s.NewMkLines("options.mk", + mkrcsid, + "WRKSRC:=${.CURDIR}") + + mklines.Check() + + c.Check(s.Output(), equals, "") // Don't warn that ".CURDIR should not be evaluated at load time." +} + +func (s *Suite) Test_MkLineChecker_WarnVaruseLocalbase(c *check.C) { + mkline := NewMkLine(NewLine("options.mk", 56, "PKGNAME=${LOCALBASE}", nil)) + + MkLineChecker{mkline}.WarnVaruseLocalbase() + + c.Check(s.Output(), equals, "WARN: options.mk:56: The LOCALBASE variable should not be used by packages.\n") +} + +func (s *Suite) Test_MkLineChecker_CheckRelativePkgdir(c *check.C) { + mkline := NewMkLine(NewLine("Makefile", 46, "# dummy", nil)) + + MkLineChecker{mkline}.CheckRelativePkgdir("../pkgbase") + + c.Check(s.Output(), equals, ""+ + "ERROR: Makefile:46: \"../pkgbase\" does not exist.\n"+ + "WARN: Makefile:46: \"../pkgbase\" is not a valid relative package directory.\n") +} + +// PR pkg/46570, item 2 +func (s *Suite) Test_MkLineChecker__unclosed_varuse(c *check.C) { + mkline := NewMkLine(NewLine("Makefile", 93, "EGDIRS=${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d", nil)) + + MkLineChecker{mkline}.checkVarassign() + + c.Check(s.Output(), equals, ""+ + "WARN: Makefile:93: Pkglint parse error in MkLine.Tokenize at \"${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".\n"+ + "WARN: Makefile:93: Pkglint parse error in ShTokenizer.ShAtom at \"${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\" (quoting=plain)\n"+ + "WARN: Makefile:93: EGDIRS is defined but not used. Spelling mistake?\n") +} + +func (s *Suite) Test_MkLineChecker__Varuse_Modifier_L(c *check.C) { + s.Init(c) + s.UseCommandLine("-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("x11/xkeyboard-config/Makefile", + "FILES_SUBST+=XKBCOMP_SYMLINK=${${XKBBASE}/xkbcomp:L:Q}") + + MkLineChecker{G.Mk.mklines[0]}.Check() + + c.Check(s.Output(), equals, "") // Don't warn that ${XKBBASE}/xkbcomp is used but not defined. +} + +func (s *Suite) Test_MkLineChecker_CheckCond__comparison_with_shell_command(c *check.C) { + s.Init(c) + s.UseCommandLine("-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("security/openssl/Makefile", + mkrcsid, + ".if ${PKGSRC_COMPILER} == \"gcc\" && ${CC} == \"cc\"", + ".endif") + + G.Mk.Check() + + // Don't warn about unknown shell command "cc". + c.Check(s.Output(), equals, "WARN: security/openssl/Makefile:2: Use ${PKGSRC_COMPILER:Mgcc} instead of the == operator.\n") +} + +func (s *Suite) Test_MkLine_CheckCond_comparing_PKGSRC_COMPILER_with_eqeq(c *check.C) { + s.Init(c) + s.UseCommandLine("-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("audio/pulseaudio/Makefile", + mkrcsid, + ".if ${OPSYS} == \"Darwin\" && ${PKGSRC_COMPILER} == \"clang\"", + ".endif") + + G.Mk.Check() + + c.Check(s.Output(), equals, "WARN: audio/pulseaudio/Makefile:2: Use ${PKGSRC_COMPILER:Mclang} instead of the == operator.\n") +} + +func (s *Suite) Test_MkLineChecker_CheckVartype__CFLAGS_with_backticks(c *check.C) { + s.Init(c) + s.UseCommandLine("-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("chat/pidgin-icb/Makefile", + mkrcsid, + "CFLAGS+=\t`pkg-config pidgin --cflags`") + mkline := G.Mk.mklines[1] + + words, rest := splitIntoMkWords(mkline.Line, mkline.Value()) + + c.Check(words, deepEquals, []string{"`pkg-config pidgin --cflags`"}) + c.Check(rest, equals, "") + + MkLineChecker{G.Mk.mklines[1]}.CheckVartype("CFLAGS", opAssignAppend, "`pkg-config pidgin --cflags`", "") + + c.Check(s.Output(), equals, "") // No warning about "`pkg-config" being an unknown CFlag. +} + +// See PR 46570, Ctrl+F "4. Shell quoting". +// Pkglint is correct, since the shell sees this definition for +// CPPFLAGS as three words, not one word. +func (s *Suite) Test_MkLineChecker_CheckVartype_CFLAGS(c *check.C) { + G.globalData.InitVartypes() + mklines := s.NewMkLines("Makefile", + mkrcsid, + "CPPFLAGS.SunOS+=\t-DPIPECOMMAND=\\\"/usr/sbin/sendmail -bs %s\\\"") + + mklines.Check() + + c.Check(s.Output(), equals, ""+ + "WARN: Makefile:2: Unknown compiler flag \"-bs\".\n"+ + "WARN: Makefile:2: Compiler flag \"%s\\\\\\\"\" should start with a hyphen.\n") +} diff --git a/pkgtools/pkglint/files/mklines.go b/pkgtools/pkglint/files/mklines.go index 3d5714656f5..fc61f158227 100644 --- a/pkgtools/pkglint/files/mklines.go +++ b/pkgtools/pkglint/files/mklines.go @@ -1,6 +1,7 @@ package main import ( + "netbsd.org/pkglint/trace" "path" "strings" ) @@ -8,7 +9,7 @@ import ( // MkLines contains data for the Makefile (or *.mk) that is currently checked. type MkLines struct { mklines []*MkLine - lines []*Line + lines []Line forVars map[string]bool // The variables currently used in .for loops target string // Current make(1) target vardef map[string]*MkLine // varname => line; for all variables that are defined in the current file @@ -21,7 +22,7 @@ type MkLines struct { indentation Indentation // Indentation depth of preprocessing directives } -func NewMkLines(lines []*Line) *MkLines { +func NewMkLines(lines []Line) *MkLines { mklines := make([]*MkLine, len(lines)) for i, line := range lines { mklines[i] = NewMkLine(line) @@ -76,8 +77,8 @@ func (mklines *MkLines) VarValue(varname string) (value string, found bool) { } func (mklines *MkLines) Check() { - if G.opts.Debug { - defer tracecall1(mklines.lines[0].Fname)() + if trace.Tracing { + defer trace.Call1(mklines.lines[0].Filename())() } G.Mk = mklines @@ -99,14 +100,15 @@ func (mklines *MkLines) Check() { // In the second pass, the actual checks are done. - mklines.lines[0].CheckRcsid(`#\s+`, "# ") + LineChecker{mklines.lines[0]}.CheckRcsid(`#\s+`, "# ") var substcontext SubstContext var varalign VaralignBlock indentation := &mklines.indentation indentation.Push(0) for _, mkline := range mklines.mklines { - mkline.Check() + ck := MkLineChecker{mkline} + ck.Check() varalign.Check(mkline) switch { @@ -128,10 +130,10 @@ func (mklines *MkLines) Check() { } case mkline.IsCond(): - mkline.checkCond(mklines.forVars) + ck.checkCond(mklines.forVars) case mkline.IsDependency(): - mkline.checkDependencyRule(allowedTargets) + ck.checkDependencyRule(allowedTargets) mklines.target = mkline.Targets() } } @@ -142,15 +144,15 @@ func (mklines *MkLines) Check() { ChecklinesTrailingEmptyLines(mklines.lines) if indentation.Len() != 1 && indentation.Depth() != 0 { - lastMkline.Line.Errorf("Directive indentation is not 0, but %d.", indentation.Depth()) + lastMkline.Errorf("Directive indentation is not 0, but %d.", indentation.Depth()) } SaveAutofixChanges(mklines.lines) } func (mklines *MkLines) determineDefinedVariables() { - if G.opts.Debug { - defer tracecall0()() + if trace.Tracing { + defer trace.Call0()() } for _, mkline := range mklines.mklines { @@ -163,16 +165,16 @@ func (mklines *MkLines) determineDefinedVariables() { case "BUILD_DEFS", "PKG_GROUPS_VARS", "PKG_USERS_VARS": for _, varname := range splitOnSpace(mkline.Value()) { mklines.buildDefs[varname] = true - if G.opts.Debug { - traceStep1("%q is added to BUILD_DEFS.", varname) + if trace.Tracing { + trace.Step1("%q is added to BUILD_DEFS.", varname) } } case "PLIST_VARS": for _, id := range splitOnSpace(mkline.Value()) { mklines.plistVars["PLIST."+id] = true - if G.opts.Debug { - traceStep1("PLIST.%s is added to PLIST_VARS.", id) + if trace.Tracing { + trace.Step1("PLIST.%s is added to PLIST_VARS.", id) } mklines.UseVar(mkline, "PLIST."+id) } @@ -188,16 +190,16 @@ func (mklines *MkLines) determineDefinedVariables() { for _, tool := range splitOnSpace(tools) { tool = strings.Split(tool, ":")[0] mklines.tools[tool] = true - if G.opts.Debug { - traceStep1("%s is added to USE_TOOLS.", tool) + if trace.Tracing { + trace.Step1("%s is added to USE_TOOLS.", tool) } } case "SUBST_VARS.*": for _, svar := range splitOnSpace(mkline.Value()) { mklines.UseVar(mkline, varnameCanon(svar)) - if G.opts.Debug { - traceStep1("varuse %s", svar) + if trace.Tracing { + trace.Step1("varuse %s", svar) } } @@ -224,8 +226,8 @@ func (mklines *MkLines) DetermineUsedVariables() { func (mklines *MkLines) setSeenBsdPrefsMk() { if !mklines.SeenBsdPrefsMk { mklines.SeenBsdPrefsMk = true - if G.opts.Debug { - traceStep("Mk.setSeenBsdPrefsMk") + if trace.Tracing { + trace.Stepf("Mk.setSeenBsdPrefsMk") } } } @@ -333,15 +335,15 @@ func (va *VaralignBlock) fixalign(mkline *MkLine, prefix, oldalign string) { return } - if !mkline.Line.AutofixReplace(prefix+oldalign, prefix+newalign) { + if !mkline.AutofixReplace(prefix+oldalign, prefix+newalign) { wrongColumn := tabLength(prefix+oldalign) != tabLength(prefix+newalign) switch { case hasSpace && wrongColumn: - mkline.Line.Notef("This variable value should be aligned with tabs, not spaces, to column %d.", goodWidth+1) + mkline.Notef("This variable value should be aligned with tabs, not spaces, to column %d.", goodWidth+1) case hasSpace: - mkline.Line.Notef("Variable values should be aligned with tabs, not spaces.") + mkline.Notef("Variable values should be aligned with tabs, not spaces.") case wrongColumn: - mkline.Line.Notef("This variable value should be aligned to column %d.", goodWidth+1) + mkline.Notef("This variable value should be aligned to column %d.", goodWidth+1) } if wrongColumn { Explain( diff --git a/pkgtools/pkglint/files/mklines_test.go b/pkgtools/pkglint/files/mklines_test.go index 24b63385a33..4f6a035c006 100644 --- a/pkgtools/pkglint/files/mklines_test.go +++ b/pkgtools/pkglint/files/mklines_test.go @@ -49,10 +49,10 @@ func (s *Suite) Test_MkLines_Check__unusual_target(c *check.C) { c.Check(s.Output(), equals, "WARN: Makefile:3: Unusual target \"echo\".\n") } -func (s *Suite) Test_MkLine_checklineInclude_Makefile(c *check.C) { +func (s *Suite) Test_MkLineChecker_checkInclude__Makefile(c *check.C) { mkline := NewMkLine(NewLine("Makefile", 2, ".include \"../../other/package/Makefile\"", nil)) - mkline.checkInclude() + MkLineChecker{mkline}.checkInclude() c.Check(s.Output(), equals, ""+ "ERROR: Makefile:2: \"/other/package/Makefile\" does not exist.\n"+ diff --git a/pkgtools/pkglint/files/mkparser.go b/pkgtools/pkglint/files/mkparser.go index 883dbc2aaac..4159ea5fdd6 100644 --- a/pkgtools/pkglint/files/mkparser.go +++ b/pkgtools/pkglint/files/mkparser.go @@ -1,6 +1,8 @@ package main import ( + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/trace" "strings" ) @@ -8,7 +10,7 @@ type MkParser struct { *Parser } -func NewMkParser(line *Line, text string, emitWarnings bool) *MkParser { +func NewMkParser(line Line, text string, emitWarnings bool) *MkParser { return &MkParser{NewParser(line, text, emitWarnings)} } @@ -28,9 +30,9 @@ func (p *MkParser) MkTokens() []*MkToken { } again: - dollar := strings.IndexByte(repl.rest, '$') + dollar := strings.IndexByte(repl.Rest(), '$') if dollar == -1 { - dollar = len(repl.rest) + dollar = len(repl.Rest()) } repl.Skip(dollar) if repl.AdvanceStr("$$") { @@ -71,7 +73,7 @@ func (p *MkParser) VarUse() *MkVarUse { } } - for p.VarUse() != nil || repl.AdvanceRegexp(RegexPattern(`^([^$:`+closing+`]|\$\$)+`)) { + for p.VarUse() != nil || repl.AdvanceRegexp(regex.RegexPattern(`^([^$:`+closing+`]|\$\$)+`)) { } rest := p.Rest() if hasPrefix(rest, ":L") || hasPrefix(rest, ":?") { @@ -91,7 +93,7 @@ func (p *MkParser) VarUse() *MkVarUse { return &MkVarUse{"<", nil} } if repl.PeekByte() == '$' && repl.AdvanceRegexp(`^\$(\w)`) { - varname := repl.m[1] + varname := repl.Group(1) if p.EmitWarnings { p.Line.Warnf("$%[1]s is ambiguous. Use ${%[1]s} if you mean a Makefile variable or $$%[1]s if you mean a shell variable.", varname) } @@ -116,7 +118,7 @@ func (p *MkParser) VarUseModifiers(varname, closing string) []string { continue } if repl.AdvanceStr("ts") { - rest := repl.rest + rest := repl.Rest() if len(rest) >= 2 && (rest[1] == closing[0] || rest[1] == ':') { repl.Skip(1) } else if len(rest) >= 1 && (rest[0] == closing[0] || rest[0] == ':') { @@ -130,7 +132,7 @@ func (p *MkParser) VarUseModifiers(varname, closing string) []string { case '=', 'D', 'M', 'N', 'U': if repl.AdvanceRegexp(`^[=DMNU]`) { - for p.VarUse() != nil || repl.AdvanceRegexp(RegexPattern(`^([^$:`+closing+`]|\$\$)+`)) { + for p.VarUse() != nil || repl.AdvanceRegexp(regex.RegexPattern(`^([^$:`+closing+`]|\$\$)+`)) { } modifiers = append(modifiers, repl.Since(modifierMark)) continue @@ -138,9 +140,9 @@ func (p *MkParser) VarUseModifiers(varname, closing string) []string { case 'C', 'S': if repl.AdvanceRegexp(`^[CS]([%,/:;@^|])`) { - separator := repl.m[1] + separator := repl.Group(1) repl.AdvanceStr("^") - re := RegexPattern(`^([^\` + separator + `$` + closing + `\\]|\$\$|\\.)+`) + re := regex.RegexPattern(`^([^\` + separator + `$` + closing + `\\]|\$\$|\\.)+`) for p.VarUse() != nil || repl.AdvanceRegexp(re) { } repl.AdvanceStr("$") @@ -158,8 +160,8 @@ func (p *MkParser) VarUseModifiers(varname, closing string) []string { case '@': if repl.AdvanceRegexp(`^@([\w.]+)@`) { - loopvar := repl.m[1] - for p.VarUse() != nil || repl.AdvanceRegexp(RegexPattern(`^([^$:@`+closing+`\\]|\$\$|\\.)+`)) { + loopvar := repl.Group(1) + for p.VarUse() != nil || repl.AdvanceRegexp(regex.RegexPattern(`^([^$:@`+closing+`\\]|\$\$|\\.)+`)) { } if !repl.AdvanceStr("@") && p.EmitWarnings { p.Line.Warnf("Modifier ${%s:@%s@...@} is missing the final \"@\".", varname, loopvar) @@ -176,7 +178,7 @@ func (p *MkParser) VarUseModifiers(varname, closing string) []string { case '?': repl.AdvanceStr("?") - re := RegexPattern(`^([^$:` + closing + `]|\$\$)+`) + re := regex.RegexPattern(`^([^$:` + closing + `]|\$\$)+`) for p.VarUse() != nil || repl.AdvanceRegexp(re) { } if repl.AdvanceStr(":") { @@ -188,7 +190,7 @@ func (p *MkParser) VarUseModifiers(varname, closing string) []string { } repl.Reset(modifierMark) - for p.VarUse() != nil || repl.AdvanceRegexp(RegexPattern(`^([^:$`+closing+`]|\$\$)+`)) { + for p.VarUse() != nil || repl.AdvanceRegexp(regex.RegexPattern(`^([^:$`+closing+`]|\$\$)+`)) { } if suffixSubst := repl.Since(modifierMark); contains(suffixSubst, "=") { modifiers = append(modifiers, suffixSubst) @@ -253,8 +255,8 @@ func (p *MkParser) mkCondAnd() *Tree { } func (p *MkParser) mkCondAtom() *Tree { - if G.opts.Debug { - defer tracecall1(p.Rest())() + if trace.Tracing { + defer trace.Call1(p.Rest())() } repl := p.repl @@ -288,7 +290,7 @@ func (p *MkParser) mkCondAtom() *Tree { } } case repl.AdvanceRegexp(`^(commands|exists|make|target)\s*\(`): - funcname := repl.m[1] + funcname := repl.Group(1) argMark := repl.Mark() for p.VarUse() != nil || repl.AdvanceRegexp(`^[^$)]+`) { } @@ -308,14 +310,14 @@ func (p *MkParser) mkCondAtom() *Tree { } if lhs != nil { if repl.AdvanceRegexp(`^\s*(<|<=|==|!=|>=|>)\s*(\d+(?:\.\d+)?)`) { - return NewTree("compareVarNum", *lhs, repl.m[1], repl.m[2]) + return NewTree("compareVarNum", *lhs, repl.Group(1), repl.Group(2)) } if repl.AdvanceRegexp(`^\s*(<|<=|==|!=|>=|>)\s*`) { - op := repl.m[1] + op := repl.Group(1) if (op == "!=" || op == "==") && repl.AdvanceRegexp(`^"([^"\$\\]*)"`) { - return NewTree("compareVarStr", *lhs, op, repl.m[1]) + return NewTree("compareVarStr", *lhs, op, repl.Group(1)) } else if repl.AdvanceRegexp(`^\w+`) { - return NewTree("compareVarStr", *lhs, op, repl.m[0]) + return NewTree("compareVarStr", *lhs, op, repl.Group(0)) } else if rhs := p.VarUse(); rhs != nil { return NewTree("compareVarVar", *lhs, op, *rhs) } else if repl.PeekByte() == '"' { @@ -334,7 +336,7 @@ func (p *MkParser) mkCondAtom() *Tree { } } if repl.AdvanceRegexp(`^\d+(?:\.\d+)?`) { - return NewTree("literalNum", repl.m[0]) + return NewTree("literalNum", repl.Group(0)) } } repl.Reset(mark) diff --git a/pkgtools/pkglint/files/mkshparser.go b/pkgtools/pkglint/files/mkshparser.go index 583bf1d22d4..bf2130617f3 100644 --- a/pkgtools/pkglint/files/mkshparser.go +++ b/pkgtools/pkglint/files/mkshparser.go @@ -2,12 +2,13 @@ package main import ( "fmt" + "netbsd.org/pkglint/trace" "strconv" ) -func parseShellProgram(line *Line, program string) (list *MkShList, err error) { - if G.opts.Debug { - defer tracecall(program)() +func parseShellProgram(line Line, program string) (list *MkShList, err error) { + if trace.Tracing { + defer trace.Call(program)() } tokens, rest := splitIntoShellTokens(line, program) @@ -54,16 +55,16 @@ func (lex *ShellLexer) Lex(lval *shyySymType) (ttype int) { return 0 } - if G.opts.Debug { + if trace.Tracing { defer func() { tname := shyyTokname(shyyTok2[ttype-shyyPrivate]) switch ttype { case tkWORD, tkASSIGNMENT_WORD: - traceStep("lex %v %q", tname, lval.Word.MkText) + trace.Stepf("lex %v %q", tname, lval.Word.MkText) case tkIO_NUMBER: - traceStep("lex %v %v", tname, lval.IONum) + trace.Stepf("lex %v %v", tname, lval.IONum) default: - traceStep("lex %v", tname) + trace.Stepf("lex %v", tname) } }() } diff --git a/pkgtools/pkglint/files/mkshtypes.go b/pkgtools/pkglint/files/mkshtypes.go index 5bee1c276c9..203837e0d66 100644 --- a/pkgtools/pkglint/files/mkshtypes.go +++ b/pkgtools/pkglint/files/mkshtypes.go @@ -1,6 +1,9 @@ package main -import "fmt" +import ( + "fmt" + "netbsd.org/pkglint/regex" +) type MkShList struct { AndOrs []*MkShAndOr @@ -144,7 +147,7 @@ func (c *StrCommand) HasOption(opt string) bool { return false } -func (c *StrCommand) AnyArgMatches(pattern RegexPattern) bool { +func (c *StrCommand) AnyArgMatches(pattern regex.RegexPattern) bool { for _, arg := range c.Args { if matches(arg, pattern) { return true diff --git a/pkgtools/pkglint/files/package.go b/pkgtools/pkglint/files/package.go index f8ff29ac886..040c1232211 100644 --- a/pkgtools/pkglint/files/package.go +++ b/pkgtools/pkglint/files/package.go @@ -3,6 +3,8 @@ package main import ( "fmt" "netbsd.org/pkglint/pkgver" + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/trace" "path" "regexp" "strconv" @@ -26,9 +28,9 @@ type Package struct { vardef map[string]*MkLine // (varname, varcanon) => line varuse map[string]*MkLine // (varname, varcanon) => line - bl3 map[string]*Line // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. + bl3 map[string]Line // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. plistSubstCond map[string]bool // varname => true; list of all variables that are used as conditionals (@comment or nothing) in PLISTs. - included map[string]*Line // fname => line + included map[string]Line // fname => line seenMakefileCommon bool // Does the package have any .includes? loadTimeTools map[string]bool // true=ok, false=not ok, absent=not mentioned in USE_TOOLS. conditionalIncludes map[string]*MkLine @@ -40,9 +42,9 @@ func NewPackage(pkgpath string) *Package { Pkgpath: pkgpath, vardef: make(map[string]*MkLine), varuse: make(map[string]*MkLine), - bl3: make(map[string]*Line), + bl3: make(map[string]Line), plistSubstCond: make(map[string]bool), - included: make(map[string]*Line), + included: make(map[string]Line), loadTimeTools: make(map[string]bool), conditionalIncludes: make(map[string]*MkLine), unconditionalIncludes: make(map[string]*MkLine), @@ -79,15 +81,15 @@ func (pkg *Package) varValue(varname string) (string, bool) { func (pkg *Package) setSeenBsdPrefsMk() { if !pkg.SeenBsdPrefsMk { pkg.SeenBsdPrefsMk = true - if G.opts.Debug { - traceStep("Pkg.setSeenBsdPrefsMk") + if trace.Tracing { + trace.Stepf("Pkg.setSeenBsdPrefsMk") } } } func (pkg *Package) checkPossibleDowngrade() { - if G.opts.Debug { - defer tracecall0()() + if trace.Tracing { + defer trace.Call0()() } m, _, pkgversion := match2(pkg.EffectivePkgname, rePkgname) @@ -99,16 +101,16 @@ func (pkg *Package) checkPossibleDowngrade() { change := G.globalData.LastChange[pkg.Pkgpath] if change == nil { - if G.opts.Debug { - traceStep1("No change log for package %q", pkg.Pkgpath) + if trace.Tracing { + trace.Step1("No change log for package %q", pkg.Pkgpath) } return } if change.Action == "Updated" { - changeVersion := regcomp(`nb\d+$`).ReplaceAllString(change.Version, "") + changeVersion := regex.Compile(`nb\d+$`).ReplaceAllString(change.Version, "") if pkgver.Compare(pkgversion, changeVersion) < 0 { - mkline.Line.Warnf("The package is being downgraded from %s (see %s) to %s", change.Version, change.Line.ReferenceFrom(mkline.Line), pkgversion) + mkline.Warnf("The package is being downgraded from %s (see %s) to %s", change.Version, change.Line.ReferenceFrom(mkline.Line), pkgversion) Explain( "The files in doc/CHANGES-*, in which all version changes are", "recorded, have a higher version number than what the package says.", @@ -119,8 +121,8 @@ func (pkg *Package) checkPossibleDowngrade() { } func (pkg *Package) checklinesBuildlink3Inclusion(mklines *MkLines) { - if G.opts.Debug { - defer tracecall0()() + if trace.Tracing { + defer trace.Call0()() } // Collect all the included buildlink3.mk files from the file. @@ -137,18 +139,18 @@ func (pkg *Package) checklinesBuildlink3Inclusion(mklines *MkLines) { } } - if G.opts.Debug { + if trace.Tracing { for packageBl3 := range pkg.bl3 { if includedFiles[packageBl3] == nil { - traceStep1("%s/buildlink3.mk is included by the package but not by the buildlink3.mk file.", packageBl3) + trace.Step1("%s/buildlink3.mk is included by the package but not by the buildlink3.mk file.", packageBl3) } } } } func checkdirPackage(pkgpath string) { - if G.opts.Debug { - defer tracecall1(pkgpath)() + if trace.Tracing { + defer trace.Call1(pkgpath)() } G.Pkg = NewPackage(pkgpath) @@ -218,8 +220,8 @@ func checkdirPackage(pkgpath string) { } func (pkg *Package) loadPackageMakefile(fname string) *MkLines { - if G.opts.Debug { - defer tracecall1(fname)() + if trace.Tracing { + defer trace.Call1(fname)() } mainLines, allLines := NewMkLines(nil), NewMkLines(nil) @@ -250,19 +252,19 @@ func (pkg *Package) loadPackageMakefile(fname string) *MkLines { } } - if G.opts.Debug { - traceStep1("DISTINFO_FILE=%s", pkg.DistinfoFile) - traceStep1("FILESDIR=%s", pkg.Filesdir) - traceStep1("PATCHDIR=%s", pkg.Patchdir) - traceStep1("PKGDIR=%s", pkg.Pkgdir) + if trace.Tracing { + trace.Step1("DISTINFO_FILE=%s", pkg.DistinfoFile) + trace.Step1("FILESDIR=%s", pkg.Filesdir) + trace.Step1("PATCHDIR=%s", pkg.Patchdir) + trace.Step1("PKGDIR=%s", pkg.Pkgdir) } return mainLines } func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkLines, includingFnameForUsedCheck string) bool { - if G.opts.Debug { - defer tracecall1(fname)() + if trace.Tracing { + defer trace.Call1(fname)() } fileLines := LoadNonemptyLines(fname, true) @@ -300,8 +302,8 @@ func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkL if path.Base(fname) != "buildlink3.mk" { if m, bl3File := match1(includeFile, `^\.\./\.\./(.*)/buildlink3\.mk$`); m { G.Pkg.bl3[bl3File] = line - if G.opts.Debug { - traceStep1("Buildlink3 file in package: %q", bl3File) + if trace.Tracing { + trace.Step1("Buildlink3 file in package: %q", bl3File) } } } @@ -316,8 +318,8 @@ func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkL } if path.Base(fname) == "Makefile" && !hasPrefix(incDir, "../../mk/") && incBase != "buildlink3.mk" && incBase != "builtin.mk" && incBase != "options.mk" { - if G.opts.Debug { - traceStep1("Including %q sets seenMakefileCommon.", includeFile) + if trace.Tracing { + trace.Step1("Including %q sets seenMakefileCommon.", includeFile) } G.Pkg.seenMakefileCommon = true } @@ -340,8 +342,8 @@ func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkL } } - if G.opts.Debug { - traceStep1("Including %q.", dirname+"/"+includeFile) + if trace.Tracing { + trace.Step1("Including %q.", dirname+"/"+includeFile) } includingFname := ifelseStr(incBase == "Makefile.common" && incDir != "", fname, "") if !pkg.readMakefile(dirname+"/"+includeFile, mainLines, allLines, includingFname) { @@ -354,8 +356,8 @@ func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkL varname, op, value := mkline.Varname(), mkline.Op(), mkline.Value() if op != opAssignDefault || G.Pkg.vardef[varname] == nil { - if G.opts.Debug { - traceStep("varassign(%q, %q, %q)", varname, op, value) + if trace.Tracing { + trace.Stepf("varassign(%q, %q, %q)", varname, op, value) } G.Pkg.vardef[varname] = mkline } @@ -370,8 +372,8 @@ func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkL } func (pkg *Package) checkfilePackageMakefile(fname string, mklines *MkLines) { - if G.opts.Debug { - defer tracecall1(fname)() + if trace.Tracing { + defer trace.Call1(fname)() } vardef := pkg.vardef @@ -394,7 +396,7 @@ func (pkg *Package) checkfilePackageMakefile(fname string, mklines *MkLines) { } if perlLine, noconfLine := vardef["REPLACE_PERL"], vardef["NO_CONFIGURE"]; perlLine != nil && noconfLine != nil { - perlLine.Warnf("REPLACE_PERL is ignored when NO_CONFIGURE is set (in %s)", noconfLine.Line.ReferenceFrom(perlLine.Line)) + perlLine.Warnf("REPLACE_PERL is ignored when NO_CONFIGURE is set (in %s)", noconfLine.ReferenceFrom(perlLine.Line)) } if vardef["LICENSE"] == nil && vardef["META_PACKAGE"] == nil { @@ -421,7 +423,7 @@ func (pkg *Package) checkfilePackageMakefile(fname string, mklines *MkLines) { } if imake, x11 := vardef["USE_IMAKE"], vardef["USE_X11"]; imake != nil && x11 != nil { - if !hasSuffix(x11.Line.Fname, "/mk/x11.buildlink3.mk") { + if !hasSuffix(x11.Line.Filename(), "/mk/x11.buildlink3.mk") { imake.Line.Notef("USE_IMAKE makes USE_X11 in %s superfluous.", x11.Line.ReferenceFrom(imake.Line)) } } @@ -486,8 +488,8 @@ func (pkg *Package) determineEffectivePkgVars() { } } if pkg.EffectivePkgnameLine != nil { - if G.opts.Debug { - traceStep("Effective name=%q base=%q version=%q", + if trace.Tracing { + trace.Stepf("Effective name=%q base=%q version=%q", pkg.EffectivePkgname, pkg.EffectivePkgbase, pkg.EffectivePkgversion) } } @@ -497,14 +499,14 @@ func (pkg *Package) pkgnameFromDistname(pkgname, distname string) string { tokens := NewMkParser(dummyLine, pkgname, false).MkTokens() subst := func(str, smod string) (result string) { - if G.opts.Debug { - defer tracecall(str, smod, ref(result))() + if trace.Tracing { + defer trace.Call(str, smod, trace.Ref(result))() } qsep := regexp.QuoteMeta(smod[1:2]) - if m, left, from, right, to, flags := match5(smod, RegexPattern(`^S`+qsep+`(\^?)([^:]*?)(\$?)`+qsep+`([^:]*)`+qsep+`([1g]*)$`)); m { + if m, left, from, right, to, flags := regex.Match5(smod, regex.RegexPattern(`^S`+qsep+`(\^?)([^:]*?)(\$?)`+qsep+`([^:]*)`+qsep+`([1g]*)$`)); m { result := mkopSubst(str, left != "", from, right != "", to, flags) - if G.opts.Debug { - traceStep("subst %q %q => %q", str, smod, result) + if trace.Tracing { + trace.Stepf("subst %q %q => %q", str, smod, result) } return result } @@ -544,8 +546,8 @@ func (pkg *Package) expandVariableWithDefault(varname, defaultValue string) stri if containsVarRef(value) { value = resolveVariableRefs(value) } - if G.opts.Debug { - traceStep2("Expanded %q to %q", varname, value) + if trace.Tracing { + trace.Step2("Expanded %q to %q", varname, value) } return value } @@ -580,8 +582,8 @@ func (pkg *Package) checkUpdate() { } func (pkg *Package) ChecklinesPackageMakefileVarorder(mklines *MkLines) { - if G.opts.Debug { - defer tracecall0()() + if trace.Tracing { + defer trace.Call0()() } if !G.opts.WarnOrder || pkg.seenMakefileCommon { @@ -692,10 +694,10 @@ func (pkg *Package) ChecklinesPackageMakefileVarorder(mklines *MkLines) { for lineno < len(mklines.lines) { mkline := mklines.mklines[lineno] line := mklines.lines[lineno] - text := line.Text + text := line.Text() - if G.opts.Debug { - traceStep("[varorder] section %d variable %d vars %v", sectindex, varindex, vars) + if trace.Tracing { + trace.Stepf("[varorder] section %d variable %d vars %v", sectindex, varindex, vars) } if nextSection { @@ -784,13 +786,13 @@ func (mklines *MkLines) checkForUsedComment(relativeName string) { expected := "# used by " + relativeName for _, line := range lines { - if line.Text == expected { + if line.Text() == expected { return } } i := 0 - for i < 2 && hasPrefix(lines[i].Text, "#") { + for i < 2 && hasPrefix(lines[i].Text(), "#") { i++ } @@ -811,8 +813,8 @@ func (mklines *MkLines) checkForUsedComment(relativeName string) { } func (pkg *Package) checkLocallyModified(fname string) { - if G.opts.Debug { - defer tracecall(fname)() + if trace.Tracing { + defer trace.Call(fname)() } ownerLine := pkg.vardef["OWNER"] @@ -830,8 +832,8 @@ func (pkg *Package) checkLocallyModified(fname string) { } username := G.CurrentUsername - if G.opts.Debug { - traceStep("user=%q owner=%q maintainer=%q", username, owner, maintainer) + if trace.Tracing { + trace.Stepf("user=%q owner=%q maintainer=%q", username, owner, maintainer) } if username == strings.Split(owner, "@")[0] || username == strings.Split(maintainer, "@")[0] { @@ -860,22 +862,22 @@ func (pkg *Package) CheckInclude(mkline *MkLine, indentation *Indentation) { mkline.data = includeLine } - if path.Dir(abspath(mkline.Line.Fname)) == abspath(G.CurrentDir) { + if path.Dir(abspath(mkline.Filename())) == abspath(G.CurrentDir) { includefile := mkline.Includefile() if indentation.IsConditional() { pkg.conditionalIncludes[includefile] = mkline if other := pkg.unconditionalIncludes[includefile]; other != nil { dependingOn := mkline.data.(mkLineInclude).conditionVars - mkline.Line.Warnf("%q is included conditionally here (depending on %s) and unconditionally in %s.", - cleanpath(includefile), dependingOn, other.Line.ReferenceFrom(mkline.Line)) + mkline.Warnf("%q is included conditionally here (depending on %s) and unconditionally in %s.", + cleanpath(includefile), dependingOn, other.ReferenceFrom(mkline.Line)) } } else { pkg.unconditionalIncludes[includefile] = mkline if other := pkg.conditionalIncludes[includefile]; other != nil { dependingOn := other.data.(mkLineInclude).conditionVars - mkline.Line.Warnf("%q is included unconditionally here and conditionally in %s (depending on %s).", - cleanpath(includefile), other.Line.ReferenceFrom(mkline.Line), dependingOn) + mkline.Warnf("%q is included unconditionally here and conditionally in %s (depending on %s).", + cleanpath(includefile), other.ReferenceFrom(mkline.Line), dependingOn) } } } diff --git a/pkgtools/pkglint/files/parser.go b/pkgtools/pkglint/files/parser.go index b02689923de..c74ec401ea2 100644 --- a/pkgtools/pkglint/files/parser.go +++ b/pkgtools/pkglint/files/parser.go @@ -1,25 +1,26 @@ package main import ( + "netbsd.org/pkglint/textproc" "strings" ) type Parser struct { - Line *Line - repl *PrefixReplacer + Line Line + repl *textproc.PrefixReplacer EmitWarnings bool } -func NewParser(line *Line, s string, emitWarnings bool) *Parser { - return &Parser{line, NewPrefixReplacer(s), emitWarnings} +func NewParser(line Line, s string, emitWarnings bool) *Parser { + return &Parser{line, textproc.NewPrefixReplacer(s), emitWarnings} } func (p *Parser) EOF() bool { - return p.repl.rest == "" + return p.repl.EOF() } func (p *Parser) Rest() string { - return p.repl.rest + return p.repl.Rest() } func (p *Parser) PkgbasePattern() (pkgbase string) { @@ -29,7 +30,7 @@ func (p *Parser) PkgbasePattern() (pkgbase string) { if repl.AdvanceRegexp(`^\$\{\w+\}`) || repl.AdvanceRegexp(`^[\w.*+,{}]+`) || repl.AdvanceRegexp(`^\[[\d-]+\]`) { - pkgbase += repl.m[0] + pkgbase += repl.Group(0) continue } @@ -49,12 +50,12 @@ func (p *Parser) PkgbasePattern() (pkgbase string) { } type DependencyPattern struct { - pkgbase string // "freeciv-client", "{gcc48,gcc48-libs}", "${EMACS_REQD}" - lowerOp string // ">=", ">" - lower string // "2.5.0", "${PYVER}" - upperOp string // "<", "<=" - upper string // "3.0", "${PYVER}" - wildcard string // "[0-9]*", "1.5.*", "${PYVER}" + Pkgbase string // "freeciv-client", "{gcc48,gcc48-libs}", "${EMACS_REQD}" + LowerOp string // ">=", ">" + Lower string // "2.5.0", "${PYVER}" + UpperOp string // "<", "<=" + Upper string // "3.0", "${PYVER}" + Wildcard string // "[0-9]*", "1.5.*", "${PYVER}" } func (p *Parser) Dependency() *DependencyPattern { @@ -62,43 +63,43 @@ func (p *Parser) Dependency() *DependencyPattern { var dp DependencyPattern mark := repl.Mark() - dp.pkgbase = p.PkgbasePattern() - if dp.pkgbase == "" { + dp.Pkgbase = p.PkgbasePattern() + if dp.Pkgbase == "" { return nil } mark2 := repl.Mark() if repl.AdvanceStr(">=") || repl.AdvanceStr(">") { - op := repl.s + op := repl.Str() if repl.AdvanceRegexp(`^(?:(?:\$\{\w+\})+|\d[\w.]*)`) { - dp.lowerOp = op - dp.lower = repl.m[0] + dp.LowerOp = op + dp.Lower = repl.Group(0) } else { repl.Reset(mark2) } } if repl.AdvanceStr("<=") || repl.AdvanceStr("<") { - op := repl.s + op := repl.Str() if repl.AdvanceRegexp(`^(?:(?:\$\{\w+\})+|\d[\w.]*)`) { - dp.upperOp = op - dp.upper = repl.m[0] + dp.UpperOp = op + dp.Upper = repl.Group(0) } else { repl.Reset(mark2) } } - if dp.lowerOp != "" || dp.upperOp != "" { + if dp.LowerOp != "" || dp.UpperOp != "" { return &dp } - if repl.AdvanceStr("-") && repl.rest != "" { - dp.wildcard = repl.AdvanceRest() + if repl.AdvanceStr("-") && !repl.EOF() { + dp.Wildcard = repl.AdvanceRest() return &dp } - if hasPrefix(dp.pkgbase, "${") && hasSuffix(dp.pkgbase, "}") { + if hasPrefix(dp.Pkgbase, "${") && hasSuffix(dp.Pkgbase, "}") { return &dp } - if hasSuffix(dp.pkgbase, "-*") { - dp.pkgbase = strings.TrimSuffix(dp.pkgbase, "-*") - dp.wildcard = "*" + if hasSuffix(dp.Pkgbase, "-*") { + dp.Pkgbase = strings.TrimSuffix(dp.Pkgbase, "-*") + dp.Wildcard = "*" return &dp } diff --git a/pkgtools/pkglint/files/patches.go b/pkgtools/pkglint/files/patches.go index 4debf06e5cd..e9c5270407f 100644 --- a/pkgtools/pkglint/files/patches.go +++ b/pkgtools/pkglint/files/patches.go @@ -3,20 +3,21 @@ package main // Checks for patch files. import ( + "netbsd.org/pkglint/trace" "path" "strings" ) -func ChecklinesPatch(lines []*Line) { - if G.opts.Debug { - defer tracecall1(lines[0].Fname)() +func ChecklinesPatch(lines []Line) { + if trace.Tracing { + defer trace.Call1(lines[0].Filename())() } (&PatchChecker{lines, NewExpecter(lines), false, false}).Check() } type PatchChecker struct { - lines []*Line + lines []Line exp *Expecter seenDocumentation bool previousLineEmpty bool @@ -29,11 +30,11 @@ const ( ) func (ck *PatchChecker) Check() { - if G.opts.Debug { - defer tracecall0()() + if trace.Tracing { + defer trace.Call0()() } - if ck.lines[0].CheckRcsid(``, "") { + if (LineChecker{ck.lines[0]}).CheckRcsid(``, "") { ck.exp.Advance() } ck.previousLineEmpty = ck.exp.ExpectEmptyLine() @@ -76,16 +77,16 @@ func (ck *PatchChecker) Check() { } ck.exp.Advance() - ck.previousLineEmpty = ck.isEmptyLine(line.Text) + ck.previousLineEmpty = ck.isEmptyLine(line.Text()) if !ck.previousLineEmpty { ck.seenDocumentation = true } } if patchedFiles > 1 { - NewLineWhole(ck.lines[0].Fname).Warnf("Contains patches for %d files, should be only one.", patchedFiles) + NewLineWhole(ck.lines[0].Filename()).Warnf("Contains patches for %d files, should be only one.", patchedFiles) } else if patchedFiles == 0 { - NewLineWhole(ck.lines[0].Fname).Errorf("Contains no patch.") + NewLineWhole(ck.lines[0].Filename()).Errorf("Contains no patch.") } ChecklinesTrailingEmptyLines(ck.lines) @@ -94,13 +95,13 @@ func (ck *PatchChecker) Check() { // See http://www.gnu.org/software/diffutils/manual/html_node/Detailed-Unified.html func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { - if G.opts.Debug { - defer tracecall0()() + if trace.Tracing { + defer trace.Call0()() } patchedFileType := guessFileType(ck.exp.CurrentLine(), patchedFile) - if G.opts.Debug { - traceStep("guessFileType(%q) = %s", patchedFile, patchedFileType) + if trace.Tracing { + trace.Stepf("guessFileType(%q) = %s", patchedFile, patchedFileType) } hasHunks := false @@ -108,15 +109,15 @@ func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { hasHunks = true linesToDel := toInt(ck.exp.m[2], 1) linesToAdd := toInt(ck.exp.m[4], 1) - if G.opts.Debug { - traceStep("hunk -%d +%d", linesToDel, linesToAdd) + if trace.Tracing { + trace.Stepf("hunk -%d +%d", linesToDel, linesToAdd) } ck.checktextUniHunkCr() - for linesToDel > 0 || linesToAdd > 0 || hasPrefix(ck.exp.CurrentLine().Text, "\\") { + for linesToDel > 0 || linesToAdd > 0 || hasPrefix(ck.exp.CurrentLine().Text(), "\\") { line := ck.exp.CurrentLine() ck.exp.Advance() - text := line.Text + text := line.Text() switch { case text == "": linesToDel-- @@ -143,7 +144,7 @@ func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { } if !ck.exp.EOF() { line := ck.exp.CurrentLine() - if !ck.isEmptyLine(line.Text) && !matches(line.Text, rePatchUniFileDel) { + if !ck.isEmptyLine(line.Text()) && !matches(line.Text(), rePatchUniFileDel) { line.Warnf("Empty line or end of file expected.") Explain( "This empty line makes the end of the patch clearly visible.", @@ -153,9 +154,9 @@ func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { } } -func (ck *PatchChecker) checkBeginDiff(line *Line, patchedFiles int) { - if G.opts.Debug { - defer tracecall0()() +func (ck *PatchChecker) checkBeginDiff(line Line, patchedFiles int) { + if trace.Tracing { + defer trace.Call0()() } if !ck.seenDocumentation && patchedFiles == 0 { @@ -181,8 +182,8 @@ func (ck *PatchChecker) checkBeginDiff(line *Line, patchedFiles int) { } func (ck *PatchChecker) checklineContext(text string, patchedFileType FileType) { - if G.opts.Debug { - defer tracecall2(text, patchedFileType.String())() + if trace.Tracing { + defer trace.Call2(text, patchedFileType.String())() } if G.opts.WarnExtra { @@ -193,8 +194,8 @@ func (ck *PatchChecker) checklineContext(text string, patchedFileType FileType) } func (ck *PatchChecker) checklineAdded(addedText string, patchedFileType FileType) { - if G.opts.Debug { - defer tracecall2(addedText, patchedFileType.String())() + if trace.Tracing { + defer trace.Call2(addedText, patchedFileType.String())() } ck.checktextRcsid(addedText) @@ -222,12 +223,12 @@ func (ck *PatchChecker) checklineAdded(addedText string, patchedFileType FileTyp } func (ck *PatchChecker) checktextUniHunkCr() { - if G.opts.Debug { - defer tracecall0()() + if trace.Tracing { + defer trace.Call0()() } line := ck.exp.PreviousLine() - if hasSuffix(line.Text, "\r") { + if hasSuffix(line.Text(), "\r") { if !line.AutofixReplace("\r\n", "\n") { line.Errorf("The hunk header must not end with a CR character.") Explain( @@ -282,9 +283,9 @@ func (ft FileType) String() string { } // This is used to select the proper subroutine for detecting absolute pathnames. -func guessFileType(line *Line, fname string) (fileType FileType) { - if G.opts.Debug { - defer tracecall(fname, "=>", &fileType)() +func guessFileType(line Line, fname string) (fileType FileType) { + if trace.Tracing { + defer trace.Call(fname, "=>", &fileType)() } basename := path.Base(fname) @@ -309,15 +310,15 @@ func guessFileType(line *Line, fname string) (fileType FileType) { return ftUnknown } - if G.opts.Debug { - traceStep1("Unknown file type for %q", fname) + if trace.Tracing { + trace.Step1("Unknown file type for %q", fname) } return ftUnknown } -func checkwordAbsolutePathname(line *Line, word string) { - if G.opts.Debug { - defer tracecall1(word)() +func checkwordAbsolutePathname(line Line, word string) { + if trace.Tracing { + defer trace.Call1(word)() } switch { @@ -344,13 +345,13 @@ func checkwordAbsolutePathname(line *Line, word string) { } // Looks for strings like "/dev/cd0" appearing in source code -func checklineSourceAbsolutePathname(line *Line, text string) { +func checklineSourceAbsolutePathname(line Line, text string) { if !strings.ContainsAny(text, "\"'") { return } if matched, before, _, str := match3(text, `^(.*)(["'])(/\w[^"']*)["']`); matched { - if G.opts.Debug { - traceStep2("checklineSourceAbsolutePathname: before=%q, str=%q", before, str) + if trace.Tracing { + trace.Step2("checklineSourceAbsolutePathname: before=%q, str=%q", before, str) } switch { @@ -366,9 +367,9 @@ func checklineSourceAbsolutePathname(line *Line, text string) { } } -func checklineOtherAbsolutePathname(line *Line, text string) { - if G.opts.Debug { - defer tracecall1(text)() +func checklineOtherAbsolutePathname(line Line, text string) { + if trace.Tracing { + defer trace.Call1(text)() } if hasPrefix(text, "#") && !hasPrefix(text, "#!") { @@ -383,8 +384,8 @@ func checklineOtherAbsolutePathname(line *Line, text string) { case hasSuffix(before, "."): // Example: ../dir // XXX new: case matches(before, `s.$`): // Example: sed -e s,/usr,@PREFIX@, default: - if G.opts.Debug { - traceStep1("before=%q", before) + if trace.Tracing { + trace.Step1("before=%q", before) } checkwordAbsolutePathname(line, path) } diff --git a/pkgtools/pkglint/files/patches_test.go b/pkgtools/pkglint/files/patches_test.go index c45bd39b497..bae14f27588 100644 --- a/pkgtools/pkglint/files/patches_test.go +++ b/pkgtools/pkglint/files/patches_test.go @@ -106,7 +106,7 @@ func (s *Suite) Test_ChecklinesPatch__git_without_comment(c *check.C) { func (s *Suite) Test_checklineOtherAbsolutePathname(c *check.C) { line := NewLine("patch-ag", 1, "+$install -s -c ./bin/rosegarden ${DESTDIR}$BINDIR", nil) - checklineOtherAbsolutePathname(line, line.Text) + checklineOtherAbsolutePathname(line, line.Text()) c.Check(s.Output(), equals, "") } diff --git a/pkgtools/pkglint/files/pkglint.go b/pkgtools/pkglint/files/pkglint.go index 2589fe45e9c..63716abe28d 100644 --- a/pkgtools/pkglint/files/pkglint.go +++ b/pkgtools/pkglint/files/pkglint.go @@ -4,6 +4,9 @@ import ( "fmt" "io" "netbsd.org/pkglint/getopt" + "netbsd.org/pkglint/histogram" + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/trace" "os" "os/user" "path" @@ -16,7 +19,7 @@ const confMake = "@BMAKE@" const confVersion = "@VERSION@" func main() { - G.logOut, G.logErr, G.debugOut = os.Stdout, os.Stderr, os.Stdout + G.logOut, G.logErr, trace.Out = os.Stdout, os.Stderr, os.Stdout os.Exit(new(Pkglint).Main(os.Args...)) } @@ -45,10 +48,12 @@ func (pkglint *Pkglint) Main(args ...string) (exitcode int) { pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() - G.rematch = NewHistogram() - G.renomatch = NewHistogram() - G.retime = NewHistogram() - G.loghisto = NewHistogram() + regex.Profiling = true + G.loghisto = histogram.New() + defer func() { + G.loghisto.PrintStats("loghisto", G.logOut, 0) + regex.PrintStats() + }() } for _, arg := range G.opts.args { @@ -63,7 +68,7 @@ func (pkglint *Pkglint) Main(args ...string) (exitcode int) { currentUser, err := user.Current() if err == nil { // On Windows, this is `Computername\Username`. - G.CurrentUsername = regcomp(`^.*\\`).ReplaceAllString(currentUser.Username, "") + G.CurrentUsername = regex.Compile(`^.*\\`).ReplaceAllString(currentUser.Username, "") } for len(G.Todo) != 0 { @@ -74,12 +79,6 @@ func (pkglint *Pkglint) Main(args ...string) (exitcode int) { checkToplevelUnusedLicenses() pkglint.PrintSummary() - if G.opts.Profiling { - G.loghisto.PrintStats("loghisto", G.logOut, 0) - G.rematch.PrintStats("rematch", G.logOut, 10) - G.renomatch.PrintStats("renomatch", G.logOut, 10) - G.retime.PrintStats("retime", G.logOut, 10) - } if G.errors != 0 { return 1 } @@ -91,7 +90,7 @@ func (pkglint *Pkglint) ParseCommandLine(args []string) *int { opts := getopt.NewOptions() check := opts.AddFlagGroup('C', "check", "check,...", "enable or disable specific checks") - opts.AddFlagVar('d', "debug", &gopts.Debug, false, "log verbose call traces for debugging") + opts.AddFlagVar('d', "debug", &trace.Tracing, false, "log verbose call traces for debugging") opts.AddFlagVar('e', "explain", &gopts.Explain, false, "explain the diagnostics or give further help") opts.AddFlagVar('f', "show-autofix", &gopts.PrintAutofix, false, "show what pkglint can fix automatically") opts.AddFlagVar('F', "autofix", &gopts.Autofix, false, "try to automatically fix some errors (experimental)") @@ -178,8 +177,8 @@ func (pkglint *Pkglint) PrintSummary() { } func (pkglint *Pkglint) CheckDirent(fname string) { - if G.opts.Debug { - defer tracecall1(fname)() + if trace.Tracing { + defer trace.Call1(fname)() } st, err := os.Lstat(fname) @@ -222,7 +221,7 @@ func (pkglint *Pkglint) CheckDirent(fname string) { // Returns the pkgsrc top-level directory, relative to the given file or directory. func findPkgsrcTopdir(fname string) string { - for _, dir := range []string{".", "..", "../..", "../../.."} { + for _, dir := range [...]string{".", "..", "../..", "../../.."} { if fileExists(fname + "/" + dir + "/mk/bsd.pkg.mk") { return dir } @@ -231,15 +230,15 @@ func findPkgsrcTopdir(fname string) string { } func resolveVariableRefs(text string) string { - if G.opts.Debug { - defer tracecall1(text)() + if trace.Tracing { + defer trace.Call1(text)() } visited := make(map[string]bool) // To prevent endless loops str := text for { - replaced := regcomp(`\$\{([\w.]+)\}`).ReplaceAllStringFunc(str, func(m string) string { + replaced := regex.Compile(`\$\{([\w.]+)\}`).ReplaceAllStringFunc(str, func(m string) string { varname := m[2 : len(m)-1] if !visited[varname] { visited[varname] = true @@ -264,8 +263,8 @@ func resolveVariableRefs(text string) string { } func CheckfileExtra(fname string) { - if G.opts.Debug { - defer tracecall1(fname)() + if trace.Tracing { + defer trace.Call1(fname)() } if lines := LoadNonemptyLines(fname, false); lines != nil { @@ -273,16 +272,16 @@ func CheckfileExtra(fname string) { } } -func ChecklinesDescr(lines []*Line) { - if G.opts.Debug { - defer tracecall1(lines[0].Fname)() +func ChecklinesDescr(lines []Line) { + if trace.Tracing { + defer trace.Call1(lines[0].Filename())() } for _, line := range lines { - line.CheckLength(80) - line.CheckTrailingWhitespace() - line.CheckValidCharacters(`[\t -~]`) - if contains(line.Text, "${") { + LineChecker{line}.CheckLength(80) + LineChecker{line}.CheckTrailingWhitespace() + LineChecker{line}.CheckValidCharacters(`[\t -~]`) + if contains(line.Text(), "${") { line.Notef("Variables are not expanded in the DESCR file.") } } @@ -301,9 +300,9 @@ func ChecklinesDescr(lines []*Line) { SaveAutofixChanges(lines) } -func ChecklinesMessage(lines []*Line) { - if G.opts.Debug { - defer tracecall1(lines[0].Fname)() +func ChecklinesMessage(lines []Line) { + if trace.Tracing { + defer trace.Call1(lines[0].Filename())() } explainMessage := func() { @@ -322,17 +321,17 @@ func ChecklinesMessage(lines []*Line) { } hline := strings.Repeat("=", 75) - if line := lines[0]; line.Text != hline { + if line := lines[0]; line.Text() != hline { line.Warnf("Expected a line of exactly 75 \"=\" characters.") explainMessage() } - lines[1].CheckRcsid(``, "") + LineChecker{lines[1]}.CheckRcsid(``, "") for _, line := range lines { - line.CheckLength(80) - line.CheckTrailingWhitespace() - line.CheckValidCharacters(`[\t -~]`) + LineChecker{line}.CheckLength(80) + LineChecker{line}.CheckTrailingWhitespace() + LineChecker{line}.CheckValidCharacters(`[\t -~]`) } - if lastLine := lines[len(lines)-1]; lastLine.Text != hline { + if lastLine := lines[len(lines)-1]; lastLine.Text() != hline { lastLine.Warnf("Expected a line of exactly 75 \"=\" characters.") explainMessage() } @@ -340,8 +339,8 @@ func ChecklinesMessage(lines []*Line) { } func CheckfileMk(fname string) { - if G.opts.Debug { - defer tracecall1(fname)() + if trace.Tracing { + defer trace.Call1(fname)() } lines := LoadNonemptyLines(fname, true) @@ -354,8 +353,8 @@ func CheckfileMk(fname string) { } func Checkfile(fname string) { - if G.opts.Debug { - defer tracecall1(fname)() + if trace.Tracing { + defer trace.Call1(fname)() } basename := path.Base(fname) @@ -447,8 +446,8 @@ func Checkfile(fname string) { } case matches(fname, `(?:^|/)patches/manual[^/]*$`): - if G.opts.Debug { - traceStep1("Unchecked file %q.", fname) + if trace.Tracing { + trace.Step1("Unchecked file %q.", fname) } case matches(fname, `(?:^|/)patches/[^/]*$`): @@ -487,10 +486,10 @@ func Checkfile(fname string) { } } -func ChecklinesTrailingEmptyLines(lines []*Line) { +func ChecklinesTrailingEmptyLines(lines []Line) { max := len(lines) last := max - for last > 1 && lines[last-1].Text == "" { + for last > 1 && lines[last-1].Text() == "" { last-- } if last != max { diff --git a/pkgtools/pkglint/files/pkglint_test.go b/pkgtools/pkglint/files/pkglint_test.go index e0e709b9f5c..17c9fe00bd0 100644 --- a/pkgtools/pkglint/files/pkglint_test.go +++ b/pkgtools/pkglint/files/pkglint_test.go @@ -4,6 +4,7 @@ import ( "strings" check "gopkg.in/check.v1" + "netbsd.org/pkglint/trace" "os" ) @@ -30,12 +31,12 @@ func (s *Suite) Test_Pkglint_Main_no_args(c *check.C) { // go test -c -covermode count // pkgsrcdir=... -// env PKGLINT_TESTCMDLINE="$pkgsrcdir -r" ./pkglint.test -test.coverprofile pkglint.cov -check.f TestRunPkglint +// env PKGLINT_TESTCMDLINE="$pkgsrcdir -r" ./pkglint.test -test.coverprofile pkglint.cov // go tool cover -html=pkglint.cov -o coverage.html func (s *Suite) Test_Pkglint_coverage(c *check.C) { cmdline := os.Getenv("PKGLINT_TESTCMDLINE") if cmdline != "" { - G.logOut, G.logErr, G.debugOut = os.Stdout, os.Stderr, os.Stdout + G.logOut, G.logErr, trace.Out = os.Stdout, os.Stderr, os.Stdout new(Pkglint).Main(append([]string{"pkglint"}, splitOnSpace(cmdline)...)...) } } diff --git a/pkgtools/pkglint/files/pkgver/vercmp_test.go b/pkgtools/pkglint/files/pkgver/vercmp_test.go index e44a88c15d5..a63db7a1b94 100644 --- a/pkgtools/pkglint/files/pkgver/vercmp_test.go +++ b/pkgtools/pkglint/files/pkgver/vercmp_test.go @@ -26,7 +26,7 @@ func (s *Suite) Test_newVersion(c *check.C) { c.Check(newVersion("0pre20160620"), check.DeepEquals, &version{[]int{0, -1, 20160620}, 0}) } -func (s *Suite) Test_pkgverCmp(c *check.C) { +func (s *Suite) Test_Compare(c *check.C) { var versions = [][]string{ {"0pre20160620"}, {"0"}, diff --git a/pkgtools/pkglint/files/plist.go b/pkgtools/pkglint/files/plist.go index 36207dded12..b4891b68413 100644 --- a/pkgtools/pkglint/files/plist.go +++ b/pkgtools/pkglint/files/plist.go @@ -1,17 +1,19 @@ package main import ( + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/trace" "path" "sort" "strings" ) -func ChecklinesPlist(lines []*Line) { - if G.opts.Debug { - defer tracecall1(lines[0].Fname)() +func ChecklinesPlist(lines []Line) { + if trace.Tracing { + defer trace.Call1(lines[0].Filename())() } - lines[0].CheckRcsid(`@comment `, "@comment ") + LineChecker{lines[0]}.CheckRcsid(`@comment `, "@comment ") if len(lines) == 1 { lines[0].Warnf("PLIST files shouldn't be empty.") @@ -41,16 +43,16 @@ type PlistChecker struct { } type PlistLine struct { - line *Line + line Line conditional string // e.g. PLIST.docs text string // Like line.text, without the conditional } -func (ck *PlistChecker) Check(plainLines []*Line) { +func (ck *PlistChecker) Check(plainLines []Line) { plines := ck.NewLines(plainLines) ck.collectFilesAndDirs(plines) - if fname := plines[0].line.Fname; path.Base(fname) == "PLIST.common_end" { + if fname := plines[0].line.Filename(); path.Base(fname) == "PLIST.common_end" { commonLines, err := readLines(strings.TrimSuffix(fname, "_end"), false) if err == nil { ck.collectFilesAndDirs(ck.NewLines(commonLines)) @@ -74,10 +76,10 @@ func (ck *PlistChecker) Check(plainLines []*Line) { } } -func (ck *PlistChecker) NewLines(lines []*Line) []*PlistLine { +func (ck *PlistChecker) NewLines(lines []Line) []*PlistLine { plines := make([]*PlistLine, len(lines)) for i, line := range lines { - conditional, text := "", line.Text + conditional, text := "", line.Text() if hasPrefix(text, "${PLIST.") { if m, cond, rest := match2(text, `^\$\{(PLIST\.[\w-.]+)\}(.*)`); m { conditional, text = cond, rest @@ -290,7 +292,7 @@ func (ck *PlistChecker) checkpathLib(pline *PlistLine, dirname, basename string) func (ck *PlistChecker) checkpathMan(pline *PlistLine) { line := pline.line - m, catOrMan, section, manpage, ext, gz := match5(pline.text, `^man/(cat|man)(\w+)/(.*?)\.(\w+)(\.gz)?$`) + m, catOrMan, section, manpage, ext, gz := regex.Match5(pline.text, `^man/(cat|man)(\w+)/(.*?)\.(\w+)(\.gz)?$`) if !m { // maybe: line.Warnf("Invalid filename %q for manual page.", text) return @@ -469,14 +471,14 @@ func (pline *PlistLine) warnImakeMannewsuffix() { type plistLineSorter struct { first *PlistLine plines []*PlistLine - lines []*Line - after map[*PlistLine][]*Line + lines []Line + after map[*PlistLine][]Line swapped bool autofixed bool } func NewPlistLineSorter(plines []*PlistLine) *plistLineSorter { - s := &plistLineSorter{first: plines[0], after: make(map[*PlistLine][]*Line)} + s := &plistLineSorter{first: plines[0], after: make(map[*PlistLine][]Line)} prev := plines[0] for _, pline := range plines[1:] { if hasPrefix(pline.text, "@") || contains(pline.text, "$") { @@ -510,10 +512,8 @@ func (s *plistLineSorter) Sort() { } firstLine := s.first.line - firstLine.RememberAutofix("Sorting the whole file.") - firstLine.logAutofix() - firstLine.changed = true // Otherwise the changes won't be saved - lines := []*Line{firstLine} + firstLine.AutofixMark("Sorting the whole file.") + lines := []Line{firstLine} lines = append(lines, s.after[s.first]...) for _, pline := range s.plines { lines = append(lines, pline.line) diff --git a/pkgtools/pkglint/files/regex.go b/pkgtools/pkglint/files/regex.go deleted file mode 100644 index 5e77c01069b..00000000000 --- a/pkgtools/pkglint/files/regex.go +++ /dev/null @@ -1,112 +0,0 @@ -package main - -import ( - "fmt" - "regexp" - "time" -) - -type RegexPattern string - -func regcomp(re RegexPattern) *regexp.Regexp { - if G.res == nil { - G.res = make(map[RegexPattern]*regexp.Regexp) - } - cre := G.res[re] - if cre == nil { - cre = regexp.MustCompile(string(re)) - G.res[re] = cre - } - return cre -} - -func match(s string, re RegexPattern) []string { - if !G.opts.Profiling { - return regcomp(re).FindStringSubmatch(s) - } - - before := time.Now() - immediatelyBefore := time.Now() - m := regcomp(re).FindStringSubmatch(s) - after := time.Now() - - delay := immediatelyBefore.UnixNano() - before.UnixNano() - timeTaken := after.UnixNano() - immediatelyBefore.UnixNano() - delay - - G.retime.Add(string(re), int(timeTaken)) - if m != nil { - G.rematch.Add(string(re), 1) - } else { - G.renomatch.Add(string(re), 1) - } - return m -} - -func matches(s string, re RegexPattern) bool { - matches := regcomp(re).MatchString(s) - if G.opts.Profiling { - if matches { - G.rematch.Add(string(re), 1) - } else { - G.renomatch.Add(string(re), 1) - } - } - return matches -} - -func matchn(s string, re RegexPattern, n int) []string { - if m := match(s, re); m != nil { - if len(m) != 1+n { - panic(fmt.Sprintf("expected match%d, got match%d for %q", len(m)-1, n, re)) - } - return m - } - return nil -} - -func match1(s string, re RegexPattern) (matched bool, m1 string) { - if m := matchn(s, re, 1); m != nil { - return true, m[1] - } - return -} -func match2(s string, re RegexPattern) (matched bool, m1, m2 string) { - if m := matchn(s, re, 2); m != nil { - return true, m[1], m[2] - } - return -} -func match3(s string, re RegexPattern) (matched bool, m1, m2, m3 string) { - if m := matchn(s, re, 3); m != nil { - return true, m[1], m[2], m[3] - } - return -} -func match4(s string, re RegexPattern) (matched bool, m1, m2, m3, m4 string) { - if m := matchn(s, re, 4); m != nil { - return true, m[1], m[2], m[3], m[4] - } - return -} -func match5(s string, re RegexPattern) (matched bool, m1, m2, m3, m4, m5 string) { - if m := matchn(s, re, 5); m != nil { - return true, m[1], m[2], m[3], m[4], m[5] - } - return -} - -func replaceFirst(s string, re RegexPattern, replacement string) ([]string, string) { - if G.opts.Debug { - defer tracecall(s, re, replacement)() - } - - if m := regcomp(re).FindStringSubmatchIndex(s); m != nil { - replaced := s[:m[0]] + replacement + s[m[1]:] - mm := make([]string, len(m)/2) - for i := 0; i < len(m); i += 2 { - mm[i/2] = s[negToZero(m[i]):negToZero(m[i+1])] - } - return mm, replaced - } - return nil, s -} diff --git a/pkgtools/pkglint/files/regex/regex.go b/pkgtools/pkglint/files/regex/regex.go new file mode 100644 index 00000000000..c7addc854b9 --- /dev/null +++ b/pkgtools/pkglint/files/regex/regex.go @@ -0,0 +1,146 @@ +package regex + +import ( + "fmt" + "netbsd.org/pkglint/histogram" + "os" + "regexp" + "time" +) + +type RegexPattern string + +var ( + Profiling bool +) + +var ( + res map[RegexPattern]*regexp.Regexp + rematch *histogram.Histogram + renomatch *histogram.Histogram + retime *histogram.Histogram +) + +func Compile(re RegexPattern) *regexp.Regexp { + if res == nil { + res = make(map[RegexPattern]*regexp.Regexp) + } + cre := res[re] + if cre == nil { + cre = regexp.MustCompile(string(re)) + res[re] = cre + } + return cre +} + +func Match(s string, re RegexPattern) []string { + if !Profiling { + return Compile(re).FindStringSubmatch(s) + } + + before := time.Now() + immediatelyBefore := time.Now() + m := Compile(re).FindStringSubmatch(s) + after := time.Now() + + delay := immediatelyBefore.UnixNano() - before.UnixNano() + timeTaken := after.UnixNano() - immediatelyBefore.UnixNano() - delay + + if retime == nil { + retime = histogram.New() + rematch = histogram.New() + renomatch = histogram.New() + } + + retime.Add(string(re), int(timeTaken)) + if m != nil { + rematch.Add(string(re), 1) + } else { + renomatch.Add(string(re), 1) + } + return m +} + +func Matches(s string, re RegexPattern) bool { + matches := Compile(re).MatchString(s) + if Profiling { + if matches { + rematch.Add(string(re), 1) + } else { + renomatch.Add(string(re), 1) + } + } + return matches +} + +func Match1(s string, re RegexPattern) (matched bool, m1 string) { + if m := matchn(s, re, 1); m != nil { + return true, m[1] + } + return +} + +func Match2(s string, re RegexPattern) (matched bool, m1, m2 string) { + if m := matchn(s, re, 2); m != nil { + return true, m[1], m[2] + } + return +} + +func Match3(s string, re RegexPattern) (matched bool, m1, m2, m3 string) { + if m := matchn(s, re, 3); m != nil { + return true, m[1], m[2], m[3] + } + return +} + +func Match4(s string, re RegexPattern) (matched bool, m1, m2, m3, m4 string) { + if m := matchn(s, re, 4); m != nil { + return true, m[1], m[2], m[3], m[4] + } + return +} + +func Match5(s string, re RegexPattern) (matched bool, m1, m2, m3, m4, m5 string) { + if m := matchn(s, re, 5); m != nil { + return true, m[1], m[2], m[3], m[4], m[5] + } + return +} + +func ReplaceFirst(s string, re RegexPattern, replacement string) ([]string, string) { + if m := Compile(re).FindStringSubmatchIndex(s); m != nil { + replaced := s[:m[0]] + replacement + s[m[1]:] + mm := make([]string, len(m)/2) + for i := 0; i < len(m); i += 2 { + mm[i/2] = s[max0(m[i]):max0(m[i+1])] + } + return mm, replaced + } + return nil, s +} + +func PrintStats() { + if Profiling { + rematch.PrintStats("rematch", os.Stdout, 10) + renomatch.PrintStats("renomatch", os.Stdout, 10) + retime.PrintStats("retime", os.Stdout, 10) + } +} + +func matchn(s string, re RegexPattern, n int) []string { + if m := Match(s, re); m != nil { + if len(m) != 1+n { + panic(fmt.Sprintf("expected match%d, got match%d for %q", len(m)-1, n, re)) + } + return m + } + return nil +} + +func max0(a int) int { + if a >= 0 { + return a + } + return 0 +} diff --git a/pkgtools/pkglint/files/shell.go b/pkgtools/pkglint/files/shell.go index 9d07649ada2..0287e7ce519 100644 --- a/pkgtools/pkglint/files/shell.go +++ b/pkgtools/pkglint/files/shell.go @@ -3,6 +3,8 @@ package main // Parsing and checking shell commands embedded in Makefiles import ( + "netbsd.org/pkglint/textproc" + "netbsd.org/pkglint/trace" "path" "strings" ) @@ -15,7 +17,7 @@ const ( ) type ShellLine struct { - line *Line + line Line mkline *MkLine } @@ -27,8 +29,8 @@ var shellcommandsContextType = &Vartype{lkNone, BtShellCommands, []AclEntry{{"*" var shellwordVuc = &VarUseContext{shellcommandsContextType, vucTimeUnknown, vucQuotPlain, false} func (shline *ShellLine) CheckWord(token string, checkQuoting bool) { - if G.opts.Debug { - defer tracecall(token, checkQuoting)() + if trace.Tracing { + defer trace.Call(token, checkQuoting)() } if token == "" || hasPrefix(token, "#") { @@ -39,7 +41,7 @@ func (shline *ShellLine) CheckWord(token string, checkQuoting bool) { p := NewMkParser(line, token, false) if varuse := p.VarUse(); varuse != nil && p.EOF() { - shline.mkline.CheckVaruse(varuse, shellwordVuc) + MkLineChecker{shline.mkline}.CheckVaruse(varuse, shellwordVuc) return } @@ -55,8 +57,8 @@ func (shline *ShellLine) CheckWord(token string, checkQuoting bool) { quoting := shqPlain outer: for !parser.EOF() { - if G.opts.Debug { - traceStep("shell state %s: %q", quoting, parser.Rest()) + if trace.Tracing { + trace.Stepf("shell state %s: %q", quoting, parser.Rest()) } switch { @@ -87,7 +89,7 @@ outer: case repl.AdvanceRegexp(`^\$\$([0-9A-Z_a-z]+|#)`), repl.AdvanceRegexp(`^\$\$\{([0-9A-Z_a-z]+|#)\}`), repl.AdvanceRegexp(`^\$\$(\$)\$`): - shvarname := repl.m[1] + shvarname := repl.Group(1) if G.opts.WarnQuoting && checkQuoting && shline.variableNeedsQuoting(shvarname) { line.Warnf("Unquoted shell variable %q.", shvarname) Explain( @@ -171,8 +173,8 @@ outer: } func (shline *ShellLine) checkVaruseToken(parser *MkParser, quoting ShQuoting) bool { - if G.opts.Debug { - defer tracecall(parser.Rest(), quoting)() + if trace.Tracing { + defer trace.Call(parser.Rest(), quoting)() } varuse := parser.VarUse() @@ -207,7 +209,7 @@ func (shline *ShellLine) checkVaruseToken(parser *MkParser, quoting ShQuoting) b if varname != "@" { vucstate := quoting.ToVarUseContext() vuc := &VarUseContext{shellcommandsContextType, vucTimeUnknown, vucstate, true} - shline.mkline.CheckVaruse(varuse, vuc) + MkLineChecker{shline.mkline}.CheckVaruse(varuse, vuc) } return true } @@ -217,13 +219,13 @@ func (shline *ShellLine) checkVaruseToken(parser *MkParser, quoting ShQuoting) b // before a dollar, a backslash or a backtick. // // See http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_06_03 -func (shline *ShellLine) unescapeBackticks(shellword string, repl *PrefixReplacer, quoting ShQuoting) (unescaped string, newQuoting ShQuoting) { - if G.opts.Debug { - defer tracecall(shellword, quoting, "=>", ref(&unescaped))() +func (shline *ShellLine) unescapeBackticks(shellword string, repl *textproc.PrefixReplacer, quoting ShQuoting) (unescaped string, newQuoting ShQuoting) { + if trace.Tracing { + defer trace.Call(shellword, quoting, "=>", trace.Ref(&unescaped))() } line := shline.line - for repl.rest != "" { + for !repl.EOF() { switch { case repl.AdvanceStr("`"): if quoting == shqBackt { @@ -234,7 +236,7 @@ func (shline *ShellLine) unescapeBackticks(shellword string, repl *PrefixReplace return unescaped, quoting case repl.AdvanceRegexp("^\\\\([\"\\\\`$])"): - unescaped += repl.m[1] + unescaped += repl.Group(1) case repl.AdvanceStr("\\"): line.Warnf("Backslashes should be doubled inside backticks.") @@ -249,13 +251,13 @@ func (shline *ShellLine) unescapeBackticks(shellword string, repl *PrefixReplace "http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html") case repl.AdvanceRegexp("^([^\\\\`]+)"): - unescaped += repl.m[1] + unescaped += repl.Group(1) default: - line.Errorf("Internal pkglint error in ShellLine.unescapeBackticks at %q (rest=%q)", shellword, repl.rest) + line.Errorf("Internal pkglint error in ShellLine.unescapeBackticks at %q (rest=%q)", shellword, repl.Rest()) } } - line.Errorf("Unfinished backquotes: rest=%q", repl.rest) + line.Errorf("Unfinished backquotes: rest=%q", repl.Rest()) return unescaped, quoting } @@ -270,8 +272,8 @@ func (shline *ShellLine) variableNeedsQuoting(shvarname string) bool { } func (shline *ShellLine) CheckShellCommandLine(shelltext string) { - if G.opts.Debug { - defer tracecall1(shelltext)() + if trace.Tracing { + defer trace.Call1(shelltext)() } line := shline.line @@ -303,10 +305,10 @@ func (shline *ShellLine) CheckShellCommandLine(shelltext string) { line.Notef("You don't need to use \"-\" before %q.", cmd) } - repl := NewPrefixReplacer(shelltext) + repl := textproc.NewPrefixReplacer(shelltext) repl.AdvanceRegexp(`^\s+`) if repl.AdvanceRegexp(`^[-@]+`) { - shline.checkHiddenAndSuppress(repl.m[0], repl.rest) + shline.checkHiddenAndSuppress(repl.Group(0), repl.Rest()) } setE := false if repl.AdvanceStr("${RUN}") { @@ -315,12 +317,12 @@ func (shline *ShellLine) CheckShellCommandLine(shelltext string) { repl.AdvanceStr("${_PKG_SILENT}${_PKG_DEBUG}") } - shline.CheckShellCommand(repl.rest, &setE) + shline.CheckShellCommand(repl.Rest(), &setE) } func (shline *ShellLine) CheckShellCommand(shellcmd string, pSetE *bool) { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } program, err := parseShellProgram(shline.line, shellcmd) @@ -368,8 +370,8 @@ func (shline *ShellLine) CheckShellCommands(shellcmds string) { } func (shline *ShellLine) checkHiddenAndSuppress(hiddenAndSuppress, rest string) { - if G.opts.Debug { - defer tracecall(hiddenAndSuppress, rest)() + if trace.Tracing { + defer trace.Call(hiddenAndSuppress, rest)() } switch { @@ -430,8 +432,8 @@ func NewSimpleCommandChecker(shline *ShellLine, cmd *MkShSimpleCommand) *SimpleC } func (scc *SimpleCommandChecker) Check() { - if G.opts.Debug { - defer tracecall(scc.strcmd)() + if trace.Tracing { + defer trace.Call(scc.strcmd)() } scc.checkCommandStart() @@ -443,8 +445,8 @@ func (scc *SimpleCommandChecker) Check() { } func (scc *SimpleCommandChecker) checkCommandStart() { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } shellword := scc.strcmd.Name @@ -469,8 +471,8 @@ func (scc *SimpleCommandChecker) checkCommandStart() { } func (scc *SimpleCommandChecker) handleTool() bool { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } shellword := scc.strcmd.Name @@ -495,8 +497,8 @@ func (scc *SimpleCommandChecker) handleTool() bool { } func (scc *SimpleCommandChecker) handleForbiddenCommand() bool { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } shellword := scc.strcmd.Name @@ -513,8 +515,8 @@ func (scc *SimpleCommandChecker) handleForbiddenCommand() bool { } func (scc *SimpleCommandChecker) handleCommandVariable() bool { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } shellword := scc.strcmd.Name @@ -543,13 +545,13 @@ func (scc *SimpleCommandChecker) handleCommandVariable() bool { } func (scc *SimpleCommandChecker) handleComment() bool { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } shellword := scc.strcmd.Name - if G.opts.Debug { - defer tracecall1(shellword)() + if trace.Tracing { + defer trace.Call1(shellword)() } if !hasPrefix(shellword, "#") { @@ -585,15 +587,15 @@ func (scc *SimpleCommandChecker) handleComment() bool { } func (scc *SimpleCommandChecker) checkAbsolutePathnames() { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } cmdname := scc.strcmd.Name isSubst := false for _, arg := range scc.strcmd.Args { if !isSubst { - scc.shline.line.CheckAbsolutePathname(arg) + LineChecker{scc.shline.line}.CheckAbsolutePathname(arg) } if false && isSubst && !matches(arg, `"^[\"\'].*[\"\']$`) { scc.shline.line.Warnf("Substitution commands like %q should always be quoted.", arg) @@ -607,8 +609,8 @@ func (scc *SimpleCommandChecker) checkAbsolutePathnames() { } func (scc *SimpleCommandChecker) checkAutoMkdirs() { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } cmdname := scc.strcmd.Name @@ -644,8 +646,8 @@ func (scc *SimpleCommandChecker) checkAutoMkdirs() { } func (scc *SimpleCommandChecker) checkInstallMulti() { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } cmd := scc.strcmd @@ -673,8 +675,8 @@ func (scc *SimpleCommandChecker) checkInstallMulti() { } func (scc *SimpleCommandChecker) checkPaxPe() { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } if scc.strcmd.Name == "${PAX}" && scc.strcmd.HasOption("-pe") { @@ -687,8 +689,8 @@ func (scc *SimpleCommandChecker) checkPaxPe() { } func (scc *SimpleCommandChecker) checkEchoN() { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } if scc.strcmd.Name == "${ECHO}" && scc.strcmd.HasOption("-n") { @@ -701,8 +703,8 @@ type ShellProgramChecker struct { } func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } getSimple := func(list *MkShList) *MkShSimpleCommand { @@ -743,8 +745,8 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { } func (spc *ShellProgramChecker) checkWords(words []*ShToken, checkQuoting bool) { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } for _, word := range words { @@ -753,16 +755,16 @@ func (spc *ShellProgramChecker) checkWords(words []*ShToken, checkQuoting bool) } func (spc *ShellProgramChecker) checkWord(word *ShToken, checkQuoting bool) { - if G.opts.Debug { - defer tracecall(word.MkText)() + if trace.Tracing { + defer trace.Call(word.MkText)() } spc.shline.CheckWord(word.MkText, checkQuoting) } -func (scc *ShellProgramChecker) checkPipeExitcode(line *Line, pipeline *MkShPipeline) { - if G.opts.Debug { - defer tracecall()() +func (scc *ShellProgramChecker) checkPipeExitcode(line Line, pipeline *MkShPipeline) { + if trace.Tracing { + defer trace.Call()() } if G.opts.WarnExtra && len(pipeline.Cmds) > 1 { @@ -777,8 +779,8 @@ func (scc *ShellProgramChecker) checkPipeExitcode(line *Line, pipeline *MkShPipe } func (scc *ShellProgramChecker) checkSetE(list *MkShList, eflag *bool) { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } // Disabled until the shell parser can recognize "command || exit 1" reliably. @@ -803,8 +805,8 @@ func (scc *ShellProgramChecker) checkSetE(list *MkShList, eflag *bool) { // Some shell commands should not be used in the install phase. func (shline *ShellLine) checkCommandUse(shellcmd string) { - if G.opts.Debug { - defer tracecall()() + if trace.Tracing { + defer trace.Call()() } if G.Mk == nil || !matches(G.Mk.target, `^(?:pre|do|post)-install$`) { @@ -846,9 +848,9 @@ func (shline *ShellLine) checkCommandUse(shellcmd string) { } // Example: "word1 word2;;;" => "word1", "word2", ";;", ";" -func splitIntoShellTokens(line *Line, text string) (tokens []string, rest string) { - if G.opts.Debug { - defer tracecall(line, text)() +func splitIntoShellTokens(line Line, text string) (tokens []string, rest string) { + if trace.Tracing { + defer trace.Call(line, text)() } word := "" @@ -878,9 +880,9 @@ func splitIntoShellTokens(line *Line, text string) (tokens []string, rest string // Example: "word1 word2;;;" => "word1", "word2;;;" // Compare devel/bmake/files/str.c, function brk_string. -func splitIntoMkWords(line *Line, text string) (words []string, rest string) { - if G.opts.Debug { - defer tracecall(line, text)() +func splitIntoMkWords(line Line, text string) (words []string, rest string) { + if trace.Tracing { + defer trace.Call(line, text)() } p := NewShTokenizer(line, text, false) diff --git a/pkgtools/pkglint/files/shell_test.go b/pkgtools/pkglint/files/shell_test.go index 163edbba0ce..d1656cc518c 100644 --- a/pkgtools/pkglint/files/shell_test.go +++ b/pkgtools/pkglint/files/shell_test.go @@ -2,6 +2,7 @@ package main import ( check "gopkg.in/check.v1" + "netbsd.org/pkglint/textproc" ) func (s *Suite) Test_splitIntoShellTokens__line_continuation(c *check.C) { @@ -237,9 +238,6 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__nofix(c *check.C) { "\techo ${PKGNAME:Q}") shline := NewShellLine(G.Mk.mklines[0]) - c.Check(shline.line.raw[0].textnl, equals, "\techo ${PKGNAME:Q}\n") - c.Check(shline.line.raw[0].Lineno, equals, 1) - shline.CheckShellCommandLine("echo ${PKGNAME:Q}") c.Check(s.Output(), equals, ""+ @@ -375,7 +373,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__echo(c *check.C) { "# dummy") mkline := NewMkLine(NewLine("fname", 3, "# dummy", nil)) - mkline.checkText("echo \"hello, world\"") + MkLineChecker{mkline}.checkText("echo \"hello, world\"") c.Check(s.Output(), equals, "") @@ -510,11 +508,11 @@ func (s *Suite) Test_ShellLine_unescapeBackticks(c *check.C) { shline := NewShellLine(NewMkLine(dummyLine)) // foobar="`echo \"foo bar\"`" text := "foobar=\"`echo \\\"foo bar\\\"`\"" - repl := NewPrefixReplacer(text) + repl := textproc.NewPrefixReplacer(text) repl.AdvanceStr("foobar=\"`") backtCommand, newQuoting := shline.unescapeBackticks(text, repl, shqDquotBackt) c.Check(backtCommand, equals, "echo \"foo bar\"") c.Check(newQuoting, equals, shqDquot) - c.Check(repl.rest, equals, "\"") + c.Check(repl.Rest(), equals, "\"") } diff --git a/pkgtools/pkglint/files/shtokenizer.go b/pkgtools/pkglint/files/shtokenizer.go index c4e48ed32d8..ebadec0f788 100644 --- a/pkgtools/pkglint/files/shtokenizer.go +++ b/pkgtools/pkglint/files/shtokenizer.go @@ -1,11 +1,13 @@ package main +import "netbsd.org/pkglint/textproc" + type ShTokenizer struct { parser *Parser mkp *MkParser } -func NewShTokenizer(line *Line, text string, emitWarnings bool) *ShTokenizer { +func NewShTokenizer(line Line, text string, emitWarnings bool) *ShTokenizer { p := NewParser(line, text, emitWarnings) mkp := &MkParser{p} return &ShTokenizer{p, mkp} @@ -52,7 +54,7 @@ func (p *ShTokenizer) ShAtom(quoting ShQuoting) *ShAtom { if atom == nil { repl.Reset(mark) - p.parser.Line.Warnf("Pkglint parse error in ShTokenizer.ShAtom at %q (quoting=%s)", repl.rest, quoting) + p.parser.Line.Warnf("Pkglint parse error in ShTokenizer.ShAtom at %q (quoting=%s)", repl.Rest(), quoting) } return atom } @@ -65,19 +67,19 @@ func (p *ShTokenizer) shAtomPlain() *ShAtom { repl := p.parser.repl switch { case repl.AdvanceHspace(): - return &ShAtom{shtSpace, repl.s, q, nil} + return &ShAtom{shtSpace, repl.Str(), q, nil} case repl.AdvanceStr("\""): - return &ShAtom{shtWord, repl.s, shqDquot, nil} + return &ShAtom{shtWord, repl.Str(), shqDquot, nil} case repl.AdvanceStr("'"): - return &ShAtom{shtWord, repl.s, shqSquot, nil} + return &ShAtom{shtWord, repl.Str(), shqSquot, nil} case repl.AdvanceStr("`"): - return &ShAtom{shtWord, repl.s, shqBackt, nil} + return &ShAtom{shtWord, repl.Str(), shqBackt, nil} case repl.AdvanceRegexp(`^#.*`): - return &ShAtom{shtComment, repl.m[0], q, nil} + return &ShAtom{shtComment, repl.Group(0), q, nil} case repl.AdvanceStr("$$("): - return &ShAtom{shtSubshell, repl.s, q, nil} + return &ShAtom{shtSubshell, repl.Str(), q, nil} case repl.AdvanceRegexp(`^(?:[!#%*+,\-./0-9:=?@A-Z\[\]^_a-z{}~]+|\\[^$]|` + reShDollar + `)+`): - return &ShAtom{shtWord, repl.m[0], q, nil} + return &ShAtom{shtWord, repl.Group(0), q, nil} } return nil } @@ -86,11 +88,11 @@ func (p *ShTokenizer) shAtomDquot() *ShAtom { repl := p.parser.repl switch { case repl.AdvanceStr("\""): - return &ShAtom{shtWord, repl.s, shqPlain, nil} + return &ShAtom{shtWord, repl.Str(), shqPlain, nil} case repl.AdvanceStr("`"): - return &ShAtom{shtWord, repl.s, shqDquotBackt, nil} + return &ShAtom{shtWord, repl.Str(), shqDquotBackt, nil} case repl.AdvanceRegexp(`^(?:[\t !#%&'()*+,\-./0-9:;<=>?@A-Z\[\]^_a-z{|}~]+|\\[^$]|` + reShDollar + `)+`): - return &ShAtom{shtWord, repl.m[0], shqDquot, nil} // XXX: unescape? + return &ShAtom{shtWord, repl.Group(0), shqDquot, nil} // XXX: unescape? } return nil } @@ -99,9 +101,9 @@ func (p *ShTokenizer) shAtomSquot() *ShAtom { repl := p.parser.repl switch { case repl.AdvanceStr("'"): - return &ShAtom{shtWord, repl.s, shqPlain, nil} + return &ShAtom{shtWord, repl.Str(), shqPlain, nil} case repl.AdvanceRegexp(`^([\t !"#%&()*+,\-./0-9:;<=>?@A-Z\[\\\]^_` + "`" + `a-z{|}~]+|\$\$)+`): - return &ShAtom{shtWord, repl.m[0], shqSquot, nil} + return &ShAtom{shtWord, repl.Group(0), shqSquot, nil} } return nil } @@ -114,17 +116,17 @@ func (p *ShTokenizer) shAtomBackt() *ShAtom { repl := p.parser.repl switch { case repl.AdvanceStr("\""): - return &ShAtom{shtWord, repl.s, shqBacktDquot, nil} + return &ShAtom{shtWord, repl.Str(), shqBacktDquot, nil} case repl.AdvanceStr("`"): - return &ShAtom{shtWord, repl.s, shqPlain, nil} + return &ShAtom{shtWord, repl.Str(), shqPlain, nil} case repl.AdvanceStr("'"): - return &ShAtom{shtWord, repl.s, shqBacktSquot, nil} + return &ShAtom{shtWord, repl.Str(), shqBacktSquot, nil} case repl.AdvanceHspace(): - return &ShAtom{shtSpace, repl.s, q, nil} + return &ShAtom{shtSpace, repl.Str(), q, nil} case repl.AdvanceRegexp("^#[^`]*"): - return &ShAtom{shtComment, repl.s, q, nil} + return &ShAtom{shtComment, repl.Str(), q, nil} case repl.AdvanceRegexp(`^(?:[!#%*+,\-./0-9:=?@A-Z\[\]_a-z~]+|\\[^$]|` + reShDollar + `)+`): - return &ShAtom{shtWord, repl.s, q, nil} + return &ShAtom{shtWord, repl.Str(), q, nil} } return nil } @@ -143,17 +145,17 @@ func (p *ShTokenizer) shAtomSub() *ShAtom { case repl.AdvanceHspace(): return atom(shtSpace) case repl.AdvanceStr("\""): - //return &ShAtom{shtWord, repl.s, shqDquot, nil} + //return &ShAtom{shtWord, repl.Str(), shqDquot, nil} case repl.AdvanceStr("'"): - return &ShAtom{shtWord, repl.s, shqSubshSquot, nil} + return &ShAtom{shtWord, repl.Str(), shqSubshSquot, nil} case repl.AdvanceStr("`"): - //return &ShAtom{shtWord, repl.s, shqBackt, nil} + //return &ShAtom{shtWord, repl.Str(), shqBackt, nil} case repl.AdvanceRegexp(`^#.*`): - return &ShAtom{shtComment, repl.m[0], q, nil} + return &ShAtom{shtComment, repl.Group(0), q, nil} case repl.AdvanceStr(")"): - return &ShAtom{shtWord, repl.s, shqPlain, nil} + return &ShAtom{shtWord, repl.Str(), shqPlain, nil} case repl.AdvanceRegexp(`^(?:[!#%*+,\-./0-9:=?@A-Z\[\]^_a-z{}~]+|\\[^$]|` + reShDollar + `)+`): - return &ShAtom{shtWord, repl.m[0], q, nil} + return &ShAtom{shtWord, repl.Group(0), q, nil} } return nil } @@ -166,17 +168,17 @@ func (p *ShTokenizer) shAtomDquotBackt() *ShAtom { repl := p.parser.repl switch { case repl.AdvanceStr("`"): - return &ShAtom{shtWord, repl.s, shqDquot, nil} + return &ShAtom{shtWord, repl.Str(), shqDquot, nil} case repl.AdvanceStr("\""): - return &ShAtom{shtWord, repl.s, shqDquotBacktDquot, nil} + return &ShAtom{shtWord, repl.Str(), shqDquotBacktDquot, nil} case repl.AdvanceStr("'"): - return &ShAtom{shtWord, repl.s, shqDquotBacktSquot, nil} + return &ShAtom{shtWord, repl.Str(), shqDquotBacktSquot, nil} case repl.AdvanceRegexp("^#[^`]*"): - return &ShAtom{shtComment, repl.s, q, nil} + return &ShAtom{shtComment, repl.Str(), q, nil} case repl.AdvanceRegexp(`^(?:[!#%*+,\-./0-9:=?@A-Z\[\]_a-z~]+|\\[^$]|` + reShDollar + `)+`): - return &ShAtom{shtWord, repl.s, q, nil} + return &ShAtom{shtWord, repl.Str(), q, nil} case repl.AdvanceHspace(): - return &ShAtom{shtSpace, repl.s, q, nil} + return &ShAtom{shtSpace, repl.Str(), q, nil} } return nil } @@ -185,9 +187,9 @@ func (p *ShTokenizer) shAtomBacktDquot() *ShAtom { repl := p.parser.repl switch { case repl.AdvanceStr("\""): - return &ShAtom{shtWord, repl.s, shqBackt, nil} + return &ShAtom{shtWord, repl.Str(), shqBackt, nil} case repl.AdvanceRegexp(`^(?:[\t !%&()*+,\-./0-9:;<=>?@A-Z\[\]^_a-z{|}~]+|\\[^$]|` + reShDollar + `)+`): - return &ShAtom{shtWord, repl.m[0], shqBacktDquot, nil} + return &ShAtom{shtWord, repl.Group(0), shqBacktDquot, nil} } return nil } @@ -197,9 +199,9 @@ func (p *ShTokenizer) shAtomBacktSquot() *ShAtom { repl := p.parser.repl switch { case repl.AdvanceStr("'"): - return &ShAtom{shtWord, repl.s, shqBackt, nil} + return &ShAtom{shtWord, repl.Str(), shqBackt, nil} case repl.AdvanceRegexp(`^([\t !"#%&()*+,\-./0-9:;<=>?@A-Z\[\\\]^_` + "`" + `a-z{|}~]+|\$\$)+`): - return &ShAtom{shtWord, repl.m[0], q, nil} + return &ShAtom{shtWord, repl.Group(0), q, nil} } return nil } @@ -209,9 +211,9 @@ func (p *ShTokenizer) shAtomSubshSquot() *ShAtom { repl := p.parser.repl switch { case repl.AdvanceStr("'"): - return &ShAtom{shtWord, repl.s, shqSubsh, nil} + return &ShAtom{shtWord, repl.Str(), shqSubsh, nil} case repl.AdvanceRegexp(`^([\t !"#%&()*+,\-./0-9:;<=>?@A-Z\[\\\]^_` + "`" + `a-z{|}~]+|\$\$)+`): - return &ShAtom{shtWord, repl.m[0], q, nil} + return &ShAtom{shtWord, repl.Group(0), q, nil} } return nil } @@ -221,9 +223,9 @@ func (p *ShTokenizer) shAtomDquotBacktDquot() *ShAtom { repl := p.parser.repl switch { case repl.AdvanceStr("\""): - return &ShAtom{shtWord, repl.s, shqDquotBackt, nil} + return &ShAtom{shtWord, repl.Str(), shqDquotBackt, nil} case repl.AdvanceRegexp(`^(?:[\t !%&()*+,\-./0-9:;<=>?@A-Z\[\]^_a-z{|}~]+|\\[^$]|` + reShDollar + `)+`): - return &ShAtom{shtWord, repl.m[0], q, nil} + return &ShAtom{shtWord, repl.Group(0), q, nil} } return nil } @@ -232,9 +234,9 @@ func (p *ShTokenizer) shAtomDquotBacktSquot() *ShAtom { repl := p.parser.repl switch { case repl.AdvanceStr("'"): - return &ShAtom{shtWord, repl.s, shqDquotBackt, nil} + return &ShAtom{shtWord, repl.Str(), shqDquotBackt, nil} case repl.AdvanceRegexp(`^(?:[\t !"#%()*+,\-./0-9:;<=>?@A-Z\[\]^_a-z{|}~]+|\\[^$]|\\\$\$|\$\$)+`): - return &ShAtom{shtWord, repl.m[0], shqDquotBacktSquot, nil} + return &ShAtom{shtWord, repl.Group(0), shqDquotBacktSquot, nil} } return nil } @@ -251,9 +253,9 @@ func (p *ShTokenizer) shOperator(q ShQuoting) *ShAtom { repl.AdvanceStr(")"), repl.AdvanceStr("|"), repl.AdvanceStr("&"): - return &ShAtom{shtOperator, repl.s, q, nil} + return &ShAtom{shtOperator, repl.Str(), q, nil} case repl.AdvanceRegexp(`^\d*(?:<<-|<<|<&|<>|>>|>&|>\||<|>)`): - return &ShAtom{shtOperator, repl.m[0], q, nil} + return &ShAtom{shtOperator, repl.Group(0), q, nil} } return nil } @@ -319,11 +321,11 @@ nextatom: return NewShToken(repl.Since(inimark), atoms...) } -func (p *ShTokenizer) Mark() PrefixReplacerMark { +func (p *ShTokenizer) Mark() textproc.PrefixReplacerMark { return p.parser.repl.Mark() } -func (p *ShTokenizer) Reset(mark PrefixReplacerMark) { +func (p *ShTokenizer) Reset(mark textproc.PrefixReplacerMark) { p.parser.repl.Reset(mark) } diff --git a/pkgtools/pkglint/files/substcontext.go b/pkgtools/pkglint/files/substcontext.go index 2eef82b5101..8220c820ede 100644 --- a/pkgtools/pkglint/files/substcontext.go +++ b/pkgtools/pkglint/files/substcontext.go @@ -96,7 +96,7 @@ func (ctx *SubstContext) Finish(mkline *MkLine) { mkline.Warnf("Incomplete SUBST block: %s missing.", ctx.varname("SUBST_FILES")) } if len(ctx.sed) == 0 && len(ctx.vars) == 0 && ctx.filterCmd == "" { - mkline.Line.Warnf("Incomplete SUBST block: %s, %s or %s missing.", + mkline.Warnf("Incomplete SUBST block: %s, %s or %s missing.", ctx.varname("SUBST_SED"), ctx.varname("SUBST_VARS"), ctx.varname("SUBST_FILTER_CMD")) } ctx.id = "" diff --git a/pkgtools/pkglint/files/textproc/prefixreplacer.go b/pkgtools/pkglint/files/textproc/prefixreplacer.go new file mode 100644 index 00000000000..535a78dea62 --- /dev/null +++ b/pkgtools/pkglint/files/textproc/prefixreplacer.go @@ -0,0 +1,136 @@ +package textproc + +import ( + "fmt" + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/trace" + "strings" +) + +var Testing bool + +type PrefixReplacerMark string + +type PrefixReplacer struct { + rest string + s string + m []string +} + +func NewPrefixReplacer(s string) *PrefixReplacer { + return &PrefixReplacer{s, "", nil} +} + +func (pr *PrefixReplacer) EOF() bool { + return pr.rest == "" +} + +func (pr *PrefixReplacer) Rest() string { + return pr.rest +} + +// Match returns a matching group from the last matched AdvanceRegexp. +func (pr *PrefixReplacer) Group(index int) string { + return pr.m[index] +} + +// Rest returns the last match from AdvanceStr, AdvanceBytesFunc or AdvanceHspace. +func (pr *PrefixReplacer) Str() string { + return pr.s +} + +func (pr *PrefixReplacer) AdvanceStr(prefix string) bool { + pr.m = nil + pr.s = "" + if strings.HasPrefix(pr.rest, prefix) { + if trace.Tracing { + trace.Stepf("PrefixReplacer.AdvanceStr(%q, %q)", pr.rest, prefix) + } + pr.s = prefix + pr.rest = pr.rest[len(prefix):] + return true + } + return false +} + +func (pr *PrefixReplacer) AdvanceBytesFunc(fn func(c byte) bool) bool { + i := 0 + for i < len(pr.rest) && fn(pr.rest[i]) { + i++ + } + if i != 0 { + pr.s = pr.rest[:i] + pr.rest = pr.rest[i:] + return true + } + return false +} + +func (pr *PrefixReplacer) AdvanceHspace() bool { + i := 0 + rest := pr.rest + for i < len(rest) && (rest[i] == ' ' || rest[i] == '\t') { + i++ + } + if i != 0 { + pr.s = pr.rest[:i] + pr.rest = pr.rest[i:] + return true + } + return false +} + +func (pr *PrefixReplacer) AdvanceRegexp(re regex.RegexPattern) bool { + pr.m = nil + pr.s = "" + if !strings.HasPrefix(string(re), "^") { + panic(fmt.Sprintf("PrefixReplacer.AdvanceRegexp: regular expression %q must have prefix %q.", re, "^")) + } + if Testing && regex.Matches("", re) { + panic(fmt.Sprintf("PrefixReplacer.AdvanceRegexp: the empty string must not match the regular expression %q.", re)) + } + if m := regex.Match(pr.rest, re); m != nil { + if trace.Tracing { + trace.Stepf("PrefixReplacer.AdvanceRegexp(%q, %q, %q)", pr.rest, re, m[0]) + } + pr.rest = pr.rest[len(m[0]):] + pr.m = m + pr.s = m[0] + return true + } + return false +} + +func (pr *PrefixReplacer) PeekByte() int { + rest := pr.rest + if rest == "" { + return -1 + } + return int(rest[0]) +} + +func (pr *PrefixReplacer) Mark() PrefixReplacerMark { + return PrefixReplacerMark(pr.rest) +} + +func (pr *PrefixReplacer) Reset(mark PrefixReplacerMark) { + pr.rest = string(mark) +} + +func (pr *PrefixReplacer) Skip(n int) { + pr.rest = pr.rest[n:] +} + +func (pr *PrefixReplacer) SkipSpace() { + pr.rest = strings.TrimLeft(pr.rest, " \t") +} + +func (pr *PrefixReplacer) Since(mark PrefixReplacerMark) string { + return string(mark[:len(mark)-len(pr.rest)]) +} + +func (pr *PrefixReplacer) AdvanceRest() string { + rest := pr.rest + pr.rest = "" + return rest +} diff --git a/pkgtools/pkglint/files/toplevel.go b/pkgtools/pkglint/files/toplevel.go index 623d41fc532..d7790b5d3cd 100644 --- a/pkgtools/pkglint/files/toplevel.go +++ b/pkgtools/pkglint/files/toplevel.go @@ -1,13 +1,15 @@ package main +import "netbsd.org/pkglint/trace" + type Toplevel struct { previousSubdir string subdirs []string } func CheckdirToplevel() { - if G.opts.Debug { - defer tracecall1(G.CurrentDir)() + if trace.Tracing { + defer trace.Call1(G.CurrentDir)() } ctx := new(Toplevel) @@ -19,7 +21,7 @@ func CheckdirToplevel() { } for _, line := range lines { - if m, commentedOut, indentation, subdir, comment := match4(line.Text, `^(#?)SUBDIR\s*\+=(\s*)(\S+)\s*(?:#\s*(.*?)\s*|)$`); m { + if m, commentedOut, indentation, subdir, comment := match4(line.Text(), `^(#?)SUBDIR\s*\+=(\s*)(\S+)\s*(?:#\s*(.*?)\s*|)$`); m { ctx.checkSubdir(line, commentedOut == "#", indentation, subdir, comment) } } @@ -35,7 +37,7 @@ func CheckdirToplevel() { } } -func (ctx *Toplevel) checkSubdir(line *Line, commentedOut bool, indentation, subdir, comment string) { +func (ctx *Toplevel) checkSubdir(line Line, commentedOut bool, indentation, subdir, comment string) { if commentedOut && comment == "" { line.Warnf("%q commented out without giving a reason.", subdir) } diff --git a/pkgtools/pkglint/files/trace/tracing.go b/pkgtools/pkglint/files/trace/tracing.go new file mode 100644 index 00000000000..d5bd032f58b --- /dev/null +++ b/pkgtools/pkglint/files/trace/tracing.go @@ -0,0 +1,114 @@ +// Package trace traces function calls and steps in-between. +package trace + +import ( + "fmt" + "io" + "reflect" + "runtime" + "strings" +) + +var ( + Tracing bool + Out io.Writer +) + +var traceDepth int + +func Ref(rv interface{}) ref { + return ref{rv} +} + +func (r ref) String() string { + ptr := reflect.ValueOf(r.intf) + ref := reflect.Indirect(ptr) + return fmt.Sprintf("%v", ref) +} + +func Stepf(format string, args ...interface{}) { + if Tracing { + msg := fmt.Sprintf(format, args...) + io.WriteString(Out, fmt.Sprintf("TRACE: %s %s\n", traceIndent(), msg)) + } +} + +func Step1(format string, arg0 string) { + Stepf(format, arg0) +} + +func Step2(format string, arg0, arg1 string) { + Stepf(format, arg0, arg1) +} + +func Call0() func() { + return traceCall() +} + +func Call1(arg1 string) func() { + return traceCall(arg1) +} + +func Call2(arg1, arg2 string) func() { + return traceCall(arg1, arg2) +} + +func Call(args ...interface{}) func() { + return traceCall(args...) +} + +type ref struct { + intf interface{} +} + +// http://stackoverflow.com/questions/13476349/check-for-nil-and-nil-interface-in-go +func isNil(a interface{}) bool { + defer func() { + recover() + }() + return a == nil || reflect.ValueOf(a).IsNil() +} + +func argsStr(args ...interface{}) string { + argsStr := "" + for i, arg := range args { + if i != 0 { + argsStr += ", " + } + if str, ok := arg.(fmt.Stringer); ok && !isNil(str) { + argsStr += str.String() + } else { + argsStr += fmt.Sprintf("%#v", arg) + } + } + return argsStr +} + +func traceIndent() string { + indent := "" + for i := 0; i < traceDepth; i++ { + indent += fmt.Sprintf("%d ", i+1) + } + return indent +} + +func traceCall(args ...interface{}) func() { + if !Tracing { + panic("Internal pkglint error: calls to trace.Call must only occur in tracing mode") + } + + funcname := "?" + if pc, _, _, ok := runtime.Caller(2); ok { + if fn := runtime.FuncForPC(pc); fn != nil { + funcname = strings.TrimPrefix(fn.Name(), "netbsd.org/pkglint.") + } + } + indent := traceIndent() + io.WriteString(Out, fmt.Sprintf("TRACE: %s+ %s(%s)\n", indent, funcname, argsStr(args...))) + traceDepth++ + + return func() { + traceDepth-- + io.WriteString(Out, fmt.Sprintf("TRACE: %s- %s(%s)\n", indent, funcname, argsStr(args...))) + } +} diff --git a/pkgtools/pkglint/files/util.go b/pkgtools/pkglint/files/util.go index 88d42801a9c..0b5547f45f2 100644 --- a/pkgtools/pkglint/files/util.go +++ b/pkgtools/pkglint/files/util.go @@ -2,24 +2,43 @@ package main import ( "fmt" - "io" "io/ioutil" + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/trace" "os" "path" "path/filepath" - "reflect" "regexp" - "runtime" - "sort" "strconv" "strings" "time" ) // Short names for commonly used functions. -func contains(s, substr string) bool { return strings.Contains(s, substr) } -func hasPrefix(s, prefix string) bool { return strings.HasPrefix(s, prefix) } -func hasSuffix(s, suffix string) bool { return strings.HasSuffix(s, suffix) } +func contains(s, substr string) bool { + return strings.Contains(s, substr) +} +func hasPrefix(s, prefix string) bool { + return strings.HasPrefix(s, prefix) +} +func hasSuffix(s, suffix string) bool { + return strings.HasSuffix(s, suffix) +} +func matches(s string, re regex.RegexPattern) bool { + return regex.Matches(s, re) +} +func match1(s string, re regex.RegexPattern) (matched bool, m1 string) { + return regex.Match1(s, re) +} +func match2(s string, re regex.RegexPattern) (matched bool, m1, m2 string) { + return regex.Match2(s, re) +} +func match3(s string, re regex.RegexPattern) (matched bool, m1, m2, m3 string) { + return regex.Match3(s, re) +} +func match4(s string, re regex.RegexPattern) (matched bool, m1, m2, m3, m4 string) { + return regex.Match4(s, re) +} func ifelseStr(cond bool, a, b string) string { if cond { @@ -35,8 +54,8 @@ func imax(a, b int) int { return b } -func mustMatch(s string, re RegexPattern) []string { - if m := match(s, re); m != nil { +func mustMatch(s string, re regex.RegexPattern) []string { + if m := regex.Match(s, re); m != nil { return m } panic(fmt.Sprintf("mustMatch %q %q", s, re)) @@ -81,7 +100,7 @@ func isCommitted(fname string) bool { lines := loadCvsEntries(fname) needle := "/" + path.Base(fname) + "/" for _, line := range lines { - if hasPrefix(line.Text, needle) { + if hasPrefix(line.Text(), needle) { return true } } @@ -92,8 +111,8 @@ func isLocallyModified(fname string) bool { lines := loadCvsEntries(fname) needle := "/" + path.Base(fname) + "/" for _, line := range lines { - if hasPrefix(line.Text, needle) { - cvsModTime, err := time.Parse(time.ANSIC, strings.Split(line.Text, "/")[3]) + if hasPrefix(line.Text(), needle) { + cvsModTime, err := time.Parse(time.ANSIC, strings.Split(line.Text(), "/")[3]) if err != nil { return false } @@ -103,9 +122,10 @@ func isLocallyModified(fname string) bool { } // https://msdn.microsoft.com/en-us/library/windows/desktop/ms724290(v=vs.85).aspx + // (System Services > Windows System Information > Time > About Time > File Times) delta := cvsModTime.Unix() - st.ModTime().Unix() - if G.opts.Debug { - traceStep("cvs.time=%v fs.time=%v delta=%v", cvsModTime, st.ModTime(), delta) + if trace.Tracing { + trace.Stepf("cvs.time=%v fs.time=%v delta=%v", cvsModTime, st.ModTime(), delta) } return !(-2 <= delta && delta <= 2) } @@ -113,7 +133,7 @@ func isLocallyModified(fname string) bool { return false } -func loadCvsEntries(fname string) []*Line { +func loadCvsEntries(fname string) []Line { dir := path.Dir(fname) if dir == G.CvsEntriesDir { return G.CvsEntriesLines @@ -195,7 +215,7 @@ func varIsUsed(varname string) bool { } func splitOnSpace(s string) []string { - return regcomp(`\s+`).Split(s, -1) + return regex.Compile(`\s+`).Split(s, -1) } func fileExists(fname string) bool { @@ -208,109 +228,6 @@ func dirExists(fname string) bool { return err == nil && st.Mode().IsDir() } -type PrefixReplacerMark string - -type PrefixReplacer struct { - rest string - s string // The last match from a prefix - m []string // The last match from a regular expression -} - -func NewPrefixReplacer(s string) *PrefixReplacer { - return &PrefixReplacer{s, "", nil} -} - -func (pr *PrefixReplacer) AdvanceStr(prefix string) bool { - pr.m = nil - pr.s = "" - if hasPrefix(pr.rest, prefix) { - if G.opts.Debug { - traceStep("PrefixReplacer.AdvanceStr(%q, %q)", pr.rest, prefix) - } - pr.s = prefix - pr.rest = pr.rest[len(prefix):] - return true - } - return false -} - -func (pr *PrefixReplacer) AdvanceBytesFunc(fn func(c byte) bool) bool { - i := 0 - for i < len(pr.rest) && fn(pr.rest[i]) { - i++ - } - if i != 0 { - pr.s = pr.rest[:i] - pr.rest = pr.rest[i:] - return true - } - return false -} - -func (pr *PrefixReplacer) AdvanceHspace() bool { - i := 0 - rest := pr.rest - for i < len(rest) && (rest[i] == ' ' || rest[i] == '\t') { - i++ - } - if i != 0 { - pr.s = pr.rest[:i] - pr.rest = pr.rest[i:] - return true - } - return false -} - -func (pr *PrefixReplacer) AdvanceRegexp(re RegexPattern) bool { - pr.m = nil - pr.s = "" - if !hasPrefix(string(re), "^") { - panic(fmt.Sprintf("PrefixReplacer.AdvanceRegexp: regular expression %q must have prefix %q.", re, "^")) - } - if G.Testing && matches("", re) { - panic(fmt.Sprintf("PrefixReplacer.AdvanceRegexp: the empty string must not match the regular expression %q.", re)) - } - if m := match(pr.rest, re); m != nil { - if G.opts.Debug { - traceStep("PrefixReplacer.AdvanceRegexp(%q, %q, %q)", pr.rest, re, m[0]) - } - pr.rest = pr.rest[len(m[0]):] - pr.m = m - pr.s = m[0] - return true - } - return false -} - -func (pr *PrefixReplacer) PeekByte() int { - rest := pr.rest - if len(rest) == 0 { - return -1 - } - return int(rest[0]) -} - -func (pr *PrefixReplacer) Mark() PrefixReplacerMark { - return PrefixReplacerMark(pr.rest) -} -func (pr *PrefixReplacer) Reset(mark PrefixReplacerMark) { - pr.rest = string(mark) -} -func (pr *PrefixReplacer) Skip(n int) { - pr.rest = pr.rest[n:] -} -func (pr *PrefixReplacer) SkipSpace() { - pr.rest = strings.TrimLeft(pr.rest, " \t") -} -func (pr *PrefixReplacer) Since(mark PrefixReplacerMark) string { - return string(mark[:len(mark)-len(pr.rest)]) -} -func (pr *PrefixReplacer) AdvanceRest() string { - rest := pr.rest - pr.rest = "" - return rest -} - // Useful in combination with regex.Find*Index func negToZero(i int) int { if i >= 0 { @@ -338,104 +255,15 @@ func dirglob(dirname string) []string { return fnames } -// http://stackoverflow.com/questions/13476349/check-for-nil-and-nil-interface-in-go -func isNil(a interface{}) bool { - defer func() { recover() }() - return a == nil || reflect.ValueOf(a).IsNil() -} - -func argsStr(args ...interface{}) string { - argsStr := "" - for i, arg := range args { - if i != 0 { - argsStr += ", " - } - if str, ok := arg.(fmt.Stringer); ok && !isNil(str) { - argsStr += str.String() - } else { - argsStr += fmt.Sprintf("%#v", arg) - } - } - return argsStr -} - -func traceIndent() string { - indent := "" - for i := 0; i < G.traceDepth; i++ { - indent += fmt.Sprintf("%d ", i+1) - } - return indent -} - -func traceStep(format string, args ...interface{}) { - if G.opts.Debug { - msg := fmt.Sprintf(format, args...) - io.WriteString(G.debugOut, fmt.Sprintf("TRACE: %s %s\n", traceIndent(), msg)) - } -} -func traceStep1(format string, arg0 string) { - traceStep(format, arg0) -} -func traceStep2(format string, arg0, arg1 string) { - traceStep(format, arg0, arg1) -} - -func tracecallInternal(args ...interface{}) func() { - if !G.opts.Debug { - panic("Internal pkglint error: calls to tracecall must only occur in tracing mode") - } - - funcname := "?" - if pc, _, _, ok := runtime.Caller(2); ok { - if fn := runtime.FuncForPC(pc); fn != nil { - funcname = strings.TrimPrefix(fn.Name(), "netbsd.org/pkglint.") - } - } - indent := traceIndent() - io.WriteString(G.debugOut, fmt.Sprintf("TRACE: %s+ %s(%s)\n", indent, funcname, argsStr(args...))) - G.traceDepth++ - - return func() { - G.traceDepth-- - io.WriteString(G.debugOut, fmt.Sprintf("TRACE: %s- %s(%s)\n", indent, funcname, argsStr(args...))) - } -} -func tracecall0() func() { - return tracecallInternal() -} -func tracecall1(arg1 string) func() { - return tracecallInternal(arg1) -} -func tracecall2(arg1, arg2 string) func() { - return tracecallInternal(arg1, arg2) -} -func tracecall(args ...interface{}) func() { - return tracecallInternal(args...) -} - -type Ref struct { - intf interface{} -} - -func ref(rv interface{}) Ref { - return Ref{rv} -} - -func (r Ref) String() string { - ptr := reflect.ValueOf(r.intf) - ref := reflect.Indirect(ptr) - return fmt.Sprintf("%v", ref) -} - // Emulates make(1)'s :S substitution operator. func mkopSubst(s string, left bool, from string, right bool, to string, flags string) string { - if G.opts.Debug { - defer tracecall(s, left, from, right, to, flags)() + if trace.Tracing { + defer trace.Call(s, left, from, right, to, flags)() } - re := RegexPattern(ifelseStr(left, "^", "") + regexp.QuoteMeta(from) + ifelseStr(right, "$", "")) + re := regex.RegexPattern(ifelseStr(left, "^", "") + regexp.QuoteMeta(from) + ifelseStr(right, "$", "")) done := false gflag := contains(flags, "g") - return regcomp(re).ReplaceAllStringFunc(s, func(match string) string { + return regex.Compile(re).ReplaceAllStringFunc(s, func(match string) string { if gflag || !done { done = !gflag return to @@ -449,11 +277,12 @@ func relpath(from, to string) string { absTo, err2 := filepath.Abs(to) rel, err3 := filepath.Rel(absFrom, absTo) if err1 != nil || err2 != nil || err3 != nil { - panic("relpath" + argsStr(from, to, err1, err2, err3)) + trace.Stepf("relpath.panic", from, to, err1, err2, err3) + panic("relpath") } result := filepath.ToSlash(rel) - if G.opts.Debug { - traceStep("relpath from %q to %q = %q", from, to, result) + if trace.Tracing { + trace.Stepf("relpath from %q to %q = %q", from, to, result) } return result } @@ -489,68 +318,14 @@ func containsVarRef(s string) bool { return contains(s, "${") } -func reReplaceRepeatedly(from string, re RegexPattern, to string) string { - replaced := regcomp(re).ReplaceAllString(from, to) +func reReplaceRepeatedly(from string, re regex.RegexPattern, to string) string { + replaced := regex.Compile(re).ReplaceAllString(from, to) if replaced != from { return reReplaceRepeatedly(replaced, re, to) } return replaced } -type Histogram struct { - histo map[string]int -} - -func NewHistogram() *Histogram { - h := new(Histogram) - h.histo = make(map[string]int) - return h -} - -func (h *Histogram) Add(s string, n int) { - if G.opts.Profiling { - h.histo[s] += n - } -} - -func (h *Histogram) PrintStats(caption string, out io.Writer, limit int) { - entries := make([]HistogramEntry, len(h.histo)) - - i := 0 - for s, count := range h.histo { - entries[i] = HistogramEntry{s, count} - i++ - } - - sort.Sort(ByCountDesc(entries)) - - for i, entry := range entries { - fmt.Fprintf(out, "%s %6d %s\n", caption, entry.count, entry.s) - if limit > 0 && i >= limit { - break - } - } -} - -type HistogramEntry struct { - s string - count int -} -type ByCountDesc []HistogramEntry - -func (a ByCountDesc) Len() int { - return len(a) -} -func (a ByCountDesc) Swap(i, j int) { - a[i], a[j] = a[j], a[i] -} -func (a ByCountDesc) Less(i, j int) bool { - if a[j].count < a[i].count { - return true - } - return a[i].count == a[j].count && a[i].s < a[j].s -} - func hasAlnumPrefix(s string) bool { if s == "" { return false diff --git a/pkgtools/pkglint/files/util_test.go b/pkgtools/pkglint/files/util_test.go index 9f9e1c20272..214577e8ffa 100644 --- a/pkgtools/pkglint/files/util_test.go +++ b/pkgtools/pkglint/files/util_test.go @@ -2,6 +2,8 @@ package main import ( check "gopkg.in/check.v1" + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/textproc" "testing" ) @@ -33,7 +35,7 @@ func (s *Suite) Test_MkopSubst__gflag(c *check.C) { } func (s *Suite) Test_replaceFirst(c *check.C) { - m, rest := replaceFirst("a+b+c+d", `(\w)(.)(\w)`, "X") + m, rest := regex.ReplaceFirst("a+b+c+d", `(\w)(.)(\w)`, "X") c.Assert(m, check.NotNil) c.Check(m, check.DeepEquals, []string{"a+b", "a", "+", "b"}) @@ -83,7 +85,7 @@ func (s *Suite) Test_isEmptyDir_and_getSubdirs(c *check.C) { } func (s *Suite) Test_PrefixReplacer_Since(c *check.C) { - repl := NewPrefixReplacer("hello, world") + repl := textproc.NewPrefixReplacer("hello, world") mark := repl.Mark() repl.AdvanceRegexp(`^\w+`) c.Check(repl.Since(mark), equals, "hello") diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go index 309a502d0f0..7748886e31f 100644 --- a/pkgtools/pkglint/files/vardefs.go +++ b/pkgtools/pkglint/files/vardefs.go @@ -13,8 +13,6 @@ import ( // * lkSpace is a list whose elements are split by whitespace // // See vartypecheck.go for how these types are checked. -// -// Last synced with mk/defaults/mk.conf revision 1.118 func (gd *GlobalData) InitVartypes() { @@ -46,29 +44,43 @@ func (gd *GlobalData) InitVartypes() { acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk:; *: use-loadtime, use") } + jvms := enum("openjdk8 oracle-jdk8 openjdk7 sun-jdk7 sun-jdk6 jdk16 jdk15 kaffe") // See mk/java-vm.mk:/_PKG_JVMS/ + + // Last synced with mk/defaults/mk.conf revision 1.269 + usr("USE_CWRAPPERS", lkNone, enum("yes no auto")) usr("ALLOW_VULNERABLE_PACKAGES", lkNone, BtYes) + usr("AUDIT_PACKAGES_FLAGS", lkShell, BtShellWord) usr("MANINSTALL", lkShell, enum("maninstall catinstall")) usr("MANZ", lkNone, BtYes) usr("GZIP", lkShell, BtShellWord) + usr("MAKE_JOBS", lkNone, BtInteger) usr("MKCRYPTO", lkNone, BtYesNo) usr("OBJHOSTNAME", lkNone, BtYes) usr("OBJMACHINE", lkNone, BtYes) - usr("PKG_SUFX", lkNone, BtFilename) + usr("SIGN_PACKAGES", lkNone, enum("gpg x509")) + usr("X509_KEY", lkNone, BtPathname) + usr("X509_CERTIFICATE", lkNone, BtPathname) + usr("PATCH_DEBUG", lkNone, BtYes) + usr("PKG_COMPRESSION", lkNone, enum("gzip bzip2 none")) usr("PKGSRC_LOCKTYPE", lkNone, enum("none sleep once")) usr("PKGSRC_SLEEPSECS", lkNone, BtInteger) - usr("USETBL", lkNone, BtYes) usr("ABI", lkNone, enum("32 64")) usr("PKG_DEVELOPER", lkNone, BtYesNo) usr("USE_ABI_DEPENDS", lkNone, BtYesNo) usr("PKG_REGISTER_SHELLS", lkNone, enum("YES NO")) - usr("PKGSRC_COMPILER", lkShell, enum("ccache ccc clang distcc f2c gcc hp icc ido gcc mipspro mipspro-ucode pcc sunpro xlc")) + usr("PKGSRC_COMPILER", lkShell, enum("ccache ccc clang distcc f2c gcc hp icc ido mipspro mipspro-ucode pcc sunpro xlc")) + usr("PKGSRC_KEEP_BIN_PKGS", lkNone, BtYesNo) usr("PKGSRC_MESSAGE_RECIPIENTS", lkShell, BtMailAddress) usr("PKGSRC_SHOW_BUILD_DEFS", lkNone, BtYesNo) - usr("PKGSRC_SHOW_PATCH_ERRORMSG", lkNone, BtYesNo) usr("PKGSRC_RUN_TEST", lkNone, BtYesNo) + usr("PKGSRC_MKPIE", lkNone, BtYesNo) + usr("PKGSRC_USE_FORTIFY", lkNone, BtYesNo) + usr("PKGSRC_USE_RELRO", lkNone, BtYesNo) + usr("PKGSRC_USE_SSP", lkNone, enum("no yes strong all")) usr("PREFER_PKGSRC", lkShell, BtIdentifier) usr("PREFER_NATIVE", lkShell, BtIdentifier) usr("PREFER_NATIVE_PTHREADS", lkNone, BtYesNo) + usr("WRKOBJDIR", lkNone, BtPathname) usr("LOCALBASE", lkNone, BtPathname) usr("CROSSBASE", lkNone, BtPathname) usr("VARBASE", lkNone, BtPathname) @@ -77,22 +89,25 @@ func (gd *GlobalData) InitVartypes() { usr("MOTIFBASE", lkNone, BtPathname) usr("PKGINFODIR", lkNone, BtPathname) usr("PKGMANDIR", lkNone, BtPathname) - usr("USE_XPKGWEDGE", lkNone, BtYesNo) + usr("PKGGNUDIR", lkNone, BtPathname) usr("BSDSRCDIR", lkNone, BtPathname) usr("BSDXSRCDIR", lkNone, BtPathname) usr("DISTDIR", lkNone, BtPathname) usr("DIST_PATH", lkNone, BtPathlist) - usr("DEFAULT_VIEW", lkNone, BtUnknown) + usr("DEFAULT_VIEW", lkNone, BtUnknown) // XXX: deprecate? pkgviews has been removed usr("FETCH_CMD", lkNone, BtShellCommand) - usr("FETCH_USING", lkNone, enum("curl custom fetch ftp manual wget")) + usr("FETCH_USING", lkNone, enum("auto curl custom fetch ftp manual wget")) + usr("FETCH_BEFORE_ARGS", lkShell, BtShellWord) + usr("FETCH_AFTER_ARGS", lkShell, BtShellWord) usr("FETCH_RESUME_ARGS", lkShell, BtShellWord) usr("FETCH_OUTPUT_ARGS", lkShell, BtShellWord) + usr("FIX_SYSTEM_HEADERS", lkNone, BtYes) usr("LIBTOOLIZE_PLIST", lkNone, BtYesNo) usr("PKG_RESUME_TRANSFERS", lkNone, BtYesNo) usr("PKG_SYSCONFBASE", lkNone, BtPathname) + usr("INIT_SYSTEM", lkNone, enum("rc.d smf")) usr("RCD_SCRIPTS_DIR", lkNone, BtPathname) usr("PACKAGES", lkNone, BtPathname) - usr("PKGVULNDIR", lkNone, BtPathname) usr("PASSIVE_FETCH", lkNone, BtYes) usr("PATCH_FUZZ_FACTOR", lkNone, enum("-F0 -F1 -F2 -F3")) usr("ACCEPTABLE_LICENSES", lkShell, BtIdentifier) @@ -101,24 +116,221 @@ func (gd *GlobalData) InitVartypes() { usr("HOST_SPECIFIC_PKGS", lkShell, BtPkgPath) usr("GROUP_SPECIFIC_PKGS", lkShell, BtPkgPath) usr("USER_SPECIFIC_PKGS", lkShell, BtPkgPath) + usr("EXTRACT_USING", lkNone, enum("bsdtar gtar nbtar pax")) usr("FAILOVER_FETCH", lkNone, BtYes) usr("MASTER_SORT", lkShell, BtUnknown) usr("MASTER_SORT_REGEX", lkShell, BtUnknown) + usr("MASTER_SORT_RANDOM", lkNone, BtYes) usr("PATCH_DEBUG", lkNone, BtYes) usr("PKG_FC", lkNone, BtShellCommand) - usr("IMAKE", lkNone, BtShellCommand) usr("IMAKEOPTS", lkShell, BtShellWord) usr("PRE_ROOT_CMD", lkNone, BtShellCommand) - pkg("USE_GAMESGROUP", lkNone, BtYesNo) usr("SU_CMD", lkNone, BtShellCommand) usr("SU_CMD_PATH_APPEND", lkNone, BtPathlist) usr("FATAL_OBJECT_FMT_SKEW", lkNone, BtYesNo) usr("WARN_NO_OBJECT_FMT", lkNone, BtYesNo) usr("SMART_MESSAGES", lkNone, BtYes) usr("BINPKG_SITES", lkShell, BtURL) - usr("BIN_INSTALL_FLAG", lkShell, BtShellWord) + usr("BIN_INSTALL_FLAGS", lkShell, BtShellWord) usr("LOCALPATCHES", lkNone, BtPathname) + // The remaining variables from mk/defaults/mk.conf follow the + // naming conventions from MkLine.getVariableType, furthermore + // they may be redefined by packages. Therefore they cannot be + // defined as user-defined. + if false { + usr("ACROREAD_FONTPATH", lkNone, BtPathlist) + usr("AMANDA_USER", lkNone, BtUserGroupName) + usr("AMANDA_TMP", lkNone, BtPathname) + usr("AMANDA_VAR", lkNone, BtPathname) + usr("APACHE_USER", lkNone, BtUserGroupName) + usr("APACHE_GROUP", lkNone, BtUserGroupName) + usr("APACHE_SUEXEC_CONFIGURE_ARGS", lkShell, BtShellWord) + usr("APACHE_SUEXEC_DOCROOT", lkShell, BtPathname) + usr("ARLA_CACHE", lkNone, BtPathname) + usr("BIND_DIR", lkNone, BtPathname) + usr("BIND_GROUP", lkNone, BtUserGroupName) + usr("BIND_USER", lkNone, BtUserGroupName) + usr("CACTI_GROUP", lkNone, BtUserGroupName) + usr("CACTI_USER", lkNone, BtUserGroupName) + usr("CANNA_GROUP", lkNone, BtUserGroupName) + usr("CANNA_USER", lkNone, BtUserGroupName) + usr("CDRECORD_CONF", lkNone, BtPathname) + usr("CLAMAV_GROUP", lkNone, BtUserGroupName) + usr("CLAMAV_USER", lkNone, BtUserGroupName) + usr("CLAMAV_DBDIR", lkNone, BtPathname) + usr("CONSERVER_DEFAULTHOST", lkNone, BtIdentifier) + usr("CONSERVER_DEFAULTPORT", lkNone, BtInteger) + usr("CUPS_GROUP", lkNone, BtUserGroupName) + usr("CUPS_USER", lkNone, BtUserGroupName) + usr("CUPS_SYSTEM_GROUPS", lkShell, BtUserGroupName) + usr("CYRUS_IDLE", lkNone, enum("poll idled no")) + usr("CYRUS_GROUP", lkNone, BtUserGroupName) + usr("CYRUS_USER", lkNone, BtUserGroupName) + usr("DBUS_GROUP", lkNone, BtUserGroupName) + usr("DBUS_USER", lkNone, BtUserGroupName) + usr("DEFANG_GROUP", lkNone, BtUserGroupName) + usr("DEFANG_USER", lkNone, BtUserGroupName) + usr("DEFANG_SPOOLDIR", lkNone, BtPathname) + usr("DEFAULT_IRC_SERVER", lkNone, BtIdentifier) + usr("DEFAULT_SERIAL_DEVICE", lkNone, BtPathname) + usr("DIALER_GROUP", lkNone, BtUserGroupName) + usr("DT_LAYOUT", lkNone, enum("US FI FR GER DV")) + usr("ELK_GUI", lkShell, enum("none xaw motif")) + usr("EMACS_TYPE", lkNone, enum("emacs25 emacs25nox emacs24 emacs24nox emacs23 emacs23nox emacs22 emacs22nox emacs21 emacs21nox emacs20 xemacs214 xemacs215")) + usr("EXIM_GROUP", lkNone, BtUserGroupName) + usr("EXIM_USER", lkNone, BtUserGroupName) + usr("FLUXBOX_USE_XINERAMA", lkNone, enum("YES NO")) + usr("FLUXBOX_USE_KDE", lkNone, enum("YES NO")) + usr("FLUXBOX_USE_GNOME", lkNone, enum("YES NO")) + usr("FLUXBOX_USE_XFT", lkNone, enum("YES NO")) + usr("FOX_USE_XUNICODE", lkNone, enum("YES NO")) + usr("FREEWNN_USER", lkNone, BtUserGroupName) + usr("FREEWNN_GROUP", lkNone, BtUserGroupName) + usr("GAMES_USER", lkNone, BtUserGroupName) + usr("GAMES_GROUP", lkNone, BtUserGroupName) + usr("GAMEMODE", lkNone, BtFileMode) + usr("GAMEDIRMODE", lkNone, BtFileMode) + usr("GAMEDATAMODE", lkNone, BtFileMode) + usr("GAMEGRP", lkNone, BtUserGroupName) + usr("GAMEOWN", lkNone, BtUserGroupName) + usr("GRUB_NETWORK_CARDS", lkNone, BtIdentifier) + usr("GRUB_PRESET_COMMAND", lkNone, enum("bootp dhcp rarp")) + usr("GRUB_SCAN_ARGS", lkShell, BtShellWord) + usr("HASKELL_COMPILER", lkNone, enum("ghc")) + usr("HOWL_GROUP", lkNone, BtUserGroupName) + usr("HOWL_USER", lkNone, BtUserGroupName) + usr("ICECAST_CHROOTDIR", lkNone, BtPathname) + usr("ICECAST_CHUNKLEN", lkNone, BtInteger) + usr("ICECAST_SOURCE_BUFFSIZE", lkNone, BtInteger) + usr("IMAP_UW_CCLIENT_MBOX_FMT", lkNone, enum("mbox mbx mh mmdf mtx mx news phile tenex unix")) + usr("IMAP_UW_MAILSPOOLHOME", lkNone, BtFilename) + usr("IMDICTDIR", lkNone, BtPathname) + usr("INN_DATA_DIR", lkNone, BtPathname) + usr("INN_USER", lkNone, BtUserGroupName) + usr("INN_GROUP", lkNone, BtUserGroupName) + usr("IRCD_HYBRID_NICLEN", lkNone, BtInteger) + usr("IRCD_HYBRID_TOPICLEN", lkNone, BtInteger) + usr("IRCD_HYBRID_SYSLOG_EVENTS", lkNone, BtUnknown) + usr("IRCD_HYBRID_SYSLOG_FACILITY", lkNone, BtIdentifier) + usr("IRCD_HYBRID_MAXCONN", lkNone, BtInteger) + usr("IRCD_HYBRID_IRC_USER", lkNone, BtUserGroupName) + usr("IRCD_HYBRID_IRC_GROUP", lkNone, BtUserGroupName) + usr("IRRD_USE_PGP", lkNone, enum("5 2")) + usr("JABBERD_USER", lkNone, BtUserGroupName) + usr("JABBERD_GROUP", lkNone, BtUserGroupName) + usr("JABBERD_LOGDIR", lkNone, BtPathname) + usr("JABBERD_SPOOLDIR", lkNone, BtPathname) + usr("JABBERD_PIDDIR", lkNone, BtPathname) + usr("JAKARTA_HOME", lkNone, BtPathname) + usr("KERBEROS", lkNone, BtYes) + usr("KERMIT_SUID_UUCP", lkNone, BtYes) + usr("KJS_USE_PCRE", lkNone, BtYes) + usr("KNEWS_DOMAIN_FILE", lkNone, BtPathname) + usr("KNEWS_DOMAIN_NAME", lkNone, BtIdentifier) + usr("LIBDVDCSS_HOMEPAGE", lkNone, BtHomepage) + usr("LIBDVDCSS_MASTER_SITES", lkShell, BtFetchURL) + usr("LATEX2HTML_ICONPATH", lkNone, BtURL) + usr("LEAFNODE_DATA_DIR", lkNone, BtPathname) + usr("LEAFNODE_USER", lkNone, BtUserGroupName) + usr("LEAFNODE_GROUP", lkNone, BtUserGroupName) + usr("LINUX_LOCALES", lkShell, BtIdentifier) + usr("MAILAGENT_DOMAIN", lkNone, BtIdentifier) + usr("MAILAGENT_EMAIL", lkNone, BtMailAddress) + usr("MAILAGENT_FQDN", lkNone, BtIdentifier) + usr("MAILAGENT_ORGANIZATION", lkNone, BtUnknown) + usr("MAJORDOMO_HOMEDIR", lkNone, BtPathname) + usr("MAKEINFO_ARGS", lkShell, BtShellWord) + usr("MECAB_CHARSET", lkNone, BtIdentifier) + usr("MEDIATOMB_GROUP", lkNone, BtUserGroupName) + usr("MEDIATOMB_USER", lkNone, BtUserGroupName) + usr("MLDONKEY_GROUP", lkNone, BtUserGroupName) + usr("MLDONKEY_HOME", lkNone, BtPathname) + usr("MLDONKEY_USER", lkNone, BtUserGroupName) + usr("MONOTONE_GROUP", lkNone, BtUserGroupName) + usr("MONOTONE_USER", lkNone, BtUserGroupName) + usr("MOTIF_TYPE", lkNone, enum("motif openmotif lesstif dt")) + usr("MOTIF_TYPE_DEFAULT", lkNone, enum("motif openmotif lesstif dt")) + usr("MTOOLS_ENABLE_FLOPPYD", lkNone, BtYesNo) + usr("MYSQL_USER", lkNone, BtUserGroupName) + usr("MYSQL_GROUP", lkNone, BtUserGroupName) + usr("MYSQL_DATADIR", lkNone, BtPathname) + usr("MYSQL_CHARSET", lkNone, BtIdentifier) + usr("MYSQL_EXTRA_CHARSET", lkShell, BtIdentifier) + usr("NAGIOS_GROUP", lkNone, BtUserGroupName) + usr("NAGIOS_USER", lkNone, BtUserGroupName) + usr("NAGIOSCMD_GROUP", lkNone, BtUserGroupName) + usr("NAGIOSDIR", lkNone, BtPathname) + usr("NBPAX_PROGRAM_PREFIX", lkNone, BtIdentifier) + usr("NMH_EDITOR", lkNone, BtIdentifier) + usr("NMH_MTA", lkNone, enum("smtp sendmail")) + usr("NMH_PAGER", lkNone, BtIdentifier) + usr("NS_PREFERRED", lkNone, enum("communicator navigator mozilla")) + usr("OPENSSH_CHROOT", lkNone, BtPathname) + usr("OPENSSH_USER", lkNone, BtUserGroupName) + usr("OPENSSH_GROUP", lkNone, BtUserGroupName) + usr("P4USER", lkNone, BtUserGroupName) + usr("P4GROUP", lkNone, BtUserGroupName) + usr("P4ROOT", lkNone, BtPathname) + usr("P4PORT", lkNone, BtInteger) + usr("PALMOS_DEFAULT_SDK", lkNone, enum("1 2 3.1 3.5")) + usr("PAPERSIZE", lkNone, enum("A4 Letter")) + usr("PGGROUP", lkNone, BtUserGroupName) + usr("PGUSER", lkNone, BtUserGroupName) + usr("PGHOME", lkNone, BtPathname) + usr("PILRC_USE_GTK", lkNone, BtYesNo) + usr("PKG_JVM_DEFAULT", lkNone, jvms) + usr("POPTOP_USE_MPPE", lkNone, BtYes) + usr("PROCMAIL_MAILSPOOLHOME", lkNone, BtFilename) + usr("PROCMAIL_TRUSTED_IDS", lkShell, BtIdentifier) + usr("PVM_SSH", lkNone, BtPathname) + usr("QMAILDIR", lkNone, BtPathname) + usr("QMAIL_QFILTER_TMPDIR", lkNone, BtPathname) + usr("QMAIL_QUEUE_DIR", lkNone, BtPathname) + usr("QMAIL_QUEUE_EXTRA", lkNone, BtMailAddress) + usr("QPOPPER_FAC", lkNone, BtIdentifier) + usr("QPOPPER_USER", lkNone, BtUserGroupName) + usr("QPOPPER_SPOOL_DIR", lkNone, BtPathname) + usr("RASMOL_DEPTH", lkNone, enum("8 16 32")) + usr("RELAY_CTRL_DIR", lkNone, BtPathname) + usr("RPM_DB_PREFIX", lkNone, BtPathname) + usr("RSSH_SCP_PATH", lkNone, BtPathname) + usr("RSSH_SFTP_SERVER_PATH", lkNone, BtPathname) + usr("RSSH_CVS_PATH", lkNone, BtPathname) + usr("RSSH_RDIST_PATH", lkNone, BtPathname) + usr("RSSH_RSYNC_PATH", lkNone, BtPathname) + usr("SAWFISH_THEMES", lkShell, BtFilename) + usr("SCREWS_GROUP", lkNone, BtUserGroupName) + usr("SCREWS_USER", lkNone, BtUserGroupName) + usr("SDIST_PAWD", lkNone, enum("pawd pwd")) + usr("SERIAL_DEVICES", lkShell, BtPathname) + usr("SILC_CLIENT_WITH_PERL", lkNone, BtYesNo) + usr("SSH_SUID", lkNone, BtYesNo) + usr("SSYNC_PAWD", lkNone, enum("pawd pwd")) + usr("SUSE_PREFER", lkNone, enum("13.1 12.1 10.0")) + usr("TEX_DEFAULT", lkNone, enum("teTeX3")) + usr("TEXMFSITE", lkNone, BtPathname) + usr("THTTPD_LOG_FACILITY", lkNone, BtIdentifier) + usr("UNPRIVILEGED", lkNone, BtYesNo) + usr("USE_CROSS_COMPILE", lkNone, BtYesNo) + usr("USE_CRYPTO", lkNone, BtYesNo) + usr("USERPPP_GROUP", lkNone, BtUserGroupName) + usr("UUCP_GROUP", lkNone, BtUserGroupName) + usr("UUCP_USER", lkNone, BtUserGroupName) + usr("VIM_EXTRA_OPTS", lkShell, BtShellWord) + usr("WCALC_HTMLDIR", lkNone, BtPathname) + usr("WCALC_HTMLPATH", lkNone, BtPathname) // URL path + usr("WCALC_CGIDIR", lkNone, BtPrefixPathname) + usr("WCALC_CGIPATH", lkNone, BtPathname) // URL path + usr("WDM_MANAGERS", lkShell, BtIdentifier) + usr("WINDOWMAKER_OPTIONS", lkShell, enum("gnome kde openlook lite")) + usr("WINDOWMAKER_THEMES", lkShell, BtPkgName) + usr("X10_PORT", lkNone, BtPathname) + usr("XAW_TYPE", lkNone, enum("standard 3d xpm neXtaw")) + usr("XLOCK_DEFAULT_MODE", lkNone, BtIdentifier) + usr("ZSH_STATIC", lkNone, BtYes) + } + // some other variables, sorted alphabetically acl(".CURDIR", lkNone, BtPathname, "buildlink3.mk:; *: use, use-loadtime") @@ -377,6 +589,7 @@ func (gd *GlobalData) InitVartypes() { pkglist("HEADER_TEMPLATES", lkShell, BtPathname) pkg("HOMEPAGE", lkNone, BtHomepage) acl("IGNORE_PKG.*", lkNone, BtYes, "*: set, use-loadtime") + sys("IMAKE", lkNone, BtShellCommand) acl("INCOMPAT_CURSES", lkSpace, BtMachinePlatformPattern, "Makefile: set, append") acl("INCOMPAT_ICONV", lkSpace, BtMachinePlatformPattern, "") acl("INFO_DIR", lkNone, BtPathname, "") // relative to PREFIX @@ -591,7 +804,6 @@ func (gd *GlobalData) InitVartypes() { acl("PKG_HACKS", lkShell, BtIdentifier, "hacks.mk: append") sys("PKG_INFO", lkNone, BtShellCommand) sys("PKG_JAVA_HOME", lkNone, BtPathname) - jvms := enum("openjdk8 oracle-jdk8 openjdk7 sun-jdk7 sun-jdk6 jdk16 jdk15 kaffe") // See mk/java-vm.mk:/_PKG_JVMS/ sys("PKG_JVM", lkNone, jvms) acl("PKG_JVMS_ACCEPTED", lkShell, jvms, "Makefile: set; Makefile.common: default, set") usr("PKG_JVM_DEFAULT", lkNone, jvms) @@ -738,6 +950,7 @@ func (gd *GlobalData) InitVartypes() { pkg("USE_CMAKE", lkNone, BtYes) usr("USE_DESTDIR", lkNone, BtYes) pkglist("USE_FEATURES", lkShell, BtIdentifier) + acl("USE_GAMESGROUP", lkNone, BtYesNo, "buildlink3.mk, builtin.mk:; *: set, default, use") pkg("USE_GCC_RUNTIME", lkNone, BtYesNo) pkg("USE_GNU_CONFIGURE_HOST", lkNone, BtYesNo) acl("USE_GNU_ICONV", lkNone, BtYes, "Makefile, Makefile.common, options.mk: set") diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go index b96fdd70dc0..ccd6523edc1 100644 --- a/pkgtools/pkglint/files/vartypecheck.go +++ b/pkgtools/pkglint/files/vartypecheck.go @@ -1,6 +1,8 @@ package main import ( + "netbsd.org/pkglint/regex" + "netbsd.org/pkglint/trace" "path" "sort" "strings" @@ -8,7 +10,7 @@ import ( type VartypeCheck struct { MkLine *MkLine - Line *Line + Line Line Varname string Op MkOperator Value string @@ -17,6 +19,18 @@ type VartypeCheck struct { Guessed bool // Whether the type definition is guessed (based on the variable name) or explicitly defined (see vardefs.go). } +// NewVartypeCheckValue creates a VartypeCheck context by copying all +// fields except the value. This is typically used when checking parts +// of composite types. +func NewVartypeCheckValue(vc *VartypeCheck, value string) *VartypeCheck { + valueNoVar := vc.MkLine.withoutMakeVariables(value) + + copy := *vc + copy.Value = value + copy.ValueNoVar = valueNoVar + return © +} + type MkOperator uint8 const ( @@ -107,14 +121,14 @@ var ( ) func (cv *VartypeCheck) AwkCommand() { - if G.opts.Debug { - traceStep1("Unchecked AWK command: %q", cv.Value) + if trace.Tracing { + trace.Step1("Unchecked AWK command: %q", cv.Value) } } func (cv *VartypeCheck) BasicRegularExpression() { - if G.opts.Debug { - traceStep1("Unchecked basic regular expression: %q", cv.Value) + if trace.Tracing { + trace.Step1("Unchecked basic regular expression: %q", cv.Value) } } @@ -199,6 +213,19 @@ func (cv *VartypeCheck) ConfFiles() { if len(words)%2 != 0 { cv.Line.Warnf("Values for %s should always be pairs of paths.", cv.Varname) } + + for i, word := range words { + NewVartypeCheckValue(cv, word).Pathname() + + if i%2 == 1 && !hasPrefix(word, "${") { + cv.Line.Warnf("The destination file %q should start with a variable reference.", word) + Explain( + "Since pkgsrc can be installed in different locations, the", + "configuration files will also end up in different locations.", + "Typical variables that are used for configuration files are", + "PKG_SYSCONFDIR, PKG_SYSCONFBASE, PREFIX, VARBASE.") + } + } } func (cv *VartypeCheck) Dependency() { @@ -206,7 +233,7 @@ func (cv *VartypeCheck) Dependency() { parser := NewParser(line, value, false) deppat := parser.Dependency() - if deppat != nil && deppat.wildcard == "" && (parser.Rest() == "{,nb*}" || parser.Rest() == "{,nb[0-9]*}") { + if deppat != nil && deppat.Wildcard == "" && (parser.Rest() == "{,nb*}" || parser.Rest() == "{,nb[0-9]*}") { line.Warnf("Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.") Explain( "The \"{,nb*}\" extension is only necessary for dependencies of the", @@ -226,7 +253,7 @@ func (cv *VartypeCheck) Dependency() { return } - wildcard := deppat.wildcard + wildcard := deppat.Wildcard if m, inside := match1(wildcard, `^\[(.*)\]\*$`); m { if inside != "0-9" { line.Warnf("Only [0-9]* is allowed in the numeric part of a dependency.") @@ -248,14 +275,14 @@ func (cv *VartypeCheck) Dependency() { } } else if wildcard == "*" { - line.Warnf("Please use \"%[1]s-[0-9]*\" instead of \"%[1]s-*\".", deppat.pkgbase) + line.Warnf("Please use \"%[1]s-[0-9]*\" instead of \"%[1]s-*\".", deppat.Pkgbase) Explain( "If you use a * alone, the package specification may match other", "packages that have the same prefix, but a longer name. For example,", "foo-* matches foo-1.2, but also foo-client-1.2 and foo-server-1.2.") } - if nocclasses := regcomp(`\[[\d-]+\]`).ReplaceAllString(wildcard, ""); contains(nocclasses, "-") { + if nocclasses := regex.Compile(`\[[\d-]+\]`).ReplaceAllString(wildcard, ""); contains(nocclasses, "-") { line.Warnf("The version pattern %q should not contain a hyphen.", wildcard) Explain( "Pkgsrc interprets package names with version numbers like this:", @@ -274,7 +301,7 @@ func (cv *VartypeCheck) DependencyWithPath() { } if m, pattern, relpath, pkg := match3(value, `(.*):(\.\./\.\./[^/]+/([^/]+))$`); m { - cv.MkLine.CheckRelativePkgdir(relpath) + MkLineChecker{cv.MkLine}.CheckRelativePkgdir(relpath) switch pkg { case "msgfmt", "gettext": @@ -285,7 +312,7 @@ func (cv *VartypeCheck) DependencyWithPath() { line.Warnf("Please use USE_TOOLS+=gmake instead of this dependency.") } - cv.MkLine.CheckVartypePrimitive(cv.Varname, BtDependency, cv.Op, pattern, cv.MkComment, cv.Guessed) + MkLineChecker{cv.MkLine}.CheckVartypePrimitive(cv.Varname, BtDependency, cv.Op, pattern, cv.MkComment, cv.Guessed) return } @@ -350,7 +377,7 @@ func (cv *VartypeCheck) EmulPlatform() { } func (cv *VartypeCheck) FetchURL() { - cv.MkLine.CheckVartypePrimitive(cv.Varname, BtURL, cv.Op, cv.Value, cv.MkComment, cv.Guessed) + MkLineChecker{cv.MkLine}.CheckVartypePrimitive(cv.Varname, BtURL, cv.Op, cv.Value, cv.MkComment, cv.Guessed) for siteURL, siteName := range G.globalData.MasterSiteURLToVar { if hasPrefix(cv.Value, siteURL) { @@ -410,7 +437,7 @@ func (cv *VartypeCheck) FileMode() { } func (cv *VartypeCheck) Homepage() { - cv.MkLine.CheckVartypePrimitive(cv.Varname, BtURL, cv.Op, cv.Value, cv.MkComment, cv.Guessed) + MkLineChecker{cv.MkLine}.CheckVartypePrimitive(cv.Varname, BtURL, cv.Op, cv.Value, cv.MkComment, cv.Guessed) if m, wrong, sitename, subdir := match3(cv.Value, `^(\$\{(MASTER_SITE\w+)(?::=([\w\-/]+))?\})`); m { baseURL := G.globalData.MasterSiteVarToURL[sitename] @@ -591,8 +618,8 @@ func (cv *VartypeCheck) Option() { line, value, valueNovar := cv.Line, cv.Value, cv.ValueNoVar if value != valueNovar { - if G.opts.Debug { - traceStep1("Unchecked option name: %q", value) + if trace.Tracing { + trace.Step1("Unchecked option name: %q", value) } return } @@ -620,7 +647,7 @@ func (cv *VartypeCheck) Option() { // The PATH environment variable func (cv *VartypeCheck) Pathlist() { if !contains(cv.Value, ":") && cv.Guessed { - cv.MkLine.CheckVartypePrimitive(cv.Varname, BtPathname, cv.Op, cv.Value, cv.MkComment, cv.Guessed) + MkLineChecker{cv.MkLine}.CheckVartypePrimitive(cv.Varname, BtPathname, cv.Op, cv.Value, cv.MkComment, cv.Guessed) return } @@ -648,7 +675,7 @@ func (cv *VartypeCheck) Pathmask() { if !matches(cv.ValueNoVar, `^[#\-0-9A-Za-z._~+%*?/\[\]]*`) { cv.Line.Warnf("%q is not a valid pathname mask.", cv.Value) } - cv.Line.CheckAbsolutePathname(cv.Value) + LineChecker{cv.Line}.CheckAbsolutePathname(cv.Value) } // Like Filename, but including slashes @@ -660,7 +687,7 @@ func (cv *VartypeCheck) Pathname() { if !matches(cv.ValueNoVar, `^[#\-0-9A-Za-z._~+%/]*$`) { cv.Line.Warnf("%q is not a valid pathname.", cv.Value) } - cv.Line.CheckAbsolutePathname(cv.Value) + LineChecker{cv.Line}.CheckAbsolutePathname(cv.Value) } func (cv *VartypeCheck) Perl5Packlist() { @@ -683,7 +710,7 @@ func (cv *VartypeCheck) PkgName() { } func (cv *VartypeCheck) PkgOptionsVar() { - cv.MkLine.CheckVartypePrimitive(cv.Varname, BtVariableName, cv.Op, cv.Value, cv.MkComment, cv.Guessed) + MkLineChecker{cv.MkLine}.CheckVartypePrimitive(cv.Varname, BtVariableName, cv.Op, cv.Value, cv.MkComment, cv.Guessed) if matches(cv.Value, `\$\{PKGBASE[:\}]`) { cv.Line.Errorf("PKGBASE must not be used in PKG_OPTIONS_VAR.") Explain( @@ -701,14 +728,14 @@ func (cv *VartypeCheck) PkgOptionsVar() { // A directory name relative to the top-level pkgsrc directory. // Despite its name, it is more similar to RelativePkgDir than to RelativePkgPath. func (cv *VartypeCheck) PkgPath() { - cv.MkLine.CheckRelativePkgdir(G.CurPkgsrcdir + "/" + cv.Value) + MkLineChecker{cv.MkLine}.CheckRelativePkgdir(G.CurPkgsrcdir + "/" + cv.Value) } func (cv *VartypeCheck) PkgRevision() { if !matches(cv.Value, `^[1-9]\d*$`) { cv.Line.Warnf("%s must be a positive integer number.", cv.Varname) } - if path.Base(cv.Line.Fname) != "Makefile" { + if path.Base(cv.Line.Filename()) != "Makefile" { cv.Line.Errorf("%s only makes sense directly in the package Makefile.", cv.Varname) Explain( "Usually, different packages using the same Makefile.common have", @@ -797,12 +824,12 @@ func (cv *VartypeCheck) PythonDependency() { // Refers to a package directory, e.g. ../../category/pkgbase. func (cv *VartypeCheck) RelativePkgDir() { - cv.MkLine.CheckRelativePkgdir(cv.Value) + MkLineChecker{cv.MkLine}.CheckRelativePkgdir(cv.Value) } // Refers to a file or directory, e.g. ../../category/pkgbase, ../../category/pkgbase/Makefile. func (cv *VartypeCheck) RelativePkgPath() { - cv.MkLine.CheckRelativePath(cv.Value, true) + MkLineChecker{cv.MkLine}.CheckRelativePath(cv.Value, true) } func (cv *VartypeCheck) Restricted() { @@ -820,11 +847,10 @@ func (cv *VartypeCheck) SedCommand() { func (cv *VartypeCheck) SedCommands() { line := cv.Line - mkline := cv.MkLine tokens, rest := splitIntoShellTokens(line, cv.Value) if rest != "" { - if strings.Contains(line.Text, "#") { + if strings.Contains(line.Text(), "#") { line.Errorf("Invalid shell words %q in sed commands.", rest) Explain( "When sed commands have embedded \"#\" characters, they need to be", @@ -858,7 +884,7 @@ func (cv *VartypeCheck) SedCommands() { "", "This way, short sed commands cannot be hidden at the end of a line.") } - mkline.CheckVartypePrimitive(cv.Varname, BtSedCommand, cv.Op, tokens[i], cv.MkComment, cv.Guessed) + MkLineChecker{cv.MkLine}.CheckVartypePrimitive(cv.Varname, BtSedCommand, cv.Op, tokens[i], cv.MkComment, cv.Guessed) } else { line.Errorf("The -e option to sed requires an argument.") } @@ -1002,7 +1028,7 @@ func (cv *VartypeCheck) WrapperTransform() { } func (cv *VartypeCheck) WrkdirSubdirectory() { - cv.MkLine.CheckVartypePrimitive(cv.Varname, BtPathname, cv.Op, cv.Value, cv.MkComment, cv.Guessed) + MkLineChecker{cv.MkLine}.CheckVartypePrimitive(cv.Varname, BtPathname, cv.Op, cv.Value, cv.MkComment, cv.Guessed) } // A directory relative to ${WRKSRC}, for use in CONFIGURE_DIRS and similar variables. diff --git a/pkgtools/pkglint/files/vartypecheck_test.go b/pkgtools/pkglint/files/vartypecheck_test.go index 5f95a532cec..a0246937325 100644 --- a/pkgtools/pkglint/files/vartypecheck_test.go +++ b/pkgtools/pkglint/files/vartypecheck_test.go @@ -85,13 +85,16 @@ func (s *Suite) Test_VartypeCheck_Comment(c *check.C) { func (s *Suite) Test_VartypeCheck_ConfFiles(c *check.C) { runVartypeChecks("CONF_FILES", opAssignAppend, (*VartypeCheck).ConfFiles, "single/file", - "share/etc/config etc/config", - "share/etc/config etc/config file", - "share/etc/config etc/config share/etc/config2 etc/config2") + "share/etc/config ${PKG_SYSCONFDIR}/etc/config", + "share/etc/config ${PKG_SYSCONFBASE}/etc/config file", + "share/etc/config ${PREFIX}/etc/config share/etc/config2 ${VARBASE}/config2", + "share/etc/bootrc /etc/bootrc") c.Check(s.Output(), equals, ""+ "WARN: fname:1: Values for CONF_FILES should always be pairs of paths.\n"+ - "WARN: fname:3: Values for CONF_FILES should always be pairs of paths.\n") + "WARN: fname:3: Values for CONF_FILES should always be pairs of paths.\n"+ + "WARN: fname:5: Found absolute pathname: /etc/bootrc\n"+ + "WARN: fname:5: The destination file \"/etc/bootrc\" should start with a variable reference.\n") } func (s *Suite) Test_VartypeCheck_Dependency(c *check.C) { |