diff options
| author | rillig <rillig> | 2016-06-05 11:24:32 +0000 |
|---|---|---|
| committer | rillig <rillig> | 2016-06-05 11:24:32 +0000 |
| commit | 4d3acc3264d943af5c30a32e831aa0c855294262 (patch) | |
| tree | f4e136572d070360ac77a47171a58d80fb17aa4f /pkgtools | |
| parent | c478ed9a603b045e5d2066afcdb2625f747ead72 (diff) | |
| download | pkgsrc-4d3acc3264d943af5c30a32e831aa0c855294262.tar.gz | |
Updated pkglint to 5.4.0.
Changes since 5.3.7:
* Replaced the -D... debug options with a single -d
* Omitted duplicate diagnostics
* Marked the :Q operator unnecessary for some variables
* Improved detection of whether bsd.prefs.mk has been included,
which fixed unwarranted warnings about load time evaluation of
some variables like ${ECHO} and ${SED}
* Improved detection for $(VAR) with round parentheses
* Fixed allowed locations for several variables
* Improved detection for used variables (still not perfect)
* Added warning that MASTER_SITES should not be used in HOMEPAGE
* Fixed warning about manual patches not being in distinfo
* Added a check for missing MASTER_SITE_* variables
* Added a check for unfinished url2pkg work
* Fixed several wrong warnings
Diffstat (limited to 'pkgtools')
53 files changed, 5397 insertions, 1952 deletions
diff --git a/pkgtools/pkglint/Makefile b/pkgtools/pkglint/Makefile index e4be2c76727..c0bfb270d0f 100644 --- a/pkgtools/pkglint/Makefile +++ b/pkgtools/pkglint/Makefile @@ -1,6 +1,6 @@ -# $NetBSD: Makefile,v 1.484 2016/04/10 16:59:37 joerg Exp $ +# $NetBSD: Makefile,v 1.485 2016/06/05 11:24:32 rillig Exp $ -PKGNAME= pkglint-5.3.7 +PKGNAME= pkglint-5.4.0 DISTFILES= # none CATEGORIES= pkgtools @@ -19,7 +19,7 @@ GO_SRCPATH= netbsd.org/pkglint SUBST_CLASSES+= pkglint SUBST_STAGE.pkglint= post-configure -SUBST_FILES.pkglint+= main.go +SUBST_FILES.pkglint+= main.go package_test.go SUBST_SED.pkglint+= -e s\|@VERSION@\|${PKGNAME:S/pkglint-//}\|g SUBST_SED.pkglint+= -e s\|@BMAKE@\|${MAKE:Q}\|g diff --git a/pkgtools/pkglint/files/buildlink3.go b/pkgtools/pkglint/files/buildlink3.go index 5f93cb48d24..5a4254620f2 100644 --- a/pkgtools/pkglint/files/buildlink3.go +++ b/pkgtools/pkglint/files/buildlink3.go @@ -5,7 +5,7 @@ import ( ) func ChecklinesBuildlink3Mk(mklines *MkLines) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(mklines.lines[0].Fname)() } @@ -118,7 +118,7 @@ func ChecklinesBuildlink3Mk(mklines *MkLines) { if varname == "BUILDLINK_ABI_DEPENDS."+pkgbase { abiLine = line - parser := NewParser(line, value) + parser := NewParser(line, value, false) if dp := parser.Dependency(); dp != nil && parser.EOF() { abi = dp } @@ -126,7 +126,7 @@ func ChecklinesBuildlink3Mk(mklines *MkLines) { } if varname == "BUILDLINK_API_DEPENDS."+pkgbase { apiLine = line - parser := NewParser(line, value) + parser := NewParser(line, value, false) if dp := parser.Dependency(); dp != nil && parser.EOF() { api = dp } @@ -174,8 +174,8 @@ func ChecklinesBuildlink3Mk(mklines *MkLines) { } } else { - if G.opts.DebugUnchecked { - exp.CurrentLine().Debugf("Unchecked line in third paragraph.") + if G.opts.Debug { + traceStep1("Unchecked line %s in third paragraph.", exp.CurrentLine().linenos()) } exp.Advance() } diff --git a/pkgtools/pkglint/files/category.go b/pkgtools/pkglint/files/category.go index cbdfe60a2fd..a0b5908b2a5 100644 --- a/pkgtools/pkglint/files/category.go +++ b/pkgtools/pkglint/files/category.go @@ -5,7 +5,7 @@ import ( ) func CheckdirCategory() { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(G.CurrentDir)() } diff --git a/pkgtools/pkglint/files/check_test.go b/pkgtools/pkglint/files/check_test.go index da58cd729df..38e6c0e571d 100644 --- a/pkgtools/pkglint/files/check_test.go +++ b/pkgtools/pkglint/files/check_test.go @@ -77,7 +77,8 @@ func (s *Suite) NewMkLines(fname string, lines ...string) *MkLines { func (s *Suite) DebugToStdout() { G.debugOut = os.Stdout - G.opts.DebugTrace = true + G.logOut = os.Stdout + G.opts.Debug = true } func (s *Suite) UseCommandLine(c *check.C, args ...string) { @@ -85,21 +86,35 @@ func (s *Suite) UseCommandLine(c *check.C, args ...string) { if exitcode != nil && *exitcode != 0 { c.Fatalf("Cannot parse command line: %#v", args) } + G.opts.LogVerbose = true // See SetUpTest } -func (s *Suite) RegisterTool(toolname, varname string, varRequired bool) { - if G.globalData.Tools == nil { - G.globalData.Tools = make(map[string]bool) - G.globalData.Vartools = make(map[string]string) - G.globalData.toolsVarRequired = make(map[string]bool) - G.globalData.PredefinedTools = make(map[string]bool) +func (s *Suite) RegisterMasterSite(varname string, urls ...string) { + name2url := &G.globalData.MasterSiteVarToURL + url2name := &G.globalData.MasterSiteURLToVar + if *name2url == nil { + *name2url = make(map[string]string) + *url2name = make(map[string]string) } - G.globalData.Tools[toolname] = true - G.globalData.Vartools[toolname] = varname - if varRequired { - G.globalData.toolsVarRequired[toolname] = true + (*name2url)[varname] = urls[0] + for _, url := range urls { + (*url2name)[url] = varname + } +} + +func (s *Suite) RegisterTool(tool *Tool) { + reg := G.globalData.Tools + + if len(reg.byName) == 0 && len(reg.byVarname) == 0 { + reg = NewToolRegistry() + G.globalData.Tools = reg + } + if tool.Name != "" { + reg.byName[tool.Name] = tool + } + if tool.Varname != "" { + reg.byVarname[tool.Varname] = tool } - G.globalData.PredefinedTools[toolname] = true } func (s *Suite) CreateTmpFile(c *check.C, relFname, content string) (absFname string) { @@ -140,9 +155,10 @@ func (s *Suite) ExpectFatalError(action func()) { } func (s *Suite) SetUpTest(c *check.C) { - G = GlobalVars{TestingData: &TestingData{VerifiedBits: make(map[string]bool)}} + G = GlobalVars{Testing: true} G.logOut, G.logErr, G.debugOut = &s.stdout, &s.stderr, &s.stdout s.UseCommandLine(c /* no arguments */) + G.opts.LogVerbose = true // To detect duplicate work being done } func (s *Suite) TearDownTest(c *check.C) { diff --git a/pkgtools/pkglint/files/dir.go b/pkgtools/pkglint/files/dir.go index d62385020f1..3507b3bb831 100644 --- a/pkgtools/pkglint/files/dir.go +++ b/pkgtools/pkglint/files/dir.go @@ -6,13 +6,13 @@ import ( ) func CheckDirent(fname string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(fname)() } st, err := os.Lstat(fname) if err != nil || !st.Mode().IsDir() && !st.Mode().IsRegular() { - Errorf(fname, noLines, "No such file or directory.") + NewLineWhole(fname).Errorf("No such file or directory.") return } isDir := st.Mode().IsDir() @@ -24,7 +24,7 @@ func CheckDirent(fname string) { G.Infrastructure = matches(absCurrentDir, `/mk/|/mk$`) G.CurPkgsrcdir = findPkgsrcTopdir(G.CurrentDir) if G.CurPkgsrcdir == "" { - Errorf(fname, noLines, "Cannot determine the pkgsrc root directory for %q.", G.CurrentDir) + NewLineWhole(fname).Error1("Cannot determine the pkgsrc root directory for %q.", G.CurrentDir) return } @@ -44,6 +44,6 @@ func CheckDirent(fname string) { case ".": CheckdirToplevel() default: - Errorf(fname, noLines, "Cannot check directories outside a pkgsrc tree.") + NewLineWhole(fname).Error0("Cannot check directories outside a pkgsrc tree.") } } diff --git a/pkgtools/pkglint/files/distinfo.go b/pkgtools/pkglint/files/distinfo.go index 07e34551142..048ebdd5faa 100644 --- a/pkgtools/pkglint/files/distinfo.go +++ b/pkgtools/pkglint/files/distinfo.go @@ -9,7 +9,7 @@ import ( ) func ChecklinesDistinfo(lines []*Line) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(lines[0].Fname)() } @@ -20,8 +20,8 @@ func ChecklinesDistinfo(lines []*Line) { } else if G.Pkg != nil && dirExists(G.CurrentDir+"/"+G.Pkg.Patchdir) { patchesDir = G.Pkg.Patchdir } - if G.opts.DebugMisc { - Debugf(fname, noLines, "patchesDir=%q", patchesDir) + if G.opts.Debug { + traceStep1("patchesDir=%q", patchesDir) } ck := &distinfoLinesChecker{ @@ -124,16 +124,16 @@ func (ck *distinfoLinesChecker) checkPatchSha1(line *Line, patchFname, distinfoS func (ck *distinfoLinesChecker) checkUnrecordedPatches() { files, err := ioutil.ReadDir(G.CurrentDir + "/" + ck.patchdir) if err != nil { - if G.opts.DebugUnchecked { - Debugf(ck.distinfoFilename, noLines, "Cannot read patchesDir %q: %s", ck.patchdir, err) + if G.opts.Debug { + traceStep("Cannot read patchesDir %q: %s", ck.patchdir, err) } return } for _, file := range files { patch := file.Name() - if file.Mode().IsRegular() && !ck.patches[patch] { - Errorf(ck.distinfoFilename, noLines, "patch %q is not recorded. Run \"%s makepatchsum\".", ck.patchdir+"/"+patch, confMake) + if file.Mode().IsRegular() && !ck.patches[patch] && hasPrefix(patch, "patch-") { + NewLineWhole(ck.distinfoFilename).Errorf("patch %q is not recorded. Run \"%s makepatchsum\".", ck.patchdir+"/"+patch, confMake) } } } diff --git a/pkgtools/pkglint/files/distinfo_test.go b/pkgtools/pkglint/files/distinfo_test.go index 008bf4ad28a..97d7a863a52 100644 --- a/pkgtools/pkglint/files/distinfo_test.go +++ b/pkgtools/pkglint/files/distinfo_test.go @@ -83,3 +83,17 @@ func (s *Suite) TestChecklinesDistinfo_UnrecordedPatches(c *check.C) { "ERROR: ~/distinfo: patch \"patches/patch-aa\" is not recorded. Run \""+confMake+" makepatchsum\".\n"+ "ERROR: ~/distinfo: patch \"patches/patch-src-Makefile\" is not recorded. Run \""+confMake+" makepatchsum\".\n") } + +func (s *Suite) TestChecklinesDistinfo_ManualPatches(c *check.C) { + s.CreateTmpFile(c, "patches/manual-libtool.m4", + "") + G.CurrentDir = s.tmpdir + + ChecklinesDistinfo(s.NewLines(s.tmpdir+"/distinfo", + "$"+"NetBSD$", + "", + "SHA1 (patch-aa) = ...")) + + c.Check(s.Output(), equals, ""+ + "WARN: ~/distinfo:3: Patch file \"patch-aa\" does not exist in directory \"patches\".\n") +} diff --git a/pkgtools/pkglint/files/expecter.go b/pkgtools/pkglint/files/expecter.go index f7ceb9470d8..bf236038d66 100644 --- a/pkgtools/pkglint/files/expecter.go +++ b/pkgtools/pkglint/files/expecter.go @@ -38,7 +38,7 @@ func (exp *Expecter) StepBack() { } func (exp *Expecter) AdvanceIfMatches(re string) bool { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall2(exp.CurrentLine().Text, re)() } @@ -53,7 +53,7 @@ func (exp *Expecter) AdvanceIfMatches(re string) bool { } func (exp *Expecter) AdvanceIfPrefix(prefix string) bool { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall2(exp.CurrentLine().Text, prefix)() } @@ -61,7 +61,7 @@ func (exp *Expecter) AdvanceIfPrefix(prefix string) bool { } func (exp *Expecter) AdvanceIfEquals(text string) bool { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall2(exp.CurrentLine().Text, text)() } diff --git a/pkgtools/pkglint/files/files.go b/pkgtools/pkglint/files/files.go index 7006aee75bc..8e9529f8dd5 100644 --- a/pkgtools/pkglint/files/files.go +++ b/pkgtools/pkglint/files/files.go @@ -3,18 +3,17 @@ package main import ( "io/ioutil" "os" - "strconv" "strings" ) func LoadNonemptyLines(fname string, joinContinuationLines bool) []*Line { lines, err := readLines(fname, joinContinuationLines) if err != nil { - Errorf(fname, noLines, "Cannot be read.") + NewLineWhole(fname).Error0("Cannot be read.") return nil } if len(lines) == 0 { - Errorf(fname, noLines, "Must not be empty.") + NewLineWhole(fname).Error0("Must not be empty.") return nil } return lines @@ -23,10 +22,10 @@ func LoadNonemptyLines(fname string, joinContinuationLines bool) []*Line { func LoadExistingLines(fname string, foldBackslashLines bool) []*Line { lines, err := readLines(fname, foldBackslashLines) if err != nil { - Fatalf(fname, noLines, "Cannot be read.") + NewLineWhole(fname).Fatalf("Cannot be read.") } if lines == nil { - Fatalf(fname, noLines, "Must not be empty.") + NewLineWhole(fname).Fatalf("Must not be empty.") } return lines } @@ -133,7 +132,7 @@ func convertToLogicalLines(fname string, rawText string, joinContinuationLines b } if 0 < len(rawLines) && !hasSuffix(rawLines[len(rawLines)-1].textnl, "\n") { - Errorf(fname, strconv.Itoa(rawLines[len(rawLines)-1].Lineno), "File must end with a newline.") + NewLineEOF(fname).Error0("File must end with a newline.") } return loglines @@ -149,33 +148,34 @@ func SaveAutofixChanges(lines []*Line) (autofixed bool) { return } - changes := make(map[string][]*RawLine) + changes := make(map[string][]string) changed := make(map[string]bool) for _, line := range lines { if line.changed { changed[line.Fname] = true } - changes[line.Fname] = append(changes[line.Fname], line.rawLines()...) + changes[line.Fname] = append(changes[line.Fname], line.modifiedLines()...) } for fname := range changed { - rawLines := changes[fname] + changedLines := changes[fname] tmpname := fname + ".pkglint.tmp" text := "" - for _, rawLine := range rawLines { - text += rawLine.textnl + for _, changedLine := range changedLines { + text += changedLine } err := ioutil.WriteFile(tmpname, []byte(text), 0666) if err != nil { - Errorf(tmpname, noLines, "Cannot write.") + NewLineWhole(tmpname).Error0("Cannot write.") continue } err = os.Rename(tmpname, fname) if err != nil { - Errorf(fname, noLines, "Cannot overwrite with auto-fixed content.") + NewLineWhole(fname).Error0("Cannot overwrite with auto-fixed content.") continue } - autofixf(fname, noLines, "Has been auto-fixed. Please re-run pkglint.") + msg := "Has been auto-fixed. Please re-run pkglint." + logs(llAutofix, fname, "", msg, msg) autofixed = true } return diff --git a/pkgtools/pkglint/files/files_test.go b/pkgtools/pkglint/files/files_test.go index ae588b851d0..928c9858dac 100644 --- a/pkgtools/pkglint/files/files_test.go +++ b/pkgtools/pkglint/files/files_test.go @@ -37,7 +37,7 @@ func (s *Suite) TestConvertToLogicalLines_contInLastLine(c *check.C) { c.Check(lines, check.HasLen, 1) c.Check(lines[0].String(), equals, "fname_contlast:1: last line\\") - c.Check(s.Stdout(), equals, "ERROR: fname_contlast:1: File must end with a newline.\n") + c.Check(s.Stdout(), equals, "ERROR: fname_contlast:EOF: File must end with a newline.\n") } func (s *Suite) TestSplitRawLine(c *check.C) { diff --git a/pkgtools/pkglint/files/getopt.go b/pkgtools/pkglint/files/getopt.go index 7f3eae73366..02ecd743e63 100644 --- a/pkgtools/pkglint/files/getopt.go +++ b/pkgtools/pkglint/files/getopt.go @@ -128,8 +128,10 @@ optchar: argarg := optchars[ai+utf8.RuneLen(optchar):] if argarg != "" { return 0, data.parse(string([]rune{'-', optchar}), argarg) - } else { + } else if i+1 < len(args) { return 1, data.parse(string([]rune{'-', optchar}), args[i+1]) + } else { + return 0, optErr("option requires an argument: -" + string([]rune{optchar})) } } } diff --git a/pkgtools/pkglint/files/getopt_test.go b/pkgtools/pkglint/files/getopt_test.go index 52e7f7fceeb..4ccb76179c1 100644 --- a/pkgtools/pkglint/files/getopt_test.go +++ b/pkgtools/pkglint/files/getopt_test.go @@ -43,6 +43,10 @@ func (s *Suite) TestGetopt_UnknownFlagInGroup(c *check.C) { _, err = opts.Parse([]string{"progname", "--warnings=all", "--warnings=no-error"}) c.Check(err.Error(), equals, "progname: unknown option: --warnings=no-error") + + _, err = opts.Parse([]string{"progname", "-W"}) + + c.Check(err.Error(), equals, "progname: option requires an argument: -W") } func (s *Suite) TestGetopt_AbbreviatedLong(c *check.C) { diff --git a/pkgtools/pkglint/files/globaldata.go b/pkgtools/pkglint/files/globaldata.go index 5d2d33a4433..753cbb36ec4 100644 --- a/pkgtools/pkglint/files/globaldata.go +++ b/pkgtools/pkglint/files/globaldata.go @@ -10,16 +10,11 @@ import ( // GlobalData contains data describing pkgsrc as a whole. type GlobalData struct { Pkgsrcdir string // Relative to the current working directory. - MasterSiteUrls map[string]string // "https://github.com/" => "MASTER_SITE_GITHUB" - MasterSiteVars map[string]bool // "MASTER_SITE_GITHUB" => true + MasterSiteURLToVar map[string]string // "https://github.com/" => "MASTER_SITE_GITHUB" + MasterSiteVarToURL map[string]string // "MASTER_SITE_GITHUB" => "https://github.com/" PkgOptions map[string]string // "x11" => "Provides X11 support" - Tools map[string]bool // Known tool names, e.g. "sed" and "gm4". - Vartools map[string]string // Maps tool names to their respective variable, e.g. "sed" => "SED", "gzip" => "GZIP_CMD". - PredefinedTools map[string]bool // Tools that a package does not need to add to USE_TOOLS explicitly because they are used by the pkgsrc infrastructure, too. - VarnameToToolname map[string]string // Maps the tool variable names to the tool name they use, e.g. "GZIP_CMD" => "gzip" and "SED" => "sed". + Tools ToolRegistry // SystemBuildDefs map[string]bool // The set of user-defined variables that are added to BUILD_DEFS within the bsd.pkg.mk file. - toolvarsVarRequired map[string]bool // Tool variable names that may not be converted to their "direct" form, that is: ${CP} may not be written as cp. - toolsVarRequired map[string]bool // Tools that need to be written in variable form, e.g. "echo"; see Vartools. suggestedUpdates []SuggestedUpdate // suggestedWipUpdates []SuggestedUpdate // LastChange map[string]*Change // @@ -72,14 +67,16 @@ func (gd *GlobalData) loadDistSites() { fname := gd.Pkgsrcdir + "/mk/fetch/sites.mk" lines := LoadExistingLines(fname, true) - names := make(map[string]bool) + name2url := make(map[string]string) url2name := make(map[string]string) for _, line := range lines { if m, varname, _, _, urls, _ := MatchVarassign(line.Text); m { if hasPrefix(varname, "MASTER_SITE_") && varname != "MASTER_SITE_BACKUP" { - names[varname] = true for _, url := range splitOnSpace(urls) { if matches(url, `^(?:http://|https://|ftp://)`) { + if name2url[varname] == "" { + name2url[varname] = url + } url2name[url] = varname } } @@ -88,14 +85,13 @@ func (gd *GlobalData) loadDistSites() { } // Explicitly allowed, although not defined in mk/fetch/sites.mk. - names["MASTER_SITE_SUSE_UPD"] = true - names["MASTER_SITE_LOCAL"] = true + name2url["MASTER_SITE_LOCAL"] = "ftp://ftp.NetBSD.org/pub/pkgsrc/distfiles/LOCAL_PORTS/" - if G.opts.DebugMisc { - Debugf(fname, noLines, "Loaded %d MASTER_SITE_* URLs.", len(url2name)) + if G.opts.Debug { + traceStep("Loaded %d MASTER_SITE_* URLs.", len(url2name)) } - gd.MasterSiteUrls = url2name - gd.MasterSiteVars = names + gd.MasterSiteURLToVar = url2name + gd.MasterSiteVarToURL = name2url } func (gd *GlobalData) loadPkgOptions() { @@ -124,15 +120,18 @@ func (gd *GlobalData) loadTools() { } } } + if len(toolFiles) <= 1 { + NewLineWhole(fname).Fatalf("Too few tool files.") + } } - if len(toolFiles) <= 1 { - Fatalf(toolFiles[0], noLines, "Too few tool files.") - } - tools := make(map[string]bool) - vartools := make(map[string]string) - predefinedTools := make(map[string]bool) - varnameToToolname := make(map[string]string) + reg := NewToolRegistry() + reg.RegisterTool(&Tool{"echo", "ECHO", true, true, true}) + reg.RegisterTool(&Tool{"echo -n", "ECHO_N", true, true, true}) + reg.RegisterTool(&Tool{"false", "FALSE", true /*why?*/, true, false}) + reg.RegisterTool(&Tool{"test", "TEST", true, true, true}) + reg.RegisterTool(&Tool{"true", "TRUE", true /*why?*/, true, true}) + systemBuildDefs := make(map[string]bool) for _, basename := range toolFiles { @@ -141,27 +140,25 @@ func (gd *GlobalData) loadTools() { for _, line := range lines { if m, varname, _, _, value, _ := MatchVarassign(line.Text); m { if varname == "TOOLS_CREATE" && (value == "[" || matches(value, `^?[-\w.]+$`)) { - tools[value] = true - } else if m, toolname := match1(varname, `^(?:_TOOLS_VARNAME)\.([-\w.]+|\[)$`); m { - tools[toolname] = true - vartools[toolname] = value - varnameToToolname[value] = toolname + reg.Register(value) + + } else if m, toolname := match1(varname, `^_TOOLS_VARNAME\.([-\w.]+|\[)$`); m { + reg.RegisterVarname(toolname, value) } else if m, toolname := match1(varname, `^(?:TOOLS_PATH|_TOOLS_DEPMETHOD)\.([-\w.]+|\[)$`); m { - tools[toolname] = true + reg.Register(toolname) } else if m, toolname := match1(varname, `_TOOLS\.(.*)`); m { - tools[toolname] = true + reg.Register(toolname) for _, tool := range splitOnSpace(value) { - tools[tool] = true + reg.Register(tool) } } } } } - { - basename := "bsd.pkg.mk" + for _, basename := range []string{"bsd.prefs.mk", "bsd.pkg.mk"} { fname := G.globalData.Pkgsrcdir + "/mk/" + basename condDepth := 0 @@ -171,14 +168,18 @@ func (gd *GlobalData) loadTools() { if m, varname, _, _, value, _ := MatchVarassign(text); m { if varname == "USE_TOOLS" { - if G.opts.DebugTools { - line.Debugf("[condDepth=%d] %s", condDepth, value) + if G.opts.Debug { + traceStep("[condDepth=%d] %s", condDepth, value) } - if condDepth == 0 { - for _, tool := range splitOnSpace(value) { - if !containsVarRef(tool) && tools[tool] { - predefinedTools[tool] = true - predefinedTools["TOOLS_"+tool] = true + if condDepth == 0 || condDepth == 1 && basename == "bsd.prefs.mk" { + for _, toolname := range splitOnSpace(value) { + if !containsVarRef(toolname) { + for _, tool := range []*Tool{reg.Register(toolname), reg.Register("TOOLS_" + toolname)} { + tool.Predefined = true + if basename == "bsd.prefs.mk" { + tool.UsableAtLoadtime = true + } + } } } } @@ -200,14 +201,11 @@ func (gd *GlobalData) loadTools() { } } - if G.opts.DebugTools { - dummyLine.Debugf("tools: %v", stringBoolMapKeys(tools)) - dummyLine.Debugf("vartools: %v", stringStringMapKeys(vartools)) - dummyLine.Debugf("predefinedTools: %v", stringBoolMapKeys(predefinedTools)) - dummyLine.Debugf("varnameToToolname: %v", stringStringMapKeys(varnameToToolname)) + if G.opts.Debug { + reg.Trace() } - if G.opts.DebugMisc { - dummyLine.Debugf("systemBuildDefs: %v", systemBuildDefs) + if G.opts.Debug { + traceStep("systemBuildDefs: %v", systemBuildDefs) } // Some user-defined variables do not influence the binary @@ -224,24 +222,8 @@ func (gd *GlobalData) loadTools() { systemBuildDefs["GAMEOWN"] = true systemBuildDefs["GAMEGRP"] = true - gd.Tools = tools - gd.Vartools = vartools - gd.PredefinedTools = predefinedTools - gd.VarnameToToolname = varnameToToolname + gd.Tools = reg gd.SystemBuildDefs = systemBuildDefs - gd.toolvarsVarRequired = map[string]bool{ - "ECHO": true, - "ECHO_N": true, - "FALSE": true, - "TEST": true, - "TRUE": true, - } - gd.toolsVarRequired = map[string]bool{ - "echo": true, - "false": true, - "test": true, - "true": true, - } } func loadSuggestedUpdates(fname string) []SuggestedUpdate { @@ -345,7 +327,7 @@ func (gd *GlobalData) loadDocChanges() { docdir := G.globalData.Pkgsrcdir + "/doc" files, err := ioutil.ReadDir(docdir) if err != nil { - Fatalf(docdir, noLines, "Cannot be read.") + NewLineWhole(docdir).Fatalf("Cannot be read.") } var fnames []string @@ -537,3 +519,62 @@ func (gd *GlobalData) loadDeprecatedVars() { "SUBST_POSTCMD.*": "Has been removed, as it seemed unused.", } } + +// See `mk/tools/`. +type Tool struct { + Name string // e.g. "sed", "gzip" + Varname string // e.g. "SED", "GZIP_CMD" + MustUseVarForm bool // True for `echo`, because of many differing implementations. + Predefined bool // This tool is used by the pkgsrc infrastructure, therefore the package does not need to add it to `USE_TOOLS` explicitly. + UsableAtLoadtime bool // May be used after including `bsd.prefs.mk`. +} + +type ToolRegistry struct { + byName map[string]*Tool + byVarname map[string]*Tool +} + +func NewToolRegistry() ToolRegistry { + return ToolRegistry{make(map[string]*Tool), make(map[string]*Tool)} +} + +func (tr *ToolRegistry) Register(toolname string) *Tool { + tool := tr.byName[toolname] + if tool == nil { + tool = &Tool{Name: toolname} + tr.byName[toolname] = tool + } + return tool +} + +func (tr *ToolRegistry) RegisterVarname(toolname, varname string) *Tool { + tool := tr.Register(toolname) + tool.Varname = varname + tr.byVarname[varname] = tool + return tool +} + +func (tr *ToolRegistry) RegisterTool(tool *Tool) { + if tool.Name != "" && tr.byName[tool.Name] == nil { + tr.byName[tool.Name] = tool + } + if tool.Varname != "" && tr.byVarname[tool.Varname] == nil { + tr.byVarname[tool.Varname] = tool + } +} + +func (tr *ToolRegistry) Trace() { + if G.opts.Debug { + defer tracecall0()() + } + + var keys []string + for k := range tr.byName { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, toolname := range keys { + traceStep("tool %+v", tr.byName[toolname]) + } +} diff --git a/pkgtools/pkglint/files/globaldata_test.go b/pkgtools/pkglint/files/globaldata_test.go index c5b6483be3e..8c4c1d840f6 100644 --- a/pkgtools/pkglint/files/globaldata_test.go +++ b/pkgtools/pkglint/files/globaldata_test.go @@ -31,31 +31,47 @@ func (s *Suite) TestParselinesSuggestedUpdates(c *check.C) { } func (s *Suite) TestGlobalData_LoadTools(c *check.C) { - s.UseCommandLine(c, "-Dtools") - s.CreateTmpFile(c, "mk/tools/bsd.tools.mk", ""+ - ".include \"flex.mk\"\n"+ - ".include \"gettext.mk\"\n") - s.CreateTmpFile(c, "mk/tools/defaults.mk", ""+ - "_TOOLS_VARNAME.chown=CHOWN\n"+ - "_TOOLS_VARNAME.mv=MV\n"+ - "_TOOLS_VARNAME.gawk=AWK\n") - s.CreateTmpFile(c, "mk/tools/flex.mk", ""+ - "# empty\n") - s.CreateTmpFile(c, "mk/tools/gettext.mk", ""+ - "USE_TOOLS+=msgfmt\n"+ - "TOOLS_CREATE+=msgfmt\n") - s.CreateTmpFile(c, "mk/bsd.pkg.mk", "# empty\n") + s.CreateTmpFileLines(c, "mk/tools/bsd.tools.mk", + ".include \"flex.mk\"", + ".include \"gettext.mk\"") + s.CreateTmpFileLines(c, "mk/tools/defaults.mk", + "_TOOLS_VARNAME.chown=CHOWN", + "_TOOLS_VARNAME.gawk=AWK", + "_TOOLS_VARNAME.mv=MV", + "_TOOLS_VARNAME.pwd=PWD") + s.CreateTmpFileLines(c, "mk/tools/flex.mk", + "# empty") + s.CreateTmpFileLines(c, "mk/tools/gettext.mk", + "USE_TOOLS+=msgfmt", + "TOOLS_CREATE+=msgfmt") + s.CreateTmpFileLines(c, "mk/bsd.prefs.mk", + "USE_TOOLS+=\tpwd") + s.CreateTmpFileLines(c, "mk/bsd.pkg.mk", + "USE_TOOLS+=\tmv") G.globalData.Pkgsrcdir = s.tmpdir G.CurrentDir = s.tmpdir G.CurPkgsrcdir = "." G.globalData.loadTools() + G.opts.Debug = true + G.globalData.Tools.Trace() + c.Check(s.Output(), equals, ""+ - "DEBUG: tools: [chown gawk msgfmt mv]\n"+ - "DEBUG: vartools: [chown gawk mv]\n"+ - "DEBUG: predefinedTools: []\n"+ - "DEBUG: varnameToToolname: [AWK CHOWN MV]\n") + "TRACE: + (*ToolRegistry).Trace()\n"+ + "TRACE: 1 tool &{Name:TOOLS_mv Varname: MustUseVarForm:false Predefined:true UsableAtLoadtime:false}\n"+ + "TRACE: 1 tool &{Name:TOOLS_pwd Varname: MustUseVarForm:false Predefined:true UsableAtLoadtime:true}\n"+ + "TRACE: 1 tool &{Name:chown Varname:CHOWN MustUseVarForm:false Predefined:false UsableAtLoadtime:false}\n"+ + "TRACE: 1 tool &{Name:echo Varname:ECHO MustUseVarForm:true Predefined:true UsableAtLoadtime:true}\n"+ + "TRACE: 1 tool &{Name:echo -n Varname:ECHO_N MustUseVarForm:true Predefined:true UsableAtLoadtime:true}\n"+ + "TRACE: 1 tool &{Name:false Varname:FALSE MustUseVarForm:true Predefined:true UsableAtLoadtime:false}\n"+ + "TRACE: 1 tool &{Name:gawk Varname:AWK MustUseVarForm:false Predefined:false UsableAtLoadtime:false}\n"+ + "TRACE: 1 tool &{Name:msgfmt Varname: MustUseVarForm:false Predefined:false UsableAtLoadtime:false}\n"+ + "TRACE: 1 tool &{Name:mv Varname:MV MustUseVarForm:false Predefined:true UsableAtLoadtime:false}\n"+ + "TRACE: 1 tool &{Name:pwd Varname:PWD MustUseVarForm:false Predefined:true UsableAtLoadtime:true}\n"+ + "TRACE: 1 tool &{Name:test Varname:TEST MustUseVarForm:true Predefined:true UsableAtLoadtime:true}\n"+ + "TRACE: 1 tool &{Name:true Varname:TRUE MustUseVarForm:true Predefined:true UsableAtLoadtime:true}\n"+ + "TRACE: - (*ToolRegistry).Trace()\n") } func (s *Suite) TestGlobalData_loadDocChanges(c *check.C) { @@ -84,7 +100,7 @@ func (s *Suite) TestGlobalData_deprecated(c *check.C) { G.globalData.loadDeprecatedVars() line := NewLine("Makefile", 5, "USE_PERL5=\tyes", nil) - NewMkLine(line).CheckVarassign() + NewMkLine(line).checkVarassign() c.Check(s.Output(), equals, "WARN: Makefile:5: Definition of USE_PERL5 is deprecated. Use USE_TOOLS+=perl or USE_TOOLS+=perl:run instead.\n") } diff --git a/pkgtools/pkglint/files/globalvars.go b/pkgtools/pkglint/files/globalvars.go index 51dc69905e5..cc94335eb86 100644 --- a/pkgtools/pkglint/files/globalvars.go +++ b/pkgtools/pkglint/files/globalvars.go @@ -11,18 +11,19 @@ type GlobalVars struct { Pkg *Package // The package that is currently checked. Mk *MkLines // The Makefile (or fragment) that is currently checked. - Todo []string // The files or directories that still need to be checked. - CurrentDir string // The currently checked directory, relative to the cwd - CurPkgsrcdir string // The pkgsrc directory, relative to currentDir - Wip bool // Is the currently checked directory from pkgsrc-wip? - Infrastructure bool // Is the currently checked item from the pkgsrc infrastructure? - TestingData *TestingData // Is pkglint in self-testing mode (only during development)? + Todo []string // The files or directories that still need to be checked. + CurrentDir string // The currently checked directory, relative to the cwd + CurPkgsrcdir string // The pkgsrc directory, relative to currentDir + Wip bool // Is the currently checked directory from pkgsrc-wip? + Infrastructure bool // Is the currently checked item from the pkgsrc infrastructure? + Testing bool // Is pkglint in self-testing mode (only during development)? Hash map[string]*Hash // Maps "alg:fname" => hash (inter-package check). UsedLicenses map[string]bool // Maps "license name" => true (inter-package check). errors int warnings int + logged map[string]bool explanationsAvailable bool explanationsGiven map[string]bool autofixAvailable bool @@ -52,18 +53,6 @@ type CmdOpts struct { CheckPatches, CheckPlist bool - DebugInclude, - DebugMisc, - DebugPatches, - DebugQuoting, - DebugShell, - DebugTools, - DebugTrace, - DebugUnchecked, - DebugUnused, - DebugVartypes, - DebugVaruse bool - WarnAbsname, WarnDirectcmd, WarnExtra, @@ -82,9 +71,11 @@ type CmdOpts struct { PrintHelp, DumpMakefile, Import, + LogVerbose, Profiling, Quiet, Recursive, + Debug, PrintAutofix, PrintSource, PrintVersion bool @@ -97,8 +88,4 @@ type Hash struct { line *Line } -type TestingData struct { - VerifiedBits map[string]bool -} - var G GlobalVars diff --git a/pkgtools/pkglint/files/licenses.go b/pkgtools/pkglint/files/licenses.go index b4459d414ae..38eb999cde0 100644 --- a/pkgtools/pkglint/files/licenses.go +++ b/pkgtools/pkglint/files/licenses.go @@ -23,7 +23,7 @@ func checktoplevelUnusedLicenses() { licensepath := licensedir + "/" + licensename if fileExists(licensepath) { if !G.UsedLicenses[licensename] { - Warnf(licensepath, noLines, "This license seems to be unused.") + NewLineWhole(licensepath).Warn0("This license seems to be unused.") } } } diff --git a/pkgtools/pkglint/files/line.go b/pkgtools/pkglint/files/line.go index 3af10a65846..836118cab65 100644 --- a/pkgtools/pkglint/files/line.go +++ b/pkgtools/pkglint/files/line.go @@ -38,8 +38,8 @@ type Line struct { Text string raw []*RawLine changed bool - before []*RawLine - after []*RawLine + before []string + after []string autofixMessage *string } @@ -57,10 +57,19 @@ func NewLineEOF(fname string) *Line { return NewLineMulti(fname, -1, 0, "", nil) } -func (line *Line) rawLines() []*RawLine { - switch { // prevent inlining +// NewLineWhole creates a dummy line for logging messages that affect a file as a whole. +func NewLineWhole(fname string) *Line { + return NewLine(fname, 0, "", nil) +} + +func (line *Line) modifiedLines() []string { + var result []string + result = append(result, line.before...) + for _, raw := range line.raw { + result = append(result, raw.textnl) } - return append(append(append([]*RawLine(nil), line.before...), line.raw...), line.after...) + result = append(result, line.after...) + return result } func (line *Line) linenos() string { @@ -90,7 +99,10 @@ func (line *Line) IsMultiline() bool { func (line *Line) printSource(out io.Writer) { if G.opts.PrintSource { io.WriteString(out, "\n") - for _, rawLine := range line.rawLines() { + for _, before := range line.before { + io.WriteString(out, "+ "+before) + } + for _, rawLine := range line.raw { if rawLine.textnl != rawLine.orignl { if rawLine.orignl != "" { io.WriteString(out, "- "+rawLine.orignl) @@ -102,17 +114,20 @@ func (line *Line) printSource(out io.Writer) { io.WriteString(out, "> "+rawLine.orignl) } } + for _, after := range line.after { + io.WriteString(out, "+ "+after) + } } } func (line *Line) Fatalf(format string, args ...interface{}) { line.printSource(G.logErr) - Fatalf(line.Fname, line.linenos(), format, args...) + logs(llFatal, line.Fname, line.linenos(), format, fmt.Sprintf(format, args...)) } func (line *Line) Errorf(format string, args ...interface{}) { line.printSource(G.logOut) - Errorf(line.Fname, line.linenos(), format, args...) + logs(llError, line.Fname, line.linenos(), format, fmt.Sprintf(format, args...)) line.logAutofix() } func (line *Line) Error0(format string) { line.Errorf(format) } @@ -121,7 +136,7 @@ func (line *Line) Error2(format, arg1, arg2 string) { line.Errorf(format, arg1, func (line *Line) Warnf(format string, args ...interface{}) { line.printSource(G.logOut) - Warnf(line.Fname, line.linenos(), format, args...) + logs(llWarn, line.Fname, line.linenos(), format, fmt.Sprintf(format, args...)) line.logAutofix() } func (line *Line) Warn0(format string) { line.Warnf(format) } @@ -130,42 +145,34 @@ func (line *Line) Warn2(format, arg1, arg2 string) { line.Warnf(format, arg1, ar func (line *Line) Notef(format string, args ...interface{}) { line.printSource(G.logOut) - Notef(line.Fname, line.linenos(), format, args...) + logs(llNote, line.Fname, line.linenos(), format, fmt.Sprintf(format, args...)) line.logAutofix() } func (line *Line) Note0(format string) { line.Notef(format) } func (line *Line) Note1(format, arg1 string) { line.Notef(format, arg1) } func (line *Line) Note2(format, arg1, arg2 string) { line.Notef(format, arg1, arg2) } -func (line *Line) Debugf(format string, args ...interface{}) { - line.printSource(G.logOut) - Debugf(line.Fname, line.linenos(), format, args...) - line.logAutofix() -} -func (line *Line) Debug1(format, arg1 string) { line.Debugf(format, arg1) } -func (line *Line) Debug2(format, arg1, arg2 string) { line.Debugf(format, arg1, arg2) } - func (line *Line) String() string { return line.Fname + ":" + line.linenos() + ": " + line.Text } func (line *Line) logAutofix() { if line.autofixMessage != nil { - autofixf(line.Fname, line.linenos(), "%s", *line.autofixMessage) + logs(llAutofix, line.Fname, line.linenos(), "%s", *line.autofixMessage) line.autofixMessage = nil } } func (line *Line) AutofixInsertBefore(text string) bool { if G.opts.PrintAutofix || G.opts.Autofix { - line.before = append(line.before, &RawLine{0, "", text + "\n"}) + line.before = append(line.before, text+"\n") } return line.RememberAutofix("Inserting a line %q before this line.", text) } func (line *Line) AutofixInsertAfter(text string) bool { if G.opts.PrintAutofix || G.opts.Autofix { - line.after = append(line.after, &RawLine{0, "", text + "\n"}) + line.after = append(line.after, text+"\n") } return line.RememberAutofix("Inserting a line %q after this line.", text) } @@ -213,7 +220,7 @@ func (line *Line) RememberAutofix(format string, args ...interface{}) (hasBeenFi } line.changed = true if G.opts.Autofix { - autofixf(line.Fname, line.linenos(), format, args...) + logs(llAutofix, line.Fname, line.linenos(), format, fmt.Sprintf(format, args...)) return true } if G.opts.PrintAutofix { @@ -224,7 +231,7 @@ func (line *Line) RememberAutofix(format string, args ...interface{}) (hasBeenFi } func (line *Line) CheckAbsolutePathname(text string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(text)() } @@ -275,7 +282,7 @@ func (line *Line) CheckTrailingWhitespace() { } func (line *Line) CheckRcsid(prefixRe, suggestedPrefix string) bool { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall2(prefixRe, suggestedPrefix)() } diff --git a/pkgtools/pkglint/files/line_test.go b/pkgtools/pkglint/files/line_test.go index 864269fcfbf..9a31f1944c9 100644 --- a/pkgtools/pkglint/files/line_test.go +++ b/pkgtools/pkglint/files/line_test.go @@ -10,25 +10,25 @@ func (s *Suite) TestLineModify(c *check.C) { line := NewLine("fname", 1, "dummy", s.NewRawLines(1, "original\n")) c.Check(line.changed, equals, false) - c.Check(line.rawLines(), check.DeepEquals, s.NewRawLines(1, "original\n")) + c.Check(line.raw, check.DeepEquals, s.NewRawLines(1, "original\n")) line.AutofixReplaceRegexp(`(.)(.*)(.)`, "$3$2$1") c.Check(line.changed, equals, true) - c.Check(line.rawLines(), check.DeepEquals, s.NewRawLines(1, "original\n", "lriginao\n")) + c.Check(line.raw, check.DeepEquals, s.NewRawLines(1, "original\n", "lriginao\n")) line.changed = false line.AutofixReplace("i", "u") c.Check(line.changed, equals, true) - c.Check(line.rawLines(), check.DeepEquals, s.NewRawLines(1, "original\n", "lruginao\n")) + c.Check(line.raw, check.DeepEquals, s.NewRawLines(1, "original\n", "lruginao\n")) c.Check(line.raw[0].textnl, equals, "lruginao\n") line.changed = false line.AutofixReplace("lruginao", "middle") c.Check(line.changed, equals, true) - c.Check(line.rawLines(), check.DeepEquals, s.NewRawLines(1, "original\n", "middle\n")) + c.Check(line.raw, check.DeepEquals, s.NewRawLines(1, "original\n", "middle\n")) c.Check(line.raw[0].textnl, equals, "middle\n") line.AutofixInsertBefore("before") @@ -36,21 +36,21 @@ func (s *Suite) TestLineModify(c *check.C) { line.AutofixInsertAfter("between middle and after") line.AutofixInsertAfter("after") - c.Check(line.rawLines(), check.DeepEquals, s.NewRawLines( - 0, "", "before\n", - 0, "", "between before and middle\n", - 1, "original\n", "middle\n", - 0, "", "between middle and after\n", - 0, "", "after\n")) + c.Check(line.modifiedLines(), check.DeepEquals, []string{ + "before\n", + "between before and middle\n", + "middle\n", + "between middle and after\n", + "after\n"}) line.AutofixDelete() - c.Check(line.rawLines(), check.DeepEquals, s.NewRawLines( - 0, "", "before\n", - 0, "", "between before and middle\n", - 1, "original\n", "", - 0, "", "between middle and after\n", - 0, "", "after\n")) + c.Check(line.modifiedLines(), check.DeepEquals, []string{ + "before\n", + "between before and middle\n", + "", + "between middle and after\n", + "after\n"}) } func (s *Suite) TestLine_CheckAbsolutePathname(c *check.C) { diff --git a/pkgtools/pkglint/files/logging.go b/pkgtools/pkglint/files/logging.go index 071470cb313..d5aca5c8fdf 100644 --- a/pkgtools/pkglint/files/logging.go +++ b/pkgtools/pkglint/files/logging.go @@ -3,12 +3,10 @@ package main import ( "fmt" "io" + "path" "strings" ) -const noFile = "" -const noLines = "" - type LogLevel struct { TraditionalName string GccName string @@ -19,26 +17,42 @@ var ( llError = &LogLevel{"ERROR", "error"} llWarn = &LogLevel{"WARN", "warning"} llNote = &LogLevel{"NOTE", "note"} - llDebug = &LogLevel{"DEBUG", "debug"} llAutofix = &LogLevel{"AUTOFIX", "autofix"} ) -var dummyLine = NewLine(noFile, 0, "", nil) +var dummyLine = NewLine("", 0, "", nil) + +func shallBeLogged(fname, lineno, msg string) bool { + uniq := path.Clean(fname) + ":" + lineno + ":" + msg + if G.logged[uniq] { + return false + } -func logf(out io.Writer, level *LogLevel, fname, lineno, format string, args ...interface{}) bool { - if fname != noFile { + if G.logged == nil { + G.logged = make(map[string]bool) + } + G.logged[uniq] = true + return true +} + +func logs(level *LogLevel, fname, lineno, format, msg string) bool { + if fname != "" { fname = cleanpath(fname) } + if !G.opts.LogVerbose && !shallBeLogged(fname, lineno, msg) { + return false + } + var text, sep string if !G.opts.GccOutput { text += sep + level.TraditionalName + ":" sep = " " } - if fname != noFile { + if fname != "" { text += sep + fname sep = ": " - if lineno != noLines { + if lineno != "" { text += ":" + lineno } } @@ -49,31 +63,24 @@ func logf(out io.Writer, level *LogLevel, fname, lineno, format string, args ... if G.opts.Profiling { G.loghisto.Add(format, 1) } - text += sep + fmt.Sprintf(format, args...) + "\n" + text += sep + msg + "\n" + + out := G.logOut + if level == llFatal { + out = G.logErr + } + io.WriteString(out, text) - return true -} -func Fatalf(fname, lineno, format string, args ...interface{}) { - logf(G.logErr, llFatal, fname, lineno, format, args...) - panic(pkglintFatal{}) -} -func Errorf(fname, lineno, format string, args ...interface{}) bool { - G.errors++ - return logf(G.logOut, llError, fname, lineno, format, args...) -} -func Warnf(fname, lineno, format string, args ...interface{}) bool { - G.warnings++ - return logf(G.logOut, llWarn, fname, lineno, format, args...) -} -func Notef(fname, lineno, format string, args ...interface{}) bool { - return logf(G.logOut, llNote, fname, lineno, format, args...) -} -func autofixf(fname, lineno, format string, args ...interface{}) bool { - return logf(G.logOut, llAutofix, fname, lineno, format, args...) -} -func Debugf(fname, lineno, format string, args ...interface{}) bool { - return logf(G.debugOut, llDebug, fname, lineno, format, args...) + switch level { + case llFatal: + panic(pkglintFatal{}) + case llError: + G.errors++ + case llWarn: + G.warnings++ + } + return true } func Explain(explanation ...string) { @@ -92,7 +99,7 @@ func Explain(explanation ...string) { io.WriteString(G.logOut, "\t"+explanationLine+"\n") } io.WriteString(G.logOut, "\n") - } else if G.TestingData != nil { + } else if G.Testing { for _, s := range explanation { if l := tabLength(s); l > 68 && contains(s, " ") { print(fmt.Sprintf("Long explanation line (%d): %s\n", l, s)) diff --git a/pkgtools/pkglint/files/main.go b/pkgtools/pkglint/files/main.go index 959c7a851f7..c12fb988dba 100644 --- a/pkgtools/pkglint/files/main.go +++ b/pkgtools/pkglint/files/main.go @@ -86,7 +86,7 @@ func (pkglint *Pkglint) ParseCommandLine(args []string) *int { opts := NewOptions() check := opts.AddFlagGroup('C', "check", "check,...", "enable or disable specific checks") - debug := opts.AddFlagGroup('D', "debugging", "debug,...", "enable or disable debugging categories") + opts.AddFlagVar('d', "debug", &gopts.Debug, false, "log verbose call traces for debugging") opts.AddFlagVar('e', "explain", &gopts.Explain, false, "explain the diagnostics or give further help") opts.AddFlagVar('f', "show-autofix", &gopts.PrintAutofix, false, "show what pkglint can fix automatically") opts.AddFlagVar('F', "autofix", &gopts.Autofix, false, "try to automatically fix some errors (experimental)") @@ -94,6 +94,7 @@ func (pkglint *Pkglint) ParseCommandLine(args []string) *int { opts.AddFlagVar('h', "help", &gopts.PrintHelp, false, "print a detailed usage message") opts.AddFlagVar('I', "dumpmakefile", &gopts.DumpMakefile, false, "dump the Makefile after parsing") opts.AddFlagVar('i', "import", &gopts.Import, false, "prepare the import of a wip package") + opts.AddFlagVar('m', "log-verbose", &gopts.LogVerbose, false, "allow the same log message more than once") opts.AddFlagVar('p', "profiling", &gopts.Profiling, false, "profile the executing program") opts.AddFlagVar('q', "quiet", &gopts.Quiet, false, "don't print a summary line when finishing") opts.AddFlagVar('r', "recursive", &gopts.Recursive, false, "check subdirectories, too") @@ -114,18 +115,6 @@ func (pkglint *Pkglint) ParseCommandLine(args []string) *int { check.AddFlagVar("patches", &gopts.CheckPatches, true, "check patches") check.AddFlagVar("PLIST", &gopts.CheckPlist, true, "check PLIST files") - debug.AddFlagVar("include", &gopts.DebugInclude, false, "included files") - debug.AddFlagVar("misc", &gopts.DebugMisc, false, "all things that didn't fit elsewhere") - debug.AddFlagVar("patches", &gopts.DebugPatches, false, "the states of the patch parser") - debug.AddFlagVar("quoting", &gopts.DebugQuoting, false, "additional information about quoting") - debug.AddFlagVar("shell", &gopts.DebugShell, false, "the parsers for shell words and shell commands") - debug.AddFlagVar("tools", &gopts.DebugTools, false, "the tools framework") - debug.AddFlagVar("trace", &gopts.DebugTrace, false, "follow subroutine calls") - debug.AddFlagVar("unchecked", &gopts.DebugUnchecked, false, "show the current limitations of pkglint") - debug.AddFlagVar("unused", &gopts.DebugUnused, false, "unused variables") - debug.AddFlagVar("vartypes", &gopts.DebugVartypes, false, "additional type information") - debug.AddFlagVar("varuse", &gopts.DebugVaruse, false, "contexts where variables are used") - warn.AddFlagVar("absname", &gopts.WarnAbsname, true, "warn about use of absolute file names") warn.AddFlagVar("directcmd", &gopts.WarnDirectcmd, true, "warn about use of direct command names instead of Make variables") warn.AddFlagVar("extra", &gopts.WarnExtra, false, "enable some extra warnings") diff --git a/pkgtools/pkglint/files/mkline.go b/pkgtools/pkglint/files/mkline.go index 96677eb7b21..0953797f768 100644 --- a/pkgtools/pkglint/files/mkline.go +++ b/pkgtools/pkglint/files/mkline.go @@ -5,6 +5,7 @@ package main import ( "fmt" "os" + "path" "strconv" "strings" ) @@ -23,14 +24,12 @@ type MkLine struct { xcomment string } -func (mkline *MkLine) Error1(format, arg1 string) { mkline.Line.Error1(format, arg1) } -func (mkline *MkLine) Warn0(format string) { mkline.Line.Warn0(format) } -func (mkline *MkLine) Warn1(format, arg1 string) { mkline.Line.Warn1(format, arg1) } -func (mkline *MkLine) Warn2(format, arg1, arg2 string) { mkline.Line.Warn2(format, arg1, arg2) } -func (mkline *MkLine) Note0(format string) { mkline.Line.Note0(format) } -func (mkline *MkLine) Note2(format, arg1, arg2 string) { mkline.Line.Note2(format, arg1, arg2) } -func (mkline *MkLine) Debug1(format, arg1 string) { mkline.Line.Debug1(format, arg1) } -func (mkline *MkLine) Debug2(format, arg1, arg2 string) { mkline.Line.Debug2(format, arg1, arg2) } +func (mkline *MkLine) Error1(format, arg1 string) { mkline.Line.Error1(format, arg1) } +func (mkline *MkLine) Warn0(format string) { mkline.Line.Warn0(format) } +func (mkline *MkLine) Warn1(format, arg1 string) { mkline.Line.Warn1(format, arg1) } +func (mkline *MkLine) Warn2(format, arg1, arg2 string) { mkline.Line.Warn2(format, arg1, arg2) } +func (mkline *MkLine) Note0(format string) { mkline.Line.Note0(format) } +func (mkline *MkLine) Note2(format, arg1, arg2 string) { mkline.Line.Note2(format, arg1, arg2) } func NewMkLine(line *Line) (mkline *MkLine) { mkline = &MkLine{Line: line} @@ -119,6 +118,9 @@ func NewMkLine(line *Line) (mkline *MkLine) { return mkline } +func (mkline *MkLine) String() string { + return fmt.Sprintf("%s:%s", mkline.Line.Fname, mkline.Line.linenos()) +} func (mkline *MkLine) IsVarassign() bool { return mkline.xtype == 1 } func (mkline *MkLine) Varname() string { return mkline.xs1 } func (mkline *MkLine) Varcanon() string { return mkline.xs2 } @@ -143,32 +145,228 @@ func (mkline *MkLine) IsDependency() bool { return mkline.xtype == 8 } func (mkline *MkLine) Targets() string { return mkline.xs1 } func (mkline *MkLine) Sources() string { return mkline.xs2 } -func (mkline *MkLine) Tokenize(s string) { - p := NewParser(mkline.Line, s) - p.MkTokens() - if p.Rest() != "" { - mkline.Error1("Invalid Makefile syntax at %q.", p.Rest()) +func (mkline *MkLine) Check() { + mkline.Line.CheckTrailingWhitespace() + mkline.Line.CheckValidCharacters(`[\t -~]`) + + switch { + case mkline.IsVarassign(): + mkline.checkVarassign() + + case mkline.IsShellcmd(): + shellcmd := mkline.Shellcmd() + mkline.checkText(shellcmd) + NewShellLine(mkline).CheckShellCommandLine(shellcmd) + + case mkline.IsComment(): + if hasPrefix(mkline.Line.Text, "# url2pkg-marker") { + mkline.Line.Error0("This comment indicates unfinished work (url2pkg).") + } + + case mkline.IsInclude(): + mkline.checkInclude() } } -func (mkline *MkLine) CheckVardef(varname string, op MkOperator) { - if G.opts.DebugTrace { - defer tracecall(varname, op)() +func (mkline *MkLine) checkInclude() { + if G.opts.Debug { + defer tracecall0()() } - defineVar(mkline, varname) - mkline.CheckVardefPermissions(varname, op) + includefile := mkline.Includefile() + mustExist := mkline.MustExist() + if G.opts.Debug { + traceStep1("includefile=%s", includefile) + } + mkline.CheckRelativePath(includefile, mustExist) + + switch { + case hasSuffix(includefile, "/Makefile"): + mkline.Line.Error0("Other Makefiles must not be included directly.") + Explain4( + "If you want to include portions of another Makefile, extract", + "the common parts and put them into a Makefile.common. After", + "that, both this one and the other package should include the", + "Makefile.common.") + + case includefile == "../../mk/bsd.prefs.mk": + if path.Base(mkline.Line.Fname) == "buildlink3.mk" { + mkline.Note0("For efficiency reasons, please include bsd.fast.prefs.mk instead of bsd.prefs.mk.") + } + if G.Pkg != nil { + G.Pkg.setSeenBsdPrefsMk() + } + + case includefile == "../../mk/bsd.fast.prefs.mk", includefile == "../../mk/buildlink3/bsd.builtin.mk": + if G.Pkg != nil { + G.Pkg.setSeenBsdPrefsMk() + } + + case hasSuffix(includefile, "/x11-links/buildlink3.mk"): + mkline.Error1("%s must not be included directly. Include \"../../mk/x11.buildlink3.mk\" instead.", includefile) + + case hasSuffix(includefile, "/jpeg/buildlink3.mk"): + mkline.Error1("%s must not be included directly. Include \"../../mk/jpeg.buildlink3.mk\" instead.", includefile) + + case hasSuffix(includefile, "/intltool/buildlink3.mk"): + mkline.Warn0("Please write \"USE_TOOLS+= intltool\" instead of this line.") + + case hasSuffix(includefile, "/builtin.mk"): + mkline.Line.Error2("%s must not be included directly. Include \"%s/buildlink3.mk\" instead.", includefile, path.Dir(includefile)) + } +} + +func (mkline *MkLine) checkCond(indentation *Indentation, forVars map[string]bool) { + indent, directive, args := mkline.Indent(), mkline.Directive(), mkline.Args() + + switch directive { + case "endif", "endfor", "elif", "else": + if indentation.Len() > 1 { + indentation.Pop() + } else { + mkline.Error1("Unmatched .%s.", directive) + } + } + + // Check the indentation + if expected := strings.Repeat(" ", indentation.Depth()); indent != expected { + if G.opts.WarnSpace && !mkline.Line.AutofixReplace("."+indent, "."+expected) { + mkline.Line.Notef("This directive should be indented by %d spaces.", indentation.Depth()) + } + } + + if directive == "if" && matches(args, `^!defined\([\w]+_MK\)$`) { + indentation.Push(indentation.Depth()) + + } else if matches(directive, `^(?:if|ifdef|ifndef|for|elif|else)$`) { + indentation.Push(indentation.Depth() + 2) + } + + reDirectivesWithArgs := `^(?:if|ifdef|ifndef|elif|for|undef)$` + if matches(directive, reDirectivesWithArgs) && args == "" { + mkline.Error1("\".%s\" requires arguments.", directive) + + } else if !matches(directive, reDirectivesWithArgs) && args != "" { + mkline.Error1("\".%s\" does not take arguments.", directive) + + if directive == "else" { + mkline.Note0("If you meant \"else if\", use \".elif\".") + } + + } else if directive == "if" || directive == "elif" { + mkline.CheckCond() + + } else if directive == "ifdef" || directive == "ifndef" { + if matches(args, `\s`) { + mkline.Error1("The \".%s\" directive can only handle _one_ argument.", directive) + } else { + mkline.Line.Warnf("The \".%s\" directive is deprecated. Please use \".if %sdefined(%s)\" instead.", + directive, ifelseStr(directive == "ifdef", "", "!"), args) + } + + } else if directive == "for" { + if m, vars, values := match2(args, `^(\S+(?:\s*\S+)*?)\s+in\s+(.*)$`); m { + for _, forvar := range splitOnSpace(vars) { + if !G.Infrastructure && hasPrefix(forvar, "_") { + mkline.Warn1("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", forvar) + } + + if matches(forvar, `^[_a-z][_a-z0-9]*$`) { + // Fine. + } else if matches(forvar, `[A-Z]`) { + mkline.Warn0(".for variable names should not contain uppercase letters.") + } else { + mkline.Error1("Invalid variable name %q.", forvar) + } + + forVars[forvar] = true + } + + // Check if any of the value's types is not guessed. + guessed := true + for _, value := range splitOnSpace(values) { + if m, vname := match1(value, `^\$\{(.*)\}`); m { + vartype := mkline.getVariableType(vname) + if vartype != nil && !vartype.guessed { + guessed = false + } + } + } + + forLoopType := &Vartype{lkSpace, CheckvarUnchecked, []AclEntry{{"*", aclpAllRead}}, guessed} + forLoopContext := &VarUseContext{forLoopType, vucTimeParse, vucQuotFor, vucExtentWord} + for _, forLoopVar := range mkline.extractUsedVariables(values) { + mkline.CheckVaruse(&MkVarUse{forLoopVar, nil}, forLoopContext) + } + } + + } else if directive == "undef" && args != "" { + for _, uvar := range splitOnSpace(args) { + if forVars[uvar] { + mkline.Note0("Using \".undef\" after a \".for\" loop is unnecessary.") + } + } + } } -func (mkline *MkLine) CheckVardefPermissions(varname string, op MkOperator) { +func (mkline *MkLine) checkDependencyRule(allowedTargets map[string]bool) { + targets := splitOnSpace(mkline.Targets()) + sources := splitOnSpace(mkline.Sources()) + + for _, source := range sources { + if source == ".PHONY" { + for _, target := range targets { + allowedTargets[target] = true + } + } + } + + for _, target := range targets { + if target == ".PHONY" { + for _, dep := range sources { + allowedTargets[dep] = true + } + + } else if target == ".ORDER" { + // TODO: Check for spelling mistakes. + + } else if !allowedTargets[target] { + mkline.Warn1("Unusual target %q.", target) + Explain3( + "If you want to define your own targets, you can \"declare\"", + "them by inserting a \".PHONY: my-target\" line before this line. This", + "will tell make(1) to not interpret this target's name as a filename.") + } + } +} + +func (mkline *MkLine) Tokenize(s string) []*MkToken { + if G.opts.Debug { + defer tracecall(mkline, s)() + } + + p := NewMkParser(mkline.Line, s, true) + tokens := p.MkTokens() + if p.Rest() != "" { + mkline.Warn1("Pkglint parse error in MkLine.Tokenize at %q.", p.Rest()) + } + return tokens +} + +func (mkline *MkLine) checkVarassignDefPermissions() { if !G.opts.WarnPerm { return } + if G.opts.Debug { + defer tracecall()() + } + varname := mkline.Varname() + op := mkline.Op() vartype := mkline.getVariableType(varname) if vartype == nil { - if G.opts.DebugMisc { - mkline.Debug1("No type definition found for %q.", varname) + if G.opts.Debug { + traceStep1("No type definition found for %q.", varname) } return } @@ -188,8 +386,8 @@ func (mkline *MkLine) CheckVardefPermissions(varname string, op MkOperator) { case perms.Contains(needed): break case perms == aclpUnknown: - if G.opts.DebugUnchecked { - mkline.Line.Debug1("Unknown permissions for %q.", varname) + if G.opts.Debug { + traceStep1("Unknown permissions for %q.", varname) } default: alternativeActions := perms & aclpAllWrite @@ -216,33 +414,39 @@ func (mkline *MkLine) CheckVardefPermissions(varname string, op MkOperator) { } } -func (mkline *MkLine) CheckVaruse(varname string, mod string, vuc *VarUseContext) { - if G.opts.DebugTrace { - defer tracecall(mkline, varname, mod, *vuc)() +func (mkline *MkLine) CheckVaruse(varuse *MkVarUse, vuc *VarUseContext) { + if G.opts.Debug { + defer tracecall(mkline, varuse, vuc)() } + if varuse.IsExpression() { + return + } + + varname := varuse.varname vartype := mkline.getVariableType(varname) if G.opts.WarnExtra && (vartype == nil || vartype.guessed) && !varIsUsed(varname) && - !(G.Mk != nil && G.Mk.forVars[varname]) { + !(G.Mk != nil && G.Mk.forVars[varname]) && + !hasPrefix(varname, "${") { mkline.Warn1("%s is used but not defined. Spelling mistake?", varname) } - mkline.CheckVarusePermissions(varname, vuc) + mkline.CheckVarusePermissions(varname, vartype, vuc) if varname == "LOCALBASE" && !G.Infrastructure { mkline.WarnVaruseLocalbase() } - needsQuoting := mkline.variableNeedsQuoting(varname, vuc) + needsQuoting := mkline.variableNeedsQuoting(varname, vartype, vuc) if vuc.quoting == vucQuotFor { mkline.checkVaruseFor(varname, vartype, needsQuoting) } if G.opts.WarnQuoting && vuc.quoting != vucQuotUnknown && needsQuoting != nqDontKnow { - mkline.CheckVaruseShellword(varname, vartype, vuc, mod, needsQuoting) + mkline.CheckVaruseShellword(varname, vartype, vuc, varuse.Mod(), needsQuoting) } if G.globalData.UserDefinedVars[varname] != nil && !G.globalData.SystemBuildDefs[varname] && !G.Mk.buildDefs[varname] { @@ -256,19 +460,21 @@ func (mkline *MkLine) CheckVaruse(varname string, mod string, vuc *VarUseContext } } -func (mkline *MkLine) CheckVarusePermissions(varname string, vuc *VarUseContext) { +func (mkline *MkLine) CheckVarusePermissions(varname string, vartype *Vartype, vuc *VarUseContext) { if !G.opts.WarnPerm { return } + if G.opts.Debug { + defer tracecall(varname, vuc)() + } // This is the type of the variable that is being used. Not to // be confused with vuc.vartype, which is the type of the // context in which the variable is used (often a ShellCommand // or, in an assignment, the type of the left hand side variable). - vartype := mkline.getVariableType(varname) if vartype == nil { - if G.opts.DebugMisc { - mkline.Debug1("No type definition found for %q.", varname) + if G.opts.Debug { + traceStep1("No type definition found for %q.", varname) } return } @@ -293,31 +499,71 @@ func (mkline *MkLine) CheckVarusePermissions(varname string, vuc *VarUseContext) isIndirect = true } - if isLoadTime && !isIndirect { + done := false + tool := G.globalData.Tools.byVarname[varname] + + if isLoadTime && tool != nil { + done = tool.Predefined && (G.Mk == nil || G.Mk.SeenBsdPrefsMk || G.Pkg == nil || G.Pkg.SeenBsdPrefsMk) + + if !done && G.Pkg != nil && !G.Pkg.SeenBsdPrefsMk && G.Mk != nil && !G.Mk.SeenBsdPrefsMk { + mkline.Warn1("To use the tool %q at load time, bsd.prefs.mk has to be included before.", varname) + done = true + } + + if !done && G.Pkg != nil { + usable, defined := G.Pkg.loadTimeTools[tool.Name] + if usable { + done = true + } + if defined && !usable { + mkline.Warn1("To use the tool %q at load time, it has to be added to USE_TOOLS before including bsd.prefs.mk.", varname) + done = true + } + } + } + + if !done && isLoadTime && !isIndirect { mkline.Warn1("%s should not be evaluated at load time.", varname) Explain( "Many variables, especially lists of something, get their values", "incrementally. Therefore it is generally unsafe to rely on their", "value until it is clear that it will never change again. This", "point is reached when the whole package Makefile is loaded and", - "execution of the shell commands starts, in some cases earlier.", + "execution of the shell commands starts; in some cases earlier.", "", "Additionally, when using the \":=\" operator, each $$ is replaced", "with a single $, so variables that have references to shell", "variables or regular expressions are modified in a subtle way.") + done = true } - if isLoadTime && isIndirect { + if !done && isLoadTime && isIndirect { mkline.Warn1("%s should not be evaluated indirectly at load time.", varname) Explain4( "The variable on the left-hand side may be evaluated at load time,", "but the variable on the right-hand side may not. Because of the", "assignment in this line, the variable might be used indirectly", "at load time, before it is guaranteed to be properly initialized.") + done = true } if !perms.Contains(aclpUseLoadtime) && !perms.Contains(aclpUse) { - mkline.Warn1("%s may not be used in this file.", varname) + needed := aclpUse + if isLoadTime { + needed = aclpUseLoadtime + } + alternativeFiles := vartype.AllowedFiles(needed) + if alternativeFiles != "" { + mkline.Warn2("%s may not be used in this file; it would be ok in %s.", + varname, alternativeFiles) + } else { + mkline.Warn1("%s may not be used in any file; it is a write-only variable.", varname) + } + Explain4( + "The allowed actions for a variable are determined based on the file", + "name in which the variable is used or defined. The exact rules are", + "hard-coded into pkglint. If they seem to be incorrect, please ask", + "on the tech-pkg@NetBSD.org mailing list.") } } @@ -353,7 +599,7 @@ func (mkline *MkLine) WarnVaruseLocalbase() { } func (mkline *MkLine) checkVaruseFor(varname string, vartype *Vartype, needsQuoting NeedsQuoting) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall(varname, vartype, needsQuoting)() } @@ -371,6 +617,9 @@ func (mkline *MkLine) checkVaruseFor(varname string, vartype *Vartype, needsQuot } func (mkline *MkLine) CheckVaruseShellword(varname string, vartype *Vartype, vuc *VarUseContext, mod string, needsQuoting NeedsQuoting) { + if G.opts.Debug { + defer tracecall(varname, vartype, vuc, mod, needsQuoting)() + } // In GNU configure scripts, a few variables need to be // passed through the :M* operator before they reach the @@ -390,7 +639,27 @@ func (mkline *MkLine) CheckVaruseShellword(varname string, vartype *Vartype, vuc } else if needsQuoting == nqYes { correctMod := strippedMod + ifelseStr(needMstar, ":M*:Q", ":Q") - if mod != correctMod { + if correctMod == mod+":Q" && vuc.extent == vucExtentWordpart && !vartype.IsShell() { + mkline.Line.Warnf("The list variable %s should not be embedded in a word.", varname) + Explain( + "When a list variable has multiple elements, this expression expands", + "to something unexpected:", + "", + "Example: ${MASTER_SITE_SOURCEFORGE}directory/ expands to", + "", + "\thttps://mirror1.sf.net/ https://mirror2.sf.net/directory/", + "", + "The first URL is missing the directory. To fix this, write", + "\t${MASTER_SITE_SOURCEFORGE:=directory/}.", + "", + "Example: -l${LIBS} expands to", + "", + "\t-llib1 lib2", + "", + "The second library is missing the -l. To fix this, write", + "${LIBS:@lib@-l${lib}@}.") + + } else if mod != correctMod { if vuc.quoting == vucQuotPlain { if !mkline.Line.AutofixReplace("${"+varname+mod+"}", "${"+varname+correctMod+"}") { mkline.Line.Warnf("Please use ${%s%s} instead of ${%s%s}.", varname, correctMod, varname, mod) @@ -401,6 +670,18 @@ func (mkline *MkLine) CheckVaruseShellword(varname string, vartype *Vartype, vuc } Explain1( "See the pkgsrc guide, section \"quoting guideline\", for details.") + + } else if vuc.quoting != vucQuotPlain { + mkline.Line.Warnf("Please move ${%s%s} outside of any quoting characters.", varname, mod) + Explain( + "The :Q modifier only works reliably when it is used outside of any", + "quoting characters.", + "", + "Examples:", + "Instead of CFLAGS=\"${CFLAGS:Q}\",", + " write CFLAGS=${CFLAGS:Q}.", + "Instead of 's,@CFLAGS@,${CFLAGS:Q},',", + " write 's,@CFLAGS@,'${CFLAGS:Q}','.") } } @@ -432,8 +713,8 @@ func (mkline *MkLine) CheckVaruseShellword(varname string, vartype *Vartype, vuc } } -func (mkline *MkLine) CheckDecreasingOrder(varname, value string) { - if G.opts.DebugTrace { +func (mkline *MkLine) checkVarassignPythonVersions(varname, value string) { + if G.opts.Debug { defer tracecall2(varname, value)() } @@ -458,27 +739,28 @@ func (mkline *MkLine) CheckDecreasingOrder(varname, value string) { } } -func (mkline *MkLine) CheckVarassign() { - if G.opts.DebugTrace { - defer tracecall0()() - } - +func (mkline *MkLine) checkVarassign() { varname := mkline.Varname() op := mkline.Op() value := mkline.Value() comment := mkline.Comment() varcanon := varnameCanon(varname) - mkline.CheckVardef(varname, op) - mkline.CheckVarassignBsdPrefs() + if G.opts.Debug { + defer tracecall(varname, op, value)() + } + + defineVar(mkline, varname) + mkline.checkVarassignDefPermissions() + mkline.checkVarassignBsdPrefs() - mkline.CheckText(value) + mkline.checkText(value) mkline.CheckVartype(varname, op, value, comment) // If the variable is not used and is untyped, it may be a spelling mistake. if op == opAssignEval && varname == strings.ToLower(varname) { - if G.opts.DebugUnchecked { - mkline.Debug1("%s might be unused unless it is an argument to a procedure file.", varname) + if G.opts.Debug { + traceStep1("%s might be unused unless it is an argument to a procedure file.", varname) } } else if !varIsUsed(varname) { @@ -491,7 +773,127 @@ func (mkline *MkLine) CheckVarassign() { } } - if matches(value, `/etc/rc\.d`) { + mkline.checkVarassignSpecific() + + if varname == "EVAL_PREFIX" { + if m, evalVarname := match1(value, `^([\w_]+)=`); m { + + // The variables mentioned in EVAL_PREFIX will later be + // defined by find-prefix.mk. Therefore, they are marked + // as known in the current file. + G.Mk.vardef[evalVarname] = mkline + } + } + + if varname == "USE_TOOLS" { + for _, fullToolname := range splitOnSpace(value) { + toolname := strings.Split(fullToolname, ":")[0] + if G.Pkg != nil { + if !G.Pkg.SeenBsdPrefsMk { + G.Pkg.loadTimeTools[toolname] = true + if G.opts.Debug { + traceStep1("loadTimeTool %q", toolname) + } + } else if !G.Pkg.loadTimeTools[toolname] { + G.Pkg.loadTimeTools[toolname] = false + if G.opts.Debug { + traceStep1("too late for loadTimeTool %q", toolname) + } + } + } + } + } + + if fix := G.globalData.Deprecated[varname]; fix != "" { + mkline.Warn2("Definition of %s is deprecated. %s", varname, fix) + } else if fix := G.globalData.Deprecated[varcanon]; fix != "" { + mkline.Warn2("Definition of %s is deprecated. %s", varname, fix) + } + + mkline.checkVarassignPlistComment(varname, value) + mkline.checkVarassignVaruse() +} + +func (mkline *MkLine) checkVarassignVaruse() { + if G.opts.Debug { + defer tracecall()() + } + + op := mkline.Op() + + time := vucTimeRun + if op == opAssignEval || op == opAssignShell { + time = vucTimeParse + } + + vartype := mkline.getVariableType(mkline.Varname()) + if op == opAssignShell { + vartype = shellcommandsContextType + } + + if vartype != nil && vartype.IsShell() { + mkline.checkVarassignVaruseShell(vartype, time) + } else { + mkline.checkVarassignVaruseMk(vartype, time) + } +} + +func (mkline *MkLine) checkVarassignVaruseMk(vartype *Vartype, time vucTime) { + if G.opts.Debug { + defer tracecall(vartype, time)() + } + tokens := NewMkParser(mkline.Line, mkline.Value(), false).MkTokens() + for i, token := range tokens { + if token.Varuse != nil { + spaceLeft := i-1 < 0 || matches(tokens[i-1].Text, `\s$`) + spaceRight := i+1 >= len(tokens) || matches(tokens[i+1].Text, `^\s`) + extent := vucExtentWordpart + if spaceLeft && spaceRight { + extent = vucExtentWord + } + + vuc := &VarUseContext{vartype, time, vucQuotPlain, extent} + mkline.CheckVaruse(token.Varuse, vuc) + } + } +} + +func (mkline *MkLine) checkVarassignVaruseShell(vartype *Vartype, time vucTime) { + if G.opts.Debug { + defer tracecall(vartype, time)() + } + + extent := func(tokens []*ShAtom, i int) vucExtent { + if i-1 >= 0 { + typ := tokens[i-1].Type + if typ != shtSpace && typ != shtSemicolon && typ != shtParenOpen && typ != shtParenClose { + return vucExtentWordpart + } + } + if i+1 < len(tokens) { + typ := tokens[i+1].Type + if typ != shtSpace && typ != shtSemicolon && typ != shtParenOpen && typ != shtParenClose { + return vucExtentWordpart + } + } + return vucExtentWord + } + + atoms := NewShTokenizer(mkline.Line, mkline.Value(), false).ShAtoms() + for i, atom := range atoms { + if atom.Type == shtVaruse { + extent := extent(atoms, i) + vuc := &VarUseContext{vartype, time, atom.Quoting.ToVarUseContext(), extent} + mkline.CheckVaruse(atom.Data.(*MkVarUse), vuc) + } + } +} + +func (mkline *MkLine) checkVarassignSpecific() { + varname := mkline.Varname() + value := mkline.Value() + + if contains(value, "/etc/rc.d") { mkline.Warn0("Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") } @@ -510,7 +912,7 @@ func (mkline *MkLine) CheckVarassign() { } } - if varname == "CONFIGURE_ARGS" && matches(value, `=\$\{PREFIX\}/share/kde`) { + if varname == "CONFIGURE_ARGS" && contains(value, "=${PREFIX}/share/kde") { mkline.Note0("Please .include \"../../meta-pkgs/kde3/kde3.mk\" instead of this line.") Explain3( "That file does many things automatically and consistently that this", @@ -518,21 +920,11 @@ func (mkline *MkLine) CheckVarassign() { "out some explicit dependencies.") } - if varname == "EVAL_PREFIX" { - if m, evalVarname := match1(value, `^([\w_]+)=`); m { - - // The variables mentioned in EVAL_PREFIX will later be - // defined by find-prefix.mk. Therefore, they are marked - // as known in the current file. - G.Mk.vardef[evalVarname] = mkline - } - } - if varname == "PYTHON_VERSIONS_ACCEPTED" { - mkline.CheckDecreasingOrder(varname, value) + mkline.checkVarassignPythonVersions(varname, value) } - if comment == "# defined" && !hasSuffix(varname, "_MK") && !hasSuffix(varname, "_COMMON") { + if mkline.Comment() == "# defined" && !hasSuffix(varname, "_MK") && !hasSuffix(varname, "_COMMON") { mkline.Note0("Please use \"# empty\", \"# none\" or \"yes\" instead of \"# defined\".") Explain( "The value #defined says something about the state of the variable,", @@ -548,31 +940,12 @@ func (mkline *MkLine) CheckVarassign() { } } - if fix := G.globalData.Deprecated[varname]; fix != "" { - mkline.Warn2("Definition of %s is deprecated. %s", varname, fix) - } else if fix := G.globalData.Deprecated[varcanon]; fix != "" { - mkline.Warn2("Definition of %s is deprecated. %s", varname, fix) - } - if hasPrefix(varname, "SITES_") { mkline.Warn0("SITES_* is deprecated. Please use SITES.* instead.") } - - mkline.CheckVarassignPlistComment(varname, value) - - time := vucTimeRun - if op == opAssignEval || op == opAssignShell { - time = vucTimeParse - } - - usedVars := mkline.extractUsedVariables(value) - vuc := &VarUseContext{mkline.getVariableType(varname), time, vucQuotUnknown, vucExtentUnknown} - for _, usedVar := range usedVars { - mkline.CheckVaruse(usedVar, "", vuc) - } } -func (mkline *MkLine) CheckVarassignBsdPrefs() { +func (mkline *MkLine) checkVarassignBsdPrefs() { if G.opts.WarnExtra && mkline.Op() == opAssignDefault && G.Pkg != nil && !G.Pkg.SeenBsdPrefsMk { switch mkline.Varcanon() { case "BUILDLINK_PKGSRCDIR.*", "BUILDLINK_DEPMETHOD.*", "BUILDLINK_ABI_DEPENDS.*": @@ -593,7 +966,7 @@ func (mkline *MkLine) CheckVarassignBsdPrefs() { } } -func (mkline *MkLine) CheckVarassignPlistComment(varname, value string) { +func (mkline *MkLine) checkVarassignPlistComment(varname, value string) { if false && // This is currently neither correct nor helpful contains(value, "@comment") && !matches(value, `="@comment "`) { mkline.Warn1("Please don't use @comment in %s.", varname) @@ -621,63 +994,63 @@ func (mkline *MkLine) CheckVarassignPlistComment(varname, value string) { } } -const reVarnamePlural = "^(?:" + - ".*[Ss]" + - "|.*LIST" + - "|.*_AWK" + - "|.*_ENV" + - "|.*_OVERRIDE" + - "|.*_PREREQ" + - "|.*_REQD" + - "|.*_SED" + - "|.*_SKIP" + - "|.*_SRC" + - "|.*_SUBST" + - "|.*_TARGET" + - "|.*_TMPL" + - "|BROKEN_EXCEPT_ON_PLATFORM" + - "|BROKEN_ON_PLATFORM" + - "|BUILDLINK_DEPMETHOD" + - "|BUILDLINK_LDADD" + - "|BUILDLINK_TRANSFORM" + - "|COMMENT" + - "|CRYPTO" + - "|DEINSTALL_TEMPLATE" + - "|EVAL_PREFIX" + - "|EXTRACT_ONLY" + - "|FETCH_MESSAGE" + - "|FIX_RPATH" + - "|GENERATE_PLIST" + - "|INSTALL_TEMPLATE" + - "|INTERACTIVE_STAGE" + - "|LICENSE" + - "|MASTER_SITE_.*" + - "|MASTER_SORT_REGEX" + - "|NOT_FOR_COMPILER" + - "|NOT_FOR_PLATFORM" + - "|ONLY_FOR_COMPILER" + - "|ONLY_FOR_PLATFORM" + - "|PERL5_PACKLIST" + - "|PLIST_CAT" + - "|PLIST_PRE" + - "|PKG_FAIL_REASON" + - "|PKG_SKIP_REASON" + - "|PREPEND_PATH" + - "|PYTHON_VERSIONS_INCOMPATIBLE" + - "|REPLACE_INTERPRETER" + - "|REPLACE_PERL" + - "|REPLACE_RUBY" + - "|RESTRICTED" + - "|SITES_.*" + - "|TOOLS_ALIASES\\.*" + - "|TOOLS_BROKEN" + - "|TOOLS_CREATE" + - "|TOOLS_GNU_MISSING" + - "|TOOLS_NOOP" + - ")$" +const reVarnamePlural = `^(?:` + + `.*[Ss]` + + `|.*LIST` + + `|.*_AWK` + + `|.*_ENV` + + `|.*_OVERRIDE` + + `|.*_PREREQ` + + `|.*_REQD` + + `|.*_SED` + + `|.*_SKIP` + + `|.*_SRC` + + `|.*_SUBST` + + `|.*_TARGET` + + `|.*_TMPL` + + `|BROKEN_EXCEPT_ON_PLATFORM` + + `|BROKEN_ON_PLATFORM` + + `|BUILDLINK_DEPMETHOD` + + `|BUILDLINK_LDADD` + + `|BUILDLINK_TRANSFORM` + + `|COMMENT` + + `|CRYPTO` + + `|DEINSTALL_TEMPLATE` + + `|EVAL_PREFIX` + + `|EXTRACT_ONLY` + + `|FETCH_MESSAGE` + + `|FIX_RPATH` + + `|GENERATE_PLIST` + + `|INSTALL_TEMPLATE` + + `|INTERACTIVE_STAGE` + + `|LICENSE` + + `|MASTER_SITE_.*` + + `|MASTER_SORT_REGEX` + + `|NOT_FOR_COMPILER` + + `|NOT_FOR_PLATFORM` + + `|ONLY_FOR_COMPILER` + + `|ONLY_FOR_PLATFORM` + + `|PERL5_PACKLIST` + + `|PLIST_CAT` + + `|PLIST_PRE` + + `|PKG_FAIL_REASON` + + `|PKG_SKIP_REASON` + + `|PREPEND_PATH` + + `|PYTHON_VERSIONS_INCOMPATIBLE` + + `|REPLACE_INTERPRETER` + + `|REPLACE_PERL` + + `|REPLACE_RUBY` + + `|RESTRICTED` + + `|SITES_.+` + + `|TOOLS_ALIASES\..+` + + `|TOOLS_BROKEN` + + `|TOOLS_CREATE` + + `|TOOLS_GNU_MISSING` + + `|TOOLS_NOOP` + + `)$` func (mkline *MkLine) CheckVartype(varname string, op MkOperator, value, comment string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall(varname, op, value, comment)() } @@ -701,29 +1074,30 @@ func (mkline *MkLine) CheckVartype(varname string, op MkOperator, value, comment switch { case vartype == nil: // Cannot check anything if the type is not known. - if G.opts.DebugUnchecked { - mkline.Debug1("Unchecked variable assignment for %s.", varname) + if G.opts.Debug { + traceStep1("Unchecked variable assignment for %s.", varname) } case op == opAssignShell: - if G.opts.DebugMisc { - mkline.Debug1("Use of !=: %q", value) + if G.opts.Debug { + traceStep1("Unchecked use of !=: %q", value) } case vartype.kindOfList == lkNone: mkline.CheckVartypePrimitive(varname, vartype.checker, op, value, comment, vartype.IsConsideredList(), vartype.guessed) + case value == "": + break + case vartype.kindOfList == lkSpace: for _, word := range splitOnSpace(value) { mkline.CheckVartypePrimitive(varname, vartype.checker, op, word, comment, true, vartype.guessed) } case vartype.kindOfList == lkShell: - shline := NewShellLine(mkline) - words, _ := splitIntoShellWords(mkline.Line, value) + words, _ := splitIntoMkWords(mkline.Line, value) for _, word := range words { mkline.CheckVartypePrimitive(varname, vartype.checker, op, word, comment, true, vartype.guessed) - shline.CheckToken(word, true) } } } @@ -731,13 +1105,12 @@ func (mkline *MkLine) CheckVartype(varname string, op MkOperator, value, comment // For some variables (like `BuildlinkDepth`), `op` influences the valid values. // The `comment` parameter comes from a variable assignment, when a part of the line is commented out. func (mkline *MkLine) CheckVartypePrimitive(varname string, checker *VarChecker, op MkOperator, value, comment string, isList bool, guessed bool) { - if G.opts.DebugTrace { - defer tracecall(varname, op, value, comment, isList, guessed)() + if G.opts.Debug { + defer tracecall(varname, checker.name, op, value, comment, isList, guessed)() } - ctx := &VartypeCheck{mkline, mkline.Line, varname, op, value, "", comment, isList, guessed} - ctx.valueNovar = mkline.withoutMakeVariables(value, isList) - + valueNoVar := mkline.withoutMakeVariables(value, isList) + ctx := &VartypeCheck{mkline, mkline.Line, varname, op, value, valueNoVar, comment, isList, guessed} checker.checker(ctx) } @@ -756,19 +1129,11 @@ func (mkline *MkLine) withoutMakeVariables(value string, qModifierAllowed bool) } } -func (mkline *MkLine) CheckText(text string) { - if G.opts.DebugTrace { +func (mkline *MkLine) checkText(text string) { + if G.opts.Debug { defer tracecall1(text)() } - if m, varname := match1(text, `^(?:[^#]*[^\$])?\$(\w+)`); m { - mkline.Warn1("$%[1]s is ambiguous. Use ${%[1]s} if you mean a Makefile variable or $$%[1]s if you mean a shell variable.", varname) - } - - if mkline.Line.firstLine == 1 { - mkline.Line.CheckRcsid(`# `, "# ") - } - if contains(text, "${WRKSRC}/..") { mkline.Warn0("Building the package should take place entirely inside ${WRKSRC}, not \"${WRKSRC}/..\".") Explain( @@ -812,11 +1177,11 @@ func (mkline *MkLine) CheckText(text string) { } func (mkline *MkLine) CheckCond() { - if G.opts.DebugTrace { - defer tracecall0()() + if G.opts.Debug { + defer tracecall1(mkline.Args())() } - p := NewParser(mkline.Line, mkline.Args()) + p := NewMkParser(mkline.Line, mkline.Args(), false) cond := p.MkCond() if !p.EOF() { mkline.Warn1("Invalid conditional %q.", mkline.Args()) @@ -981,46 +1346,20 @@ const ( nqDontKnow ) -func (mkline *MkLine) variableNeedsQuoting(varname string, vuc *VarUseContext) (needsQuoting NeedsQuoting) { - if G.opts.DebugTrace { - defer tracecall(varname, *vuc, "=>", needsQuoting)() +func (nq NeedsQuoting) String() string { + return [...]string{"no", "yes", "doesn't matter", "don't known"}[nq] +} + +func (mkline *MkLine) variableNeedsQuoting(varname string, vartype *Vartype, vuc *VarUseContext) (needsQuoting NeedsQuoting) { + if G.opts.Debug { + defer tracecall(varname, vartype, vuc, "=>", &needsQuoting)() } - vartype := mkline.getVariableType(varname) if vartype == nil || vuc.vartype == nil { return nqDontKnow } - isPlainWord := vartype.checker.IsEnum() - switch vartype.checker { - case CheckvarBuildlinkDepmethod, - CheckvarCategory, - CheckvarDistSuffix, - CheckvarEmulPlatform, - CheckvarFileMode, - CheckvarFilename, - CheckvarIdentifier, - CheckvarInteger, - CheckvarOption, - CheckvarPathname, - CheckvarPerl5Packlist, - CheckvarPkgName, - CheckvarPkgOptionsVar, - CheckvarPkgPath, - CheckvarPkgRevision, - CheckvarPrefixPathname, - CheckvarPythonDependency, - CheckvarRelativePkgDir, - CheckvarRelativePkgPath, - CheckvarStage, - CheckvarUserGroupName, - CheckvarVersion, - CheckvarWrkdirSubdirectory, - CheckvarYesNo, - CheckvarYesNoIndirectly: - isPlainWord = true - } - if isPlainWord { + if vartype.checker.IsEnum() || vartype.IsBasicSafe() { if vartype.kindOfList == lkNone { return nqDoesntMatter } @@ -1037,12 +1376,10 @@ func (mkline *MkLine) variableNeedsQuoting(varname string, vuc *VarUseContext) ( } // Determine whether the context expects a list of shell words or not. - wantList := vuc.vartype.IsConsideredList() && (vuc.quoting == vucQuotBackt || vuc.extent != vucExtentWordpart) + wantList := vuc.vartype.IsConsideredList() haveList := vartype.IsConsideredList() - - if G.opts.DebugQuoting { - mkline.Line.Debugf("variableNeedsQuoting: varname=%q, context=%v, type=%v, wantList=%v, haveList=%v", - varname, vuc, vartype, wantList, haveList) + if G.opts.Debug { + traceStep("wantList=%v, haveList=%v", wantList, haveList) } // A shell word may appear as part of a shell word, for example COMPILER_RPATH_FLAG. @@ -1052,9 +1389,18 @@ func (mkline *MkLine) variableNeedsQuoting(varname string, vuc *VarUseContext) ( } } + // Both of these can be correct, depending on the situation: + // 1. echo ${PERL5:Q} + // 2. xargs ${PERL5} + if vuc.extent == vucExtentWord && vuc.quoting == vucQuotPlain { + if wantList && haveList { + return nqDontKnow + } + } + // Assuming the tool definitions don't include very special characters, // so they can safely be used inside any quotes. - if G.globalData.VarnameToToolname[varname] != "" { + if G.globalData.Tools.byVarname[varname] != nil { switch vuc.quoting { case vucQuotPlain: if vuc.extent != vucExtentWordpart { @@ -1071,23 +1417,42 @@ func (mkline *MkLine) variableNeedsQuoting(varname string, vuc *VarUseContext) ( // to be quoted. An exception is in the case of backticks, // because the whole backticks expression is parsed as a single // shell word by pkglint. - if vuc.extent == vucExtentWordpart && vuc.quoting != vucQuotBackt { + if vuc.extent == vucExtentWordpart && vuc.vartype != nil && vuc.vartype.IsShell() && vuc.quoting != vucQuotBackt { return nqYes } + // SUBST_MESSAGE.perl= Replacing in ${REPLACE_PERL} + if vuc.vartype != nil && vuc.vartype.IsPlainString() { + return nqNo + } + // Assigning lists to lists does not require any quoting, though // there may be cases like "CONFIGURE_ARGS+= -libs ${LDFLAGS:Q}" // where quoting is necessary. - if wantList && haveList { + if wantList && haveList && vuc.extent != vucExtentWordpart { return nqDoesntMatter } if wantList != haveList { + if vuc.vartype != nil && vartype != nil { + if vuc.vartype.checker == CheckvarFetchURL && vartype.checker == CheckvarHomepage { + return nqNo + } + if vuc.vartype.checker == CheckvarHomepage && vartype.checker == CheckvarFetchURL { + return nqNo // Just for HOMEPAGE=${MASTER_SITE_*:=subdir/}. + } + } + return nqYes + } + + // Bad: LDADD += -l${LIBS} + // Good: LDADD += ${LIBS:@lib@-l${lib} @} + if wantList && haveList && vuc.extent == vucExtentWordpart { return nqYes } - if G.opts.DebugQuoting { - mkline.Line.Debug1("Don't know whether :Q is needed for %q", varname) + if G.opts.Debug { + traceStep1("Don't know whether :Q is needed for %q", varname) } return nqDontKnow } @@ -1095,6 +1460,10 @@ func (mkline *MkLine) variableNeedsQuoting(varname string, vuc *VarUseContext) ( // Returns the type of the variable (maybe guessed based on the variable name), // or nil if the type cannot even be guessed. func (mkline *MkLine) getVariableType(varname string) *Vartype { + if G.opts.Debug { + defer tracecall1(varname)() + } + if vartype := G.globalData.vartypes[varname]; vartype != nil { return vartype } @@ -1102,11 +1471,20 @@ func (mkline *MkLine) getVariableType(varname string) *Vartype { return vartype } - if G.globalData.VarnameToToolname[varname] != "" { - return &Vartype{lkNone, CheckvarShellCommand, []AclEntry{{"*", aclpUse}}, false} + if tool := G.globalData.Tools.byVarname[varname]; tool != nil { + perms := aclpUse + if G.opts.Debug { + traceStep("Use of tool %+v", tool) + } + if tool.UsableAtLoadtime { + if G.Pkg == nil || G.Pkg.SeenBsdPrefsMk || G.Pkg.loadTimeTools[tool.Name] { + perms |= aclpUseLoadtime + } + } + return &Vartype{lkNone, CheckvarShellCommand, []AclEntry{{"*", perms}}, false} } - if m, toolvarname := match1(varname, `^TOOLS_(.*)`); m && G.globalData.VarnameToToolname[toolvarname] != "" { + if m, toolvarname := match1(varname, `^TOOLS_(.*)`); m && G.globalData.Tools.byVarname[toolvarname] != nil { return &Vartype{lkNone, CheckvarPathname, []AclEntry{{"*", aclpUse}}, false} } @@ -1149,11 +1527,11 @@ func (mkline *MkLine) getVariableType(varname string) *Vartype { gtype = &Vartype{lkNone, CheckvarYes, allowAll, true} } - if G.opts.DebugVartypes { + if G.opts.Debug { if gtype != nil { - mkline.Line.Debug2("The guessed type of %q is %q.", varname, gtype.String()) + traceStep2("The guessed type of %q is %q.", varname, gtype.String()) } else { - mkline.Line.Debug1("No type definition found for %q.", varname) + traceStep1("No type definition found for %q.", varname) } } return gtype @@ -1176,8 +1554,8 @@ func (mkline *MkLine) extractUsedVariables(text string) []string { } } - if rest != "" && G.opts.DebugMisc { - mkline.Debug1("extractUsedVariables: rest=%q", rest) + if G.opts.Debug && rest != "" { + traceStep1("extractUsedVariables: rest=%q", rest) } return result } @@ -1212,7 +1590,7 @@ func (mkline *MkLine) determineUsedVariables() (varnames []string) { } rest = rest[min:] - m := regcomp(`(?:\$\{|\$\(|defined\(|empty\()([0-9+.A-Z_a-z]+)[:})]`).FindStringSubmatchIndex(rest) + m := regcomp(`(?:\$\{|\$\(|defined\(|empty\()([*+\-.0-9A-Z_a-z]+)[:})]`).FindStringSubmatchIndex(rest) if m == nil { return } @@ -1286,19 +1664,25 @@ func (q vucQuoting) String() string { type vucExtent uint8 const ( - vucExtentUnknown vucExtent = iota - vucExtentWord // Example: echo ${LOCALBASE} - vucExtentWordpart // Example: echo LOCALBASE=${LOCALBASE} + vucExtentWord vucExtent = iota // Example: echo ${LOCALBASE} + vucExtentWordpart // Example: echo LOCALBASE=${LOCALBASE} ) -func (e vucExtent) String() string { - return [...]string{"unknown", "word", "wordpart"}[e] -} +func (e vucExtent) String() string { return [...]string{"word", "wordpart"}[e] } func (vuc *VarUseContext) String() string { typename := "no-type" if vuc.vartype != nil { typename = vuc.vartype.String() } - return fmt.Sprintf("(%s %s %s %s)", vuc.time, typename, vuc.quoting, vuc.extent) + return fmt.Sprintf("(%s time:%s quoting:%s extent:%s)", typename, vuc.time, vuc.quoting, vuc.extent) } + +type Indentation struct { + data []int +} + +func (ind *Indentation) Len() int { return len(ind.data) } +func (ind *Indentation) Depth() int { return ind.data[len(ind.data)-1] } +func (ind *Indentation) Pop() { ind.data = ind.data[:len(ind.data)-1] } +func (ind *Indentation) Push(indent int) { ind.data = append(ind.data, indent) } diff --git a/pkgtools/pkglint/files/mkline_test.go b/pkgtools/pkglint/files/mkline_test.go index baace91dfce..62c6fd57183 100644 --- a/pkgtools/pkglint/files/mkline_test.go +++ b/pkgtools/pkglint/files/mkline_test.go @@ -5,7 +5,7 @@ import ( ) func (s *Suite) TestChecklineMkVartype_SimpleType(c *check.C) { - s.UseCommandLine(c, "-Wtypes", "-Dunchecked") + s.UseCommandLine(c, "-Wtypes") G.globalData.InitVartypes() mkline := NewMkLine(NewLine("fname", 1, "COMMENT=\tA nice package", nil)) @@ -56,19 +56,19 @@ func (s *Suite) TestMkLine_CheckVaralign_Autofix(c *check.C) { varalign.Finish() c.Check(lines[0].changed, equals, true) - c.Check(lines[0].rawLines()[0].String(), equals, "1:VAR=\tvalue\n") + c.Check(lines[0].raw[0].String(), equals, "1:VAR=\tvalue\n") c.Check(lines[2].changed, equals, true) - c.Check(lines[2].rawLines()[0].String(), equals, "3:VAR=\tvalue\n") + c.Check(lines[2].raw[0].String(), equals, "3:VAR=\tvalue\n") c.Check(lines[4].changed, equals, true) - c.Check(lines[4].rawLines()[0].String(), equals, "5:VAR=\tvalue\n") + c.Check(lines[4].raw[0].String(), equals, "5:VAR=\tvalue\n") c.Check(lines[6].changed, equals, true) - c.Check(lines[6].rawLines()[0].String(), equals, "7:VAR=\tvalue\n") + c.Check(lines[6].raw[0].String(), equals, "7:VAR=\tvalue\n") c.Check(lines[8].changed, equals, true) - c.Check(lines[8].rawLines()[0].String(), equals, "9:VAR=\tvalue\n") + c.Check(lines[8].raw[0].String(), equals, "9:VAR=\tvalue\n") c.Check(lines[10].changed, equals, true) - c.Check(lines[10].rawLines()[0].String(), equals, "11:VAR=\tvalue\n") + c.Check(lines[10].raw[0].String(), equals, "11:VAR=\tvalue\n") c.Check(lines[12].changed, equals, false) - c.Check(lines[12].rawLines()[0].String(), equals, "13:VAR=\tvalue\n") + c.Check(lines[12].raw[0].String(), equals, "13:VAR=\tvalue\n") c.Check(s.Output(), equals, ""+ "NOTE: file.mk:1: This variable value should be aligned with tabs, not spaces, to column 9.\n"+ "AUTOFIX: file.mk:1: Replacing \"VAR= \" with \"VAR=\\t\".\n"+ @@ -149,93 +149,6 @@ func (s *Suite) TestMkLine_CheckVaralign_OnlySpaces(c *check.C) { "NOTE: file.mk:4: This variable value should be aligned with tabs, not spaces, to column 33.\n") } -func (s *Suite) TestMkLine_CheckVaralign_Advanced(c *check.C) { - s.UseCommandLine(c, "-Wspace") - fname := s.CreateTmpFileLines(c, "Makefile", - "# $"+"NetBSD$", - "", - "VAR= \\", // In continuation lines, indenting with spaces is ok - "\tvalue", - "", - "VAR= indented with one space", // Exactly one space is ok in general - "VAR= indented with two spaces", // Two spaces are uncommon - "", - "BLOCK=\tindented with tab", - "BLOCK_LONGVAR= indented with space", // This is ok, to prevent the block from being indented further - "", - "BLOCK=\tshort", - "BLOCK_LONGVAR=\tlong", - "", - "GRP_A= avalue", // The values in a block should be aligned - "GRP_AA= value", - "GRP_AAA= value", - "GRP_AAAA= value", - "", - "VAR=\t${VAR}${BLOCK}${BLOCK_LONGVAR} # suppress warnings about unused variables", - "VAR=\t${GRP_A}${GRP_AA}${GRP_AAA}${GRP_AAAA}") - mklines := NewMkLines(LoadExistingLines(fname, true)) - - mklines.Check() - - c.Check(s.Output(), equals, ""+ - "NOTE: ~/Makefile:6: This variable value should be aligned with tabs, not spaces, to column 9.\n"+ - "NOTE: ~/Makefile:7: This variable value should be aligned with tabs, not spaces, to column 9.\n"+ - "NOTE: ~/Makefile:12: This variable value should be aligned to column 17.\n"+ - "NOTE: ~/Makefile:15: This variable value should be aligned with tabs, not spaces, to column 17.\n"+ - "NOTE: ~/Makefile:16: This variable value should be aligned with tabs, not spaces, to column 17.\n"+ - "NOTE: ~/Makefile:17: This variable value should be aligned with tabs, not spaces, to column 17.\n"+ - "NOTE: ~/Makefile:18: This variable value should be aligned with tabs, not spaces, to column 17.\n") - - s.UseCommandLine(c, "-Wspace", "--autofix") - - mklines.Check() - - c.Check(s.Output(), equals, ""+ - "AUTOFIX: ~/Makefile:6: Replacing \"VAR= \" with \"VAR=\\t\".\n"+ - "AUTOFIX: ~/Makefile:7: Replacing \"VAR= \" with \"VAR=\\t\".\n"+ - "AUTOFIX: ~/Makefile:12: Replacing \"BLOCK=\\t\" with \"BLOCK=\\t\\t\".\n"+ - "AUTOFIX: ~/Makefile:15: Replacing \"GRP_A= \" with \"GRP_A=\\t\\t\".\n"+ - "AUTOFIX: ~/Makefile:16: Replacing \"GRP_AA= \" with \"GRP_AA=\\t\\t\".\n"+ - "AUTOFIX: ~/Makefile:17: Replacing \"GRP_AAA= \" with \"GRP_AAA=\\t\".\n"+ - "AUTOFIX: ~/Makefile:18: Replacing \"GRP_AAAA= \" with \"GRP_AAAA=\\t\".\n"+ - "AUTOFIX: ~/Makefile: Has been auto-fixed. Please re-run pkglint.\n") - c.Check(s.LoadTmpFile(c, "Makefile"), equals, ""+ - "# $"+"NetBSD$\n"+ - "\n"+ - "VAR= \\\n"+ - "\tvalue\n"+ - "\n"+ - "VAR=\tindented with one space\n"+ - "VAR=\tindented with two spaces\n"+ - "\n"+ - "BLOCK=\tindented with tab\n"+ - "BLOCK_LONGVAR= indented with space\n"+ - "\n"+ - "BLOCK=\t\tshort\n"+ - "BLOCK_LONGVAR=\tlong\n"+ - "\n"+ - "GRP_A=\t\tavalue\n"+ - "GRP_AA=\t\tvalue\n"+ - "GRP_AAA=\tvalue\n"+ - "GRP_AAAA=\tvalue\n"+ - "\n"+ - "VAR=\t${VAR}${BLOCK}${BLOCK_LONGVAR} # suppress warnings about unused variables\n"+ - "VAR=\t${GRP_A}${GRP_AA}${GRP_AAA}${GRP_AAAA}\n") -} - -func (s *Suite) TestMkLine_CheckVaralign_Misc(c *check.C) { - s.UseCommandLine(c, "-Wspace") - mklines := s.NewMkLines("Makefile", - "# $"+"NetBSD$", - "", - "VAR= space", - "VAR=\ttab ${VAR}") - - mklines.Check() - - c.Check(s.Output(), equals, "NOTE: Makefile:3: Variable values should be aligned with tabs, not spaces.\n") -} - func (s *Suite) TestMkLine_fields(c *check.C) { mklines := NewMkLines(s.NewLines("test.mk", "VARNAME.param?=value # varassign comment", @@ -300,7 +213,7 @@ func (s *Suite) TestMkLine_checkVarassign(c *check.C) { G.globalData.InitVartypes() mkline := NewMkLine(NewLine("fname", 10, "MASTER_SITES=http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=", nil)) - mkline.CheckVarassign() + mkline.checkVarassign() c.Check(s.Output(), equals, "") } @@ -337,22 +250,17 @@ func (s *Suite) TestChecklineMkCondition(c *check.C) { NewMkLine(NewLine("fname", 1, ".if ${EMUL_PLATFORM} == \"linux-x386\"", nil)).CheckCond() - c.Check(s.Output(), equals, "WARN: fname:1: \"x386\" is not valid for the hardware architecture part of EMUL_PLATFORM. Use one of { alpha amd64 arc arm arm32 cobalt convex dreamcast hpcmips hpcsh hppa i386 ia64 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 ns32k pc532 pmax powerpc rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } instead.\n") + c.Check(s.Output(), equals, "WARN: fname:1: \"x386\" is not valid for the hardware architecture part of EMUL_PLATFORM. Use one of { aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } instead.\n") NewMkLine(NewLine("fname", 1, ".if ${EMUL_PLATFORM:Mlinux-x386}", nil)).CheckCond() - c.Check(s.Output(), equals, "WARN: fname:1: The pattern \"x386\" cannot match any of { alpha amd64 arc arm arm32 cobalt convex dreamcast hpcmips hpcsh hppa i386 ia64 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 ns32k pc532 pmax powerpc rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } for the hardware architecture part of EMUL_PLATFORM.\n") -} - -func (s *Suite) TestMkLine_variableNeedsQuoting(c *check.C) { - mkline := NewMkLine(NewLine("fname", 1, "PKGNAME := ${UNKNOWN}", nil)) - G.globalData.InitVartypes() - pkgnameType := G.globalData.vartypes["PKGNAME"] + c.Check(s.Output(), equals, "WARN: fname:1: The pattern \"x386\" cannot match any of { aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } for the hardware architecture part of EMUL_PLATFORM.\n") - vuc := &VarUseContext{pkgnameType, vucTimeParse, vucQuotUnknown, vucExtentUnknown} - nq := mkline.variableNeedsQuoting("UNKNOWN", vuc) + NewMkLine(NewLine("fname", 98, ".if ${MACHINE_PLATFORM:MUnknownOS-*-*} || ${MACHINE_ARCH:Mx86}", nil)).CheckCond() - c.Check(nq, equals, nqDontKnow) + c.Check(s.Output(), equals, ""+ + "WARN: fname:98: The pattern \"UnknownOS\" cannot match any of { AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare } for the operating system part of MACHINE_PLATFORM.\n"+ + "WARN: fname:98: The pattern \"x86\" cannot match any of { aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } for MACHINE_ARCH.\n") } func (s *Suite) TestMkLine_variableNeedsQuoting_Varbase(c *check.C) { @@ -376,7 +284,7 @@ func (s *Suite) TestVarUseContext_ToString(c *check.C) { vartype := mkline.getVariableType("PKGNAME") vuc := &VarUseContext{vartype, vucTimeUnknown, vucQuotBackt, vucExtentWord} - c.Check(vuc.String(), equals, "(unknown PkgName backt word)") + c.Check(vuc.String(), equals, "(PkgName time:unknown quoting:backt extent:word)") } func (s *Suite) TestMkLine_(c *check.C) { @@ -387,8 +295,8 @@ func (s *Suite) TestMkLine_(c *check.C) { "ac_cv_libpari_libs+=\t-L${BUILDLINK_PREFIX.pari}/lib", // From math/clisp-pari/Makefile, rev. 1.8 "var+=value") - G.Mk.mklines[1].CheckVarassign() - G.Mk.mklines[2].CheckVarassign() + G.Mk.mklines[1].checkVarassign() + G.Mk.mklines[2].checkVarassign() c.Check(s.Output(), equals, ""+ "WARN: Makefile:2: ac_cv_libpari_libs is defined but not used. Spelling mistake?\n"+ @@ -408,6 +316,11 @@ func (s *Suite) TestParselineMk(c *check.C) { line2 := NewMkLine(NewLine("fname", 1, "\tsed -e 's,\\#,hash,g'", nil)) c.Check(line2.Shellcmd(), equals, "sed -e 's,\\#,hash,g'") + + // From shells/zsh/Makefile.common, rev. 1.78 + NewMkLine(NewLine("fname", 1, "\t# $ sha1 patches/patch-ac", nil)) + + c.Check(s.Output(), equals, "") } func (s *Suite) TestMkLine_LeadingSpace(c *check.C) { @@ -416,7 +329,7 @@ func (s *Suite) TestMkLine_LeadingSpace(c *check.C) { c.Check(s.Output(), equals, "WARN: rubyversion.mk:427: Makefile lines should not start with space characters.\n") } -func (s *Suite) TestMkLine_CheckVardefPermissions(c *check.C) { +func (s *Suite) TestMkLine_checkVarassignDefPermissions(c *check.C) { s.UseCommandLine(c, "-Wall") G.globalData.InitVartypes() mklines := s.NewMkLines("options.mk", @@ -451,6 +364,18 @@ func (s *Suite) TestMkLine_CheckVarusePermissions(c *check.C) { "NOTE: options.mk:4: This variable value should be aligned to column 17.\n") } +func (s *Suite) Test_MkLine_CheckVarusePermissions_LoadTime(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + mklines := s.NewMkLines("options.mk", + "# $"+"NetBSD$", + "WRKSRC:=${.CURDIR}") + + mklines.Check() + + c.Check(s.Output(), equals, "") // Don’t warn that “.CURDIR should not be evaluated at load time.” +} + func (s *Suite) TestMkLine_WarnVaruseLocalbase(c *check.C) { mkline := NewMkLine(NewLine("options.mk", 56, "PKGNAME=${LOCALBASE}", nil)) @@ -498,12 +423,351 @@ func (s *Suite) TestMkLine_CheckRelativePkgdir(c *check.C) { // PR pkg/46570, item 2 func (s *Suite) TestMkLine_UnfinishedVaruse(c *check.C) { - s.UseCommandLine(c, "-Dunchecked") mkline := NewMkLine(NewLine("Makefile", 93, "EGDIRS=${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d", nil)) - mkline.CheckVarassign() + mkline.checkVarassign() c.Check(s.Output(), equals, ""+ - "ERROR: Makefile:93: Invalid Makefile syntax at \"${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".\n"+ + "WARN: Makefile:93: Pkglint parse error in MkLine.Tokenize at \"${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".\n"+ + "WARN: Makefile:93: Pkglint parse error in ShTokenizer.ShAtom at \"${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\" (quoting=plain)\n"+ "WARN: Makefile:93: EGDIRS is defined but not used. Spelling mistake?\n") } + +func (s *Suite) TestMkLine_variableNeedsQuoting_1(c *check.C) { + mkline := NewMkLine(NewLine("fname", 1, "PKGNAME := ${UNKNOWN}", nil)) + G.globalData.InitVartypes() + + vuc := &VarUseContext{G.globalData.vartypes["PKGNAME"], vucTimeParse, vucQuotUnknown, vucExtentWord} + nq := mkline.variableNeedsQuoting("UNKNOWN", nil, vuc) + + c.Check(nq, equals, nqDontKnow) +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_2(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + s.RegisterMasterSite("MASTER_SITE_SOURCEFORGE", "http://downloads.sourceforge.net/sourceforge/") + mkline := NewMkLine(NewLine("Makefile", 95, "MASTER_SITES=\t${HOMEPAGE}", nil)) + + vuc := &VarUseContext{G.globalData.vartypes["MASTER_SITES"], vucTimeRun, vucQuotPlain, vucExtentWord} + nq := mkline.variableNeedsQuoting("HOMEPAGE", G.globalData.vartypes["HOMEPAGE"], vuc) + + c.Check(nq, equals, nqNo) + + mkline.checkVarassign() + + c.Check(s.Output(), equals, "") // Up to pkglint 5.3.6, it warned about a missing :Q here, which was wrong. +} + +// Assigning lists to lists is ok. +func (s *Suite) TestMkLine_variableNeedsQuoting_3(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + s.RegisterMasterSite("MASTER_SITE_SOURCEFORGE", "http://downloads.sourceforge.net/sourceforge/") + mkline := NewMkLine(NewLine("Makefile", 96, "MASTER_SITES=\t${MASTER_SITE_SOURCEFORGE:=squirrel-sql/}", nil)) + + mkline.checkVarassign() + + c.Check(s.Output(), equals, "") +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_4(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + mkline := NewMkLine(NewLine("builtin.mk", 3, "USE_BUILTIN.Xfixes!=\t${PKG_ADMIN} pmatch 'pkg-[0-9]*' ${BUILTIN_PKG.Xfixes:Q}", nil)) + + mkline.checkVarassign() + + c.Check(s.Output(), equals, ""+ + "WARN: builtin.mk:3: PKG_ADMIN should not be evaluated at load time.\n"+ + "NOTE: builtin.mk:3: The :Q operator isn't necessary for ${BUILTIN_PKG.Xfixes} here.\n") +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_5(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + mkline := NewMkLine(NewLine("Makefile", 3, "SUBST_SED.hpath=\t-e 's|^\\(INSTALL[\t:]*=\\).*|\\1${INSTALL}|'", nil)) + + mkline.checkVarassign() + + c.Check(s.Output(), equals, "WARN: Makefile:3: Please use ${INSTALL:Q} instead of ${INSTALL} and make sure the variable appears outside of any quoting characters.\n") +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_6(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + s.RegisterTool(&Tool{Name: "find", Varname: "FIND", Predefined: true}) + s.RegisterTool(&Tool{Name: "sort", Varname: "SORT", Predefined: true}) + G.Pkg = NewPackage("category/pkgbase") + G.Mk = s.NewMkLines("Makefile", + "# $"+"NetBSD$", + "GENERATE_PLIST= cd ${DESTDIR}${PREFIX}; ${FIND} * \\( -type f -or -type l \\) | ${SORT};") + + G.Mk.determineDefinedVariables() + G.Mk.mklines[1].Check() + + c.Check(s.Output(), equals, ""+ + "WARN: Makefile:2: The exitcode of the left-hand-side command of the pipe operator is ignored.\n") +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_7(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("Makefile", + "# $"+"NetBSD$", + "EGDIR=\t${EGDIR}/${MACHINE_GNU_PLATFORM}") + + G.Mk.mklines[1].Check() + + c.Check(s.Output(), equals, "") +} + +// Based on graphics/circos/Makefile. +func (s *Suite) TestMkLine_variableNeedsQuoting_8(c *check.C) { + s.UseCommandLine(c, "-Wall") + s.RegisterTool(&Tool{Name: "perl", Varname: "PERL5", Predefined: true}) + s.RegisterTool(&Tool{Name: "bash", Varname: "BASH", Predefined: true}) + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("Makefile", + "# $"+"NetBSD$", + "\t${RUN} cd ${WRKSRC} && ( ${ECHO} ${PERL5:Q} ; ${ECHO} ) | ${BASH} ./install", + "\t${RUN} cd ${WRKSRC} && ( ${ECHO} ${PERL5} ; ${ECHO} ) | ${BASH} ./install") + + G.Mk.mklines[1].Check() + G.Mk.mklines[2].Check() + + c.Check(s.Output(), equals, ""+ + "WARN: Makefile:2: The exitcode of the left-hand-side command of the pipe operator is ignored.\n"+ + "WARN: Makefile:3: The exitcode of the left-hand-side command of the pipe operator is ignored.\n") +} + +// Based on mail/mailfront/Makefile. +func (s *Suite) TestMkLine_variableNeedsQuoting_9(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("Makefile", + "# $"+"NetBSD$", + "MASTER_SITES=${HOMEPAGE}archive/") + + G.Mk.mklines[1].Check() + + c.Check(s.Output(), equals, "") // Don’t suggest to use ${HOMEPAGE:Q}. +} + +// Based on www/firefox31/xpi.mk. +func (s *Suite) TestMkLine_variableNeedsQuoting_10(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + s.RegisterTool(&Tool{Name: "awk", Varname: "AWK", Predefined: true}) + s.RegisterTool(&Tool{Name: "echo", Varname: "ECHO", Predefined: true}) + G.Mk = s.NewMkLines("xpi.mk", + "# $"+"NetBSD$", + "\t id=$$(${AWK} '{print}' < ${WRKSRC}/idfile) && echo \"$$id\"", + "\t id=`${AWK} '{print}' < ${WRKSRC}/idfile` && echo \"$$id\"") + + G.Mk.mklines[1].Check() + G.Mk.mklines[2].Check() + + c.Check(s.Output(), equals, "WARN: xpi.mk:2: Invoking subshells via $(...) is not portable enough.\n") // Don’t suggest to use ${AWK:Q}. +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_11(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("x11/mlterm/Makefile", + "# $"+"NetBSD$", + "SUBST_SED.link=-e 's|(LIBTOOL_LINK).*(LIBS)|& ${LDFLAGS:M*:Q}|g'", + "SUBST_SED.link=-e 's|(LIBTOOL_LINK).*(LIBS)|& '${LDFLAGS:M*:Q}'|g'") + + G.Mk.mklines[1].Check() + G.Mk.mklines[2].Check() + + c.Check(s.Output(), equals, "WARN: x11/mlterm/Makefile:2: Please move ${LDFLAGS:M*:Q} outside of any quoting characters.\n") +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_12(c *check.C) { + s.UseCommandLine(c, "-Wall") + s.RegisterMasterSite("MASTER_SITE_GITHUB", "https://github.com/") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("devel/catch/Makefile", + "# $"+"NetBSD$", + "HOMEPAGE=\t${MASTER_SITE_GITHUB:=philsquared/Catch/}", + "HOMEPAGE=\t${MASTER_SITE_GITHUB}", + "HOMEPAGE=\t${MASTER_SITES}", + "HOMEPAGE=\t${MASTER_SITES}${GITHUB_PROJECT}") + + G.Mk.Check() + + c.Check(s.Output(), equals, ""+ + "WARN: devel/catch/Makefile:2: HOMEPAGE should not be defined in terms of MASTER_SITEs. Use https://github.com/philsquared/Catch/ directly.\n"+ + "WARN: devel/catch/Makefile:3: HOMEPAGE should not be defined in terms of MASTER_SITEs. Use https://github.com/ directly.\n"+ + "WARN: devel/catch/Makefile:4: HOMEPAGE should not be defined in terms of MASTER_SITEs.\n"+ + "WARN: devel/catch/Makefile:5: HOMEPAGE should not be defined in terms of MASTER_SITEs.\n") +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_13(c *check.C) { + s.UseCommandLine(c, "-Wall") + s.RegisterTool(&Tool{Varname: "SH", Predefined: true}) + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("x11/labltk/Makefile", + "# $"+"NetBSD$", + "CONFIGURE_ARGS+=\t-tklibs \"`${SH} -c '${ECHO} $$TK_LD_FLAGS'`\"") + + G.Mk.mklines[1].Check() + + c.Check(s.Output(), equals, "") // Don’t suggest ${ECHO:Q} here. +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_14(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("x11/qt5-qtbase/Makefile.common", + "BUILDLINK_TRANSFORM+=opt:-ldl:${BUILDLINK_LDADD.dl:M*}") + + G.Mk.mklines[0].Check() + + // Note: The :M* modifier is not necessary, since this is not a GNU Configure package. + c.Check(s.Output(), equals, "WARN: x11/qt5-qtbase/Makefile.common:1: Please use ${BUILDLINK_LDADD.dl:Q} instead of ${BUILDLINK_LDADD.dl:M*}.\n") +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_15(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("benchmarks/iozone/Makefile", + "SUBST_MESSAGE.crlf=\tStripping EOL CR in ${REPLACE_PERL}") + + G.Mk.mklines[0].Check() + + c.Check(s.Output(), equals, "") // Don’t suggest ${REPLACE_PERL:Q}. +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_16(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("audio/jack-rack/Makefile", + "# $"+"NetBSD$", + "LADSPA_PLUGIN_PATH?=\t${PREFIX}/lib/ladspa", + "CPPFLAGS+=\t\t-DLADSPA_PATH=\"\\\"${LADSPA_PLUGIN_PATH}\\\"\"") + + G.Mk.Check() + + c.Check(s.Output(), equals, "WARN: audio/jack-rack/Makefile:3: The list variable LADSPA_PLUGIN_PATH should not be embedded in a word.\n") +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_17(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("x11/eterm/Makefile", + "# $"+"NetBSD$", + "DISTFILES=\t${DEFAULT_DISTFILES} ${PIXMAP_FILES}") + + G.Mk.Check() + + c.Check(s.Output(), equals, "") // Don’t warn about missing :Q operators. +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_18(c *check.C) { + s.UseCommandLine(c, "-Wall") + s.RegisterMasterSite("MASTER_SITE_GNOME", "http://ftp.gnome.org/") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("x11/gtk3/Makefile", + "# $"+"NetBSD$", + "MASTER_SITES=\tftp://ftp.gtk.org/${PKGNAME}/ ${MASTER_SITE_GNOME:=subdir/}") + + G.Mk.mklines[1].checkVarassignVaruse() + + c.Check(s.Output(), equals, "") // Don’t warn about missing :Q operators. +} + +func (s *Suite) Test_MkLine_Varuse_Modifier_L(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("x11/xkeyboard-config/Makefile", + "FILES_SUBST+=XKBCOMP_SYMLINK=${${XKBBASE}/xkbcomp:L:Q}") + + G.Mk.mklines[0].Check() + + c.Check(s.Output(), equals, "") // Don’t warn that ${XKBBASE}/xkbcomp is used but not defined. +} + +func (s *Suite) Test_MkLine_Cond_ShellCommand(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("security/openssl/Makefile", + "# $"+"NetBSD$", + ".if ${PKGSRC_COMPILER} == \"gcc\" && ${CC} == \"cc\"", + ".endif") + + G.Mk.Check() + + c.Check(s.Output(), equals, "") // Don’t warn about unknown shell command "cc". +} + +func (s *Suite) disabledTest_MkLine_Pkgmandir(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("chat/ircII/Makefile", + "# $"+"NetBSD$", + "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/man", + "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/${PKGMANDIR}") + + G.Mk.Check() + + c.Check(s.Output(), equals, "WARN: chat/ircII/Makefile:2: Please use ${PKGMANDIR} instead of \"man\".\n") +} + +func (s *Suite) Test_MkLine_Check_Cflags_Backticks(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("chat/pidgin-icb/Makefile", + "# $"+"NetBSD$", + "CFLAGS+=\t`pkg-config pidgin --cflags`") + mkline := G.Mk.mklines[1] + + words, rest := splitIntoMkWords(mkline.Line, mkline.Value()) + + c.Check(words, deepEquals, []string{"`pkg-config pidgin --cflags`"}) + c.Check(rest, equals, "") + + G.Mk.mklines[1].CheckVartype("CFLAGS", opAssignAppend, "`pkg-config pidgin --cflags`", "") + + c.Check(s.Output(), equals, "") // No warning about "`pkg-config" being an unknown CFlag. +} + +func (s *Suite) Test_MkLine_MasterSites_WordPart(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + mklines := s.NewMkLines("geography/viking/Makefile", + "# $"+"NetBSD$", + "MASTER_SITES=\t${MASTER_SITE_SOURCEFORGE:=viking/}${VERSION}/") + + mklines.Check() + + c.Check(s.Output(), equals, "WARN: geography/viking/Makefile:2: "+ + "The list variable MASTER_SITE_SOURCEFORGE should not be embedded in a word.\n") +} + +func (s *Suite) Test_MkLine_ShellCommand_WordPart(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + mklines := s.NewMkLines("x11/lablgtk1/Makefile", + "# $"+"NetBSD$", + "CONFIGURE_ENV+=\tCC=${CC}") + + mklines.Check() + + c.Check(s.Output(), equals, "WARN: x11/lablgtk1/Makefile:2: Please use ${CC:Q} instead of ${CC}.\n") +} + +func (s *Suite) Test_MkLine_shell_varuse_in_backt_dquot(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + mklines := s.NewMkLines("x11/motif/Makefile", + "# $"+"NetBSD$", + "post-patch:", + "\tfiles=`${GREP} -l \".fB$${name}.fP(3)\" *.3`") + + mklines.Check() + + c.Check(s.Output(), equals, "WARN: x11/motif/Makefile:3: Unknown shell command \"${GREP}\".\n") // No parse errors. +} diff --git a/pkgtools/pkglint/files/mklines.go b/pkgtools/pkglint/files/mklines.go index eda58e239a1..0ea785ea4b9 100644 --- a/pkgtools/pkglint/files/mklines.go +++ b/pkgtools/pkglint/files/mklines.go @@ -7,16 +7,16 @@ import ( // MkLines contains data for the Makefile (or *.mk) that is currently checked. type MkLines struct { - mklines []*MkLine - lines []*Line - forVars map[string]bool // The variables currently used in .for loops - indentation []int // Indentation depth of preprocessing directives - target string // Current make(1) target - vardef map[string]*MkLine // varname => line; for all variables that are defined in the current file - varuse map[string]*MkLine // varname => line; for all variables that are used in the current file - buildDefs map[string]bool // Variables that are registered in BUILD_DEFS, to ensure that all user-defined variables are added to it. - plistVars map[string]bool // Variables that are registered in PLIST_VARS, to ensure that all user-defined variables are added to it. - tools map[string]bool // Set of tools that are declared to be used. + mklines []*MkLine + lines []*Line + forVars map[string]bool // The variables currently used in .for loops + target string // Current make(1) target + vardef map[string]*MkLine // varname => line; for all variables that are defined in the current file + varuse map[string]*MkLine // varname => line; for all variables that are used in the current file + buildDefs map[string]bool // Variables that are registered in BUILD_DEFS, to ensure that all user-defined variables are added to it. + plistVars map[string]bool // Variables that are registered in PLIST_VARS, to ensure that all user-defined variables are added to it. + tools map[string]bool // Set of tools that are declared to be used. + SeenBsdPrefsMk bool } func NewMkLines(lines []*Line) *MkLines { @@ -25,31 +25,23 @@ func NewMkLines(lines []*Line) *MkLines { mklines[i] = NewMkLine(line) } tools := make(map[string]bool) - for tool := range G.globalData.PredefinedTools { - tools[tool] = true + for toolname, tool := range G.globalData.Tools.byName { + if tool.Predefined { + tools[toolname] = true + } } return &MkLines{ mklines, lines, make(map[string]bool), - make([]int, 1), "", make(map[string]*MkLine), make(map[string]*MkLine), make(map[string]bool), make(map[string]bool), - tools} -} - -func (mklines *MkLines) indentDepth() int { - return mklines.indentation[len(mklines.indentation)-1] -} -func (mklines *MkLines) popIndent() { - mklines.indentation = mklines.indentation[:len(mklines.indentation)-1] -} -func (mklines *MkLines) pushIndent(indent int) { - mklines.indentation = append(mklines.indentation, indent) + tools, + false} } func (mklines *MkLines) DefineVar(mkline *MkLine, varname string) { @@ -80,7 +72,7 @@ func (mklines *MkLines) VarValue(varname string) (value string, found bool) { } func (mklines *MkLines) Check() { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(mklines.lines[0].Fname)() } @@ -107,9 +99,9 @@ func (mklines *MkLines) Check() { var substcontext SubstContext var varalign VaralignBlock + indentation := Indentation{[]int{0}} // Indentation depth of preprocessing directives for _, mkline := range mklines.mklines { - mkline.Line.CheckTrailingWhitespace() - mkline.Line.CheckValidCharacters(`[\t -~]`) + mkline.Check() varalign.Check(mkline) switch { @@ -118,23 +110,21 @@ func (mklines *MkLines) Check() { case mkline.IsVarassign(): mklines.target = "" - mkline.CheckVarassign() substcontext.Varassign(mkline) - case mkline.IsShellcmd(): - shellcmd := mkline.Shellcmd() - mkline.CheckText(shellcmd) - NewShellLine(mkline).CheckShellCommandLine(shellcmd) - case mkline.IsInclude(): mklines.target = "" - mklines.checklineInclude(mkline) + switch path.Base(mkline.Includefile()) { + case "bsd.prefs.mk", "bsd.fast.prefs.mk", "bsd.builtin.mk": + mklines.setSeenBsdPrefsMk() + } case mkline.IsCond(): - mklines.checklineCond(mkline) + mkline.checkCond(&indentation, mklines.forVars) case mkline.IsDependency(): - mklines.checklineDependencyRule(mkline, mkline.Targets(), mkline.Sources(), allowedTargets) + mkline.checkDependencyRule(allowedTargets) + mklines.target = mkline.Targets() } } lastMkline := mklines.mklines[len(mklines.mklines)-1] @@ -143,14 +133,18 @@ func (mklines *MkLines) Check() { ChecklinesTrailingEmptyLines(mklines.lines) - if len(mklines.indentation) != 1 && mklines.indentDepth() != 0 { - lastMkline.Line.Errorf("Directive indentation is not 0, but %d.", mklines.indentDepth()) + if indentation.Len() != 1 && indentation.Depth() != 0 { + lastMkline.Line.Errorf("Directive indentation is not 0, but %d.", indentation.Depth()) } SaveAutofixChanges(mklines.lines) } func (mklines *MkLines) determineDefinedVariables() { + if G.opts.Debug { + defer tracecall0()() + } + for _, mkline := range mklines.mklines { if !mkline.IsVarassign() { continue @@ -161,16 +155,16 @@ func (mklines *MkLines) determineDefinedVariables() { case "BUILD_DEFS", "PKG_GROUPS_VARS", "PKG_USERS_VARS": for _, varname := range splitOnSpace(mkline.Value()) { mklines.buildDefs[varname] = true - if G.opts.DebugMisc { - mkline.Debug1("%q is added to BUILD_DEFS.", varname) + if G.opts.Debug { + traceStep1("%q is added to BUILD_DEFS.", varname) } } case "PLIST_VARS": for _, id := range splitOnSpace(mkline.Value()) { mklines.plistVars["PLIST."+id] = true - if G.opts.DebugMisc { - mkline.Debug1("PLIST.%s is added to PLIST_VARS.", id) + if G.opts.Debug { + traceStep1("PLIST.%s is added to PLIST_VARS.", id) } mklines.UseVar(mkline, "PLIST."+id) } @@ -186,16 +180,16 @@ func (mklines *MkLines) determineDefinedVariables() { for _, tool := range splitOnSpace(tools) { tool = strings.Split(tool, ":")[0] mklines.tools[tool] = true - if G.opts.DebugMisc { - mkline.Debug1("%s is added to USE_TOOLS.", tool) + if G.opts.Debug { + traceStep1("%s is added to USE_TOOLS.", tool) } } case "SUBST_VARS.*": for _, svar := range splitOnSpace(mkline.Value()) { mklines.UseVar(mkline, varnameCanon(svar)) - if G.opts.DebugMisc { - mkline.Debug1("varuse %s", svar) + if G.opts.Debug { + traceStep1("varuse %s", svar) } } @@ -217,174 +211,13 @@ func (mklines *MkLines) DetermineUsedVariables() { } } -func (mklines *MkLines) checklineCond(mkline *MkLine) { - indent, directive, args := mkline.Indent(), mkline.Directive(), mkline.Args() - - switch directive { - case "endif", "endfor", "elif", "else": - if len(mklines.indentation) > 1 { - mklines.popIndent() - } else { - mkline.Error1("Unmatched .%s.", directive) - } - } - - // Check the indentation - if expected := strings.Repeat(" ", mklines.indentDepth()); indent != expected { - if G.opts.WarnSpace && !mkline.Line.AutofixReplace("."+indent, "."+expected) { - mkline.Line.Notef("This directive should be indented by %d spaces.", mklines.indentDepth()) - } - } - - if directive == "if" && matches(args, `^!defined\([\w]+_MK\)$`) { - mklines.pushIndent(mklines.indentDepth()) - - } else if matches(directive, `^(?:if|ifdef|ifndef|for|elif|else)$`) { - mklines.pushIndent(mklines.indentDepth() + 2) - } - - reDirectivesWithArgs := `^(?:if|ifdef|ifndef|elif|for|undef)$` - if matches(directive, reDirectivesWithArgs) && args == "" { - mkline.Error1("\".%s\" requires arguments.", directive) - - } else if !matches(directive, reDirectivesWithArgs) && args != "" { - mkline.Error1("\".%s\" does not take arguments.", directive) - - if directive == "else" { - mkline.Note0("If you meant \"else if\", use \".elif\".") - } - - } else if directive == "if" || directive == "elif" { - mkline.CheckCond() - - } else if directive == "ifdef" || directive == "ifndef" { - if matches(args, `\s`) { - mkline.Error1("The \".%s\" directive can only handle _one_ argument.", directive) - } else { - mkline.Line.Warnf("The \".%s\" directive is deprecated. Please use \".if %sdefined(%s)\" instead.", - directive, ifelseStr(directive == "ifdef", "", "!"), args) - } - - } else if directive == "for" { - if m, vars, values := match2(args, `^(\S+(?:\s*\S+)*?)\s+in\s+(.*)$`); m { - for _, forvar := range splitOnSpace(vars) { - if !G.Infrastructure && hasPrefix(forvar, "_") { - mkline.Warn1("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", forvar) - } - - if matches(forvar, `^[_a-z][_a-z0-9]*$`) { - // Fine. - } else if matches(forvar, `[A-Z]`) { - mkline.Warn0(".for variable names should not contain uppercase letters.") - } else { - mkline.Error1("Invalid variable name %q.", forvar) - } - - mklines.forVars[forvar] = true - } - - // Check if any of the value's types is not guessed. - guessed := true - for _, value := range splitOnSpace(values) { - if m, vname := match1(value, `^\$\{(.*)\}`); m { - vartype := mkline.getVariableType(vname) - if vartype != nil && !vartype.guessed { - guessed = false - } - } - } - - forLoopType := &Vartype{lkSpace, CheckvarUnchecked, []AclEntry{{"*", aclpAllRead}}, guessed} - forLoopContext := &VarUseContext{forLoopType, vucTimeParse, vucQuotFor, vucExtentWord} - for _, forLoopVar := range mkline.extractUsedVariables(values) { - mkline.CheckVaruse(forLoopVar, "", forLoopContext) - } - } - - } else if directive == "undef" && args != "" { - for _, uvar := range splitOnSpace(args) { - if mklines.forVars[uvar] { - mkline.Note0("Using \".undef\" after a \".for\" loop is unnecessary.") - } - } - } -} - -func (mklines *MkLines) checklineDependencyRule(mkline *MkLine, targets, dependencies string, allowedTargets map[string]bool) { - if G.opts.DebugMisc { - mkline.Debug2("targets=%q, dependencies=%q", targets, dependencies) - } - mklines.target = targets - - for _, source := range splitOnSpace(dependencies) { - if source == ".PHONY" { - for _, target := range splitOnSpace(targets) { - allowedTargets[target] = true - } +func (mklines *MkLines) setSeenBsdPrefsMk() { + if !mklines.SeenBsdPrefsMk { + mklines.SeenBsdPrefsMk = true + if G.opts.Debug { + traceStep("Mk.setSeenBsdPrefsMk") } } - - for _, target := range splitOnSpace(targets) { - if target == ".PHONY" { - for _, dep := range splitOnSpace(dependencies) { - allowedTargets[dep] = true - } - - } else if target == ".ORDER" { - // TODO: Check for spelling mistakes. - - } else if !allowedTargets[target] { - mkline.Warn1("Unusual target %q.", target) - Explain3( - "If you want to define your own targets, you can \"declare\"", - "them by inserting a \".PHONY: my-target\" line before this line. This", - "will tell make(1) to not interpret this target's name as a filename.") - } - } -} - -func (mklines *MkLines) checklineInclude(mkline *MkLine) { - includefile := mkline.Includefile() - mustExist := mkline.MustExist() - if G.opts.DebugInclude { - mkline.Debug1("includefile=%s", includefile) - } - mkline.CheckRelativePath(includefile, mustExist) - - if hasSuffix(includefile, "/Makefile") { - mkline.Line.Error0("Other Makefiles must not be included directly.") - Explain4( - "If you want to include portions of another Makefile, extract", - "the common parts and put them into a Makefile.common. After", - "that, both this one and the other package should include the", - "Makefile.common.") - } - - if includefile == "../../mk/bsd.prefs.mk" { - if path.Base(mkline.Line.Fname) == "buildlink3.mk" { - mkline.Note0("For efficiency reasons, please include bsd.fast.prefs.mk instead of bsd.prefs.mk.") - } - if G.Pkg != nil { - G.Pkg.SeenBsdPrefsMk = true - } - } else if includefile == "../../mk/bsd.fast.prefs.mk" { - if G.Pkg != nil { - G.Pkg.SeenBsdPrefsMk = true - } - } - - if matches(includefile, `/x11-links/buildlink3\.mk$`) { - mkline.Error1("%s must not be included directly. Include \"../../mk/x11.buildlink3.mk\" instead.", includefile) - } - if matches(includefile, `/jpeg/buildlink3\.mk$`) { - mkline.Error1("%s must not be included directly. Include \"../../mk/jpeg.buildlink3.mk\" instead.", includefile) - } - if matches(includefile, `/intltool/buildlink3\.mk$`) { - mkline.Warn0("Please write \"USE_TOOLS+= intltool\" instead of this line.") - } - if m, dir := match1(includefile, `(.*)/builtin\.mk$`); m { - mkline.Line.Error2("%s must not be included directly. Include \"%s/buildlink3.mk\" instead.", includefile, dir) - } } type VaralignBlock struct { diff --git a/pkgtools/pkglint/files/mklines_test.go b/pkgtools/pkglint/files/mklines_test.go index 1efba51ba8d..750a45de08e 100644 --- a/pkgtools/pkglint/files/mklines_test.go +++ b/pkgtools/pkglint/files/mklines_test.go @@ -57,3 +57,190 @@ func (s *Suite) TestMkLines_checklineInclude_Makefile(c *check.C) { "ERROR: Makefile:2: \"/other/package/Makefile\" does not exist.\n"+ "ERROR: Makefile:2: Other Makefiles must not be included directly.\n") } + +func (s *Suite) TestMkLines_Quoting(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Pkg = NewPackage("category/pkgbase") + mklines := s.NewMkLines("Makefile", + "# $"+"NetBSD$", + "GNU_CONFIGURE=\tyes", + "CONFIGURE_ENV+=\tX_LIBS=${X11_LDFLAGS:Q}") + + mklines.Check() + + c.Check(s.Output(), equals, "WARN: Makefile:3: Please use ${X11_LDFLAGS:M*:Q} instead of ${X11_LDFLAGS:Q}.\n") +} + +func (s *Suite) Test_MkLines_Varalign_Advanced(c *check.C) { + s.UseCommandLine(c, "-Wspace") + fname := s.CreateTmpFileLines(c, "Makefile", + "# $"+"NetBSD$", + "", + "VAR= \\", // In continuation lines, indenting with spaces is ok + "\tvalue", + "", + "VAR= indented with one space", // Exactly one space is ok in general + "VAR= indented with two spaces", // Two spaces are uncommon + "", + "BLOCK=\tindented with tab", + "BLOCK_LONGVAR= indented with space", // This is ok, to prevent the block from being indented further + "", + "BLOCK=\tshort", + "BLOCK_LONGVAR=\tlong", + "", + "GRP_A= avalue", // The values in a block should be aligned + "GRP_AA= value", + "GRP_AAA= value", + "GRP_AAAA= value", + "", + "VAR=\t${VAR}${BLOCK}${BLOCK_LONGVAR} # suppress warnings about unused variables", + "VAR=\t${GRP_A}${GRP_AA}${GRP_AAA}${GRP_AAAA}") + mklines := NewMkLines(LoadExistingLines(fname, true)) + + mklines.Check() + + c.Check(s.Output(), equals, ""+ + "NOTE: ~/Makefile:6: This variable value should be aligned with tabs, not spaces, to column 9.\n"+ + "NOTE: ~/Makefile:7: This variable value should be aligned with tabs, not spaces, to column 9.\n"+ + "NOTE: ~/Makefile:12: This variable value should be aligned to column 17.\n"+ + "NOTE: ~/Makefile:15: This variable value should be aligned with tabs, not spaces, to column 17.\n"+ + "NOTE: ~/Makefile:16: This variable value should be aligned with tabs, not spaces, to column 17.\n"+ + "NOTE: ~/Makefile:17: This variable value should be aligned with tabs, not spaces, to column 17.\n"+ + "NOTE: ~/Makefile:18: This variable value should be aligned with tabs, not spaces, to column 17.\n") + + s.UseCommandLine(c, "-Wspace", "--autofix") + + mklines.Check() + + c.Check(s.Output(), equals, ""+ + "AUTOFIX: ~/Makefile:6: Replacing \"VAR= \" with \"VAR=\\t\".\n"+ + "AUTOFIX: ~/Makefile:7: Replacing \"VAR= \" with \"VAR=\\t\".\n"+ + "AUTOFIX: ~/Makefile:12: Replacing \"BLOCK=\\t\" with \"BLOCK=\\t\\t\".\n"+ + "AUTOFIX: ~/Makefile:15: Replacing \"GRP_A= \" with \"GRP_A=\\t\\t\".\n"+ + "AUTOFIX: ~/Makefile:16: Replacing \"GRP_AA= \" with \"GRP_AA=\\t\\t\".\n"+ + "AUTOFIX: ~/Makefile:17: Replacing \"GRP_AAA= \" with \"GRP_AAA=\\t\".\n"+ + "AUTOFIX: ~/Makefile:18: Replacing \"GRP_AAAA= \" with \"GRP_AAAA=\\t\".\n"+ + "AUTOFIX: ~/Makefile: Has been auto-fixed. Please re-run pkglint.\n") + c.Check(s.LoadTmpFile(c, "Makefile"), equals, ""+ + "# $"+"NetBSD$\n"+ + "\n"+ + "VAR= \\\n"+ + "\tvalue\n"+ + "\n"+ + "VAR=\tindented with one space\n"+ + "VAR=\tindented with two spaces\n"+ + "\n"+ + "BLOCK=\tindented with tab\n"+ + "BLOCK_LONGVAR= indented with space\n"+ + "\n"+ + "BLOCK=\t\tshort\n"+ + "BLOCK_LONGVAR=\tlong\n"+ + "\n"+ + "GRP_A=\t\tavalue\n"+ + "GRP_AA=\t\tvalue\n"+ + "GRP_AAA=\tvalue\n"+ + "GRP_AAAA=\tvalue\n"+ + "\n"+ + "VAR=\t${VAR}${BLOCK}${BLOCK_LONGVAR} # suppress warnings about unused variables\n"+ + "VAR=\t${GRP_A}${GRP_AA}${GRP_AAA}${GRP_AAAA}\n") +} + +func (s *Suite) Test_MkLines_Varalign_Misc(c *check.C) { + s.UseCommandLine(c, "-Wspace") + mklines := s.NewMkLines("Makefile", + "# $"+"NetBSD$", + "", + "VAR= space", + "VAR=\ttab ${VAR}") + + mklines.Check() + + c.Check(s.Output(), equals, "NOTE: Makefile:3: Variable values should be aligned with tabs, not spaces.\n") +} + +func (s *Suite) Test_MkLines_ForLoop_Multivar(c *check.C) { + s.UseCommandLine(c, "-Wall") + s.RegisterTool(&Tool{Name: "echo", Varname: "ECHO", Predefined: true}) + s.RegisterTool(&Tool{Name: "find", Varname: "FIND", Predefined: true}) + s.RegisterTool(&Tool{Name: "pax", Varname: "PAX", Predefined: true}) + mklines := s.NewMkLines("audio/squeezeboxserver/Makefile", + "# $"+"NetBSD$", + "", + ".for _list_ _dir_ in ${SBS_COPY}", + "\tcd ${WRKSRC} && ${FIND} ${${_list_}} -type f ! -name '*.orig' 2>/dev/null "+ + "| pax -rw -pm ${DESTDIR}${PREFIX}/${${_dir_}}", + ".endfor") + + mklines.Check() + + c.Check(s.Output(), equals, ""+ + "WARN: audio/squeezeboxserver/Makefile:3: Variable names starting with an underscore (_list_) are reserved for internal pkgsrc use.\n"+ + "WARN: audio/squeezeboxserver/Makefile:3: Variable names starting with an underscore (_dir_) are reserved for internal pkgsrc use.\n"+ + "WARN: audio/squeezeboxserver/Makefile:4: The exitcode of the left-hand-side command of the pipe operator is ignored.\n") +} + +func (s *Suite) Test_MkLines_Cond_Compare_YesNo(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + mklines := s.NewMkLines("databases/gdbm_compat/builtin.mk", + "# $"+"NetBSD$", + ".if ${USE_BUILTIN.gdbm} == \"no\"", + ".endif") + + mklines.Check() + + c.Check(s.Output(), equals, "WARN: databases/gdbm_compat/builtin.mk:2: "+ + "USE_BUILTIN.gdbm should be matched against \"[yY][eE][sS]\" or \"[nN][oO]\", not compared with \"no\".\n") +} + +func (s *Suite) Test_MkLines_Varuse_sh_Modifier(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + mklines := s.NewMkLines("lang/qore/module.mk", + "# $"+"NetBSD$", + "qore-version=\tqore --short-version | ${SED} -e s/-.*//", + "PLIST_SUBST+=\tQORE_VERSION=\"${qore-version:sh}\"") + + vars2 := mklines.mklines[1].determineUsedVariables() + + c.Check(vars2, deepEquals, []string{"SED"}) + + vars3 := mklines.mklines[2].determineUsedVariables() + + c.Check(vars3, deepEquals, []string{"qore-version"}) + + mklines.Check() + + c.Check(s.Output(), equals, "") // No warnings about defined but not used or vice versa +} + +func (s *Suite) Test_MkLines_Varuse_parameterized(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + mklines := s.NewMkLines("converters/wv2/Makefile", + "# $"+"NetBSD$", + "CONFIGURE_ARGS+=\t\t${CONFIGURE_ARGS.${ICONV_TYPE}-iconv}", + "CONFIGURE_ARGS.gnu-iconv=\t--with-libiconv=${BUILDLINK_PREFIX.iconv}") + + mklines.Check() + + c.Check(s.Output(), equals, "") // No warnings about defined but not used or vice versa +} + +func (s *Suite) Test_MkLines_LoopModifier(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + mklines := s.NewMkLines("chat/xchat/Makefile", + "# $"+"NetBSD$", + "GCONF_SCHEMAS=\tapps_xchat_url_handler.schemas", + "post-install:", + "\t${GCONF_SCHEMAS:@.s.@"+ + "${INSTALL_DATA} ${WRKSRC}/src/common/dbus/${.s.} ${DESTDIR}${GCONF_SCHEMAS_DIR}/@}") + + mklines.Check() + + c.Check(s.Output(), equals, ""+ // No warning about missing @ at the end + "WARN: chat/xchat/Makefile:4: Unknown shell command \"${GCONF_SCHEMAS:@.s.@"+ + "${INSTALL_DATA} ${WRKSRC}/src/common/dbus/${.s.} ${DESTDIR}${GCONF_SCHEMAS_DIR}/@}\".\n") +} diff --git a/pkgtools/pkglint/files/mkparser.go b/pkgtools/pkglint/files/mkparser.go new file mode 100644 index 00000000000..78ff5a74aac --- /dev/null +++ b/pkgtools/pkglint/files/mkparser.go @@ -0,0 +1,346 @@ +package main + +import ( + "strings" +) + +type MkParser struct { + *Parser +} + +func NewMkParser(line *Line, text string, emitWarnings bool) *MkParser { + return &MkParser{NewParser(line, text, emitWarnings)} +} + +func (p *MkParser) MkTokens() []*MkToken { + repl := p.repl + + var tokens []*MkToken + for !p.EOF() { + if repl.AdvanceStr("#") { + repl.AdvanceRest() + } + + mark := repl.Mark() + if varuse := p.VarUse(); varuse != nil { + tokens = append(tokens, &MkToken{Text: repl.Since(mark), Varuse: varuse}) + continue + } + + needsReplace := false + again: + dollar := strings.IndexByte(repl.rest, '$') + if dollar == -1 { + dollar = len(repl.rest) + } + repl.Skip(dollar) + if repl.AdvanceStr("$$") { + needsReplace = true + goto again + } + text := repl.Since(mark) + if needsReplace { + text = strings.Replace(text, "$$", "$", -1) + } + if text != "" { + tokens = append(tokens, &MkToken{Text: text}) + continue + } + + break + } + return tokens +} + +func (p *MkParser) VarUse() *MkVarUse { + repl := p.repl + + mark := repl.Mark() + if repl.AdvanceStr("${") || repl.AdvanceStr("$(") { + usingRoundParen := repl.Since(mark) == "$(" + closing := ifelseStr(usingRoundParen, ")", "}") + + varnameMark := repl.Mark() + varname := p.Varname() + if varname != "" { + if usingRoundParen && p.EmitWarnings { + p.Line.Warn1("Please use curly braces {} instead of round parentheses () for %s.", varname) + } + modifiers := p.VarUseModifiers(varname, closing) + if repl.AdvanceStr(closing) { + return &MkVarUse{varname, modifiers} + } + } + + for p.VarUse() != nil || repl.AdvanceRegexp(`^([^$:`+closing+`]|\$\$)+`) { + } + rest := p.Rest() + if hasPrefix(rest, ":L") || hasPrefix(rest, ":?") { + varexpr := repl.Since(varnameMark) + modifiers := p.VarUseModifiers(varexpr, closing) + if repl.AdvanceStr(closing) { + return &MkVarUse{varexpr, modifiers} + } + } + repl.Reset(mark) + } + + if repl.AdvanceStr("$@") { + return &MkVarUse{"@", nil} + } + if repl.AdvanceStr("$<") { + return &MkVarUse{"<", nil} + } + if repl.AdvanceRegexp(`^\$(\w)`) { + varname := repl.m[1] + if p.EmitWarnings { + p.Line.Warn1("$%[1]s is ambiguous. Use ${%[1]s} if you mean a Makefile variable or $$%[1]s if you mean a shell variable.", varname) + } + return &MkVarUse{varname, nil} + } + return nil +} + +func (p *MkParser) VarUseModifiers(varname, closing string) []string { + repl := p.repl + + var modifiers []string + mayOmitColon := false + for repl.AdvanceStr(":") || mayOmitColon { + mayOmitColon = false + modifierMark := repl.Mark() + + switch repl.PeekByte() { + case 'E', 'H', 'L', 'O', 'Q', 'R', 'T', 's', 't', 'u': + if repl.AdvanceRegexp(`^(E|H|L|Ox?|Q|R|T|sh|tA|tW|tl|tu|tw|u)`) { + modifiers = append(modifiers, repl.Since(modifierMark)) + continue + } + if repl.AdvanceStr("ts") { + rest := repl.rest + if len(rest) >= 2 && (rest[1] == closing[0] || rest[1] == ':') { + repl.Skip(1) + } else if len(rest) >= 1 && (rest[0] == closing[0] || rest[0] == ':') { + } else if repl.AdvanceRegexp(`^\\\d+`) { + } else { + break + } + modifiers = append(modifiers, repl.Since(modifierMark)) + continue + } + + case '=', 'D', 'M', 'N', 'U': + if repl.AdvanceRegexp(`^[=DMNU]`) { + for p.VarUse() != nil || repl.AdvanceRegexp(`^([^$:`+closing+`]|\$\$)+`) { + } + modifiers = append(modifiers, repl.Since(modifierMark)) + continue + } + + case 'C', 'S': + if repl.AdvanceRegexp(`^[CS]([%,/:;@^|])`) { + separator := repl.m[1] + repl.AdvanceStr("^") + re := `^([^\` + separator + `$` + closing + `\\]|\$\$|\\.)+` + for p.VarUse() != nil || repl.AdvanceRegexp(re) { + } + repl.AdvanceStr("$") + if repl.AdvanceStr(separator) { + for p.VarUse() != nil || repl.AdvanceRegexp(re) { + } + if repl.AdvanceStr(separator) { + repl.AdvanceRegexp(`^[1gW]`) + modifiers = append(modifiers, repl.Since(modifierMark)) + mayOmitColon = true + continue + } + } + } + + case '@': + if repl.AdvanceRegexp(`^@([\w.]+)@`) { + loopvar := repl.m[1] + for p.VarUse() != nil || repl.AdvanceRegexp(`^([^$:@`+closing+`\\]|\$\$|\\.)+`) { + } + if !repl.AdvanceStr("@") && p.EmitWarnings { + p.Line.Warn2("Modifier ${%s:@%s@...@} is missing the final \"@\".", varname, loopvar) + } + modifiers = append(modifiers, repl.Since(modifierMark)) + continue + } + + case '[': + if repl.AdvanceRegexp(`^\[[-.\d]+\]`) { + modifiers = append(modifiers, repl.Since(modifierMark)) + continue + } + + case '?': + repl.AdvanceStr("?") + re := `^([^$:` + closing + `]|\$\$)+` + for p.VarUse() != nil || repl.AdvanceRegexp(re) { + } + if repl.AdvanceStr(":") { + for p.VarUse() != nil || repl.AdvanceRegexp(re) { + } + modifiers = append(modifiers, repl.Since(modifierMark)) + continue + } + } + + repl.Reset(modifierMark) + for p.VarUse() != nil || repl.AdvanceRegexp(`^([^:$`+closing+`]|\$\$)+`) { + } + if suffixSubst := repl.Since(modifierMark); contains(suffixSubst, "=") { + modifiers = append(modifiers, suffixSubst) + continue + } + } + return modifiers +} + +func (p *MkParser) MkCond() *Tree { + return p.mkCondOr() +} + +func (p *MkParser) mkCondOr() *Tree { + and := p.mkCondAnd() + if and == nil { + return nil + } + + ands := append([]interface{}(nil), and) + for { + mark := p.repl.Mark() + if !p.repl.AdvanceRegexp(`^\s*\|\|\s*`) { + break + } + next := p.mkCondAnd() + if next == nil { + p.repl.Reset(mark) + break + } + ands = append(ands, next) + } + if len(ands) == 1 { + return and + } + return NewTree("or", ands...) +} + +func (p *MkParser) mkCondAnd() *Tree { + atom := p.mkCondAtom() + if atom == nil { + return nil + } + + atoms := append([]interface{}(nil), atom) + for { + mark := p.repl.Mark() + if !p.repl.AdvanceRegexp(`^\s*&&\s*`) { + break + } + next := p.mkCondAtom() + if next == nil { + p.repl.Reset(mark) + break + } + atoms = append(atoms, next) + } + if len(atoms) == 1 { + return atom + } + return NewTree("and", atoms...) +} + +func (p *MkParser) mkCondAtom() *Tree { + if G.opts.Debug { + defer tracecall1(p.Rest())() + } + + repl := p.repl + mark := repl.Mark() + repl.SkipSpace() + switch { + case repl.AdvanceStr("!"): + cond := p.mkCondAtom() + if cond != nil { + return NewTree("not", cond) + } + case repl.AdvanceStr("("): + cond := p.MkCond() + if cond != nil { + repl.SkipSpace() + if repl.AdvanceStr(")") { + return cond + } + } + case repl.AdvanceRegexp(`^defined\s*\(`): + if varname := p.Varname(); varname != "" { + if repl.AdvanceStr(")") { + return NewTree("defined", varname) + } + } + case repl.AdvanceRegexp(`^empty\s*\(`): + if varname := p.Varname(); varname != "" { + modifiers := p.VarUseModifiers(varname, ")") + if repl.AdvanceStr(")") { + return NewTree("empty", MkVarUse{varname, modifiers}) + } + } + case repl.AdvanceRegexp(`^(commands|exists|make|target)\s*\(`): + funcname := repl.m[1] + argMark := repl.Mark() + for p.VarUse() != nil || repl.AdvanceRegexp(`^[^$)]+`) { + } + arg := repl.Since(argMark) + if repl.AdvanceStr(")") { + return NewTree(funcname, arg) + } + default: + lhs := p.VarUse() + mark := repl.Mark() + if lhs == nil && repl.AdvanceStr("\"") { + if quotedLHS := p.VarUse(); quotedLHS != nil && repl.AdvanceStr("\"") { + lhs = quotedLHS + } else { + repl.Reset(mark) + } + } + if lhs != nil { + if repl.AdvanceRegexp(`^\s*(<|<=|==|!=|>=|>)\s*(\d+(?:\.\d+)?)`) { + return NewTree("compareVarNum", *lhs, repl.m[1], repl.m[2]) + } + if repl.AdvanceRegexp(`^\s*(<|<=|==|!=|>=|>)\s*`) { + op := repl.m[1] + if (op == "!=" || op == "==") && repl.AdvanceRegexp(`^"([^"\$\\]*)"`) { + return NewTree("compareVarStr", *lhs, op, repl.m[1]) + } else if repl.AdvanceRegexp(`^\w+`) { + return NewTree("compareVarStr", *lhs, op, repl.m[0]) + } else if rhs := p.VarUse(); rhs != nil { + return NewTree("compareVarVar", *lhs, op, *rhs) + } + } else { + return NewTree("not", NewTree("empty", *lhs)) // See devel/bmake/files/cond.c:/\* For \.if \$/ + } + } + if repl.AdvanceRegexp(`^\d+(?:\.\d+)?`) { + return NewTree("literalNum", repl.m[0]) + } + } + repl.Reset(mark) + return nil +} + +func (p *MkParser) Varname() string { + repl := p.repl + + mark := repl.Mark() + repl.AdvanceStr(".") + isVarnameChar := func(c byte) bool { + return 'A' <= c && c <= 'Z' || c == '_' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '+' || c == '-' || c == '.' || c == '*' + } + for p.VarUse() != nil || repl.AdvanceBytesFunc(isVarnameChar) { + } + return repl.Since(mark) +} diff --git a/pkgtools/pkglint/files/mkparser_test.go b/pkgtools/pkglint/files/mkparser_test.go new file mode 100644 index 00000000000..eb45759e56a --- /dev/null +++ b/pkgtools/pkglint/files/mkparser_test.go @@ -0,0 +1,207 @@ +package main + +import ( + check "gopkg.in/check.v1" +) + +func (s *Suite) Test_MkParser_MkTokens(c *check.C) { + checkRest := func(input string, expectedTokens []*MkToken, expectedRest string) { + p := NewMkParser(dummyLine, input, true) + actualTokens := p.MkTokens() + c.Check(actualTokens, deepEquals, expectedTokens) + for i, expectedToken := range expectedTokens { + if i < len(actualTokens) { + c.Check(*actualTokens[i], deepEquals, *expectedToken) + } + } + c.Check(p.Rest(), equals, expectedRest) + } + check := func(input string, expectedToken *MkToken) { + checkRest(input, []*MkToken{expectedToken}, "") + } + literal := func(text string) *MkToken { + return &MkToken{Text: text} + } + varuse := func(varname string, modifiers ...string) *MkToken { + text := "${" + varname + for _, modifier := range modifiers { + text += ":" + modifier + } + text += "}" + return &MkToken{Text: text, Varuse: &MkVarUse{varname: varname, modifiers: modifiers}} + } + varuseText := func(text, varname string, modifiers ...string) *MkToken { + return &MkToken{Text: text, Varuse: &MkVarUse{varname: varname, modifiers: modifiers}} + } + + check("literal", literal("literal")) + check("\\/share\\/ { print \"share directory\" }", literal("\\/share\\/ { print \"share directory\" }")) + check("find . -name \\*.orig -o -name \\*.pre", literal("find . -name \\*.orig -o -name \\*.pre")) + check("-e 's|\\$${EC2_HOME.*}|EC2_HOME}|g'", literal("-e 's|\\${EC2_HOME.*}|EC2_HOME}|g'")) + + check("${VARIABLE}", varuse("VARIABLE")) + check("${VARIABLE.param}", varuse("VARIABLE.param")) + check("${VARIABLE.${param}}", varuse("VARIABLE.${param}")) + check("${VARIABLE.hicolor-icon-theme}", varuse("VARIABLE.hicolor-icon-theme")) + check("${VARIABLE.gtk+extra}", varuse("VARIABLE.gtk+extra")) + check("${VARIABLE:S/old/new/}", varuse("VARIABLE", "S/old/new/")) + check("${GNUSTEP_LFLAGS:S/-L//g}", varuse("GNUSTEP_LFLAGS", "S/-L//g")) + check("${SUSE_VERSION:S/.//}", varuse("SUSE_VERSION", "S/.//")) + check("${MASTER_SITE_GNOME:=sources/alacarte/0.13/}", varuse("MASTER_SITE_GNOME", "=sources/alacarte/0.13/")) + check("${INCLUDE_DIRS:H:T}", varuse("INCLUDE_DIRS", "H", "T")) + check("${A.${B.${C.${D}}}}", varuse("A.${B.${C.${D}}}")) + check("${RUBY_VERSION:C/([0-9]+)\\.([0-9]+)\\.([0-9]+)/\\1/}", varuse("RUBY_VERSION", "C/([0-9]+)\\.([0-9]+)\\.([0-9]+)/\\1/")) + check("${PERL5_${_var_}:Q}", varuse("PERL5_${_var_}", "Q")) + check("${PKGNAME_REQD:C/(^.*-|^)py([0-9][0-9])-.*/\\2/}", varuse("PKGNAME_REQD", "C/(^.*-|^)py([0-9][0-9])-.*/\\2/")) + check("${PYLIB:S|/|\\\\/|g}", varuse("PYLIB", "S|/|\\\\/|g")) + check("${PKGNAME_REQD:C/ruby([0-9][0-9]+)-.*/\\1/}", varuse("PKGNAME_REQD", "C/ruby([0-9][0-9]+)-.*/\\1/")) + check("${RUBY_SHLIBALIAS:S/\\//\\\\\\//}", varuse("RUBY_SHLIBALIAS", "S/\\//\\\\\\//")) + check("${RUBY_VER_MAP.${RUBY_VER}:U${RUBY_VER}}", varuse("RUBY_VER_MAP.${RUBY_VER}", "U${RUBY_VER}")) + check("${RUBY_VER_MAP.${RUBY_VER}:U18}", varuse("RUBY_VER_MAP.${RUBY_VER}", "U18")) + check("${CONFIGURE_ARGS:S/ENABLE_OSS=no/ENABLE_OSS=yes/g}", varuse("CONFIGURE_ARGS", "S/ENABLE_OSS=no/ENABLE_OSS=yes/g")) + check("${PLIST_RUBY_DIRS:S,DIR=\"PREFIX/,DIR=\",}", varuse("PLIST_RUBY_DIRS", "S,DIR=\"PREFIX/,DIR=\",")) + check("${LDFLAGS:S/-Wl,//g:Q}", varuse("LDFLAGS", "S/-Wl,//g", "Q")) + check("${_PERL5_REAL_PACKLIST:S/^/${DESTDIR}/}", varuse("_PERL5_REAL_PACKLIST", "S/^/${DESTDIR}/")) + check("${_PYTHON_VERSION:C/^([0-9])/\\1./1}", varuse("_PYTHON_VERSION", "C/^([0-9])/\\1./1")) + check("${PKGNAME:S/py${_PYTHON_VERSION}/py${i}/}", varuse("PKGNAME", "S/py${_PYTHON_VERSION}/py${i}/")) + check("${PKGNAME:C/-[0-9].*$/-[0-9]*/}", varuse("PKGNAME", "C/-[0-9].*$/-[0-9]*/")) + check("${PKGNAME:S/py${_PYTHON_VERSION}/py${i}/:C/-[0-9].*$/-[0-9]*/}", varuse("PKGNAME", "S/py${_PYTHON_VERSION}/py${i}/", "C/-[0-9].*$/-[0-9]*/")) + check("${_PERL5_VARS:tl:S/^/-V:/}", varuse("_PERL5_VARS", "tl", "S/^/-V:/")) + check("${_PERL5_VARS_OUT:M${_var_:tl}=*:S/^${_var_:tl}=${_PERL5_PREFIX:=/}//}", varuse("_PERL5_VARS_OUT", "M${_var_:tl}=*", "S/^${_var_:tl}=${_PERL5_PREFIX:=/}//")) + check("${RUBY${RUBY_VER}_PATCHLEVEL}", varuse("RUBY${RUBY_VER}_PATCHLEVEL")) + check("${DISTFILES:M*.gem}", varuse("DISTFILES", "M*.gem")) + check("${LOCALBASE:S^/^_^}", varuse("LOCALBASE", "S^/^_^")) + check("${SOURCES:%.c=%.o}", varuse("SOURCES", "%.c=%.o")) + check("${GIT_TEMPLATES:@.t.@ ${EGDIR}/${GIT_TEMPLATEDIR}/${.t.} ${PREFIX}/${GIT_CORE_TEMPLATEDIR}/${.t.} @:M*}", + varuse("GIT_TEMPLATES", "@.t.@ ${EGDIR}/${GIT_TEMPLATEDIR}/${.t.} ${PREFIX}/${GIT_CORE_TEMPLATEDIR}/${.t.} @", "M*")) + check("${DISTNAME:C:_:-:}", varuse("DISTNAME", "C:_:-:")) + check("${CF_FILES:H:O:u:S@^@${PKG_SYSCONFDIR}/@}", varuse("CF_FILES", "H", "O", "u", "S@^@${PKG_SYSCONFDIR}/@")) + check("${ALT_GCC_RTS:S%${LOCALBASE}%%:S%/%%}", varuse("ALT_GCC_RTS", "S%${LOCALBASE}%%", "S%/%%")) + check("${PREFIX:C;///*;/;g:C;/$;;}", varuse("PREFIX", "C;///*;/;g", "C;/$;;")) + check("${GZIP_CMD:[1]:Q}", varuse("GZIP_CMD", "[1]", "Q")) + check("${DISTNAME:C/-[0-9]+$$//:C/_/-/}", varuse("DISTNAME", "C/-[0-9]+$$//", "C/_/-/")) + check("${DISTNAME:slang%=slang2%}", varuse("DISTNAME", "slang%=slang2%")) + check("${OSMAP_SUBSTVARS:@v@-e 's,\\@${v}\\@,${${v}},g' @}", varuse("OSMAP_SUBSTVARS", "@v@-e 's,\\@${v}\\@,${${v}},g' @")) + check("${BRANDELF:D${BRANDELF} -t Linux ${LINUX_LDCONFIG}:U${TRUE}}", varuse("BRANDELF", "D${BRANDELF} -t Linux ${LINUX_LDCONFIG}", "U${TRUE}")) + check("${${_var_}.*}", varuse("${_var_}.*")) + + check("${GCONF_SCHEMAS:@.s.@${INSTALL_DATA} ${WRKSRC}/src/common/dbus/${.s.} ${DESTDIR}${GCONF_SCHEMAS_DIR}/@}", + varuse("GCONF_SCHEMAS", "@.s.@${INSTALL_DATA} ${WRKSRC}/src/common/dbus/${.s.} ${DESTDIR}${GCONF_SCHEMAS_DIR}/@")) + + /* weird features */ + check("${${EMACS_VERSION_MAJOR}>22:?@comment :}", varuse("${EMACS_VERSION_MAJOR}>22", "?@comment :")) + check("${empty(CFLAGS):?:-cflags ${CFLAGS:Q}}", varuse("empty(CFLAGS)", "?:-cflags ${CFLAGS:Q}")) + check("${${PKGSRC_COMPILER}==gcc:?gcc:cc}", varuse("${PKGSRC_COMPILER}==gcc", "?gcc:cc")) + + check("${${XKBBASE}/xkbcomp:L:Q}", varuse("${XKBBASE}/xkbcomp", "L", "Q")) + check("${${PKGBASE} ${PKGVERSION}:L}", varuse("${PKGBASE} ${PKGVERSION}", "L")) + + check("${${${PKG_INFO} -E ${d} || echo:L:sh}:L:C/[^[0-9]]*/ /g:[1..3]:ts.}", + varuse("${${PKG_INFO} -E ${d} || echo:L:sh}", "L", "C/[^[0-9]]*/ /g", "[1..3]", "ts.")) + + check("${VAR:S/-//S/.//}", varuseText("${VAR:S/-//S/.//}", "VAR", "S/-//", "S/.//")) // For :S and :C, the colon can be left out. + + check("${VAR:ts}", varuse("VAR", "ts")) // The separator character can be left out. + check("${VAR:ts\\000012}", varuse("VAR", "ts\\000012")) // The separator character can be a long octal number. + check("${VAR:ts\\124}", varuse("VAR", "ts\\124")) // Or even decimal. + + check("$(GNUSTEP_USER_ROOT)", varuseText("$(GNUSTEP_USER_ROOT)", "GNUSTEP_USER_ROOT")) + c.Check(s.Output(), equals, "WARN: Please use curly braces {} instead of round parentheses () for GNUSTEP_USER_ROOT.\n") + + checkRest("${VAR)", nil, "${VAR)") // Opening brace, closing parenthesis + checkRest("$(VAR}", nil, "$(VAR}") // Opening parenthesis, closing brace + c.Check(s.Output(), equals, "WARN: Please use curly braces {} instead of round parentheses () for VAR.\n") + + check("${PLIST_SUBST_VARS:@var@${var}=${${var}:Q}@}", varuse("PLIST_SUBST_VARS", "@var@${var}=${${var}:Q}@")) + check("${PLIST_SUBST_VARS:@var@${var}=${${var}:Q}}", varuse("PLIST_SUBST_VARS", "@var@${var}=${${var}:Q}")) // Missing @ at the end + c.Check(s.Output(), equals, "WARN: Modifier ${PLIST_SUBST_VARS:@var@...@} is missing the final \"@\".\n") + + checkRest("hello, ${W:L:tl}orld", []*MkToken{ + literal("hello, "), + varuse("W", "L", "tl"), + literal("orld")}, "") + checkRest("ftp://${PKGNAME}/ ${MASTER_SITES:=subdir/}", []*MkToken{ + literal("ftp://"), + varuse("PKGNAME"), + literal("/ "), + varuse("MASTER_SITES", "=subdir/")}, "") +} + +func (s *Suite) Test_MkParser_MkCond(c *check.C) { + checkRest := func(input string, expectedTree *Tree, expectedRest string) { + p := NewMkParser(dummyLine, input, false) + actualTree := p.MkCond() + c.Check(actualTree, deepEquals, expectedTree) + c.Check(p.Rest(), equals, expectedRest) + } + check := func(input string, expectedTree *Tree) { + checkRest(input, expectedTree, "") + } + varuse := func(varname string, modifiers ...string) MkVarUse { + return MkVarUse{varname: varname, modifiers: modifiers} + } + + check("${OPSYS:MNetBSD}", + NewTree("not", NewTree("empty", varuse("OPSYS", "MNetBSD")))) + check("defined(VARNAME)", + NewTree("defined", "VARNAME")) + check("empty(VARNAME)", + NewTree("empty", varuse("VARNAME"))) + check("!empty(VARNAME)", + NewTree("not", NewTree("empty", varuse("VARNAME")))) + check("!empty(VARNAME:M[yY][eE][sS])", + NewTree("not", NewTree("empty", varuse("VARNAME", "M[yY][eE][sS]")))) + check("${VARNAME} != \"Value\"", + NewTree("compareVarStr", varuse("VARNAME"), "!=", "Value")) + check("${VARNAME:Mi386} != \"Value\"", + NewTree("compareVarStr", varuse("VARNAME", "Mi386"), "!=", "Value")) + check("${VARNAME} != Value", + NewTree("compareVarStr", varuse("VARNAME"), "!=", "Value")) + check("\"${VARNAME}\" != Value", + NewTree("compareVarStr", varuse("VARNAME"), "!=", "Value")) + check("(defined(VARNAME))", + NewTree("defined", "VARNAME")) + check("exists(/etc/hosts)", + NewTree("exists", "/etc/hosts")) + check("exists(${PREFIX}/var)", + NewTree("exists", "${PREFIX}/var")) + check("${OPSYS} == \"NetBSD\" || ${OPSYS} == \"OpenBSD\"", + NewTree("or", + NewTree("compareVarStr", varuse("OPSYS"), "==", "NetBSD"), + NewTree("compareVarStr", varuse("OPSYS"), "==", "OpenBSD"))) + check("${OPSYS} == \"NetBSD\" && ${MACHINE_ARCH} == \"i386\"", + NewTree("and", + NewTree("compareVarStr", varuse("OPSYS"), "==", "NetBSD"), + NewTree("compareVarStr", varuse("MACHINE_ARCH"), "==", "i386"))) + check("defined(A) && defined(B) || defined(C) && defined(D)", + NewTree("or", + NewTree("and", + NewTree("defined", "A"), + NewTree("defined", "B")), + NewTree("and", + NewTree("defined", "C"), + NewTree("defined", "D")))) + check("${MACHINE_ARCH:Mi386} || ${MACHINE_OPSYS:MNetBSD}", + NewTree("or", + NewTree("not", NewTree("empty", varuse("MACHINE_ARCH", "Mi386"))), + NewTree("not", NewTree("empty", varuse("MACHINE_OPSYS", "MNetBSD"))))) + + // Exotic cases + check("0", + NewTree("literalNum", "0")) + check("! ( defined(A) && empty(VARNAME) )", + NewTree("not", NewTree("and", NewTree("defined", "A"), NewTree("empty", varuse("VARNAME"))))) + check("${REQD_MAJOR} > ${MAJOR}", + NewTree("compareVarVar", varuse("REQD_MAJOR"), ">", varuse("MAJOR"))) + check("${OS_VERSION} >= 6.5", + NewTree("compareVarNum", varuse("OS_VERSION"), ">=", "6.5")) + check("${OS_VERSION} == 5.3", + NewTree("compareVarNum", varuse("OS_VERSION"), "==", "5.3")) + check("!empty(${OS_VARIANT:MIllumos})", // Probably not intended + NewTree("not", NewTree("empty", varuse("${OS_VARIANT:MIllumos}")))) + + // Errors + checkRest("!empty(PKG_OPTIONS:Msndfile) || defined(PKG_OPTIONS:Msamplerate)", + NewTree("not", NewTree("empty", varuse("PKG_OPTIONS", "Msndfile"))), + " || defined(PKG_OPTIONS:Msamplerate)") +} diff --git a/pkgtools/pkglint/files/mkshparser.go b/pkgtools/pkglint/files/mkshparser.go new file mode 100644 index 00000000000..2fcf7f79fc5 --- /dev/null +++ b/pkgtools/pkglint/files/mkshparser.go @@ -0,0 +1,655 @@ +package main + +import ( + "fmt" + "strconv" +) + +type MkShParser struct { + tok *ShTokenizer + curr *ShToken +} + +func NewMkShParser(line *Line, text string, emitWarnings bool) *MkShParser { + shp := NewShTokenizer(line, text, emitWarnings) + return &MkShParser{shp, nil} +} + +func (p *MkShParser) Program() (retval *MkShList) { + defer p.trace(&retval)() + + list := p.List() + if list == nil { + return nil + } + separator := p.Separator() + if separator == nil { + return list + } + return &MkShList{list.AndOrs, append(list.Separators, *separator)} +} + +// ::= AndOr (SeparatorOp AndOr)* +func (p *MkShParser) List() (retval *MkShList) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + var andors []*MkShAndOr + var seps []MkShSeparator + + if andor := p.AndOr(); andor != nil { + andors = append(andors, andor) + } else { + return nil + } + +next: + mark := p.mark() + if sep := p.SeparatorOp(); sep != nil { + if andor := p.AndOr(); andor != nil { + andors = append(andors, andor) + seps = append(seps, *sep) + goto next + } + } + p.reset(mark) + + ok = true + return &MkShList{andors, seps} +} + +func (p *MkShParser) AndOr() (retval *MkShAndOr) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok) + + var pipes []*MkShPipeline + var ops []string +nextpipe: + if pipe := p.Pipeline(); pipe != nil { + pipes = append(pipes, pipe) + switch op := p.peekText(); op { + case "&&", "||": + p.skip() + p.Linebreak() + ops = append(ops, op) + goto nextpipe + } + } + + if len(pipes) == len(ops) { + return nil + } + ok = true + return &MkShAndOr{pipes, ops} +} + +// ::= Command (msttPipe Linebreak Command)* +func (p *MkShParser) Pipeline() (retval *MkShPipeline) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + bang := p.eat("!") + var cmds []*MkShCommand +nextcmd: + cmd := p.Command() + if cmd == nil { + return nil + } + cmds = append(cmds, cmd) + if p.eat("|") { + p.Linebreak() + goto nextcmd + } + ok = true + return &MkShPipeline{bang, cmds} +} + +func (p *MkShParser) Command() (retval *MkShCommand) { + defer p.trace(&retval)() + + if simple := p.SimpleCommand(); simple != nil { + return &MkShCommand{Simple: simple} + } + if compound := p.CompoundCommand(); compound != nil { + redirects := p.RedirectList() + return &MkShCommand{Compound: compound, Redirects: redirects} + } + if funcdef := p.FunctionDefinition(); funcdef != nil { + return &MkShCommand{FuncDef: funcdef} + } + return nil +} + +func (p *MkShParser) CompoundCommand() (retval *MkShCompoundCommand) { + defer p.trace(&retval)() + + if brace := p.BraceGroup(); brace != nil { + return &MkShCompoundCommand{Brace: brace} + } + if subshell := p.Subshell(); subshell != nil { + return &MkShCompoundCommand{Subshell: subshell} + } + if forclause := p.ForClause(); forclause != nil { + return &MkShCompoundCommand{For: forclause} + } + if caseclause := p.CaseClause(); caseclause != nil { + return &MkShCompoundCommand{Case: caseclause} + } + if ifclause := p.IfClause(); ifclause != nil { + return &MkShCompoundCommand{If: ifclause} + } + if whileclause := p.WhileClause(); whileclause != nil { + return &MkShCompoundCommand{While: whileclause} + } + if untilclause := p.UntilClause(); untilclause != nil { + return &MkShCompoundCommand{Until: untilclause} + } + return nil +} + +func (p *MkShParser) Subshell() (retval *MkShList) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + if !p.eat("(") { + return nil + } + list := p.CompoundList() + if list == nil { + return nil + } + if !p.eat(")") { + return nil + } + ok = true + return list +} + +// ::= Newline* AndOr (Separator AndOr)* Separator? +func (p *MkShParser) CompoundList() (retval *MkShList) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + p.Linebreak() + var andors []*MkShAndOr + var separators []MkShSeparator +nextandor: + if andor := p.AndOr(); andor != nil { + andors = append(andors, andor) + if sep := p.Separator(); sep != nil { + separators = append(separators, *sep) + goto nextandor + } + } + if len(andors) == 0 { + return nil + } + ok = true + return &MkShList{andors, separators} +} + +// ::= "for" msttWORD Linebreak DoGroup +// ::= "for" msttWORD Linebreak "in" SequentialSep DoGroup +// ::= "for" msttWORD Linebreak "in" Wordlist SequentialSep DoGroup +func (p *MkShParser) ForClause() (retval *MkShForClause) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + if !p.eat("for") { + return nil + } + varword := p.Word(false) + if varword == nil || !matches(varword.MkText, `^[A-Z_a-z][0-9A-Za-z]*`) { + return nil + } + varname := varword.MkText + + var values []*ShToken + if p.eat("in") { + values = p.Wordlist() + } else { + values = []*ShToken{NewShToken("\"$$@\"", + NewShAtom(shtWord, "\"", shqDquot), + NewShAtom(shtWord, "$$@", shqDquot), + NewShAtom(shtWord, "\"", shqPlain))} + } + if values == nil || !p.SequentialSep() { + return nil + } + + p.Linebreak() + body := p.DoGroup() + if body == nil { + return nil + } + + ok = true + return &MkShForClause{varname, values, body} +} + +func (p *MkShParser) Wordlist() (retval []*ShToken) { + defer p.trace(&retval)() + + var words []*ShToken +nextword: + word := p.Word(false) + if word != nil { + words = append(words, word) + goto nextword + } + return words +} + +// ::= "case" msttWORD Linebreak "in" Linebreak CaseItem* "esac" +func (p *MkShParser) CaseClause() (retval *MkShCaseClause) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + if !p.eat("case") { + return nil + } + + panic("CaseClause") + p.Linebreak() + p.CaseItem() + return nil +} + +// ::= "("? Pattern ")" (CompoundList | Linebreak) msttDSEMI? Linebreak +func (p *MkShParser) CaseItem() (retval *MkShCaseItem) { + defer p.trace(&retval)() + + panic("CaseItem") + p.Pattern() + p.Linebreak() + p.CompoundList() + return nil +} + +// ::= msttWORD +// ::= Pattern "|" msttWORD +func (p *MkShParser) Pattern() (retval []*ShToken) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + var words []*ShToken +nextword: + word := p.Word(false) + if word == nil { + return nil + } + words = append(words, word) + if p.eat("|") { + goto nextword + + } + ok = true + return words +} + +func (p *MkShParser) IfClause() (retval *MkShIfClause) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + var conds []*MkShList + var actions []*MkShList + var elseaction *MkShList + if !p.eat("if") { + return nil + } + +nextcond: + cond := p.CompoundList() + if cond == nil || !p.eat("then") { + return nil + } + action := p.CompoundList() + if action == nil { + return nil + } + conds = append(conds, cond) + actions = append(actions, action) + if p.eat("elif") { + goto nextcond + } + if p.eat("else") { + elseaction = p.CompoundList() + if elseaction == nil { + return nil + } + } + if !p.eat("fi") { + return nil + } + ok = true + return &MkShIfClause{conds, actions, elseaction} +} + +// ::= "while" CompoundList DoGroup +func (p *MkShParser) WhileClause() (retval *MkShLoopClause) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + if !p.eat("while") { + return nil + } + + panic("WhileClause") + p.CompoundList() + p.DoGroup() + return nil +} + +// ::= "until" CompoundList DoGroup +func (p *MkShParser) UntilClause() (retval *MkShLoopClause) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + if !p.eat("until") { + return nil + } + + panic("UntilClause") + p.CompoundList() + p.DoGroup() + return nil +} + +// ::= msttNAME "(" ")" Linebreak CompoundCommand Redirect* +func (p *MkShParser) FunctionDefinition() (retval *MkShFunctionDefinition) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + funcname := p.Word(true) + if funcname == nil || !matches(funcname.MkText, `^[A-Z_a-z][0-9A-Z_a-z]*$`) { + return nil + } + + if !p.eat("(") || !p.eat(")") { + return nil + } + + p.Linebreak() + + body := p.CompoundCommand() + if body == nil { + return nil + } + + redirects := p.RedirectList() + ok = true + return &MkShFunctionDefinition{funcname.MkText, body, redirects} +} + +func (p *MkShParser) BraceGroup() (retval *MkShList) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + if !p.eat("{") { + return nil + } + list := p.CompoundList() + if list == nil { + return nil + } + if !p.eat("}") { + return nil + } + ok = true + return list +} + +func (p *MkShParser) DoGroup() (retval *MkShList) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + if !p.eat("do") { + return nil + } + list := p.CompoundList() + if list == nil { + return nil + } + if !p.eat("done") { + return nil + } + ok = true + return list +} + +func (p *MkShParser) SimpleCommand() (retval *MkShSimpleCommand) { + defer p.trace(&retval)() + ok := false + defer p.rollback(&ok)() + + var assignments []*ShToken + var name *ShToken + var args []*ShToken + var redirections []*MkShRedirection + first := true + seenName := false +nextword: + if word := p.Word(first); word != nil { + first = false + if !seenName && word.IsAssignment() { + assignments = append(assignments, word) + } else if !seenName { + name = word + seenName = true + } else { + args = append(args, word) + } + goto nextword + } + if len(assignments) == 0 && name == nil && len(args) == 0 && len(redirections) == 0 { + return nil + } + ok = true + return &MkShSimpleCommand{assignments, name, args, redirections} +} + +func (p *MkShParser) RedirectList() (retval []*MkShRedirection) { + defer p.trace(&retval)() + +nextredirect: + if redirect := p.IoRedirect(); redirect != nil { + retval = append(retval, redirect) + goto nextredirect + } + return nil +} + +func (p *MkShParser) IoRedirect() (retval *MkShRedirection) { + defer p.trace(&retval)() + + if m, redirect, fdstr, op := match3(p.peekText(), `^((\d*)\s*(<|<&|>|>&|>>|<>|>\||<<|<<-))`); m { + target := p.peekText()[len(redirect):] + _, _, _ = fdstr, op, target + + fd, err := strconv.ParseInt(fdstr, 10, 32) + if err != nil { + fd = -1 + } + p.skip() + targetToken := NewShTokenizer(p.tok.mkp.Line, target, false).ShToken() + return &MkShRedirection{int(fd), op, targetToken} + } + return nil +} + +func (p *MkShParser) NewlineList() (retval bool) { + defer p.trace(&retval)() + + ok := false + for p.eat("\n") { + ok = true + } + return ok +} + +func (p *MkShParser) Linebreak() { + for p.eat("\n") { + } +} + +func (p *MkShParser) SeparatorOp() (retval *MkShSeparator) { + defer p.trace(&retval)() + + if p.eat(";") { + op := MkShSeparator(";") + return &op + } + if p.eat("&") { + op := MkShSeparator("&") + return &op + } + return nil +} + +func (p *MkShParser) Separator() (retval *MkShSeparator) { + defer p.trace(&retval)() + + op := p.SeparatorOp() + if op == nil && p.eat("\n") { + sep := MkShSeparator('\n') + op = &sep + } + if op != nil { + p.Linebreak() + } + return op +} + +func (p *MkShParser) SequentialSep() (retval bool) { + defer p.trace(&retval)() + + if p.peekText() == ";" { + p.skip() + p.Linebreak() + return true + } else { + return p.NewlineList() + } +} + +func (p *MkShParser) Word(cmdstart bool) (retval *ShToken) { + defer p.trace(&retval)() + + if token := p.peek(); token != nil && token.IsWord() { + if cmdstart { + switch token.MkText { + case "while", "until", "for", "do", "done", + "if", "then", "else", "elif", "fi", + "{", "}": + return nil + } + } + p.skip() + return token + } + return nil +} + +func (p *MkShParser) EOF() bool { + return p.peek() == nil +} + +func (p *MkShParser) peek() *ShToken { + if p.curr == nil { + nexttoken: + p.curr = p.tok.ShToken() + if p.curr == nil && !p.tok.parser.EOF() { + p.tok.mkp.Line.Warnf("Pkglint tokenize error at " + p.tok.parser.Rest()) + p.tok.mkp.Parser.repl.AdvanceRest() + return nil + } + if p.curr != nil && hasPrefix(p.curr.MkText, "#") { + goto nexttoken + } + } + //traceStep("MkShParser.peek %v rest=%q", p.curr, p.tok.mkp.repl.rest) + return p.curr +} + +func (p *MkShParser) peekText() string { + if next := p.peek(); next != nil { + return next.MkText + } + return "" +} + +func (p *MkShParser) skip() { + p.curr = nil +} + +func (p *MkShParser) eat(s string) bool { + if p.peek() == nil { + return false + } + if p.peek().MkText == s { + p.skip() + return true + } + return false +} + +func (p *MkShParser) rollback(pok *bool) func() { + mark := p.mark() + return func() { + if !*pok { + p.reset(mark) + } + } +} + +func (p *MkShParser) trace(retval interface{}) func() { + if G.opts.Debug { + return tracecallInternal(p.peek(), p.restref(), "=>", ref(retval)) + } else { + return func() {} + } +} + +func (p *MkShParser) mark() MkShParserMark { + return MkShParserMark{p.tok.parser.repl.Mark(), p.curr} +} + +func (p *MkShParser) reset(mark MkShParserMark) { + p.tok.parser.repl.Reset(mark.rest) + p.curr = mark.curr +} + +func (p *MkShParser) restref() MkShParserRest { + return MkShParserRest{&p.tok.mkp.repl.rest} +} + +func (p *MkShParser) Rest() string { + return p.peekText() + p.tok.mkp.repl.AdvanceRest() +} + +type MkShParserMark struct { + rest PrefixReplacerMark + curr *ShToken +} + +type MkShParserRest struct { + restref *string +} + +func (rest MkShParserRest) String() string { + return fmt.Sprintf("rest=%q", *rest.restref) +} diff --git a/pkgtools/pkglint/files/mkshparser_test.go b/pkgtools/pkglint/files/mkshparser_test.go new file mode 100644 index 00000000000..bf28129bcf2 --- /dev/null +++ b/pkgtools/pkglint/files/mkshparser_test.go @@ -0,0 +1,352 @@ +package main + +import ( + check "gopkg.in/check.v1" +) + +func (s *Suite) Test_MkShParser_Program(c *check.C) { + parse := func(cmd string, expected *MkShList) { + p := NewMkShParser(dummyLine, cmd, false) + program := p.Program() + c.Check(program, deepEquals, expected) + c.Check(p.tok.parser.Rest(), equals, "") + c.Check(s.Output(), equals, "") + } + + if false { + parse(""+ + "\tcd ${WRKSRC} && ${FIND} ${${_list_}} -type f ! -name '*.orig' 2>/dev/null "+ + "| pax -rw -pm ${DESTDIR}${PREFIX}/${${_dir_}}", + NewMkShList()) + } +} + +func (s *Suite) Test_MkShParser_List(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_AndOr(c *check.C) { + parse := func(cmd string, expected *MkShAndOr) { + p := NewMkShParser(dummyLine, cmd, false) + andor := p.AndOr() + c.Check(andor, deepEquals, expected) + c.Check(p.tok.parser.Rest(), equals, "") + c.Check(s.Output(), equals, "") + } + tester := &MkShTester{c} + + parse("simplecmd", + NewMkShAndOr(NewMkShPipeline(false, tester.ParseCommand("simplecmd")))) + + expected := NewMkShAndOr(NewMkShPipeline(false, tester.ParseCommand("simplecmd1"))) + expected.Add("&&", NewMkShPipeline(false, tester.ParseCommand("simplecmd2"))) + parse("simplecmd1 && simplecmd2", expected) +} + +func (s *Suite) Test_MkShParser_Pipeline(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_Command(c *check.C) { + parse := func(cmd string, expected *MkShCommand) { + p := NewMkShParser(dummyLine, cmd, false) + command := p.Command() + c.Check(command, deepEquals, expected) + c.Check(p.tok.parser.Rest(), equals, "") + c.Check(s.Output(), equals, "") + } + tester := &MkShTester{c} + + parse("simple", + &MkShCommand{Simple: tester.ParseSimpleCommand("simple")}) +} + +func (s *Suite) Test_MkShParser_CompoundCommand(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_Subshell(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_CompoundList(c *check.C) { + parse := func(cmd string, expected *MkShList) { + p := NewMkShParser(dummyLine, cmd, false) + compoundList := p.CompoundList() + c.Check(compoundList, deepEquals, expected) + c.Check(p.tok.parser.Rest(), equals, "") + c.Check(s.Output(), equals, "") + } + tester := &MkShTester{c} + + parse("simplecmd", + NewMkShList().AddAndOr(NewMkShAndOr(NewMkShPipeline(false, tester.ParseCommand("simplecmd"))))) + + simplecmd1 := NewMkShPipeline(false, tester.ParseCommand("simplecmd1")) + simplecmd2 := NewMkShPipeline(false, tester.ParseCommand("simplecmd2")) + expected := NewMkShList().AddAndOr(NewMkShAndOr(simplecmd1).Add("&&", simplecmd2)) + parse("simplecmd1 && simplecmd2", expected) +} + +func (s *Suite) Test_MkShParser_ForClause(c *check.C) { + parse := func(cmd string, expected *MkShForClause) { + p := NewMkShParser(dummyLine, cmd, false) + forclause := p.ForClause() + c.Check(forclause, deepEquals, expected) + c.Check(p.tok.parser.Rest(), equals, "") + c.Check(s.Output(), equals, "") + } + tester := &MkShTester{c} + + params := []*ShToken{tester.Token("\"$$@\"")} + action := tester.ParseCompoundList("action;") + parse("for var; do action; done", + &MkShForClause{"var", params, action}) + + abc := []*ShToken{tester.Token("a"), tester.Token("b"), tester.Token("c")} + parse("for var in a b c; do action; done", + &MkShForClause{"var", abc, action}) + + actions := tester.ParseCompoundList("action1 && action2;") + parse("for var in a b c; do action1 && action2; done", + &MkShForClause{"var", abc, actions}) +} + +func (s *Suite) Test_MkShParser_Wordlist(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_CaseClause(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_CaseItem(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_Pattern(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_IfClause(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_WhileClause(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_UntilClause(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_FunctionDefinition(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_BraceGroup(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_DoGroup(c *check.C) { + tester := &MkShTester{c} + check := func(str string, expected *MkShList) { + p := NewMkShParser(dummyLine, str, false) + dogroup := p.DoGroup() + if c.Check(dogroup, check.NotNil) { + if !c.Check(dogroup, deepEquals, expected) { + for i, andor := range dogroup.AndOrs { + c.Check(andor, deepEquals, expected.AndOrs[i]) + } + } + } + c.Check(p.tok.parser.Rest(), equals, "") + c.Check(s.Output(), equals, "") + } + + andor := NewMkShAndOr(NewMkShPipeline(false, tester.ParseCommand("action"))) + check("do action; done", + &MkShList{[]*MkShAndOr{andor}, []MkShSeparator{";"}}) +} + +func (s *Suite) Test_MkShParser_SimpleCommand(c *check.C) { + parse := func(cmd string, builder *SimpleCommandBuilder) { + expected := builder.Cmd + p := NewMkShParser(dummyLine, cmd, false) + shcmd := p.SimpleCommand() + if c.Check(shcmd, check.NotNil) { + if !c.Check(shcmd, deepEquals, expected) { + for i, assignment := range shcmd.Assignments { + c.Check(assignment, deepEquals, expected.Assignments[i]) + } + c.Check(shcmd.Name, deepEquals, expected.Name) + for i, word := range shcmd.Args { + c.Check(word, deepEquals, expected.Args[i]) + } + for i, redirection := range shcmd.Redirections { + c.Check(redirection, deepEquals, expected.Redirections[i]) + } + } + } + c.Check(p.tok.parser.Rest(), equals, "") + c.Check(s.Output(), equals, "") + } + + fail := func(noncmd string, expectedRest string) { + p := NewMkShParser(dummyLine, noncmd, false) + shcmd := p.SimpleCommand() + c.Check(shcmd, check.IsNil) + c.Check(p.tok.parser.Rest(), equals, expectedRest) + c.Check(s.Output(), equals, "") + } + tester := &MkShTester{c} + + parse("echo ${PKGNAME:Q}", + NewSimpleCommandBuilder(). + Name(tester.Token("echo")). + Arg(tester.Token("${PKGNAME:Q}"))) + + parse("${ECHO} \"Double-quoted\" 'Single-quoted'", + NewSimpleCommandBuilder(). + Name(tester.Token("${ECHO}")). + Arg(tester.Token("\"Double-quoted\"")). + Arg(tester.Token("'Single-quoted'"))) + + parse("`cat plain` \"`cat double`\" '`cat single`'", + NewSimpleCommandBuilder(). + Name(tester.Token("`cat plain`")). + Arg(tester.Token("\"`cat double`\"")). + Arg(tester.Token("'`cat single`'"))) + + parse("`\"one word\"`", + NewSimpleCommandBuilder(). + Name(tester.Token("`\"one word\"`"))) + + parse("PAGES=\"`ls -1 | ${SED} -e 's,3qt$$,3,'`\"", + NewSimpleCommandBuilder(). + Assignment(tester.Token("PAGES=\"`ls -1 | ${SED} -e 's,3qt$$,3,'`\""))) + + parse("var=Plain var=\"Dquot\" var='Squot' var=Plain\"Dquot\"'Squot'", + NewSimpleCommandBuilder(). + Assignment(tester.Token("var=Plain")). + Assignment(tester.Token("var=\"Dquot\"")). + Assignment(tester.Token("var='Squot'")). + Assignment(tester.Token("var=Plain\"Dquot\"'Squot'"))) + + parse("${RUN} subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\"", + NewSimpleCommandBuilder(). + Name(tester.Token("${RUN}")). + Arg(tester.Token("subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\""))) + + parse("PATH=/nonexistent env PATH=${PATH:Q} true", + NewSimpleCommandBuilder(). + Assignment(tester.Token("PATH=/nonexistent")). + Name(tester.Token("env")). + Arg(tester.Token("PATH=${PATH:Q}")). + Arg(tester.Token("true"))) + + parse("{OpenGrok args", + NewSimpleCommandBuilder(). + Name(tester.Token("{OpenGrok")). + Arg(tester.Token("args"))) + + fail("if clause", "if clause") + fail("{ group; }", "{ group; }") + +} + +func (s *Suite) Test_MkShParser_RedirectList(c *check.C) { +} + +func (s *Suite) Test_MkShParser_IoRedirect(c *check.C) { +} + +func (s *Suite) Test_MkShParser_IoFile(c *check.C) { +} + +func (s *Suite) Test_MkShParser_IoHere(c *check.C) { +} + +func (s *Suite) Test_MkShParser_NewlineList(c *check.C) { +} + +func (s *Suite) Test_MkShParser_Linebreak(c *check.C) { +} + +func (s *Suite) Test_MkShParser_SeparatorOp(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_Separator(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_SequentialSep(c *check.C) { + +} + +func (s *Suite) Test_MkShParser_Word(c *check.C) { + +} + +type MkShTester struct { + c *check.C +} + +func (t *MkShTester) ParseCommand(str string) *MkShCommand { + p := NewMkShParser(dummyLine, str, false) + cmd := p.Command() + t.c.Check(cmd, check.NotNil) + t.c.Check(p.Rest(), equals, "") + return cmd +} + +func (t *MkShTester) ParseSimpleCommand(str string) *MkShSimpleCommand { + p := NewMkShParser(dummyLine, str, false) + parsed := p.SimpleCommand() + t.c.Check(parsed, check.NotNil) + t.c.Check(p.Rest(), equals, "") + return parsed +} + +func (t *MkShTester) ParseCompoundList(str string) *MkShList { + p := NewMkShParser(dummyLine, str, false) + parsed := p.CompoundList() + t.c.Check(parsed, check.NotNil) + t.c.Check(p.Rest(), equals, "") + return parsed +} + +func (t *MkShTester) Token(str string) *ShToken { + p := NewMkShParser(dummyLine, str, false) + parsed := p.peek() + p.skip() + t.c.Check(parsed, check.NotNil) + t.c.Check(p.Rest(), equals, "") + return parsed +} + +type SimpleCommandBuilder struct { + Cmd *MkShSimpleCommand +} + +func NewSimpleCommandBuilder() *SimpleCommandBuilder { + cmd := &MkShSimpleCommand{} + return &SimpleCommandBuilder{cmd} +} +func (b *SimpleCommandBuilder) Name(name *ShToken) *SimpleCommandBuilder { + b.Cmd.Name = name + return b +} +func (b *SimpleCommandBuilder) Assignment(assignment *ShToken) *SimpleCommandBuilder { + b.Cmd.Assignments = append(b.Cmd.Assignments, assignment) + return b +} +func (b *SimpleCommandBuilder) Arg(arg *ShToken) *SimpleCommandBuilder { + b.Cmd.Args = append(b.Cmd.Args, arg) + return b +} +func (b *SimpleCommandBuilder) Redirection(redirection *MkShRedirection) *SimpleCommandBuilder { + b.Cmd.Redirections = append(b.Cmd.Redirections, redirection) + return b +} diff --git a/pkgtools/pkglint/files/mkshtypes.go b/pkgtools/pkglint/files/mkshtypes.go new file mode 100644 index 00000000000..994c0450504 --- /dev/null +++ b/pkgtools/pkglint/files/mkshtypes.go @@ -0,0 +1,231 @@ +package main + +import ( + "fmt" +) + +type MkShList struct { + AndOrs []*MkShAndOr + Separators []MkShSeparator +} + +func NewMkShList() *MkShList { + return &MkShList{nil, nil} +} + +func (list *MkShList) String() string { + return fmt.Sprintf("MkShList(%v)", list.AndOrs) +} + +func (list *MkShList) AddAndOr(andor *MkShAndOr) *MkShList { + list.AndOrs = append(list.AndOrs, andor) + return list +} + +func (list *MkShList) AddSeparator(separator MkShSeparator) *MkShList { + list.Separators = append(list.Separators, separator) + return list +} + +type MkShAndOr struct { + Pipes []*MkShPipeline + Ops []string // Either "&&" or "||" +} + +func NewMkShAndOr(pipeline *MkShPipeline) *MkShAndOr { + return &MkShAndOr{[]*MkShPipeline{pipeline}, nil} +} + +func (andor *MkShAndOr) String() string { + return fmt.Sprintf("MkShAndOr(%v)", andor.Pipes) +} + +func (andor *MkShAndOr) Add(op string, pipeline *MkShPipeline) *MkShAndOr { + andor.Pipes = append(andor.Pipes, pipeline) + andor.Ops = append(andor.Ops, op) + return andor +} + +type MkShPipeline struct { + Negated bool + Cmds []*MkShCommand +} + +func NewMkShPipeline(negated bool, cmds ...*MkShCommand) *MkShPipeline { + return &MkShPipeline{negated, cmds} +} + +func (pipe *MkShPipeline) String() string { + return fmt.Sprintf("MkShPipeline(%v)", pipe.Cmds) +} + +func (pipe *MkShPipeline) Add(cmd *MkShCommand) *MkShPipeline { + pipe.Cmds = append(pipe.Cmds, cmd) + return pipe +} + +type MkShCommand struct { + Simple *MkShSimpleCommand + Compound *MkShCompoundCommand + FuncDef *MkShFunctionDefinition + Redirects []*MkShRedirection // For Compound and FuncDef +} + +func (cmd *MkShCommand) String() string { + switch { + case cmd.Simple != nil: + return cmd.Simple.String() + case cmd.Compound != nil: + return cmd.Compound.String() + case cmd.FuncDef != nil: + return cmd.FuncDef.String() + } + return "MkShCommand(?)" +} + +type MkShCompoundCommand struct { + Brace *MkShList + Subshell *MkShList + For *MkShForClause + Case *MkShCaseClause + If *MkShIfClause + While *MkShLoopClause + Until *MkShLoopClause +} + +func (cmd *MkShCompoundCommand) String() string { + switch { + case cmd.Brace != nil: + return cmd.Brace.String() + case cmd.Subshell != nil: + return cmd.Subshell.String() + case cmd.For != nil: + return cmd.For.String() + case cmd.Case != nil: + return cmd.Case.String() + case cmd.If != nil: + return cmd.If.String() + case cmd.While != nil: + return cmd.While.String() + case cmd.Until != nil: + return cmd.Until.String() + } + return "MkShCompoundCommand(?)" +} + +type MkShForClause struct { + Varname string + Values []*ShToken + Body *MkShList +} + +func (cl *MkShForClause) String() string { + return fmt.Sprintf("MkShForClause(%v, %v, %v)", cl.Varname, cl.Values, cl.Body) +} + +type MkShCaseClause struct { + Word *ShToken + Cases []*MkShCaseItem +} + +func (cl *MkShCaseClause) String() string { + return fmt.Sprintf("MkShCaseClause(...)") +} + +type MkShCaseItem struct { + Patterns []*ShToken + Action *MkShList +} + +type MkShIfClause struct { + Conds []*MkShList + Actions []*MkShList + Else *MkShList +} + +func (cl *MkShIfClause) String() string { + return "MkShIf(...)" +} + +func (cl *MkShIfClause) Prepend(cond *MkShList, action *MkShList) { + cl.Conds = append([]*MkShList{cond}, cl.Conds...) + cl.Actions = append([]*MkShList{action}, cl.Actions...) +} + +type MkShLoopClause struct { + Cond *MkShList + Action *MkShList + Until bool +} + +func (cl *MkShLoopClause) String() string { + return "MkShLoop(...)" +} + +type MkShFunctionDefinition struct { + Name string + Body *MkShCompoundCommand + Redirects []*MkShRedirection +} + +func (def *MkShFunctionDefinition) String() string { + return "MkShFunctionDef(...)" +} + +type MkShSimpleCommand struct { + Assignments []*ShToken + Name *ShToken + Args []*ShToken + Redirections []*MkShRedirection +} + +func (scmd *MkShSimpleCommand) String() string { + str := "SimpleCommand(" + first := true + sep := func() { + if first { + first = false + } else { + str += ", " + } + } + for _, word := range scmd.Assignments { + sep() + str += word.MkText + } + if word := scmd.Name; word != nil { + sep() + str += word.MkText + } + for _, word := range scmd.Args { + sep() + str += word.MkText + } + for _, redirection := range scmd.Redirections { + sep() + str += redirection.String() + } + return str + ")" +} + +type MkShRedirection struct { + Fd int // Or -1 + Op string + Target *ShToken +} + +func (r *MkShRedirection) String() string { + if r.Fd != -1 { + return fmt.Sprintf("%d%s%s", r.Fd, r.Op, r.Target.MkText) + } else { + return r.Op + r.Target.MkText + } +} + +// One of ";", "&", "\n" +type MkShSeparator string + +func (sep *MkShSeparator) String() string { + return fmt.Sprintf("%q", sep) + +} diff --git a/pkgtools/pkglint/files/mktypes.go b/pkgtools/pkglint/files/mktypes.go new file mode 100644 index 00000000000..f322bfa8e95 --- /dev/null +++ b/pkgtools/pkglint/files/mktypes.go @@ -0,0 +1,37 @@ +package main + +type MkToken struct { + Text string // Used for both literals and varuses. + Varuse *MkVarUse +} +type MkVarUse struct { + varname string + modifiers []string // E.g. "Q", "S/from/to/" +} + +func NewMkVarUse(varname string, modifiers ...string) *MkVarUse { + return &MkVarUse{varname, modifiers} +} + +func (vu *MkVarUse) Mod() string { + mod := "" + for _, modifier := range vu.modifiers { + mod += ":" + modifier + } + return mod +} + +// Whether the varname is interpreted as a variable name (the usual case) +// or as a full expression (rare). +func (vu *MkVarUse) IsExpression() bool { + if len(vu.modifiers) == 0 { + return false + } + mod := vu.modifiers[0] + return mod == "L" || hasPrefix(mod, "?") +} + +func (vu *MkVarUse) IsQ() bool { + mlen := len(vu.modifiers) + return mlen > 0 && vu.modifiers[mlen-1] == "Q" +} diff --git a/pkgtools/pkglint/files/mktypes_test.go b/pkgtools/pkglint/files/mktypes_test.go new file mode 100644 index 00000000000..161b6f95737 --- /dev/null +++ b/pkgtools/pkglint/files/mktypes_test.go @@ -0,0 +1,11 @@ +package main + +import ( + check "gopkg.in/check.v1" +) + +func (s *Suite) Test_MkVarUse_Mod(c *check.C) { + varuse := &MkVarUse{"varname", []string{"Q"}} + + c.Check(varuse.Mod(), equals, ":Q") +} diff --git a/pkgtools/pkglint/files/package.go b/pkgtools/pkglint/files/package.go index 40232102c7a..1c2d4c2c7b3 100644 --- a/pkgtools/pkglint/files/package.go +++ b/pkgtools/pkglint/files/package.go @@ -27,6 +27,7 @@ type Package struct { plistSubstCond map[string]bool // varname => true; list of all variables that are used as conditionals (@comment or nothing) in PLISTs. included map[string]*Line // fname => line seenMakefileCommon bool // Does the package have any .includes? + loadTimeTools map[string]bool // true=ok, false=not ok, absent=not mentioned in USE_TOOLS. } func NewPackage(pkgpath string) *Package { @@ -37,6 +38,7 @@ func NewPackage(pkgpath string) *Package { bl3: make(map[string]*Line), plistSubstCond: make(map[string]bool), included: make(map[string]*Line), + loadTimeTools: make(map[string]bool), } for varname, line := range G.globalData.UserDefinedVars { pkg.vardef[varname] = line @@ -55,14 +57,29 @@ func (pkg *Package) defineVar(mkline *MkLine, varname string) { } func (pkg *Package) varValue(varname string) (string, bool) { + switch varname { + case "KRB5_TYPE": + return "heimdal", true + case "PGSQL_VERSION": + return "95", true + } if mkline := pkg.vardef[varname]; mkline != nil { return mkline.Value(), true } return "", false } +func (pkg *Package) setSeenBsdPrefsMk() { + if !pkg.SeenBsdPrefsMk { + pkg.SeenBsdPrefsMk = true + if G.opts.Debug { + traceStep("Pkg.setSeenBsdPrefsMk") + } + } +} + func (pkg *Package) checkPossibleDowngrade() { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall0()() } @@ -75,8 +92,8 @@ func (pkg *Package) checkPossibleDowngrade() { change := G.globalData.LastChange[pkg.Pkgpath] if change == nil { - if G.opts.DebugMisc { - mkline.Debug1("No change log for package %q", pkg.Pkgpath) + if G.opts.Debug { + traceStep1("No change log for package %q", pkg.Pkgpath) } return } @@ -95,7 +112,7 @@ func (pkg *Package) checkPossibleDowngrade() { } func (pkg *Package) checklinesBuildlink3Inclusion(mklines *MkLines) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall0()() } @@ -113,17 +130,17 @@ func (pkg *Package) checklinesBuildlink3Inclusion(mklines *MkLines) { } } - if G.opts.DebugMisc { - for packageBl3, line := range pkg.bl3 { + if G.opts.Debug { + for packageBl3 := range pkg.bl3 { if includedFiles[packageBl3] == nil { - line.Debug1("%s/buildlink3.mk is included by the package but not by the buildlink3.mk file.", packageBl3) + traceStep1("%s/buildlink3.mk is included by the package but not by the buildlink3.mk file.", packageBl3) } } } } func checkdirPackage(pkgpath string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(pkgpath)() } @@ -183,27 +200,27 @@ func checkdirPackage(pkgpath string) { if G.opts.CheckDistinfo && G.opts.CheckPatches { if havePatches && !haveDistinfo { - Warnf(G.CurrentDir+"/"+pkg.DistinfoFile, noLines, "File not found. Please run \"%s makepatchsum\".", confMake) + NewLineWhole(G.CurrentDir+"/"+pkg.DistinfoFile).Warn1("File not found. Please run \"%s makepatchsum\".", confMake) } } if !isEmptyDir(G.CurrentDir + "/scripts") { - Warnf(G.CurrentDir+"/scripts", noLines, "This directory and its contents are deprecated! Please call the script(s) explicitly from the corresponding target(s) in the pkg's Makefile.") + NewLineWhole(G.CurrentDir + "/scripts").Warn0("This directory and its contents are deprecated! Please call the script(s) explicitly from the corresponding target(s) in the pkg's Makefile.") } } func (pkg *Package) loadPackageMakefile(fname string) *MkLines { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(fname)() } mainLines, allLines := NewMkLines(nil), NewMkLines(nil) - if !readMakefile(fname, mainLines, allLines, "") { + if !pkg.readMakefile(fname, mainLines, allLines, "") { return nil } if G.opts.DumpMakefile { - Debugf(G.CurrentDir, noLines, "Whole Makefile (with all included files) follows:") + fmt.Printf("Whole Makefile (with all included files) follows:\n") for _, line := range allLines.lines { fmt.Printf("%s\n", line.String()) } @@ -225,18 +242,18 @@ func (pkg *Package) loadPackageMakefile(fname string) *MkLines { } } - if G.opts.DebugMisc { - dummyLine.Debug1("DISTINFO_FILE=%s", pkg.DistinfoFile) - dummyLine.Debug1("FILESDIR=%s", pkg.Filesdir) - dummyLine.Debug1("PATCHDIR=%s", pkg.Patchdir) - dummyLine.Debug1("PKGDIR=%s", pkg.Pkgdir) + if G.opts.Debug { + traceStep1("DISTINFO_FILE=%s", pkg.DistinfoFile) + traceStep1("FILESDIR=%s", pkg.Filesdir) + traceStep1("PATCHDIR=%s", pkg.Patchdir) + traceStep1("PKGDIR=%s", pkg.Pkgdir) } return mainLines } -func readMakefile(fname string, mainLines *MkLines, allLines *MkLines, includingFnameForUsedCheck string) bool { - if G.opts.DebugTrace { +func (pkg *Package) readMakefile(fname string, mainLines *MkLines, allLines *MkLines, includingFnameForUsedCheck string) bool { + if G.opts.Debug { defer tracecall1(fname)() } @@ -275,8 +292,8 @@ func readMakefile(fname string, mainLines *MkLines, allLines *MkLines, including if path.Base(fname) != "buildlink3.mk" { if m, bl3File := match1(includeFile, `^\.\./\.\./(.*)/buildlink3\.mk$`); m { G.Pkg.bl3[bl3File] = line - if G.opts.DebugMisc { - line.Debug1("Buildlink3 file in package: %q", bl3File) + if G.opts.Debug { + traceStep1("Buildlink3 file in package: %q", bl3File) } } } @@ -291,8 +308,8 @@ func readMakefile(fname string, mainLines *MkLines, allLines *MkLines, including } if path.Base(fname) == "Makefile" && !hasPrefix(incDir, "../../mk/") && incBase != "buildlink3.mk" && incBase != "builtin.mk" && incBase != "options.mk" { - if G.opts.DebugInclude { - line.Debug1("Including %q sets seenMakefileCommon.", includeFile) + if G.opts.Debug { + traceStep1("Including %q sets seenMakefileCommon.", includeFile) } G.Pkg.seenMakefileCommon = true } @@ -313,11 +330,11 @@ func readMakefile(fname string, mainLines *MkLines, allLines *MkLines, including return false } - if G.opts.DebugInclude { - line.Debug1("Including %q.", dirname+"/"+includeFile) + if G.opts.Debug { + traceStep1("Including %q.", dirname+"/"+includeFile) } includingFname := ifelseStr(incBase == "Makefile.common" && incDir != "", fname, "") - if !readMakefile(dirname+"/"+includeFile, mainLines, allLines, includingFname) { + if !pkg.readMakefile(dirname+"/"+includeFile, mainLines, allLines, includingFname) { return false } } @@ -327,8 +344,8 @@ func readMakefile(fname string, mainLines *MkLines, allLines *MkLines, including varname, op, value := mkline.Varname(), mkline.Op(), mkline.Value() if op != opAssignDefault || G.Pkg.vardef[varname] == nil { - if G.opts.DebugMisc { - line.Debugf("varassign(%q, %q, %q)", varname, op, value) + if G.opts.Debug { + traceStep("varassign(%q, %q, %q)", varname, op, value) } G.Pkg.vardef[varname] = mkline } @@ -343,7 +360,7 @@ func readMakefile(fname string, mainLines *MkLines, allLines *MkLines, including } func (pkg *Package) checkfilePackageMakefile(fname string, mklines *MkLines) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(fname)() } @@ -353,16 +370,16 @@ func (pkg *Package) checkfilePackageMakefile(fname string, mklines *MkLines) { vardef["META_PACKAGE"] == nil && !fileExists(G.CurrentDir+"/"+pkg.Pkgdir+"/PLIST") && !fileExists(G.CurrentDir+"/"+pkg.Pkgdir+"/PLIST.common") { - Warnf(fname, noLines, "Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset. Are you sure PLIST handling is ok?") + NewLineWhole(fname).Warn0("Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset. Are you sure PLIST handling is ok?") } if (vardef["NO_CHECKSUM"] != nil || vardef["META_PACKAGE"] != nil) && isEmptyDir(G.CurrentDir+"/"+pkg.Patchdir) { if distinfoFile := G.CurrentDir + "/" + pkg.DistinfoFile; fileExists(distinfoFile) { - Warnf(distinfoFile, noLines, "This file should not exist if NO_CHECKSUM or META_PACKAGE is set.") + NewLineWhole(distinfoFile).Warn0("This file should not exist if NO_CHECKSUM or META_PACKAGE is set.") } } else { if distinfoFile := G.CurrentDir + "/" + pkg.DistinfoFile; !containsVarRef(distinfoFile) && !fileExists(distinfoFile) { - Warnf(distinfoFile, noLines, "File not found. Please run \"%s makesum\".", confMake) + NewLineWhole(distinfoFile).Warn1("File not found. Please run \"%s makesum\".", confMake) } } @@ -371,7 +388,7 @@ func (pkg *Package) checkfilePackageMakefile(fname string, mklines *MkLines) { } if vardef["LICENSE"] == nil { - Errorf(fname, noLines, "Each package must define its LICENSE.") + NewLineWhole(fname).Error0("Each package must define its LICENSE.") } if gnuLine, useLine := vardef["GNU_CONFIGURE"], vardef["USE_LANGUAGES"]; gnuLine != nil && useLine != nil { @@ -390,7 +407,7 @@ func (pkg *Package) checkfilePackageMakefile(fname string, mklines *MkLines) { pkg.checkPossibleDowngrade() if vardef["COMMENT"] == nil { - Warnf(fname, noLines, "No COMMENT given.") + NewLineWhole(fname).Warn0("No COMMENT given.") } if imake, x11 := vardef["USE_IMAKE"], vardef["USE_X11"]; imake != nil && x11 != nil { @@ -459,8 +476,8 @@ func (pkg *Package) determineEffectivePkgVars() { } } if pkg.EffectivePkgnameLine != nil { - if G.opts.DebugMisc { - pkg.EffectivePkgnameLine.Line.Debugf("Effective name=%q base=%q version=%q", + if G.opts.Debug { + traceStep("Effective name=%q base=%q version=%q", pkg.EffectivePkgname, pkg.EffectivePkgbase, pkg.EffectivePkgversion) } } @@ -473,8 +490,8 @@ func (pkg *Package) pkgnameFromDistname(pkgname, distname string) string { qsep := regexp.QuoteMeta(sep) if m, left, from, right, to, mod := match5(subst, `^(\^?)([^:]*)(\$?)`+qsep+`([^:]*)`+qsep+`(g?)$`); m { newPkgname := before + mkopSubst(distname, left != "", from, right != "", to, mod != "") + after - if G.opts.DebugMisc { - pkg.vardef["PKGNAME"].Debug2("pkgnameFromDistname %q => %q", pkgname, newPkgname) + if G.opts.Debug { + traceStep("%s: pkgnameFromDistname %q => %q", pkg.vardef["PKGNAME"], pkgname, newPkgname) } pkgname = newPkgname } @@ -512,7 +529,7 @@ func (pkg *Package) checkUpdate() { } func (pkg *Package) ChecklinesPackageMakefileVarorder(mklines *MkLines) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall0()() } @@ -626,8 +643,8 @@ func (pkg *Package) ChecklinesPackageMakefileVarorder(mklines *MkLines) { line := mklines.lines[lineno] text := line.Text - if G.opts.DebugMisc { - line.Debugf("[varorder] section %d variable %d vars %v", sectindex, varindex, vars) + if G.opts.Debug { + traceStep("[varorder] section %d variable %d vars %v", sectindex, varindex, vars) } if nextSection { @@ -690,6 +707,9 @@ func (pkg *Package) ChecklinesPackageMakefileVarorder(mklines *MkLines) { if vars[varindex].count == once && !maySkipSection { line.Warn1("The canonical position for the required variable %s is here.", vars[varindex].varname) Explain( + "In simple package Makefiles, some common variables should be", + "arranged in a specific order.", + "", "See doc/Makefile-example or the pkgsrc guide, section", "\"Package components\", subsection \"Makefile\" for more information.") } diff --git a/pkgtools/pkglint/files/package_test.go b/pkgtools/pkglint/files/package_test.go index cf753fe17e9..7c596c69054 100644 --- a/pkgtools/pkglint/files/package_test.go +++ b/pkgtools/pkglint/files/package_test.go @@ -142,3 +142,79 @@ func (s *Suite) TestPackage_CheckPossibleDowngrade(c *check.C) { c.Check(s.Output(), equals, "") } + +func (s *Suite) TestCheckdirPackage(c *check.C) { + s.CreateTmpFile(c, "Makefile", ""+ + "# $"+"NetBSD$\n") + G.CurrentDir = s.tmpdir + + checkdirPackage(s.tmpdir) + + c.Check(s.Output(), equals, ""+ + "WARN: ~/Makefile: Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset. Are you sure PLIST handling is ok?\n"+ + "WARN: ~/distinfo: File not found. Please run \"@BMAKE@ makesum\".\n"+ + "ERROR: ~/Makefile: Each package must define its LICENSE.\n"+ + "WARN: ~/Makefile: No COMMENT given.\n") +} + +func (s *Suite) Test_Package_Varuse_LoadTime(c *check.C) { + s.CreateTmpFileLines(c, "doc/CHANGES-2016", + "# dummy") + s.CreateTmpFileLines(c, "doc/TODO", + "# dummy") + s.CreateTmpFileLines(c, "licenses/bsd-2", + "# dummy") + s.CreateTmpFileLines(c, "mk/fetch/sites.mk", + "# dummy") + s.CreateTmpFileLines(c, "mk/bsd.pkg.mk", + "# dummy") + s.CreateTmpFileLines(c, "mk/defaults/options.description", + "option Description") + s.CreateTmpFileLines(c, "mk/defaults/mk.conf", + "# dummy") + s.CreateTmpFileLines(c, "mk/tools/bsd.tools.mk", + ".include \"defaults.mk\"") + s.CreateTmpFileLines(c, "mk/tools/defaults.mk", + "TOOLS_CREATE+=false", + "TOOLS_CREATE+=nice", + "TOOLS_CREATE+=true", + "_TOOLS_VARNAME.nice=NICE") + s.CreateTmpFileLines(c, "mk/bsd.prefs.mk", + "# dummy") + + s.CreateTmpFileLines(c, "category/pkgbase/Makefile", + "# $"+"NetBSD$", + "", + "COMMENT= Unit test", + "LICENSE= bsd-2", + "PLIST_SRC=#none", + "", + "USE_TOOLS+= echo false", + "FALSE_BEFORE!= echo false=${FALSE}", + "NICE_BEFORE!= echo nice=${NICE}", + "TRUE_BEFORE!= echo true=${TRUE}", + "", + ".include \"../../mk/bsd.prefs.mk\"", + "", + "USE_TOOLS+= nice", + "FALSE_AFTER!= echo false=${FALSE}", + "NICE_AFTER!= echo nice=${NICE}", + "TRUE_AFTER!= echo true=${TRUE}", + "", + "do-build:", + "\t${ECHO} before: ${FALSE_BEFORE} ${NICE_BEFORE} ${TRUE_BEFORE}", + "\t${ECHO} after: ${FALSE_AFTER} ${NICE_AFTER} ${TRUE_AFTER}", + "\t${ECHO}; ${FALSE}; ${NICE}; ${TRUE}", + "", + ".include \"../../mk/bsd.pkg.mk\"") + s.CreateTmpFileLines(c, "category/pkgbase/distinfo", + "$"+"NetBSD$") + + (&Pkglint{}).Main("pkglint", "-q", "-Wperm", s.tmpdir+"/category/pkgbase") + + c.Check(s.Output(), equals, ""+ + "WARN: ~/category/pkgbase/Makefile:8: To use the tool \"FALSE\" at load time, bsd.prefs.mk has to be included before.\n"+ + "WARN: ~/category/pkgbase/Makefile:9: To use the tool \"NICE\" at load time, bsd.prefs.mk has to be included before.\n"+ + "WARN: ~/category/pkgbase/Makefile:10: To use the tool \"TRUE\" at load time, bsd.prefs.mk has to be included before.\n"+ + "WARN: ~/category/pkgbase/Makefile:16: To use the tool \"NICE\" at load time, it has to be added to USE_TOOLS before including bsd.prefs.mk.\n") +} diff --git a/pkgtools/pkglint/files/parser.go b/pkgtools/pkglint/files/parser.go index f33bc1343c5..ea417eae392 100644 --- a/pkgtools/pkglint/files/parser.go +++ b/pkgtools/pkglint/files/parser.go @@ -5,12 +5,13 @@ import ( ) type Parser struct { - line *Line - repl *PrefixReplacer + Line *Line + repl *PrefixReplacer + EmitWarnings bool } -func NewParser(line *Line, s string) *Parser { - return &Parser{line, NewPrefixReplacer(s)} +func NewParser(line *Line, s string, emitWarnings bool) *Parser { + return &Parser{line, NewPrefixReplacer(s), emitWarnings} } func (p *Parser) EOF() bool { @@ -95,325 +96,3 @@ func (p *Parser) Dependency() *DependencyPattern { repl.Reset(mark) return nil } - -type MkToken struct { - literal string - varuse MkVarUse -} -type MkVarUse struct { - varname string - modifiers []string -} - -func (p *Parser) MkTokens() []*MkToken { - repl := p.repl - - var tokens []*MkToken - for !p.EOF() { - if varuse := p.VarUse(); varuse != nil { - tokens = append(tokens, &MkToken{varuse: *varuse}) - continue - } - - mark := repl.Mark() - needsReplace := false - again: - dollar := strings.IndexByte(repl.rest, '$') - if dollar == -1 { - dollar = len(repl.rest) - } - repl.Skip(dollar) - if repl.AdvanceStr("$$") { - needsReplace = true - goto again - } - literal := repl.Since(mark) - if needsReplace { - literal = strings.Replace(literal, "$$", "$", -1) - } - if literal != "" { - tokens = append(tokens, &MkToken{literal: literal}) - continue - } - - break - } - return tokens -} - -func (p *Parser) Varname() string { - repl := p.repl - - mark := repl.Mark() - repl.AdvanceStr(".") - for p.VarUse() != nil || repl.AdvanceBytes(0x00000000, 0x03ff6800, 0x87fffffe, 0x07fffffe, `[\w+\-.]`) { - } - return repl.Since(mark) -} - -func (p *Parser) VarUse() *MkVarUse { - repl := p.repl - - mark := repl.Mark() - if repl.AdvanceStr("${") || repl.AdvanceStr("$(") { - usingRoundParen := repl.Since(mark) == "$(" - closing := ifelseStr(usingRoundParen, ")", "}") - - varnameMark := repl.Mark() - varname := p.Varname() - if varname != "" { - if usingRoundParen { - p.line.Warn1("Please use curly braces {} instead of round parentheses () for %s.", varname) - } - modifiers := p.VarUseModifiers(varname, closing) - if repl.AdvanceStr(closing) { - return &MkVarUse{varname, modifiers} - } - } - - for p.VarUse() != nil || repl.AdvanceRegexp(`^([^$:`+closing+`]|\$\$)+`) { - } - rest := p.Rest() - if hasPrefix(rest, ":L") || hasPrefix(rest, ":sh") || hasPrefix(rest, ":?") { - varexpr := repl.Since(varnameMark) - modifiers := p.VarUseModifiers(varexpr, closing) - if repl.AdvanceStr(closing) { - return &MkVarUse{varexpr, modifiers} - } - } - repl.Reset(mark) - } - - return nil -} - -func (p *Parser) VarUseModifiers(varname, closing string) []string { - repl := p.repl - - var modifiers []string - mayOmitColon := false - for repl.AdvanceStr(":") || mayOmitColon { - mayOmitColon = false - modifierMark := repl.Mark() - - switch repl.PeekByte() { - case 'E', 'H', 'L', 'O', 'Q', 'R', 'T', 's', 't', 'u': - if repl.AdvanceRegexp(`^(E|H|L|Ox?|Q|R|T|sh|tA|tW|tl|tu|tw|u)`) { - modifiers = append(modifiers, repl.Since(modifierMark)) - continue - } - if repl.AdvanceStr("ts") { - rest := repl.rest - if len(rest) >= 2 && (rest[1] == closing[0] || rest[1] == ':') { - repl.Skip(1) - } else if len(rest) >= 1 && (rest[0] == closing[0] || rest[0] == ':') { - } else if repl.AdvanceRegexp(`^\\\d+`) { - } else { - break - } - modifiers = append(modifiers, repl.Since(modifierMark)) - continue - } - - case '=', 'D', 'M', 'N', 'U': - if repl.AdvanceRegexp(`^[=DMNU]`) { - for p.VarUse() != nil || repl.AdvanceRegexp(`^([^$:`+closing+`]|\$\$)+`) { - } - modifiers = append(modifiers, repl.Since(modifierMark)) - continue - } - - case 'C', 'S': - if repl.AdvanceRegexp(`^[CS]([%,/:;@^|])`) { - separator := repl.m[1] - repl.AdvanceStr("^") - re := `^([^\` + separator + `$` + closing + `\\]|\$\$|\\.)+` - for p.VarUse() != nil || repl.AdvanceRegexp(re) { - } - repl.AdvanceStr("$") - if repl.AdvanceStr(separator) { - for p.VarUse() != nil || repl.AdvanceRegexp(re) { - } - if repl.AdvanceStr(separator) { - repl.AdvanceRegexp(`^[1gW]`) - modifiers = append(modifiers, repl.Since(modifierMark)) - mayOmitColon = true - continue - } - } - } - - case '@': - if repl.AdvanceRegexp(`^@([\w.]+)@`) { - loopvar := repl.m[1] - for p.VarUse() != nil || repl.AdvanceRegexp(`^([^$:@`+closing+`\\]|\$\$|\\.)+`) { - } - if !repl.AdvanceStr("@") { - p.line.Warn2("Modifier ${%s:@%s@...@} is missing the final \"@\".", varname, loopvar) - } - modifiers = append(modifiers, repl.Since(modifierMark)) - continue - } - - case '[': - if repl.AdvanceRegexp(`^\[[-.\d]+\]`) { - modifiers = append(modifiers, repl.Since(modifierMark)) - continue - } - - case '?': - repl.AdvanceStr("?") - re := `^([^$:` + closing + `]|\$\$)+` - for p.VarUse() != nil || repl.AdvanceRegexp(re) { - } - if repl.AdvanceStr(":") { - for p.VarUse() != nil || repl.AdvanceRegexp(re) { - } - modifiers = append(modifiers, repl.Since(modifierMark)) - continue - } - } - - repl.Reset(modifierMark) - for p.VarUse() != nil || repl.AdvanceRegexp(`^([^:$`+closing+`]|\$\$)+`) { - } - if suffixSubst := repl.Since(modifierMark); contains(suffixSubst, "=") { - modifiers = append(modifiers, suffixSubst) - continue - } - } - return modifiers -} - -func (p *Parser) MkCond() *Tree { - return p.mkCondOr() -} - -func (p *Parser) mkCondOr() *Tree { - and := p.mkCondAnd() - if and == nil { - return nil - } - - ands := append([]interface{}(nil), and) - for { - mark := p.repl.Mark() - if !p.repl.AdvanceRegexp(`^\s*\|\|\s*`) { - break - } - next := p.mkCondAnd() - if next == nil { - p.repl.Reset(mark) - break - } - ands = append(ands, next) - } - if len(ands) == 1 { - return and - } - return NewTree("or", ands...) -} - -func (p *Parser) mkCondAnd() *Tree { - atom := p.mkCondAtom() - if atom == nil { - return nil - } - - atoms := append([]interface{}(nil), atom) - for { - mark := p.repl.Mark() - if !p.repl.AdvanceRegexp(`^\s*&&\s*`) { - break - } - next := p.mkCondAtom() - if next == nil { - p.repl.Reset(mark) - break - } - atoms = append(atoms, next) - } - if len(atoms) == 1 { - return atom - } - return NewTree("and", atoms...) -} - -func (p *Parser) mkCondAtom() *Tree { - if G.opts.DebugTrace { - defer tracecall1(p.Rest())() - } - - repl := p.repl - mark := repl.Mark() - repl.SkipSpace() - switch { - case repl.AdvanceStr("!"): - cond := p.mkCondAtom() - if cond != nil { - return NewTree("not", cond) - } - case repl.AdvanceStr("("): - cond := p.MkCond() - if cond != nil { - repl.SkipSpace() - if repl.AdvanceStr(")") { - return cond - } - } - case repl.AdvanceRegexp(`^defined\s*\(`): - if varname := p.Varname(); varname != "" { - if repl.AdvanceStr(")") { - return NewTree("defined", varname) - } - } - case repl.AdvanceRegexp(`^empty\s*\(`): - if varname := p.Varname(); varname != "" { - modifiers := p.VarUseModifiers(varname, ")") - if repl.AdvanceStr(")") { - return NewTree("empty", MkVarUse{varname, modifiers}) - } - } - case repl.AdvanceRegexp(`^(commands|exists|make|target)\s*\(`): - funcname := repl.m[1] - argMark := repl.Mark() - for p.VarUse() != nil || repl.AdvanceRegexp(`^[^$)]+`) { - } - arg := repl.Since(argMark) - if repl.AdvanceStr(")") { - return NewTree(funcname, arg) - } - default: - lhs := p.VarUse() - mark := repl.Mark() - if lhs == nil && repl.AdvanceStr("\"") { - if quotedLhs := p.VarUse(); quotedLhs != nil && repl.AdvanceStr("\"") { - lhs = quotedLhs - } else { - repl.Reset(mark) - } - } - if lhs != nil { - if repl.AdvanceRegexp(`^\s*(<|<=|==|!=|>=|>)\s*(\d+(?:\.\d+)?)`) { - return NewTree("compareVarNum", *lhs, repl.m[1], repl.m[2]) - } - if repl.AdvanceRegexp(`^\s*(<|<=|==|!=|>=|>)\s*`) { - op := repl.m[1] - if (op == "!=" || op == "==") && repl.AdvanceRegexp(`^"([^"\$\\]*)"`) { - return NewTree("compareVarStr", *lhs, op, repl.m[1]) - } else if repl.AdvanceRegexp(`^\w+`) { - return NewTree("compareVarStr", *lhs, op, repl.m[0]) - } else if rhs := p.VarUse(); rhs != nil { - return NewTree("compareVarVar", *lhs, op, *rhs) - } - } else { - return NewTree("not", NewTree("empty", *lhs)) // See devel/bmake/files/cond.c:/\* For \.if \$/ - } - } - if repl.AdvanceRegexp(`^\d+(?:\.\d+)?`) { - return NewTree("literalNum", repl.m[0]) - } - } - repl.Reset(mark) - return nil -} diff --git a/pkgtools/pkglint/files/parser_test.go b/pkgtools/pkglint/files/parser_test.go index d094250b682..ddb87dbbf08 100644 --- a/pkgtools/pkglint/files/parser_test.go +++ b/pkgtools/pkglint/files/parser_test.go @@ -5,222 +5,71 @@ import ( ) func (s *Suite) TestParser_PkgbasePattern(c *check.C) { - test := func(pattern, expected, rest string) { - parser := NewParser(dummyLine, pattern) + checkRest := func(pattern, expected, rest string) { + parser := NewParser(dummyLine, pattern, false) actual := parser.PkgbasePattern() c.Check(actual, equals, expected) c.Check(parser.Rest(), equals, rest) } - test("fltk", "fltk", "") - test("fltk|", "fltk", "|") - test("boost-build-1.59.*", "boost-build", "-1.59.*") - test("${PHP_PKG_PREFIX}-pdo-5.*", "${PHP_PKG_PREFIX}-pdo", "-5.*") - test("${PYPKGPREFIX}-metakit-[0-9]*", "${PYPKGPREFIX}-metakit", "-[0-9]*") + checkRest("fltk", "fltk", "") + checkRest("fltk|", "fltk", "|") + checkRest("boost-build-1.59.*", "boost-build", "-1.59.*") + checkRest("${PHP_PKG_PREFIX}-pdo-5.*", "${PHP_PKG_PREFIX}-pdo", "-5.*") + checkRest("${PYPKGPREFIX}-metakit-[0-9]*", "${PYPKGPREFIX}-metakit", "-[0-9]*") } func (s *Suite) TestParser_Dependency(c *check.C) { - testDependencyRest := func(pattern string, expected DependencyPattern, rest string) { - parser := NewParser(dummyLine, pattern) + checkRest := func(pattern string, expected DependencyPattern, rest string) { + parser := NewParser(dummyLine, pattern, false) dp := parser.Dependency() if c.Check(dp, check.NotNil) { c.Check(*dp, equals, expected) c.Check(parser.Rest(), equals, rest) } } - testDependency := func(pattern string, expected DependencyPattern) { - testDependencyRest(pattern, expected, "") + check := func(pattern string, expected DependencyPattern) { + checkRest(pattern, expected, "") } - testDependency("fltk>=1.1.5rc1<1.3", DependencyPattern{"fltk", ">=", "1.1.5rc1", "<", "1.3", ""}) - testDependency("libwcalc-1.0*", DependencyPattern{"libwcalc", "", "", "", "", "1.0*"}) - testDependency("${PHP_PKG_PREFIX}-pdo-5.*", DependencyPattern{"${PHP_PKG_PREFIX}-pdo", "", "", "", "", "5.*"}) - testDependency("${PYPKGPREFIX}-metakit-[0-9]*", DependencyPattern{"${PYPKGPREFIX}-metakit", "", "", "", "", "[0-9]*"}) - testDependency("boost-build-1.59.*", DependencyPattern{"boost-build", "", "", "", "", "1.59.*"}) - testDependency("${_EMACS_REQD}", DependencyPattern{"${_EMACS_REQD}", "", "", "", "", ""}) - testDependency("{gcc46,gcc46-libs}>=4.6.0", DependencyPattern{"{gcc46,gcc46-libs}", ">=", "4.6.0", "", "", ""}) - testDependency("perl5-*", DependencyPattern{"perl5", "", "", "", "", "*"}) - testDependency("verilog{,-current}-[0-9]*", DependencyPattern{"verilog{,-current}", "", "", "", "", "[0-9]*"}) - testDependency("mpg123{,-esound,-nas}>=0.59.18", DependencyPattern{"mpg123{,-esound,-nas}", ">=", "0.59.18", "", "", ""}) - testDependency("mysql*-{client,server}-[0-9]*", DependencyPattern{"mysql*-{client,server}", "", "", "", "", "[0-9]*"}) - testDependency("postgresql8[0-35-9]-${module}-[0-9]*", DependencyPattern{"postgresql8[0-35-9]-${module}", "", "", "", "", "[0-9]*"}) - testDependency("ncurses-${NC_VERS}{,nb*}", DependencyPattern{"ncurses", "", "", "", "", "${NC_VERS}{,nb*}"}) - testDependency("xulrunner10>=${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", DependencyPattern{"xulrunner10", ">=", "${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", "", "", ""}) - testDependencyRest("gnome-control-center>=2.20.1{,nb*}", DependencyPattern{"gnome-control-center", ">=", "2.20.1", "", "", ""}, "{,nb*}") + check("fltk>=1.1.5rc1<1.3", DependencyPattern{"fltk", ">=", "1.1.5rc1", "<", "1.3", ""}) + check("libwcalc-1.0*", DependencyPattern{"libwcalc", "", "", "", "", "1.0*"}) + check("${PHP_PKG_PREFIX}-pdo-5.*", DependencyPattern{"${PHP_PKG_PREFIX}-pdo", "", "", "", "", "5.*"}) + check("${PYPKGPREFIX}-metakit-[0-9]*", DependencyPattern{"${PYPKGPREFIX}-metakit", "", "", "", "", "[0-9]*"}) + check("boost-build-1.59.*", DependencyPattern{"boost-build", "", "", "", "", "1.59.*"}) + check("${_EMACS_REQD}", DependencyPattern{"${_EMACS_REQD}", "", "", "", "", ""}) + check("{gcc46,gcc46-libs}>=4.6.0", DependencyPattern{"{gcc46,gcc46-libs}", ">=", "4.6.0", "", "", ""}) + check("perl5-*", DependencyPattern{"perl5", "", "", "", "", "*"}) + check("verilog{,-current}-[0-9]*", DependencyPattern{"verilog{,-current}", "", "", "", "", "[0-9]*"}) + check("mpg123{,-esound,-nas}>=0.59.18", DependencyPattern{"mpg123{,-esound,-nas}", ">=", "0.59.18", "", "", ""}) + check("mysql*-{client,server}-[0-9]*", DependencyPattern{"mysql*-{client,server}", "", "", "", "", "[0-9]*"}) + check("postgresql8[0-35-9]-${module}-[0-9]*", DependencyPattern{"postgresql8[0-35-9]-${module}", "", "", "", "", "[0-9]*"}) + check("ncurses-${NC_VERS}{,nb*}", DependencyPattern{"ncurses", "", "", "", "", "${NC_VERS}{,nb*}"}) + check("xulrunner10>=${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", DependencyPattern{"xulrunner10", ">=", "${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", "", "", ""}) + checkRest("gnome-control-center>=2.20.1{,nb*}", DependencyPattern{"gnome-control-center", ">=", "2.20.1", "", "", ""}, "{,nb*}") // "{ssh{,6}-[0-9]*,openssh-[0-9]*}" is not representable using the current data structure } -func (s *Suite) TestParser_MkTokens(c *check.C) { - parse := func(input string, expectedTokens []*MkToken, expectedRest string) { - p := NewParser(dummyLine, input) - actualTokens := p.MkTokens() - c.Check(actualTokens, deepEquals, expectedTokens) - for i, expectedToken := range expectedTokens { - if i < len(actualTokens) { - c.Check(*actualTokens[i], deepEquals, *expectedToken) - } - } - c.Check(p.Rest(), equals, expectedRest) - } - token := func(input string, expectedToken MkToken) { - parse(input, []*MkToken{&expectedToken}, "") - } - literal := func(literal string) MkToken { - return MkToken{literal: literal} - } - varuse := func(varname string, modifiers ...string) MkToken { - return MkToken{varuse: MkVarUse{varname: varname, modifiers: modifiers}} - } - - token("literal", literal("literal")) - token("\\/share\\/ { print \"share directory\" }", literal("\\/share\\/ { print \"share directory\" }")) - token("find . -name \\*.orig -o -name \\*.pre", literal("find . -name \\*.orig -o -name \\*.pre")) - token("-e 's|\\$${EC2_HOME.*}|EC2_HOME}|g'", literal("-e 's|\\${EC2_HOME.*}|EC2_HOME}|g'")) - - token("${VARIABLE}", varuse("VARIABLE")) - token("${VARIABLE.param}", varuse("VARIABLE.param")) - token("${VARIABLE.${param}}", varuse("VARIABLE.${param}")) - token("${VARIABLE.hicolor-icon-theme}", varuse("VARIABLE.hicolor-icon-theme")) - token("${VARIABLE.gtk+extra}", varuse("VARIABLE.gtk+extra")) - token("${VARIABLE:S/old/new/}", varuse("VARIABLE", "S/old/new/")) - token("${GNUSTEP_LFLAGS:S/-L//g}", varuse("GNUSTEP_LFLAGS", "S/-L//g")) - token("${SUSE_VERSION:S/.//}", varuse("SUSE_VERSION", "S/.//")) - token("${MASTER_SITE_GNOME:=sources/alacarte/0.13/}", varuse("MASTER_SITE_GNOME", "=sources/alacarte/0.13/")) - token("${INCLUDE_DIRS:H:T}", varuse("INCLUDE_DIRS", "H", "T")) - token("${A.${B.${C.${D}}}}", varuse("A.${B.${C.${D}}}")) - token("${RUBY_VERSION:C/([0-9]+)\\.([0-9]+)\\.([0-9]+)/\\1/}", varuse("RUBY_VERSION", "C/([0-9]+)\\.([0-9]+)\\.([0-9]+)/\\1/")) - token("${PERL5_${_var_}:Q}", varuse("PERL5_${_var_}", "Q")) - token("${PKGNAME_REQD:C/(^.*-|^)py([0-9][0-9])-.*/\\2/}", varuse("PKGNAME_REQD", "C/(^.*-|^)py([0-9][0-9])-.*/\\2/")) - token("${PYLIB:S|/|\\\\/|g}", varuse("PYLIB", "S|/|\\\\/|g")) - token("${PKGNAME_REQD:C/ruby([0-9][0-9]+)-.*/\\1/}", varuse("PKGNAME_REQD", "C/ruby([0-9][0-9]+)-.*/\\1/")) - token("${RUBY_SHLIBALIAS:S/\\//\\\\\\//}", varuse("RUBY_SHLIBALIAS", "S/\\//\\\\\\//")) - token("${RUBY_VER_MAP.${RUBY_VER}:U${RUBY_VER}}", varuse("RUBY_VER_MAP.${RUBY_VER}", "U${RUBY_VER}")) - token("${RUBY_VER_MAP.${RUBY_VER}:U18}", varuse("RUBY_VER_MAP.${RUBY_VER}", "U18")) - token("${CONFIGURE_ARGS:S/ENABLE_OSS=no/ENABLE_OSS=yes/g}", varuse("CONFIGURE_ARGS", "S/ENABLE_OSS=no/ENABLE_OSS=yes/g")) - token("${PLIST_RUBY_DIRS:S,DIR=\"PREFIX/,DIR=\",}", varuse("PLIST_RUBY_DIRS", "S,DIR=\"PREFIX/,DIR=\",")) - token("${LDFLAGS:S/-Wl,//g:Q}", varuse("LDFLAGS", "S/-Wl,//g", "Q")) - token("${_PERL5_REAL_PACKLIST:S/^/${DESTDIR}/}", varuse("_PERL5_REAL_PACKLIST", "S/^/${DESTDIR}/")) - token("${_PYTHON_VERSION:C/^([0-9])/\\1./1}", varuse("_PYTHON_VERSION", "C/^([0-9])/\\1./1")) - token("${PKGNAME:S/py${_PYTHON_VERSION}/py${i}/}", varuse("PKGNAME", "S/py${_PYTHON_VERSION}/py${i}/")) - token("${PKGNAME:C/-[0-9].*$/-[0-9]*/}", varuse("PKGNAME", "C/-[0-9].*$/-[0-9]*/")) - token("${PKGNAME:S/py${_PYTHON_VERSION}/py${i}/:C/-[0-9].*$/-[0-9]*/}", varuse("PKGNAME", "S/py${_PYTHON_VERSION}/py${i}/", "C/-[0-9].*$/-[0-9]*/")) - token("${_PERL5_VARS:tl:S/^/-V:/}", varuse("_PERL5_VARS", "tl", "S/^/-V:/")) - token("${_PERL5_VARS_OUT:M${_var_:tl}=*:S/^${_var_:tl}=${_PERL5_PREFIX:=/}//}", varuse("_PERL5_VARS_OUT", "M${_var_:tl}=*", "S/^${_var_:tl}=${_PERL5_PREFIX:=/}//")) - token("${RUBY${RUBY_VER}_PATCHLEVEL}", varuse("RUBY${RUBY_VER}_PATCHLEVEL")) - token("${DISTFILES:M*.gem}", varuse("DISTFILES", "M*.gem")) - token("${LOCALBASE:S^/^_^}", varuse("LOCALBASE", "S^/^_^")) - token("${SOURCES:%.c=%.o}", varuse("SOURCES", "%.c=%.o")) - token("${GIT_TEMPLATES:@.t.@ ${EGDIR}/${GIT_TEMPLATEDIR}/${.t.} ${PREFIX}/${GIT_CORE_TEMPLATEDIR}/${.t.} @:M*}", - varuse("GIT_TEMPLATES", "@.t.@ ${EGDIR}/${GIT_TEMPLATEDIR}/${.t.} ${PREFIX}/${GIT_CORE_TEMPLATEDIR}/${.t.} @", "M*")) - token("${DISTNAME:C:_:-:}", varuse("DISTNAME", "C:_:-:")) - token("${CF_FILES:H:O:u:S@^@${PKG_SYSCONFDIR}/@}", varuse("CF_FILES", "H", "O", "u", "S@^@${PKG_SYSCONFDIR}/@")) - token("${ALT_GCC_RTS:S%${LOCALBASE}%%:S%/%%}", varuse("ALT_GCC_RTS", "S%${LOCALBASE}%%", "S%/%%")) - token("${PREFIX:C;///*;/;g:C;/$;;}", varuse("PREFIX", "C;///*;/;g", "C;/$;;")) - token("${GZIP_CMD:[1]:Q}", varuse("GZIP_CMD", "[1]", "Q")) - token("${DISTNAME:C/-[0-9]+$$//:C/_/-/}", varuse("DISTNAME", "C/-[0-9]+$$//", "C/_/-/")) - token("${DISTNAME:slang%=slang2%}", varuse("DISTNAME", "slang%=slang2%")) - token("${OSMAP_SUBSTVARS:@v@-e 's,\\@${v}\\@,${${v}},g' @}", varuse("OSMAP_SUBSTVARS", "@v@-e 's,\\@${v}\\@,${${v}},g' @")) - - /* weird features */ - token("${${EMACS_VERSION_MAJOR}>22:?@comment :}", varuse("${EMACS_VERSION_MAJOR}>22", "?@comment :")) - token("${${XKBBASE}/xkbcomp:L:Q}", varuse("${XKBBASE}/xkbcomp", "L", "Q")) - token("${${PKGBASE} ${PKGVERSION}:L}", varuse("${PKGBASE} ${PKGVERSION}", "L")) - token("${empty(CFLAGS):?:-cflags ${CFLAGS:Q}}", varuse("empty(CFLAGS)", "?:-cflags ${CFLAGS:Q}")) - token("${${${PKG_INFO} -E ${d} || echo:L:sh}:L:C/[^[0-9]]*/ /g:[1..3]:ts.}", - varuse("${${PKG_INFO} -E ${d} || echo:L:sh}", "L", "C/[^[0-9]]*/ /g", "[1..3]", "ts.")) - token("${VAR:S/-//S/.//}", varuse("VAR", "S/-//", "S/.//")) // For :S and :C, the colon can be left out. - token("${VAR:ts}", varuse("VAR", "ts")) // The separator character can be left out. - token("${VAR:ts\\000012}", varuse("VAR", "ts\\000012")) // The separator character can be a long octal number. - token("${VAR:ts\\124}", varuse("VAR", "ts\\124")) // Or even decimal. - - token("$(GNUSTEP_USER_ROOT)", varuse("GNUSTEP_USER_ROOT")) - c.Check(s.Output(), equals, "WARN: Please use curly braces {} instead of round parentheses () for GNUSTEP_USER_ROOT.\n") - - parse("${VAR)", nil, "${VAR)") // Opening brace, closing parenthesis - parse("$(VAR}", nil, "$(VAR}") // Opening parenthesis, closing brace - c.Check(s.Output(), equals, "WARN: Please use curly braces {} instead of round parentheses () for VAR.\n") - - token("${PLIST_SUBST_VARS:@var@${var}=${${var}:Q}@}", varuse("PLIST_SUBST_VARS", "@var@${var}=${${var}:Q}@")) - token("${PLIST_SUBST_VARS:@var@${var}=${${var}:Q}}", varuse("PLIST_SUBST_VARS", "@var@${var}=${${var}:Q}")) // Missing @ at the end - c.Check(s.Output(), equals, "WARN: Modifier ${PLIST_SUBST_VARS:@var@...@} is missing the final \"@\".\n") -} - -func (s *Suite) TestParser_MkCond_Basics(c *check.C) { - condrest := func(input string, expectedTree *Tree, expectedRest string) { - p := NewParser(dummyLine, input) - actualTree := p.MkCond() - c.Check(actualTree, deepEquals, expectedTree) - c.Check(p.Rest(), equals, expectedRest) - } - cond := func(input string, expectedTree *Tree) { - condrest(input, expectedTree, "") - } - literal := func(literal string) MkToken { - return MkToken{literal: literal} - } - varuse := func(varname string, modifiers ...string) MkVarUse { - return MkVarUse{varname: varname, modifiers: modifiers} - } - _, _ = literal, varuse - - cond("${OPSYS:MNetBSD}", - NewTree("not", NewTree("empty", varuse("OPSYS", "MNetBSD")))) - cond("defined(VARNAME)", - NewTree("defined", "VARNAME")) - cond("empty(VARNAME)", - NewTree("empty", varuse("VARNAME"))) - cond("!empty(VARNAME)", - NewTree("not", NewTree("empty", varuse("VARNAME")))) - cond("!empty(VARNAME:M[yY][eE][sS])", - NewTree("not", NewTree("empty", varuse("VARNAME", "M[yY][eE][sS]")))) - cond("${VARNAME} != \"Value\"", - NewTree("compareVarStr", varuse("VARNAME"), "!=", "Value")) - cond("${VARNAME:Mi386} != \"Value\"", - NewTree("compareVarStr", varuse("VARNAME", "Mi386"), "!=", "Value")) - cond("${VARNAME} != Value", - NewTree("compareVarStr", varuse("VARNAME"), "!=", "Value")) - cond("\"${VARNAME}\" != Value", - NewTree("compareVarStr", varuse("VARNAME"), "!=", "Value")) - cond("(defined(VARNAME))", - NewTree("defined", "VARNAME")) - cond("exists(/etc/hosts)", - NewTree("exists", "/etc/hosts")) - cond("exists(${PREFIX}/var)", - NewTree("exists", "${PREFIX}/var")) - cond("${OPSYS} == \"NetBSD\" || ${OPSYS} == \"OpenBSD\"", - NewTree("or", - NewTree("compareVarStr", varuse("OPSYS"), "==", "NetBSD"), - NewTree("compareVarStr", varuse("OPSYS"), "==", "OpenBSD"))) - cond("${OPSYS} == \"NetBSD\" && ${MACHINE_ARCH} == \"i386\"", - NewTree("and", - NewTree("compareVarStr", varuse("OPSYS"), "==", "NetBSD"), - NewTree("compareVarStr", varuse("MACHINE_ARCH"), "==", "i386"))) - cond("defined(A) && defined(B) || defined(C) && defined(D)", - NewTree("or", - NewTree("and", - NewTree("defined", "A"), - NewTree("defined", "B")), - NewTree("and", - NewTree("defined", "C"), - NewTree("defined", "D")))) - - // Exotic cases - cond("0", - NewTree("literalNum", "0")) - cond("! ( defined(A) && empty(VARNAME) )", - NewTree("not", NewTree("and", NewTree("defined", "A"), NewTree("empty", varuse("VARNAME"))))) - cond("${REQD_MAJOR} > ${MAJOR}", - NewTree("compareVarVar", varuse("REQD_MAJOR"), ">", varuse("MAJOR"))) - cond("${OS_VERSION} >= 6.5", - NewTree("compareVarNum", varuse("OS_VERSION"), ">=", "6.5")) - cond("${OS_VERSION} == 5.3", - NewTree("compareVarNum", varuse("OS_VERSION"), "==", "5.3")) - cond("!empty(${OS_VARIANT:MIllumos})", // Probably not intended - NewTree("not", NewTree("empty", varuse("${OS_VARIANT:MIllumos}")))) +// @Beta +func (s *Suite) Test_Parser_ShAst(c *check.C) { + f := func(args ...interface{}) interface{} { return nil } + Commands := f + Command := f + Arg := f + Varuse := f + Varassign := f + Subshell := f + Pipe := f + + _ = "cd ${WRKSRC}/doc/man/man3; PAGES=\"`ls -1 | ${SED} -e 's,3qt$$,3,'`\";" + + Commands( + Command("cd", + Arg(Varuse("WRKSRC"), "/doc/man/man3")), + Varassign("PAGES", Subshell( + Pipe( + Command("ls", "-1"), + Command(Varuse("SED"), "-e", "s,3qt$,3,"))))) - // Errors - condrest("!empty(PKG_OPTIONS:Msndfile) || defined(PKG_OPTIONS:Msamplerate)", - NewTree("not", NewTree("empty", varuse("PKG_OPTIONS", "Msndfile"))), - " || defined(PKG_OPTIONS:Msamplerate)") } diff --git a/pkgtools/pkglint/files/patches.go b/pkgtools/pkglint/files/patches.go index 74a8e3edb4b..7277de48d31 100644 --- a/pkgtools/pkglint/files/patches.go +++ b/pkgtools/pkglint/files/patches.go @@ -8,7 +8,7 @@ import ( ) func ChecklinesPatch(lines []*Line) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(lines[0].Fname)() } @@ -29,7 +29,7 @@ const ( ) func (ck *PatchChecker) Check() { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall0()() } @@ -83,9 +83,9 @@ func (ck *PatchChecker) Check() { } if patchedFiles > 1 { - Warnf(ck.lines[0].Fname, noLines, "Contains patches for %d files, should be only one.", patchedFiles) + NewLineWhole(ck.lines[0].Fname).Warnf("Contains patches for %d files, should be only one.", patchedFiles) } else if patchedFiles == 0 { - Errorf(ck.lines[0].Fname, noLines, "Contains no patch.") + NewLineWhole(ck.lines[0].Fname).Error0("Contains no patch.") } ChecklinesTrailingEmptyLines(ck.lines) @@ -94,13 +94,13 @@ func (ck *PatchChecker) Check() { // See http://www.gnu.org/software/diffutils/manual/html_node/Detailed-Unified.html func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall0()() } patchedFileType := guessFileType(ck.exp.CurrentLine(), patchedFile) - if G.opts.DebugMisc { - ck.exp.CurrentLine().Debugf("guessFileType(%q) = %s", patchedFile, patchedFileType) + if G.opts.Debug { + traceStep("guessFileType(%q) = %s", patchedFile, patchedFileType) } hasHunks := false @@ -108,8 +108,8 @@ func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { hasHunks = true linesToDel := toInt(ck.exp.m[2], 1) linesToAdd := toInt(ck.exp.m[4], 1) - if G.opts.DebugMisc { - ck.exp.PreviousLine().Debugf("hunk -%d +%d", linesToDel, linesToAdd) + if G.opts.Debug { + traceStep("hunk -%d +%d", linesToDel, linesToAdd) } ck.checktextUniHunkCr() @@ -154,7 +154,7 @@ func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { } func (ck *PatchChecker) checkBeginDiff(line *Line, patchedFiles int) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall0()() } @@ -181,7 +181,7 @@ func (ck *PatchChecker) checkBeginDiff(line *Line, patchedFiles int) { } func (ck *PatchChecker) checklineContext(text string, patchedFileType FileType) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall2(text, patchedFileType.String())() } @@ -193,7 +193,7 @@ func (ck *PatchChecker) checklineContext(text string, patchedFileType FileType) } func (ck *PatchChecker) checklineAdded(addedText string, patchedFileType FileType) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall2(addedText, patchedFileType.String())() } @@ -201,16 +201,10 @@ func (ck *PatchChecker) checklineAdded(addedText string, patchedFileType FileTyp line := ck.exp.PreviousLine() switch patchedFileType { - case ftShell: + case ftShell, ftIgnore: break case ftMakefile: - // This check is not as accurate as the similar one in MkLine.checkShelltext. - shellTokens, _ := splitIntoShellTokens(line, addedText) - for _, shellToken := range shellTokens { - if !hasPrefix(shellToken, "#") { - line.CheckAbsolutePathname(shellToken) - } - } + checklineOtherAbsolutePathname(line, addedText) case ftSource: checklineSourceAbsolutePathname(line, addedText) case ftConfigure: @@ -222,15 +216,13 @@ func (ck *PatchChecker) checklineAdded(addedText string, patchedFileType FileTyp "For more details, look for \"configure-scripts-override\" in", "mk/configure/gnu-configure.mk.") } - case ftIgnore: - break default: checklineOtherAbsolutePathname(line, addedText) } } func (ck *PatchChecker) checktextUniHunkCr() { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall0()() } @@ -282,7 +274,11 @@ func (ft FileType) String() string { } // This is used to select the proper subroutine for detecting absolute pathnames. -func guessFileType(line *Line, fname string) FileType { +func guessFileType(line *Line, fname string) (fileType FileType) { + if G.opts.Debug { + defer tracecall(fname, "=>", &fileType)() + } + basename := path.Base(fname) basename = strings.TrimSuffix(basename, ".in") // doesn’t influence the content type ext := strings.ToLower(strings.TrimLeft(path.Ext(basename), ".")) @@ -305,14 +301,14 @@ func guessFileType(line *Line, fname string) FileType { return ftUnknown } - if G.opts.DebugMisc { - line.Debug1("Unknown file type for %q", fname) + if G.opts.Debug { + traceStep1("Unknown file type for %q", fname) } return ftUnknown } func checkwordAbsolutePathname(line *Line, word string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(word)() } @@ -345,8 +341,8 @@ func checklineSourceAbsolutePathname(line *Line, text string) { return } if matched, before, _, str := match3(text, `^(.*)(["'])(/\w[^"']*)["']`); matched { - if G.opts.DebugMisc { - line.Debug2("checklineSourceAbsolutePathname: before=%q, str=%q", before, str) + if G.opts.Debug { + traceStep2("checklineSourceAbsolutePathname: before=%q, str=%q", before, str) } switch { @@ -363,7 +359,7 @@ func checklineSourceAbsolutePathname(line *Line, text string) { } func checklineOtherAbsolutePathname(line *Line, text string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(text)() } @@ -373,14 +369,14 @@ func checklineOtherAbsolutePathname(line *Line, text string) { } else if m, before, path, _ := match3(text, `^(.*?)((?:/[\w.]+)*/(?:bin|dev|etc|home|lib|mnt|opt|proc|sbin|tmp|usr|var)\b[\w./\-]*)(.*)$`); m { switch { case hasSuffix(before, "@"): // Example: @PREFIX@/bin - case matches(before, `[)}]$`): // Example: ${prefix}/bin + case matches(before, `[)}]$`) && !matches(before, `DESTDIR[)}]$`): // Example: ${prefix}/bin case matches(before, `\+\s*["']$`): // Example: prefix + '/lib' - case matches(before, `\w$`): // Example: libdir=$prefix/lib + case matches(before, `\$\w$`): // Example: libdir=$prefix/lib case hasSuffix(before, "."): // Example: ../dir // XXX new: case matches(before, `s.$`): // Example: sed -e s,/usr,@PREFIX@, default: - if G.opts.DebugMisc { - line.Debug1("before=%q", before) + if G.opts.Debug { + traceStep1("before=%q", before) } checkwordAbsolutePathname(line, path) } diff --git a/pkgtools/pkglint/files/patches_test.go b/pkgtools/pkglint/files/patches_test.go index 486e8105c60..6758a09d3fc 100644 --- a/pkgtools/pkglint/files/patches_test.go +++ b/pkgtools/pkglint/files/patches_test.go @@ -231,18 +231,21 @@ func (s *Suite) TestChecklinesPatch_Makefile(c *check.C) { "", "--- Makefile.orig", "+++ Makefile", - "@@ -1,3 +1,5 @@", + "@@ -1,3 +1,7 @@", " \t/bin/cp context before", "-\t/bin/cp deleted", "+\t/bin/cp added", "+#\t/bin/cp added comment", "+# added comment", + "+\t${DESTDIR}/bin/cp added", + "+\t${prefix}/bin/cp added", " \t/bin/cp context after") ChecklinesPatch(lines) c.Check(s.Output(), equals, ""+ - "WARN: patch-unified:10: Found absolute pathname: /bin/cp\n") + "WARN: patch-unified:10: Found absolute pathname: /bin/cp\n"+ + "WARN: patch-unified:13: Found absolute pathname: /bin/cp\n") G.opts.WarnExtra = true @@ -251,7 +254,8 @@ func (s *Suite) TestChecklinesPatch_Makefile(c *check.C) { c.Check(s.Output(), equals, ""+ "WARN: patch-unified:8: Found absolute pathname: /bin/cp\n"+ "WARN: patch-unified:10: Found absolute pathname: /bin/cp\n"+ - "WARN: patch-unified:13: Found absolute pathname: /bin/cp\n") + "WARN: patch-unified:13: Found absolute pathname: /bin/cp\n"+ + "WARN: patch-unified:15: Found absolute pathname: /bin/cp\n") } func (s *Suite) TestChecklinesPatch_NoNewline_withFollowingText(c *check.C) { diff --git a/pkgtools/pkglint/files/pkglint.0 b/pkgtools/pkglint/files/pkglint.0 index 895da87f366..0db26929158 100644 --- a/pkgtools/pkglint/files/pkglint.0 +++ b/pkgtools/pkglint/files/pkglint.0 @@ -14,9 +14,6 @@ DDEESSCCRRIIPPTTIIOONN --CC{{[[nnoo--]]cchheecckk,,......}} Enable or disable specific checks. For a list of checks, see below. - --DD{{[[nnoo--]]ddeebbuugg,,......}} Enable or disable debugging categories. For a list - of categories, see below. - --FF|----aauuttooffiixx Repair trivial things automatically. --II Show the _M_a_k_e_f_i_l_e that is constructed by including @@ -28,6 +25,8 @@ DDEESSCCRRIIPPTTIIOONN --WW{{[[nnoo--]]wwaarrnn,,......}} Enable or disable specific warnings. For a list of warnings, see below. + --dd|----ddeebbuugg Enable or disable verbose log for debugging pkglint. + --ee|----eexxppllaaiinn Print verbose explanations for diagnostics. --gg|----ggcccc--oouuttppuutt--ffoorrmmaatt @@ -80,41 +79,6 @@ DDEESSCCRRIIPPTTIIOONN [[nnoo--]]ppaattcchheess Check the pkgsrc specific patch files. - DDeebbuuggggiinngg OOppttiioonnss - aallll Enable all debugging options. - - nnoonnee Disable all debugging options. - - [[nnoo--]]iinncclluuddee Show the pathnames of the included Makefiles. - - [[nnoo--]]mmiisscc Some debugging stuff that hasn't made it into its own - category. - - [[nnoo--]]ppaattcchheess Print the states of the patch file parser. - - [[nnoo--]]qquuoottiinngg Additional information about why variables should be - quoted or not. - - [[nnoo--]]sshheellll Parser information from the shell word and the shell - command parsers. - - [[nnoo--]]ttoooollss Additional information about the tools from the tools - framework. - - [[nnoo--]]ttrraaccee Print the names of subroutines and their arguments as - they are entered. - - [[nnoo--]]uunncchheecckkeedd Show the things that pkglint cannot currently check. - These are mostly due to unresolved make variables. - - [[nnoo--]]uunnuusseedd Show which variables are detected as used, and so - will not generate an ``unused variable'' warning. - - [[nnoo--]]vvaarrttyyppeess Additional information about the variable types. - - [[nnoo--]]vvaarruussee Information about the contexts in which variables are - used. - WWaarrnniinnggss aallll Enable all warnings. @@ -185,4 +149,4 @@ BBUUGGSS If you don't understand the messages, feel free to ask on the <tech-pkg@NetBSD.org> mailing list. -NetBSD 7.0 January 12, 2016 NetBSD 7.0 +NetBSD 7.0 June 5, 2016 NetBSD 7.0 diff --git a/pkgtools/pkglint/files/pkglint.1 b/pkgtools/pkglint/files/pkglint.1 index e84a5a6d696..e0d93d19c9f 100644 --- a/pkgtools/pkglint/files/pkglint.1 +++ b/pkgtools/pkglint/files/pkglint.1 @@ -1,4 +1,4 @@ -.\" $NetBSD: pkglint.1,v 1.50 2016/01/12 01:02:49 rillig Exp $ +.\" $NetBSD: pkglint.1,v 1.51 2016/06/05 11:24:32 rillig Exp $ .\" From FreeBSD: portlint.1,v 1.8 1997/11/25 14:53:14 itojun Exp .\" .\" Copyright (c) 1997 by Jun-ichiro Itoh <itojun@itojun.org>. @@ -8,7 +8,7 @@ .\" Thomas Klausner <wiz@NetBSD.org>, 2012. .\" Roland Illig <rillig@NetBSD.org>, 2015, 2016. .\" -.Dd January 12, 2016 +.Dd June 5, 2016 .Dt PKGLINT 1 .Os .Sh NAME @@ -29,9 +29,6 @@ to be bugs, or that are simply deprecated. .It Fl C{[no-]check,...} Enable or disable specific checks. For a list of checks, see below. -.It Fl D{[no-]debug,...} -Enable or disable debugging categories. -For a list of categories, see below. .It Fl F Ns | Ns Fl -autofix Repair trivial things automatically. .It Fl I @@ -49,6 +46,8 @@ version number and exit. .It Fl W{[no-]warn,...} Enable or disable specific warnings. For a list of warnings, see below. +.It Fl d Ns | Ns Fl -debug +Enable or disable verbose log for debugging pkglint. .It Fl e Ns | Ns Fl -explain Print verbose explanations for diagnostics. .It Fl g Ns | Ns Fl -gcc-output-format @@ -104,39 +103,6 @@ Check Makefile fragments besides buildlink3. Check the pkgsrc specific patch files. .El .\" ======================================================================= -.Ss Debugging Options -.Bl -tag -width 18n -.It Cm all -Enable all debugging options. -.It Cm none -Disable all debugging options. -.It Cm [no-]include -Show the pathnames of the included Makefiles. -.It Cm [no-]misc -Some debugging stuff that hasn't made it into its own category. -.It Cm [no-]patches -Print the states of the patch file parser. -.It Cm [no-]quoting -Additional information about why variables should be quoted or not. -.It Cm [no-]shell -Parser information from the shell word and the shell command parsers. -.It Cm [no-]tools -Additional information about the tools from the tools framework. -.It Cm [no-]trace -Print the names of subroutines and their arguments as they are entered. -.It Cm [no-]unchecked -Show the things that pkglint cannot currently check. -These are mostly due to unresolved make variables. -.It Cm [no-]unused -Show which variables are detected as used, and so will not generate an -.Dq unused variable -warning. -.It Cm [no-]vartypes -Additional information about the variable types. -.It Cm [no-]varuse -Information about the contexts in which variables are used. -.El -.\" ======================================================================= .Ss Warnings .Bl -tag -width 18n .It Cm all diff --git a/pkgtools/pkglint/files/pkglint.go b/pkgtools/pkglint/files/pkglint.go index c9d76126ec5..417782108ac 100644 --- a/pkgtools/pkglint/files/pkglint.go +++ b/pkgtools/pkglint/files/pkglint.go @@ -22,7 +22,7 @@ func findPkgsrcTopdir(fname string) string { } func resolveVariableRefs(text string) string { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(text)() } @@ -65,14 +65,14 @@ func expandVariableWithDefault(varname, defaultValue string) string { if containsVarRef(value) { value = resolveVariableRefs(value) } - if G.opts.DebugMisc { - mkline.Debug2("Expanded %q to %q", varname, value) + if G.opts.Debug { + traceStep2("Expanded %q to %q", varname, value) } return value } func CheckfileExtra(fname string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(fname)() } @@ -82,7 +82,7 @@ func CheckfileExtra(fname string) { } func ChecklinesDescr(lines []*Line) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(lines[0].Fname)() } @@ -110,7 +110,7 @@ func ChecklinesDescr(lines []*Line) { } func ChecklinesMessage(lines []*Line) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(lines[0].Fname)() } @@ -148,7 +148,7 @@ func ChecklinesMessage(lines []*Line) { } func CheckfileMk(fname string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(fname)() } @@ -162,21 +162,21 @@ func CheckfileMk(fname string) { } func Checkfile(fname string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(fname)() } basename := path.Base(fname) if hasPrefix(basename, "work") || hasSuffix(basename, "~") || hasSuffix(basename, ".orig") || hasSuffix(basename, ".rej") { if G.opts.Import { - Errorf(fname, noLines, "Must be cleaned up before committing the package.") + NewLineWhole(fname).Error0("Must be cleaned up before committing the package.") } return } st, err := os.Lstat(fname) if err != nil { - Errorf(fname, noLines, "%s", err) + NewLineWhole(fname).Errorf("%s", err) return } @@ -198,16 +198,16 @@ func Checkfile(fname string) { case matches(fname, `(?:^|/)files/[^/]*$`): // Ok case !isEmptyDir(fname): - Warnf(fname, noLines, "Unknown directory name.") + NewLineWhole(fname).Warn0("Unknown directory name.") } case st.Mode()&os.ModeSymlink != 0: if !matches(basename, `^work`) { - Warnf(fname, noLines, "Unknown symlink name.") + NewLineWhole(fname).Warn0("Unknown symlink name.") } case !st.Mode().IsRegular(): - Errorf(fname, noLines, "Only files and directories are allowed in pkgsrc.") + NewLineWhole(fname).Error0("Only files and directories are allowed in pkgsrc.") case basename == "ALTERNATIVES": if G.opts.CheckAlternatives { @@ -255,12 +255,12 @@ func Checkfile(fname string) { } case matches(fname, `(?:^|/)patches/manual[^/]*$`): - if G.opts.DebugUnchecked { - Debugf(fname, noLines, "Unchecked file %q.", fname) + if G.opts.Debug { + traceStep1("Unchecked file %q.", fname) } case matches(fname, `(?:^|/)patches/[^/]*$`): - Warnf(fname, noLines, "Patch files should be named \"patch-\", followed by letters, '-', '_', '.', and digits only.") + NewLineWhole(fname).Warn0("Patch files should be named \"patch-\", followed by letters, '-', '_', '.', and digits only.") case matches(basename, `^(?:.*\.mk|Makefile.*)$`) && !matches(fname, `files/`) && !matches(fname, `patches/`): if G.opts.CheckMk { @@ -285,7 +285,7 @@ func Checkfile(fname string) { // Skip default: - Warnf(fname, noLines, "Unexpected file found.") + NewLineWhole(fname).Warn0("Unexpected file found.") if G.opts.CheckExtra { CheckfileExtra(fname) } @@ -402,6 +402,7 @@ func resolveVarsInRelativePath(relpath string, adjustDepth bool) string { tmp = strings.Replace(tmp, "${PHPPKGSRCDIR}", "../../lang/php55", -1) tmp = strings.Replace(tmp, "${SUSE_DIR_PREFIX}", "suse100", -1) tmp = strings.Replace(tmp, "${PYPKGSRCDIR}", "../../lang/python27", -1) + tmp = strings.Replace(tmp, "${PYPACKAGE}", "python27", -1) if G.Pkg != nil { tmp = strings.Replace(tmp, "${FILESDIR}", G.Pkg.Filesdir, -1) tmp = strings.Replace(tmp, "${PKGDIR}", G.Pkg.Pkgdir, -1) @@ -413,8 +414,8 @@ func resolveVarsInRelativePath(relpath string, adjustDepth bool) string { } } - if G.opts.DebugMisc { - dummyLine.Debug2("resolveVarsInRelativePath: %q => %q", relpath, tmp) + if G.opts.Debug { + traceStep2("resolveVarsInRelativePath: %q => %q", relpath, tmp) } return tmp } diff --git a/pkgtools/pkglint/files/plist.go b/pkgtools/pkglint/files/plist.go index 7373c05583b..7df3ca97b23 100644 --- a/pkgtools/pkglint/files/plist.go +++ b/pkgtools/pkglint/files/plist.go @@ -7,7 +7,7 @@ import ( ) func ChecklinesPlist(lines []*Line) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(lines[0].Fname)() } diff --git a/pkgtools/pkglint/files/shell.go b/pkgtools/pkglint/files/shell.go index ed79a958f10..a52cc228f14 100644 --- a/pkgtools/pkglint/files/shell.go +++ b/pkgtools/pkglint/files/shell.go @@ -8,30 +8,25 @@ import ( ) const ( - reMkShellvaruse = `(?:^|[^\$])\$\$\{?(\w+)\}?` - reVarnameDirect = `(?:[-*+.0-9A-Z_a-z{}\[]+)` - reShellToken = `^\s*(` + + reShellToken = `^\s*(` + `#.*` + // shell comment `|(?:` + `'[^']*'` + // single quoted string "|\"`[^`]+`\"" + // backticks command execution in double quotes `|"(?:\\.|[^"])*"` + // double quoted string - "|`[^`]*`" + // backticks command execution, somewhat naive + "|`[^`]*`" + // backticks command execution (very simple case) `|\\\$\$` + // a shell-escaped dollar sign `|\\[^\$]` + // other escaped characters `|\$[\w_]` + // one-character make(1) variable - `|\$\{[^{}]+\}` + // make(1) variable, ${...} - `|\$\([^()]+\)` + // make(1) variable, $(...) - `|\$[/@<^]` + // special make(1) variables `|\$\$[0-9A-Z_a-z]+` + // shell variable `|\$\$[!#?@]` + // special shell variables `|\$\$[./]` + // unescaped dollar in shell, followed by punctuation `|\$\$\$\$` + // the special pid shell variable `|\$\$\{[0-9A-Z_a-z]+[#%:]?[^}]*\}` + // shell variable in braces - `|\$\$\(` + // POSIX-style backticks replacement `|[^\(\)'\"\\\s;&\|<>` + "`" + `\$]` + // non-special character `|\$\{[^\s\"'` + "`" + `]+` + // HACK: nested make(1) variables `)+` + // any of the above may be repeated + `|\$\$\(` + // POSIX-style backticks replacement `|;;?` + `|&&?` + `|\|\|?` + @@ -41,7 +36,11 @@ const ( `|<<?` + `|>>?` + `|#.*)` - reShVarassign = `^([A-Z_a-z]\w*)=` + reShVarassign = `^([A-Z_a-z]\w*)=` + reShVarname = `(?:[!#*\-\d?@]|\$\$|[A-Za-z_]\w*)` + reShVarexpansion = `(?:(?:#|##|%|%%|:-|:=|:\?|:\+|\+)[^$\\{}]*)` + reShVaruse = `\$\$` + `(?:` + reShVarname + `|` + `\{` + reShVarname + `(?:` + reShVarexpansion + `)?` + `\})` + reShDollar = `\\\$\$|` + reShVaruse + `|\$\$[,\-/|]` ) // ShellCommandState @@ -110,22 +109,11 @@ func NewShellLine(mkline *MkLine) *ShellLine { return &ShellLine{mkline.Line, mkline} } -type ShellwordState uint8 - -const ( - swstPlain ShellwordState = iota - swstSquot - swstDquot - swstDquotBackt - swstBackt -) - -func (st ShellwordState) String() string { - return [...]string{"plain", "squot", "dquot", "dquot+backt", "backt"}[st] -} +var shellcommandsContextType = &Vartype{lkNone, CheckvarShellCommands, []AclEntry{{"*", aclpAllRuntime}}, false} +var shellwordVuc = &VarUseContext{shellcommandsContextType, vucTimeUnknown, vucQuotPlain, vucExtentWord} -func (shline *ShellLine) CheckToken(token string, checkQuoting bool) { - if G.opts.DebugTrace { +func (shline *ShellLine) CheckWord(token string, checkQuoting bool) { + if G.opts.Debug { defer tracecall(token, checkQuoting)() } @@ -134,11 +122,10 @@ func (shline *ShellLine) CheckToken(token string, checkQuoting bool) { } line := shline.line - shellcommandsContextType := &Vartype{lkNone, CheckvarShellCommands, []AclEntry{{"*", aclpAllRuntime}}, false} - shellwordVuc := &VarUseContext{shellcommandsContextType, vucTimeUnknown, vucQuotPlain, vucExtentWord} - if m, varname, mod := match2(token, `^\$\{(`+reVarnameDirect+`)(:[^{}]+)?\}$`); m { - shline.mkline.CheckVaruse(varname, mod, shellwordVuc) + p := NewMkParser(line, token, false) + if varuse := p.VarUse(); varuse != nil && p.EOF() { + shline.mkline.CheckVaruse(varuse, shellwordVuc) return } @@ -149,12 +136,13 @@ func (shline *ShellLine) CheckToken(token string, checkQuoting bool) { line.Warn0("Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") } - repl := NewPrefixReplacer(token) - state := swstPlain + parser := NewMkParser(line, token, false) + repl := parser.repl + quoting := shqPlain outer: - for repl.rest != "" { - if G.opts.DebugShell { - line.Debugf("shell state %s: %q", state, repl.rest) + for !parser.EOF() { + if G.opts.Debug { + traceStep("shell state %s: %q", quoting, parser.Rest()) } switch { @@ -162,86 +150,26 @@ outer: // reasonable to check the whole shell command // recursively, instead of splitting off the first // make(1) variable. - case state == swstBackt || state == swstDquotBackt: + case quoting == shqBackt || quoting == shqDquotBackt: var backtCommand string - backtCommand, state = shline.unescapeBackticks(token, repl, state) + backtCommand, quoting = shline.unescapeBackticks(token, repl, quoting) setE := true shline.CheckShellCommand(backtCommand, &setE) - // Make(1) variables have the same syntax, no matter in which state we are currently. - case repl.AdvanceRegexp(`^\$\{(` + reVarnameDirect + `|@)(:[^\{]+)?\}`), - repl.AdvanceRegexp(`^\$\((` + reVarnameDirect + `|@])(:[^\)]+)?\)`), - repl.AdvanceRegexp(`^\$([\w@<])()`): - varname, mod := repl.m[1], repl.m[2] - - if varname == "@" { - line := shline.line - line.Warn0("Please use \"${.TARGET}\" instead of \"$@\".") - Explain2( - "The variable $@ can easily be confused with the shell variable of", - "the same name, which has a completely different meaning.") - varname = ".TARGET" - } - - switch { - case state == swstPlain && hasSuffix(mod, ":Q"): - // Fine. - case state == swstBackt: - // Don't check anything here, to avoid false positives for tool names. - case (state == swstSquot || state == swstDquot) && matches(varname, `^(?:.*DIR|.*FILE|.*PATH|.*_VAR|PREFIX|.*BASE|PKGNAME)$`): - // This is ok if we don't allow these variables to have embedded [\$\\\"\'\`]. - case state == swstDquot && hasSuffix(mod, ":Q"): - line.Warn0("Please don't use the :Q operator in double quotes.") - Explain2( - "Either remove the :Q or the double quotes. In most cases, it is", - "more appropriate to remove the double quotes.") - } - - if varname != "@" { - vucstate := vucQuotUnknown - switch state { - case swstPlain: - vucstate = vucQuotPlain - case swstDquot: - vucstate = vucQuotDquot - case swstSquot: - vucstate = vucQuotSquot - case swstBackt: - vucstate = vucQuotBackt - } - vuc := &VarUseContext{shellcommandsContextType, vucTimeUnknown, vucstate, vucExtentWordpart} - shline.mkline.CheckVaruse(varname, mod, vuc) - } - - // The syntax of the variable modifiers can get quite - // hairy. In lack of motivation, we just skip anything - // complicated, hoping that at least the braces are balanced. - case repl.AdvanceStr("${"): - braces := 1 - skip: - for repl.rest != "" && braces > 0 { - switch { - case repl.AdvanceStr("}"): - braces-- - case repl.AdvanceStr("{"): - braces++ - case repl.AdvanceRegexp(`^[^{}]+`): - // skip - default: - break skip - } - } + // Make(1) variables have the same syntax, no matter in which state we are currently. + case shline.checkVaruseToken(parser, quoting): + break - case state == swstPlain: + case quoting == shqPlain: switch { case repl.AdvanceRegexp(`^[!#\%&\(\)*+,\-.\/0-9:;<=>?@A-Z\[\]^_a-z{|}~]+`), repl.AdvanceRegexp(`^\\(?:[ !"#'\(\)*./;?\\^{|}]|\$\$)`): case repl.AdvanceStr("'"): - state = swstSquot + quoting = shqSquot case repl.AdvanceStr("\""): - state = swstDquot + quoting = shqDquot case repl.AdvanceStr("`"): - state = swstBackt + quoting = shqBackt case repl.AdvanceRegexp(`^\$\$([0-9A-Z_a-z]+|#)`), repl.AdvanceRegexp(`^\$\$\{([0-9A-Z_a-z]+|#)\}`), repl.AdvanceRegexp(`^\$\$(\$)\$`): @@ -288,10 +216,10 @@ outer: break outer } - case state == swstSquot: + case quoting == shqSquot: switch { case repl.AdvanceRegexp(`^'`): - state = swstPlain + quoting = shqPlain case repl.AdvanceRegexp(`^[^\$\']+`): // just skip case repl.AdvanceRegexp(`^\$\$`): @@ -300,12 +228,12 @@ outer: break outer } - case state == swstDquot: + case quoting == shqDquot: switch { case repl.AdvanceStr("\""): - state = swstPlain + quoting = shqPlain case repl.AdvanceStr("`"): - state = swstDquotBackt + quoting = shqDquotBackt case repl.AdvanceRegexp("^[^$\"\\\\`]+"): break case repl.AdvanceStr("\\$$"): @@ -323,9 +251,51 @@ outer: } } - if strings.TrimSpace(repl.rest) != "" { - line.Errorf("Internal pkglint error: ShellLine.CheckToken state=%s, rest=%q, token=%q", state, repl.rest, token) + if strings.TrimSpace(parser.Rest()) != "" { + line.Warnf("Pkglint parse error in ShellLine.CheckWord at %q (quoting=%s, rest=%q)", token, quoting, parser.Rest()) + } +} + +func (shline *ShellLine) checkVaruseToken(parser *MkParser, quoting ShQuoting) bool { + if G.opts.Debug { + defer tracecall(parser.Rest(), quoting)() + } + + varuse := parser.VarUse() + if varuse == nil { + return false + } + varname := varuse.varname + + if varname == "@" { + shline.line.Warn0("Please use \"${.TARGET}\" instead of \"$@\".") + Explain2( + "The variable $@ can easily be confused with the shell variable of", + "the same name, which has a completely different meaning.") + varname = ".TARGET" + varuse = &MkVarUse{varname, varuse.modifiers} + } + + switch { + case quoting == shqPlain && varuse.IsQ(): + // Fine. + case quoting == shqBackt: + // Don't check anything here, to avoid false positives for tool names. + case (quoting == shqSquot || quoting == shqDquot) && matches(varname, `^(?:.*DIR|.*FILE|.*PATH|.*_VAR|PREFIX|.*BASE|PKGNAME)$`): + // This is ok if we don't allow these variables to have embedded [\$\\\"\'\`]. + case quoting == shqDquot && varuse.IsQ(): + shline.line.Warn0("Please don't use the :Q operator in double quotes.") + Explain2( + "Either remove the :Q or the double quotes. In most cases, it is", + "more appropriate to remove the double quotes.") + } + + if varname != "@" { + vucstate := quoting.ToVarUseContext() + vuc := &VarUseContext{shellcommandsContextType, vucTimeUnknown, vucstate, vucExtentWordpart} + shline.mkline.CheckVaruse(varuse, vuc) } + return true } // Scan for the end of the backticks, checking for single backslashes @@ -333,26 +303,30 @@ outer: // before a dollar, a backslash or a backtick. // // See http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_06_03 -func (shline *ShellLine) unescapeBackticks(shellword string, repl *PrefixReplacer, state ShellwordState) (unescaped string, newState ShellwordState) { +func (shline *ShellLine) unescapeBackticks(shellword string, repl *PrefixReplacer, quoting ShQuoting) (unescaped string, newQuoting ShQuoting) { + if G.opts.Debug { + defer tracecall(shellword, quoting, "=>", ref(&unescaped))() + } + line := shline.line for repl.rest != "" { switch { case repl.AdvanceStr("`"): - if state == swstBackt { - state = swstPlain + if quoting == shqBackt { + quoting = shqPlain } else { - state = swstDquot + quoting = shqDquot } - return unescaped, state + return unescaped, quoting - case repl.AdvanceRegexp("^\\\\([\\\\`$])"): + case repl.AdvanceRegexp("^\\\\([\"\\\\`$])"): unescaped += repl.m[1] case repl.AdvanceStr("\\"): line.Warn0("Backslashes should be doubled inside backticks.") unescaped += "\\" - case state == swstDquotBackt && repl.AdvanceStr("\""): + case quoting == shqDquotBackt && repl.AdvanceStr("\""): line.Warn0("Double quotes inside backticks inside double quotes are error prone.") Explain4( "According to the SUSv3, they produce undefined results.", @@ -364,11 +338,11 @@ func (shline *ShellLine) unescapeBackticks(shellword string, repl *PrefixReplace unescaped += repl.m[1] default: - line.Errorf("Internal pkglint error: checklineMkShellword shellword=%q rest=%q", shellword, repl.rest) + line.Errorf("Internal pkglint error in ShellLine.unescapeBackticks at %q (rest=%q)", shellword, repl.rest) } } line.Error1("Unfinished backquotes: rest=%q", repl.rest) - return unescaped, state + return unescaped, quoting } func (shline *ShellLine) variableNeedsQuoting(shvarname string) bool { @@ -388,7 +362,7 @@ type ShelltextContext struct { } func (shline *ShellLine) CheckShellCommandLine(shelltext string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(shelltext)() } @@ -437,20 +411,39 @@ func (shline *ShellLine) CheckShellCommandLine(shelltext string) { } func (shline *ShellLine) CheckShellCommand(shellcmd string, pSetE *bool) { + if false { + p := NewMkShParser(shline.line, shellcmd, false) + cmds := p.Program() + rest := p.tok.parser.Rest() + if rest != "" { + traceStep("shellcmd=%q", shellcmd) + if cmds != nil { + for _, andor := range cmds.AndOrs { + traceStep("AndOr %v", andor) + } + } + shline.line.Warnf("Pkglint parse error in ShellLine.CheckShellCommand at %q", p.peekText()+rest) + } + } + state := scstStart tokens, rest := splitIntoShellTokens(shline.line, shellcmd) + if rest != "" { + shline.line.Warnf("Pkglint parse error in ShellLine.CheckShellCommand at %q (state=%s)", rest, state) + } + prevToken := "" for _, token := range tokens { - if G.opts.DebugShell { - shline.line.Debugf("checkShellCommand state=%v token=%q", state, token) + if G.opts.Debug { + traceStep("checkShellCommand state=%v token=%q", state, token) } { - quotingNecessary := state != scstCase && - state != scstForCont && - state != scstSetCont && - !(state == scstStart && matches(token, reShVarassign)) - shline.CheckToken(token, quotingNecessary) + noQuotingNeeded := state == scstCase || + state == scstForCont || + state == scstSetCont || + (state == scstStart && matches(token, reShVarassign)) + shline.CheckWord(token, !noQuotingNeeded) } st := &ShelltextContext{shline, state, token} @@ -474,10 +467,6 @@ func (shline *ShellLine) CheckShellCommand(shellcmd string, pSetE *bool) { state = shline.nextState(state, token) prevToken = token } - - if rest != "" { - shline.line.Errorf("Internal pkglint error: ShellLine.CheckShellCommand state=%s rest=%q shellcmd=%q", state, rest, shellcmd) - } } func (shline *ShellLine) CheckShellCommands(shellcmds string) { @@ -489,7 +478,7 @@ func (shline *ShellLine) CheckShellCommands(shellcmds string) { } func (shline *ShellLine) checkHiddenAndSuppress(hiddenAndSuppress, rest string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall(hiddenAndSuppress, rest)() } @@ -537,7 +526,7 @@ func (shline *ShellLine) checkHiddenAndSuppress(hiddenAndSuppress, rest string) } func (ctx *ShelltextContext) checkCommandStart() { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall2(ctx.state.String(), ctx.shellword)() } @@ -551,14 +540,14 @@ func (ctx *ShelltextContext) checkCommandStart() { case ctx.handleForbiddenCommand(): case ctx.handleTool(): case ctx.handleCommandVariable(): - case matches(shellword, `^(?:\(|\)|:|;|;;|&&|\|\||\{|\}|break|case|cd|continue|do|done|elif|else|esac|eval|exec|exit|export|fi|for|if|read|set|shift|then|umask|unset|while)$`): - case matches(shellword, `^[\w_]+=.*$`): // Variable assignment + case matches(shellword, `^(?:\$\$\(|\(|\)|:|;|;;|&&|\|\||\{|\}|break|case|cd|continue|do|done|elif|else|esac|eval|exec|exit|export|fi|for|if|read|set|shift|then|umask|unset|while)$`): + case matches(shellword, `^\w+=`): // Variable assignment case hasPrefix(shellword, "./"): // All commands from the current directory are fine. + case hasPrefix(shellword, "${PKGSRCDIR"): // With or without the :Q modifier case ctx.handleComment(): default: if G.opts.WarnExtra { - line := ctx.shline.line - line.Warn1("Unknown shell command %q.", shellword) + ctx.shline.line.Warn1("Unknown shell command %q.", shellword) Explain3( "If you want your package to be portable to all platforms that pkgsrc", "supports, you should only use shell commands that are covered by the", @@ -568,12 +557,13 @@ func (ctx *ShelltextContext) checkCommandStart() { } func (ctx *ShelltextContext) handleTool() bool { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(ctx.shellword)() } shellword := ctx.shellword - if !G.globalData.Tools[shellword] { + tool := G.globalData.Tools.byName[shellword] + if tool == nil { return false } @@ -581,8 +571,8 @@ func (ctx *ShelltextContext) handleTool() bool { ctx.shline.line.Warn1("The %q tool is used but not added to USE_TOOLS.", shellword) } - if G.globalData.toolsVarRequired[shellword] { - ctx.shline.line.Warn2("Please use \"${%s}\" instead of %q.", G.globalData.Vartools[shellword], shellword) + if tool.MustUseVarForm { + ctx.shline.line.Warn2("Please use \"${%s}\" instead of %q.", tool.Varname, shellword) } ctx.shline.checkCommandUse(shellword) @@ -592,29 +582,27 @@ func (ctx *ShelltextContext) handleTool() bool { func (ctx *ShelltextContext) handleForbiddenCommand() bool { switch path.Base(ctx.shellword) { case "ktrace", "mktexlsr", "strace", "texconfig", "truss": - default: - return false + ctx.shline.line.Error1("%q must not be used in Makefiles.", ctx.shellword) + Explain3( + "This command must appear in INSTALL scripts, not in the package", + "Makefile, so that the package also works if it is installed as a binary", + "package via pkg_add.") + return true } - - ctx.shline.line.Error1("%q must not be used in Makefiles.", ctx.shellword) - Explain3( - "This command must appear in INSTALL scripts, not in the package", - "Makefile, so that the package also works if it is installed as a binary", - "package via pkg_add.") - return true + return false } func (ctx *ShelltextContext) handleCommandVariable() bool { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(ctx.shellword)() } shellword := ctx.shellword if m, varname := match1(shellword, `^\$\{([\w_]+)\}$`); m { - if toolname := G.globalData.VarnameToToolname[varname]; toolname != "" { - if !G.Mk.tools[toolname] { - ctx.shline.line.Warn1("The %q tool is used but not added to USE_TOOLS.", toolname) + if tool := G.globalData.Tools.byVarname[varname]; tool != nil { + if !G.Mk.tools[tool.Name] { + ctx.shline.line.Warn1("The %q tool is used but not added to USE_TOOLS.", tool.Name) } ctx.shline.checkCommandUse(shellword) return true @@ -635,11 +623,10 @@ func (ctx *ShelltextContext) handleCommandVariable() bool { } func (ctx *ShelltextContext) handleComment() bool { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(ctx.shellword)() } - line := ctx.shline.line shellword := ctx.shellword if !hasPrefix(shellword, "#") { return false @@ -649,10 +636,10 @@ func (ctx *ShelltextContext) handleComment() bool { multiline := ctx.shline.line.IsMultiline() if semicolon { - line.Warn0("A shell comment should not contain semicolons.") + ctx.shline.line.Warn0("A shell comment should not contain semicolons.") } if multiline { - line.Warn0("A shell comment does not stop at the end of line.") + ctx.shline.line.Warn0("A shell comment does not stop at the end of line.") } if semicolon || multiline { @@ -697,7 +684,7 @@ func (ctx *ShelltextContext) checkAutoMkdirs() { "relative to ${PREFIX}.") } - if (state == scstInstallDir || state == scstInstallDir2) && !matches(shellword, reMkShellvaruse) { + if (state == scstInstallDir || state == scstInstallDir2) && !contains(shellword, "$$") { if m, dirname := match1(shellword, `^(?:\$\{DESTDIR\})?\$\{PREFIX(?:|:Q)\}/(.*)`); m { line.Note1("You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= %s\" instead of this command.", dirname) Explain( @@ -928,37 +915,146 @@ func (shline *ShellLine) nextState(state scState, shellword string) scState { case state == scstEcho: return scstCont default: - if G.opts.DebugShell { - shline.line.Errorf("Internal pkglint error: shellword.nextState state=%s shellword=%q", state, shellword) + if G.opts.Debug { + traceStep("Internal pkglint error: shellword.nextState state=%s shellword=%q", state, shellword) } return scstStart } } -func splitIntoShellTokens(line *Line, text string) (words []string, rest string) { - repl := NewPrefixReplacer(text) - for repl.AdvanceRegexp(reShellToken) { - words = append(words, repl.m[1]) +// Example: "word1 word2;;;" => "word1", "word2", ";;", ";" +func splitIntoShellTokens(line *Line, text string) (tokens []string, rest string) { + if G.opts.Debug { + defer tracecall(line, text)() } - repl.AdvanceRegexp(`^\s+`) - return words, repl.rest + + word := "" + emit := func() { + if word != "" { + tokens = append(tokens, word) + word = "" + } + } + p := NewShTokenizer(line, text, false) + atoms := p.ShAtoms() + q := shqPlain + for _, atom := range atoms { + q = atom.Quoting + if atom.Type == shtSpace && q == shqPlain { + emit() + } else if atom.Type == shtWord || atom.Type == shtVaruse || atom.Quoting != shqPlain { + word += atom.Text + } else { + emit() + tokens = append(tokens, atom.Text) + } + } + emit() + return tokens, word + p.mkp.Rest() } +// Example: "word1 word2;;;" => "word1", "word2;;;" // Compare devel/bmake/files/str.c, function brk_string. -func splitIntoShellWords(line *Line, text string) (words []string, rest string) { - reShellWord := `^\s*(` + - `#.*` + // shell comment - `|(?:` + - `'[^']*'` + // single quoted string - `|"(?:\\.|[^"\\])*"` + // double quoted string - "|`[^`]*`" + // backticks command execution - "|[^\\s\"'`\\\\]+" + // normal characters - `|\\.` + // escaped character - `)+)` // any of the above may be repeated - repl := NewPrefixReplacer(text) - for repl.AdvanceRegexp(reShellWord) { - words = append(words, repl.m[1]) +func splitIntoMkWords(line *Line, text string) (words []string, rest string) { + if G.opts.Debug { + defer tracecall(line, text)() + } + + p := NewShTokenizer(line, text, false) + atoms := p.ShAtoms() + word := "" + for _, atom := range atoms { + if atom.Type == shtSpace && atom.Quoting == shqPlain { + words = append(words, word) + word = "" + } else { + word += atom.Text + } + } + if word != "" && atoms[len(atoms)-1].Quoting == shqPlain { + words = append(words, word) + word = "" + } + return words, word + p.mkp.Rest() +} + +type ShQuote struct { + repl *PrefixReplacer + q ShQuoting +} + +func NewShQuote(s string) *ShQuote { + return &ShQuote{NewPrefixReplacer(s), shqPlain} +} + +func (sq *ShQuote) Feed(str string) { + const ( + reSkip = "^[^\"'`\\\\]+" // Characters that don’t influence the quoting mode. + ) + + repl := sq.repl + repl.rest += str + for repl.rest != "" { + if repl.AdvanceRegexp(reSkip) { + continue + } + + mark := repl.Mark() + switch sq.q { + case shqPlain: + switch { + case repl.AdvanceStr("\""): + sq.q = shqDquot + case repl.AdvanceStr("'"): + sq.q = shqSquot + case repl.AdvanceStr("`"): + sq.q = shqBackt + case repl.AdvanceRegexp(`^\\.`): + } + + case shqDquot: + switch { + case repl.AdvanceStr("\""): + sq.q = shqPlain + case repl.AdvanceStr("`"): + sq.q = shqDquotBackt + case repl.AdvanceStr("'"): + case repl.AdvanceRegexp(`^\\.`): + } + case shqSquot: + switch { + case repl.AdvanceStr("'"): + sq.q = shqPlain + case repl.AdvanceRegexp(`^[^']+`): + } + case shqBackt: + switch { + case repl.AdvanceStr("`"): + sq.q = shqPlain + case repl.AdvanceStr("'"): + sq.q = shqBacktSquot + case repl.AdvanceRegexp(`^\\.`): + } + + case shqDquotBackt: + switch { + case repl.AdvanceStr("`"): + sq.q = shqDquot + case repl.AdvanceStr("'"): + sq.q = shqDquotBacktSquot + case repl.AdvanceRegexp(`^\\.`): + } + case shqDquotBacktSquot: + switch { + case repl.AdvanceStr("'"): + sq.q = shqDquotBackt + } + } + + if repl.Since(mark) == "" { + traceStep2("ShQuote.stuck stack=%s rest=%s", sq.q.String(), sq.repl.rest) + repl.AdvanceRest() + sq.q = shqUnknown + } } - repl.AdvanceRegexp(`^\s+`) - return words, repl.rest } diff --git a/pkgtools/pkglint/files/shell_test.go b/pkgtools/pkglint/files/shell_test.go index e3bb4767525..f6a94a7c8f8 100644 --- a/pkgtools/pkglint/files/shell_test.go +++ b/pkgtools/pkglint/files/shell_test.go @@ -1,6 +1,8 @@ package main import ( + "strings" + check "gopkg.in/check.v1" ) @@ -21,20 +23,77 @@ func (s *Suite) TestReShellToken(c *check.C) { c.Check(match("${file%.c}.o", re), matches) } -func (s *Suite) TestSplitIntoShellTokens_LineContinuation(c *check.C) { - line := NewLine("fname", 10, "dummy", nil) - - words, rest := splitIntoShellTokens(line, "if true; then \\") +func (s *Suite) Test_SplitIntoShellTokens_LineContinuation(c *check.C) { + words, rest := splitIntoShellTokens(dummyLine, "if true; then \\") c.Check(words, check.DeepEquals, []string{"if", "true", ";", "then"}) c.Check(rest, equals, "\\") - words, rest = splitIntoShellTokens(line, "pax -s /.*~$$//g") + c.Check(s.Output(), equals, "WARN: Pkglint parse error in ShTokenizer.ShAtom at \"\\\\\" (quoting=plain)\n") +} + +func (s *Suite) Test_SplitIntoShellTokens_DollarSlash(c *check.C) { + words, rest := splitIntoShellTokens(dummyLine, "pax -s /.*~$$//g") c.Check(words, check.DeepEquals, []string{"pax", "-s", "/.*~$$//g"}) c.Check(rest, equals, "") } +func (s *Suite) Test_SplitIntoShellTokens_DollarSubshell(c *check.C) { + words, rest := splitIntoShellTokens(dummyLine, "id=$$(${AWK} '{print}' < ${WRKSRC}/idfile) && echo \"$$id\"") + + c.Check(words, deepEquals, []string{"id=", "$$(", "${AWK}", "'{print}'", "<", "${WRKSRC}/idfile", ")", "&&", "echo", "\"$$id\""}) + c.Check(rest, equals, "") +} + +func (s *Suite) Test_SplitIntoShellTokens_Semicolons(c *check.C) { + words, rest := splitIntoShellTokens(dummyLine, "word1 word2;;;") + + c.Check(words, deepEquals, []string{"word1", "word2", ";;", ";"}) + c.Check(rest, equals, "") +} + +func (s *Suite) Test_SplitIntoShellTokens_Whitespace(c *check.C) { + text := "\t${RUN} cd ${WRKSRC}&&(${ECHO} ${PERL5:Q};${ECHO})|${BASH} ./install" + words, rest := splitIntoShellTokens(dummyLine, text) + + c.Check(words, deepEquals, []string{ + "${RUN}", + "cd", "${WRKSRC}", + "&&", "(", "${ECHO}", "${PERL5:Q}", ";", "${ECHO}", ")", + "|", "${BASH}", "./install"}) + c.Check(rest, equals, "") +} + +func (s *Suite) Test_SplitIntoShellTokens_MkVarUse(c *check.C) { + varuseWord := "${GCONF_SCHEMAS:@.s.@${INSTALL_DATA} ${WRKSRC}/src/common/dbus/${.s.} ${DESTDIR}${GCONF_SCHEMAS_DIR}/@}" + words, rest := splitIntoShellTokens(dummyLine, varuseWord) + + c.Check(words, deepEquals, []string{varuseWord}) + c.Check(rest, equals, "") +} + +func (s *Suite) Test_SplitIntoMkWords_Semicolons(c *check.C) { + words, rest := splitIntoMkWords(dummyLine, "word1 word2;;;") + + c.Check(words, deepEquals, []string{"word1", "word2;;;"}) + c.Check(rest, equals, "") +} + +func (s *Suite) Test_SplitIntoShellTokens_VaruseSpace(c *check.C) { + words, rest := splitIntoShellTokens(dummyLine, "${VAR:S/ /_/g}") + + c.Check(words, deepEquals, []string{"${VAR:S/ /_/g}"}) + c.Check(rest, equals, "") +} + +func (s *Suite) Test_SplitIntoMkWords_VaruseSpace(c *check.C) { + words, rest := splitIntoMkWords(dummyLine, "${VAR:S/ /_/g}") + + c.Check(words, deepEquals, []string{"${VAR:S/ /_/g}"}) + c.Check(rest, equals, "") +} + func (s *Suite) TestChecklineMkShellCommandLine(c *check.C) { s.UseCommandLine(c, "-Wall") G.Mk = s.NewMkLines("fname", @@ -54,8 +113,7 @@ func (s *Suite) TestChecklineMkShellCommandLine(c *check.C) { "WARN: fname:1: Unquoted shell variable \"uname\".\n"+ "WARN: fname:1: Unknown shell command \"echo\".\n") - G.globalData.Tools = map[string]bool{"echo": true} - G.globalData.PredefinedTools = map[string]bool{"echo": true} + s.RegisterTool(&Tool{Name: "echo", Predefined: true}) G.Mk = s.NewMkLines("fname", "# dummy") G.globalData.InitVartypes() @@ -63,27 +121,31 @@ func (s *Suite) TestChecklineMkShellCommandLine(c *check.C) { shline.CheckShellCommandLine("echo ${PKGNAME:Q}") // vucQuotPlain c.Check(s.Output(), equals, ""+ - "WARN: fname:1: PKGNAME may not be used in this file.\n"+ + "WARN: fname:1: PKGNAME may not be used in this file; it would be ok in Makefile, Makefile.*, *.mk.\n"+ "NOTE: fname:1: The :Q operator isn't necessary for ${PKGNAME} here.\n") shline.CheckShellCommandLine("echo \"${CFLAGS:Q}\"") // vucQuotDquot c.Check(s.Output(), equals, ""+ "WARN: fname:1: Please don't use the :Q operator in double quotes.\n"+ - "WARN: fname:1: CFLAGS may not be used in this file.\n"+ + "WARN: fname:1: CFLAGS may not be used in this file; it would be ok in Makefile, Makefile.common, options.mk, *.mk.\n"+ "WARN: fname:1: Please use ${CFLAGS:M*:Q} instead of ${CFLAGS:Q} and make sure the variable appears outside of any quoting characters.\n") shline.CheckShellCommandLine("echo '${COMMENT:Q}'") // vucQuotSquot - c.Check(s.Output(), equals, "WARN: fname:1: COMMENT may not be used in this file.\n") + c.Check(s.Output(), equals, ""+ + "WARN: fname:1: COMMENT may not be used in any file; it is a write-only variable.\n"+ + "WARN: fname:1: Please move ${COMMENT:Q} outside of any quoting characters.\n") shline.CheckShellCommandLine("echo $$@") c.Check(s.Output(), equals, "WARN: fname:1: The $@ shell variable should only be used in double quotes.\n") - shline.CheckShellCommandLine("echo \"$$\"") // As seen by make(1); the shell sees: echo $ + shline.CheckShellCommandLine("echo \"$$\"") // As seen by make(1); the shell sees: echo "$" - c.Check(s.Output(), equals, "WARN: fname:1: Unescaped $ or strange shell variable found.\n") + c.Check(s.Output(), equals, ""+ + "WARN: fname:1: Pkglint parse error in ShTokenizer.ShAtom at \"$$\\\"\" (quoting=d)\n"+ + "WARN: fname:1: Pkglint parse error in ShellLine.CheckShellCommand at \"$$\\\"\" (state=start)\n") shline.CheckShellCommandLine("echo \"\\n\"") // As seen by make(1); the shell sees: echo "\n" @@ -93,6 +155,10 @@ func (s *Suite) TestChecklineMkShellCommandLine(c *check.C) { c.Check(s.Output(), equals, "") + shline.CheckShellCommandLine("${RUN} echo $${variable+set}") + + c.Check(s.Output(), equals, "") + // Based on mail/thunderbird/Makefile, rev. 1.159 shline.CheckShellCommandLine("${RUN} subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\"") @@ -129,7 +195,7 @@ func (s *Suite) TestChecklineMkShellCommandLine(c *check.C) { "done") c.Check(s.Output(), equals, ""+ - "WARN: fname:1: WRKSRC may not be used in this file.\n"+ + "WARN: fname:1: WRKSRC may not be used in this file; it would be ok in Makefile, Makefile.*, *.mk.\n"+ "WARN: fname:1: Unknown shell command \"[\".\n"+ "WARN: fname:1: Unknown shell command \"${TOOLS_PATH.msgfmt}\".\n") @@ -147,7 +213,7 @@ func (s *Suite) TestChecklineMkShellCommandLine(c *check.C) { func (s *Suite) TestShellLine_CheckShelltext_nofix(c *check.C) { s.UseCommandLine(c, "-Wall") G.globalData.InitVartypes() - s.RegisterTool("echo", "ECHO", false) + s.RegisterTool(&Tool{Name: "echo", Predefined: true}) G.Mk = s.NewMkLines("Makefile", "\techo ${PKGNAME:Q}") shline := NewShellLine(G.Mk.mklines[0]) @@ -164,7 +230,7 @@ func (s *Suite) TestShellLine_CheckShelltext_nofix(c *check.C) { func (s *Suite) TestShellLine_CheckShelltext_showAutofix(c *check.C) { s.UseCommandLine(c, "-Wall", "--show-autofix") G.globalData.InitVartypes() - s.RegisterTool("echo", "ECHO", false) + s.RegisterTool(&Tool{Name: "echo", Predefined: true}) G.Mk = s.NewMkLines("Makefile", "\techo ${PKGNAME:Q}") shline := NewShellLine(G.Mk.mklines[0]) @@ -179,7 +245,7 @@ func (s *Suite) TestShellLine_CheckShelltext_showAutofix(c *check.C) { func (s *Suite) TestShellLine_CheckShelltext_autofix(c *check.C) { s.UseCommandLine(c, "-Wall", "--autofix") G.globalData.InitVartypes() - s.RegisterTool("echo", "ECHO", false) + s.RegisterTool(&Tool{Name: "echo", Predefined: true}) G.Mk = s.NewMkLines("Makefile", "\techo ${PKGNAME:Q}") shline := NewShellLine(G.Mk.mklines[0]) @@ -198,16 +264,17 @@ func (s *Suite) TestShellLine_CheckShelltext_InternalError1(c *check.C) { shline := NewShellLine(G.Mk.mklines[0]) // foobar="`echo \"foo bar\"`" - shline.CheckShellCommandLine("foobar=\"`echo \\\"foo bar\\\"`\"") + text := "foobar=\"`echo \\\"foo bar\\\"`\"" - c.Check(s.Output(), equals, ""+ - "WARN: fname:1: Backslashes should be doubled inside backticks.\n"+ - "WARN: fname:1: Double quotes inside backticks inside double quotes are error prone.\n"+ - "WARN: fname:1: Backslashes should be doubled inside backticks.\n"+ - "WARN: fname:1: Double quotes inside backticks inside double quotes are error prone.\n"+ - "WARN: fname:1: Unknown shell command \"echo\".\n"+ - "ERROR: fname:1: Internal pkglint error: ShellLine.CheckToken state=plain, rest=\"\\\\foo\", token=\"\\\\foo\"\n"+ - "ERROR: fname:1: Internal pkglint error: ShellLine.CheckShellCommand state=continuation rest=\"\\\\\" shellcmd=\"echo \\\\foo bar\\\\\"\n") + tokens, rest := splitIntoShellTokens(dummyLine, text) + + c.Check(tokens, deepEquals, []string{text}) + c.Check(rest, equals, "") + + shline.CheckShellCommandLine(text) + + c.Check(s.Output(), equals, ""+ // No parse errors + "WARN: fname:1: Unknown shell command \"echo\".\n") } func (s *Suite) TestShellLine_CheckShelltext_DollarWithoutVariable(c *check.C) { @@ -215,7 +282,7 @@ func (s *Suite) TestShellLine_CheckShelltext_DollarWithoutVariable(c *check.C) { G.Mk = s.NewMkLines("fname", "# dummy") shline := NewShellLine(G.Mk.mklines[0]) - s.RegisterTool("pax", "PAX", false) + s.RegisterTool(&Tool{Name: "pax", Varname: "PAX"}) G.Mk.tools["pax"] = true shline.CheckShellCommandLine("pax -rwpp -s /.*~$$//g . ${DESTDIR}${PREFIX}") @@ -223,62 +290,64 @@ func (s *Suite) TestShellLine_CheckShelltext_DollarWithoutVariable(c *check.C) { c.Check(s.Output(), equals, "") } -func (s *Suite) TestChecklineMkShellword(c *check.C) { +func (s *Suite) Test_ShellLine_CheckWord(c *check.C) { s.UseCommandLine(c, "-Wall") G.globalData.InitVartypes() shline := NewShellLine(NewMkLine(NewLine("fname", 1, "# dummy", nil))) - c.Check(matches("${list}", `^`+reVarnameDirect+`$`), equals, false) + shline.CheckWord("${${list}}", false) - shline.CheckToken("${${list}}", false) + c.Check(s.Output(), equals, "") // No warning for variables that are completely indirect. - c.Check(s.Output(), equals, "") + shline.CheckWord("${SED_FILE.${id}}", false) + + c.Check(s.Output(), equals, "WARN: fname:1: SED_FILE.${id} is used but not defined. Spelling mistake?\n") - shline.CheckToken("\"$@\"", false) + shline.CheckWord("\"$@\"", false) c.Check(s.Output(), equals, "WARN: fname:1: Please use \"${.TARGET}\" instead of \"$@\".\n") - shline.CheckToken("${COMMENT:Q}", true) + shline.CheckWord("${COMMENT:Q}", true) - c.Check(s.Output(), equals, "WARN: fname:1: COMMENT may not be used in this file.\n") + c.Check(s.Output(), equals, "WARN: fname:1: COMMENT may not be used in any file; it is a write-only variable.\n") - shline.CheckToken("\"${DISTINFO_FILE:Q}\"", true) + shline.CheckWord("\"${DISTINFO_FILE:Q}\"", true) c.Check(s.Output(), equals, ""+ - "WARN: fname:1: DISTINFO_FILE may not be used in this file.\n"+ + "WARN: fname:1: DISTINFO_FILE may not be used in this file; it would be ok in Makefile, Makefile.*, *.mk.\n"+ "NOTE: fname:1: The :Q operator isn't necessary for ${DISTINFO_FILE} here.\n") - shline.CheckToken("embed${DISTINFO_FILE:Q}ded", true) + shline.CheckWord("embed${DISTINFO_FILE:Q}ded", true) c.Check(s.Output(), equals, ""+ - "WARN: fname:1: DISTINFO_FILE may not be used in this file.\n"+ + "WARN: fname:1: DISTINFO_FILE may not be used in this file; it would be ok in Makefile, Makefile.*, *.mk.\n"+ "NOTE: fname:1: The :Q operator isn't necessary for ${DISTINFO_FILE} here.\n") - shline.CheckToken("s,\\.,,", true) + shline.CheckWord("s,\\.,,", true) c.Check(s.Output(), equals, "") - shline.CheckToken("\"s,\\.,,\"", true) + shline.CheckWord("\"s,\\.,,\"", true) c.Check(s.Output(), equals, "") } -func (s *Suite) TestShellLine_CheckToken_DollarWithoutVariable(c *check.C) { +func (s *Suite) Test_ShellLine_CheckWord_DollarWithoutVariable(c *check.C) { shline := NewShellLine(NewMkLine(NewLine("fname", 1, "# dummy", nil))) - shline.CheckToken("/.*~$$//g", false) // Typical argument to pax(1). + shline.CheckWord("/.*~$$//g", false) // Typical argument to pax(1). c.Check(s.Output(), equals, "") } func (s *Suite) TestShelltextContext_CheckCommandStart(c *check.C) { s.UseCommandLine(c, "-Wall") - s.RegisterTool("echo", "ECHO", true) + s.RegisterTool(&Tool{Name: "echo", Varname: "ECHO", MustUseVarForm: true, Predefined: true}) G.Mk = s.NewMkLines("fname", "# dummy") mkline := NewMkLine(NewLine("fname", 3, "# dummy", nil)) - mkline.CheckText("echo \"hello, world\"") + mkline.checkText("echo \"hello, world\"") c.Check(s.Output(), equals, "") @@ -289,12 +358,17 @@ func (s *Suite) TestShelltextContext_CheckCommandStart(c *check.C) { } func (s *Suite) TestShellLine_checklineMkShelltext(c *check.C) { + text := "\tfor f in *.pl; do ${SED} s,@PREFIX@,${PREFIX}, < $f > $f.tmp && ${MV} $f.tmp $f; done" - shline := NewShellLine(NewMkLine(NewLine("Makefile", 3, "# dummy", nil))) - - shline.CheckShellCommandLine("for f in *.pl; do ${SED} s,@PREFIX@,${PREFIX}, < $f > $f.tmp && ${MV} $f.tmp $f; done") + shline := NewShellLine(NewMkLine(NewLine("Makefile", 3, text, nil))) + shline.CheckShellCommandLine(text) - c.Check(s.Output(), equals, "NOTE: Makefile:3: Please use the SUBST framework instead of ${SED} and ${MV}.\n") + c.Check(s.Output(), equals, ""+ + "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Makefile variable or $$f if you mean a shell variable.\n"+ + "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Makefile variable or $$f if you mean a shell variable.\n"+ + "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Makefile variable or $$f if you mean a shell variable.\n"+ + "WARN: Makefile:3: $f is ambiguous. Use ${f} if you mean a Makefile variable or $$f if you mean a shell variable.\n"+ + "NOTE: Makefile:3: Please use the SUBST framework instead of ${SED} and ${MV}.\n") shline.CheckShellCommandLine("install -c manpage.1 ${PREFIX}/man/man1/manpage.1") @@ -321,7 +395,7 @@ func (s *Suite) TestShellLine_checkCommandUse(c *check.C) { c.Check(s.Output(), equals, "WARN: fname:1: ${CP} should not be used to install files.\n") } -func (s *Suite) TestSplitIntoShellWords(c *check.C) { +func (s *Suite) TestSplitIntoMkWords(c *check.C) { url := "http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=" words, rest := splitIntoShellTokens(dummyLine, url) // Doesn’t really make sense @@ -329,12 +403,12 @@ func (s *Suite) TestSplitIntoShellWords(c *check.C) { c.Check(words, check.DeepEquals, []string{"http://registry.gimp.org/file/fix-ca.c?action=download", "&", "id=9884", "&", "file="}) c.Check(rest, equals, "") - words, rest = splitIntoShellWords(dummyLine, url) + words, rest = splitIntoMkWords(dummyLine, url) c.Check(words, check.DeepEquals, []string{"http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file="}) c.Check(rest, equals, "") - words, rest = splitIntoShellWords(dummyLine, "a b \"c c c\" d;;d;; \"e\"''`` 'rest") + words, rest = splitIntoMkWords(dummyLine, "a b \"c c c\" d;;d;; \"e\"''`` 'rest") c.Check(words, check.DeepEquals, []string{"a", "b", "\"c c c\"", "d;;d;;", "\"e\"''``"}) c.Check(rest, equals, "'rest") @@ -389,3 +463,34 @@ func (s *Suite) TestShellLine_(c *check.C) { c.Check(s.Output(), equals, "WARN: ~/Makefile:3--4: A shell comment does not stop at the end of line.\n") } + +func (s *Suite) Test_ShQuote(c *check.C) { + traceQuoting := func(input string) (result string) { + sq := NewShQuote("") + for _, part := range strings.Split(input, "x") { + sq.Feed(part) + result += part + "[" + sq.q.String() + "]" + } + return + } + + c.Check(traceQuoting("x\"x`x`x\"x'x\"x'"), equals, "[plain]\"[d]`[db]`[d]\"[plain]'[s]\"[s]'[plain]") + c.Check(traceQuoting("x\"x`x'x'x`x\""), equals, "[plain]\"[d]`[db]'[dbs]'[db]`[d]\"[plain]") + c.Check(traceQuoting("x\\\"x\\'x\\`x\\\\"), equals, "[plain]\\\"[plain]\\'[plain]\\`[plain]\\\\[plain]") + c.Check(traceQuoting("x\"x\\\"x\\'x\\`x\\\\"), equals, "[plain]\"[d]\\\"[d]\\'[d]\\`[d]\\\\[d]") + c.Check(traceQuoting("x'x\\\"x\\'x\\`x\\\\"), equals, "[plain]'[s]\\\"[s]\\'[plain]\\`[plain]\\\\[plain]") + c.Check(traceQuoting("x`x\\\"x\\'x\\`x\\\\"), equals, "[plain]`[b]\\\"[b]\\'[b]\\`[b]\\\\[b]") +} + +func (s *Suite) Test_unescapeBackticks(c *check.C) { + shline := NewShellLine(NewMkLine(dummyLine)) + // foobar="`echo \"foo bar\"`" + text := "foobar=\"`echo \\\"foo bar\\\"`\"" + repl := NewPrefixReplacer(text) + repl.AdvanceStr("foobar=\"`") + + backtCommand, newQuoting := shline.unescapeBackticks(text, repl, shqDquotBackt) + c.Check(backtCommand, equals, "echo \"foo bar\"") + c.Check(newQuoting, equals, shqDquot) + c.Check(repl.rest, equals, "\"") +} diff --git a/pkgtools/pkglint/files/shtokenizer.go b/pkgtools/pkglint/files/shtokenizer.go new file mode 100644 index 00000000000..d21caab324b --- /dev/null +++ b/pkgtools/pkglint/files/shtokenizer.go @@ -0,0 +1,313 @@ +package main + +type ShTokenizer struct { + parser *Parser + mkp *MkParser +} + +func NewShTokenizer(line *Line, text string, emitWarnings bool) *ShTokenizer { + p := NewParser(line, text, emitWarnings) + mkp := &MkParser{p} + return &ShTokenizer{p, mkp} +} + +// See ShQuote.Feed +func (p *ShTokenizer) ShAtom(quoting ShQuoting) *ShAtom { + if p.parser.EOF() { + return nil + } + + repl := p.parser.repl + mark := repl.Mark() + + if varuse := p.mkp.VarUse(); varuse != nil { + return &ShAtom{shtVaruse, repl.Since(mark), quoting, varuse} + } + + var atom *ShAtom + switch quoting { + case shqPlain: + atom = p.shAtomPlain() + case shqDquot: + atom = p.shAtomDquot() + case shqSquot: + atom = p.shAtomSquot() + case shqBackt: + atom = p.shAtomBackt() + case shqDquotBackt: + atom = p.shAtomDquotBackt() + case shqBacktDquot: + atom = p.shAtomBacktDquot() + case shqBacktSquot: + atom = p.shAtomBacktSquot() + case shqDquotBacktDquot: + atom = p.shAtomDquotBacktDquot() + case shqDquotBacktSquot: + atom = p.shAtomDquotBacktSquot() + } + + if atom == nil { + repl.Reset(mark) + p.parser.Line.Warnf("Pkglint parse error in ShTokenizer.ShAtom at %q (quoting=%s)", repl.rest, quoting) + } + return atom +} + +func (p *ShTokenizer) shAtomPlain() *ShAtom { + q := shqPlain + repl := p.parser.repl + switch { + case repl.AdvanceHspace(): + return &ShAtom{shtSpace, repl.s, q, nil} + case repl.AdvanceStr(";;"): + return &ShAtom{shtCaseSeparator, repl.s, q, nil} + case repl.AdvanceStr(";"): + return &ShAtom{shtSemicolon, repl.s, q, nil} + case repl.AdvanceStr("("): + return &ShAtom{shtParenOpen, repl.s, q, nil} + case repl.AdvanceStr(")"): + return &ShAtom{shtParenClose, repl.s, q, nil} + case repl.AdvanceStr("||"): + return &ShAtom{shtOr, repl.s, q, nil} + case repl.AdvanceStr("&&"): + return &ShAtom{shtAnd, repl.s, q, nil} + case repl.AdvanceStr("|"): + return &ShAtom{shtPipe, repl.s, q, nil} + case repl.AdvanceStr("&"): + return &ShAtom{shtBackground, repl.s, q, nil} + case repl.AdvanceStr("\""): + return &ShAtom{shtWord, repl.s, shqDquot, nil} + case repl.AdvanceStr("'"): + return &ShAtom{shtWord, repl.s, shqSquot, nil} + case repl.AdvanceStr("`"): + return &ShAtom{shtWord, repl.s, shqBackt, nil} + case repl.AdvanceRegexp(`^(?:<|<<|>|>>|>&)`): + return &ShAtom{shtRedirect, repl.m[0], q, nil} + case repl.AdvanceRegexp(`^#.*`): + return &ShAtom{shtComment, repl.m[0], q, nil} + case repl.AdvanceStr("$$("): + return &ShAtom{shtSubshell, repl.s, q, nil} + case repl.AdvanceRegexp(`^(?:[!#%*+,\-./0-9:=?@A-Z\[\]^_a-z{}~]+|\\[^$]|` + reShDollar + `)+`): + return &ShAtom{shtWord, repl.m[0], q, nil} + } + return nil +} + +func (p *ShTokenizer) shAtomDquot() *ShAtom { + repl := p.parser.repl + switch { + case repl.AdvanceStr("\""): + return &ShAtom{shtWord, repl.s, shqPlain, nil} + case repl.AdvanceStr("`"): + return &ShAtom{shtWord, repl.s, shqDquotBackt, nil} + case repl.AdvanceRegexp(`^(?:[\t !#%&'()*+,\-./0-9:;<=>?@A-Z\[\]^_a-z{|}~]+|\\[^$]|` + reShDollar + `)+`): + return &ShAtom{shtWord, repl.m[0], shqDquot, nil} // XXX: unescape? + } + return nil +} + +func (p *ShTokenizer) shAtomSquot() *ShAtom { + repl := p.parser.repl + switch { + case repl.AdvanceStr("'"): + return &ShAtom{shtWord, repl.s, shqPlain, nil} + case repl.AdvanceRegexp(`^([\t !"#%&()*+,\-./0-9:;<=>?@A-Z\[\\\]^_` + "`" + `a-z{|}~]+|\$\$)+`): + return &ShAtom{shtWord, repl.m[0], shqSquot, nil} + } + return nil +} + +func (p *ShTokenizer) shAtomBackt() *ShAtom { + const q = shqBackt + repl := p.parser.repl + switch { + case repl.AdvanceStr("\""): + return &ShAtom{shtWord, repl.s, shqBacktDquot, nil} + case repl.AdvanceStr("`"): + return &ShAtom{shtWord, repl.s, shqPlain, nil} + case repl.AdvanceStr("'"): + return &ShAtom{shtWord, repl.s, shqBacktSquot, nil} + case repl.AdvanceHspace(): + return &ShAtom{shtSpace, repl.s, q, nil} + case repl.AdvanceStr(";;"): + return &ShAtom{shtCaseSeparator, repl.s, q, nil} + case repl.AdvanceStr(";"): + return &ShAtom{shtSemicolon, repl.s, q, nil} + case repl.AdvanceStr("("): + return &ShAtom{shtParenOpen, repl.s, q, nil} + case repl.AdvanceStr(")"): + return &ShAtom{shtParenClose, repl.s, q, nil} + case repl.AdvanceStr("||"): + return &ShAtom{shtOr, repl.s, q, nil} + case repl.AdvanceStr("&&"): + return &ShAtom{shtAnd, repl.s, q, nil} + case repl.AdvanceStr("|"): + return &ShAtom{shtPipe, repl.s, q, nil} + case repl.AdvanceStr("&"): + return &ShAtom{shtBackground, repl.s, q, nil} + case repl.AdvanceRegexp(`^(?:<|<<|>|>>|>&)`): + return &ShAtom{shtRedirect, repl.s, q, nil} + case repl.AdvanceRegexp("^#[^`]*"): + return &ShAtom{shtComment, repl.s, q, nil} + case repl.AdvanceRegexp(`^(?:[!#%*+,\-./0-9:=?@A-Z\[\]_a-z~]+|\\[^$]|` + reShDollar + `)+`): + return &ShAtom{shtWord, repl.s, q, nil} + } + return nil +} + +func (p *ShTokenizer) shAtomDquotBackt() *ShAtom { + const q = shqDquotBackt + repl := p.parser.repl + switch { + case repl.AdvanceStr("`"): + return &ShAtom{shtWord, repl.s, shqDquot, nil} + case repl.AdvanceStr("\""): + return &ShAtom{shtWord, repl.s, shqDquotBacktDquot, nil} + case repl.AdvanceStr("'"): + return &ShAtom{shtWord, repl.s, shqDquotBacktSquot, nil} + case repl.AdvanceRegexp("^#[^`]*"): + return &ShAtom{shtComment, repl.s, q, nil} + case repl.AdvanceStr(";;"): + return &ShAtom{shtCaseSeparator, repl.s, q, nil} + case repl.AdvanceStr(";"): + return &ShAtom{shtSemicolon, repl.s, q, nil} + case repl.AdvanceStr("("): + return &ShAtom{shtParenOpen, repl.s, q, nil} + case repl.AdvanceStr(")"): + return &ShAtom{shtParenClose, repl.s, q, nil} + case repl.AdvanceStr("||"): + return &ShAtom{shtOr, repl.s, q, nil} + case repl.AdvanceStr("&&"): + return &ShAtom{shtAnd, repl.s, q, nil} + case repl.AdvanceStr("|"): + return &ShAtom{shtPipe, repl.s, q, nil} + case repl.AdvanceStr("&"): + return &ShAtom{shtBackground, repl.s, q, nil} + case repl.AdvanceRegexp(`^(?:<|<<|>|>>|>&)`): + return &ShAtom{shtRedirect, repl.s, q, nil} + case repl.AdvanceRegexp(`^(?:[!#%*+,\-./0-9:=?@A-Z\[\]_a-z~]+|\\[^$]|` + reShDollar + `)+`): + return &ShAtom{shtWord, repl.s, q, nil} + case repl.AdvanceHspace(): + return &ShAtom{shtSpace, repl.s, q, nil} + } + return nil +} + +func (p *ShTokenizer) shAtomBacktDquot() *ShAtom { + repl := p.parser.repl + switch { + case repl.AdvanceStr("\""): + return &ShAtom{shtWord, repl.s, shqBackt, nil} + case repl.AdvanceRegexp(`^(?:[\t !%&()*+,\-./0-9:;<=>?@A-Z\[\]^_a-z{|}~]+|\\[^$]|` + reShDollar + `)+`): + return &ShAtom{shtWord, repl.m[0], shqBacktDquot, nil} + } + return nil +} + +func (p *ShTokenizer) shAtomBacktSquot() *ShAtom { + const q = shqBacktSquot + repl := p.parser.repl + switch { + case repl.AdvanceStr("'"): + return &ShAtom{shtWord, repl.s, shqBackt, nil} + case repl.AdvanceRegexp(`^([\t !"#%&()*+,\-./0-9:;<=>?@A-Z\[\\\]^_` + "`" + `a-z{|}~]+|\$\$)+`): + return &ShAtom{shtWord, repl.m[0], q, nil} + } + return nil +} + +func (p *ShTokenizer) shAtomDquotBacktDquot() *ShAtom { + const q = shqDquotBacktDquot + repl := p.parser.repl + switch { + case repl.AdvanceStr("\""): + return &ShAtom{shtWord, repl.s, shqDquotBackt, nil} + case repl.AdvanceRegexp(`^(?:[\t !%&()*+,\-./0-9:;<=>?@A-Z\[\]^_a-z{|}~]+|\\[^$]|` + reShDollar + `)+`): + return &ShAtom{shtWord, repl.m[0], q, nil} + } + return nil +} + +func (p *ShTokenizer) shAtomDquotBacktSquot() *ShAtom { + repl := p.parser.repl + switch { + case repl.AdvanceStr("'"): + return &ShAtom{shtWord, repl.s, shqDquotBackt, nil} + case repl.AdvanceRegexp(`^(?:[\t !"#%()*+,\-./0-9:;<=>?@A-Z\[\]^_a-z{|}~]+|\\[^$]|\\\$\$|\$\$)+`): + return &ShAtom{shtWord, repl.m[0], shqDquotBacktSquot, nil} + } + return nil +} + +func (p *ShTokenizer) ShAtoms() []*ShAtom { + var atoms []*ShAtom + q := shqPlain + for { + atom := p.ShAtom(q) + if atom == nil { + return atoms + } + atoms = append(atoms, atom) + q = atom.Quoting + } +} + +func (p *ShTokenizer) ShToken() *ShToken { + var curr *ShAtom + q := shqPlain + peek := func() *ShAtom { + if curr == nil { + curr = p.ShAtom(q) + if curr != nil { + q = curr.Quoting + } + } + return curr + } + skip := func() { + curr = nil + } + + repl := p.parser.repl + inimark := repl.Mark() + var atoms []*ShAtom + + for peek() != nil && peek().Type == shtSpace { + skip() + inimark = repl.Mark() + } + + if peek() == nil { + return nil + } + if atom := peek(); !atom.Type.IsWord() { + return NewShToken(atom.Text, atom) + } + +nextatom: + mark := repl.Mark() + atom := peek() + if atom != nil && (atom.Type.IsWord() || atom.Quoting != shqPlain) { + skip() + atoms = append(atoms, atom) + goto nextatom + } + repl.Reset(mark) + + if len(atoms) == 0 { + return nil + } + return NewShToken(repl.Since(inimark), atoms...) +} + +func (p *ShTokenizer) Mark() PrefixReplacerMark { + return p.parser.repl.Mark() +} + +func (p *ShTokenizer) Reset(mark PrefixReplacerMark) { + p.parser.repl.Reset(mark) +} + +func (p *ShTokenizer) Rest() string { + return p.parser.Rest() +} diff --git a/pkgtools/pkglint/files/shtokenizer_test.go b/pkgtools/pkglint/files/shtokenizer_test.go new file mode 100644 index 00000000000..c07bd03db6a --- /dev/null +++ b/pkgtools/pkglint/files/shtokenizer_test.go @@ -0,0 +1,443 @@ +package main + +import ( + check "gopkg.in/check.v1" +) + +// @Beta +func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { + checkRest := func(s string, expected ...*ShAtom) string { + p := NewShTokenizer(dummyLine, s, false) + q := shqPlain + for _, exp := range expected { + c.Check(p.ShAtom(q), deepEquals, exp) + q = exp.Quoting + } + return p.Rest() + } + check := func(s string, expected ...*ShAtom) { + rest := checkRest(s, expected...) + c.Check(rest, equals, "") + } + + token := func(typ ShAtomType, text string, quoting ShQuoting) *ShAtom { + return &ShAtom{typ, text, quoting, nil} + } + word := func(s string) *ShAtom { return token(shtWord, s, shqPlain) } + dquot := func(s string) *ShAtom { return token(shtWord, s, shqDquot) } + squot := func(s string) *ShAtom { return token(shtWord, s, shqSquot) } + backt := func(s string) *ShAtom { return token(shtWord, s, shqBackt) } + varuse := func(varname string, modifiers ...string) *ShAtom { + text := "${" + varname + for _, modifier := range modifiers { + text += ":" + modifier + } + text += "}" + varuse := &MkVarUse{varname: varname, modifiers: modifiers} + return &ShAtom{shtVaruse, text, shqPlain, varuse} + } + q := func(q ShQuoting, token *ShAtom) *ShAtom { + return &ShAtom{token.Type, token.Text, q, token.Data} + } + whitespace := func(s string) *ShAtom { return token(shtSpace, s, shqPlain) } + space := token(shtSpace, " ", shqPlain) + semicolon := token(shtSemicolon, ";", shqPlain) + pipe := token(shtPipe, "|", shqPlain) + + check("" /* none */) + + check("$$var", + word("$$var")) + + check("$$var$$var", + word("$$var$$var")) + + check("$$var;;", + word("$$var"), + token(shtCaseSeparator, ";;", shqPlain)) + + check("'single-quoted'", + q(shqSquot, word("'")), + q(shqSquot, word("single-quoted")), + q(shqPlain, word("'"))) + + rest := checkRest("\"" /* none */) + c.Check(rest, equals, "\"") + + check("$${file%.c}.o", + word("$${file%.c}.o")) + + check("hello", + word("hello")) + + check("hello, world", + word("hello,"), + space, + word("world")) + + check("\"", + dquot("\"")) + + check("`", + backt("`")) + + check("`cat fname`", + backt("`"), + backt("cat"), + token(shtSpace, " ", shqBackt), + backt("fname"), + word("`")) + + check("hello, \"world\"", + word("hello,"), + space, + dquot("\""), + dquot("world"), + word("\"")) + + check("set -e;", + word("set"), + space, + word("-e"), + semicolon) + + check("cd ${WRKSRC}/doc/man/man3; PAGES=\"`ls -1 | ${SED} -e 's,3qt$$,3,'`\";", + word("cd"), + space, + varuse("WRKSRC"), + word("/doc/man/man3"), + semicolon, + space, + word("PAGES="), + dquot("\""), + q(shqDquotBackt, word("`")), + q(shqDquotBackt, word("ls")), + q(shqDquotBackt, space), + q(shqDquotBackt, word("-1")), + q(shqDquotBackt, space), + token(shtPipe, "|", shqDquotBackt), + q(shqDquotBackt, space), + q(shqDquotBackt, varuse("SED")), + q(shqDquotBackt, space), + q(shqDquotBackt, word("-e")), + q(shqDquotBackt, space), + q(shqDquotBacktSquot, word("'")), + q(shqDquotBacktSquot, word("s,3qt$$,3,")), + q(shqDquotBackt, word("'")), + q(shqDquot, word("`")), + q(shqPlain, word("\"")), + semicolon) + + check("ls -1 | ${SED} -e 's,3qt$$,3,'", + word("ls"), space, word("-1"), space, + pipe, space, + varuse("SED"), space, word("-e"), space, + squot("'"), squot("s,3qt$$,3,"), word("'")) + + check("(for PAGE in $$PAGES; do ", + &ShAtom{shtParenOpen, "(", shqPlain, nil}, + word("for"), + space, + word("PAGE"), + space, + word("in"), + space, + word("$$PAGES"), + semicolon, + space, + word("do"), + space) + + check(" ${ECHO} installing ${DESTDIR}${QTPREFIX}/man/man3/$${PAGE}; ", + whitespace(" "), + varuse("ECHO"), + space, + word("installing"), + space, + varuse("DESTDIR"), + varuse("QTPREFIX"), + word("/man/man3/$${PAGE}"), + semicolon, + space) + + check(" set - X `head -1 $${PAGE}qt`; ", + whitespace(" "), + word("set"), + space, + word("-"), + space, + word("X"), + space, + backt("`"), + backt("head"), + q(shqBackt, space), + backt("-1"), + q(shqBackt, space), + backt("$${PAGE}qt"), + word("`"), + semicolon, + space) + + check("`\"one word\"`", + backt("`"), + q(shqBacktDquot, word("\"")), + q(shqBacktDquot, word("one word")), + q(shqBackt, word("\"")), + word("`")) + + check("$$var \"$$var\" '$$var' `$$var`", + word("$$var"), + space, + dquot("\""), + dquot("$$var"), + word("\""), + space, + squot("'"), + squot("$$var"), + word("'"), + space, + backt("`"), + backt("$$var"), + word("`")) + + check("\"`'echo;echo'`\"", + q(shqDquot, word("\"")), + q(shqDquotBackt, word("`")), + q(shqDquotBacktSquot, word("'")), + q(shqDquotBacktSquot, word("echo;echo")), + q(shqDquotBackt, word("'")), + q(shqDquot, word("`")), + q(shqPlain, word("\""))) + + check("cat<file", + word("cat"), + token(shtRedirect, "<", shqPlain), + word("file")) + + check("-e \"s,\\$$sysconfdir/jabberd,\\$$sysconfdir,g\"", + word("-e"), + space, + dquot("\""), + dquot("s,\\$$sysconfdir/jabberd,\\$$sysconfdir,g"), + word("\"")) + + check("echo $$,$$/", + word("echo"), + space, + word("$$,$$/")) + + rest = checkRest("COMMENT=\t\\Make $$$$ fast\"", + word("COMMENT="), + whitespace("\t"), + word("\\Make"), + space, + word("$$$$"), + space, + word("fast")) + c.Check(rest, equals, "\"") + + check("var=`echo;echo|echo&echo||echo&&echo>echo`", + q(shqPlain, word("var=")), + q(shqBackt, word("`")), + q(shqBackt, word("echo")), + q(shqBackt, semicolon), + q(shqBackt, word("echo")), + q(shqBackt, token(shtPipe, "|", shqBackt)), + q(shqBackt, word("echo")), + q(shqBackt, token(shtBackground, "&", shqBackt)), + q(shqBackt, word("echo")), + q(shqBackt, token(shtOr, "||", shqBackt)), + q(shqBackt, word("echo")), + q(shqBackt, token(shtAnd, "&&", shqBackt)), + q(shqBackt, word("echo")), + q(shqBackt, token(shtRedirect, ">", shqBackt)), + q(shqBackt, word("echo")), + q(shqPlain, word("`"))) + + check("# comment", + token(shtComment, "# comment", shqPlain)) + check("no#comment", + word("no#comment")) + check("`# comment`continue", + token(shtWord, "`", shqBackt), + token(shtComment, "# comment", shqBackt), + token(shtWord, "`", shqPlain), + token(shtWord, "continue", shqPlain)) + check("`no#comment`continue", + token(shtWord, "`", shqBackt), + token(shtWord, "no#comment", shqBackt), + token(shtWord, "`", shqPlain), + token(shtWord, "continue", shqPlain)) + + check("var=`tr 'A-Z' 'a-z'`", + token(shtWord, "var=", shqPlain), + token(shtWord, "`", shqBackt), + token(shtWord, "tr", shqBackt), + token(shtSpace, " ", shqBackt), + token(shtWord, "'", shqBacktSquot), + token(shtWord, "A-Z", shqBacktSquot), + token(shtWord, "'", shqBackt), + token(shtSpace, " ", shqBackt), + token(shtWord, "'", shqBacktSquot), + token(shtWord, "a-z", shqBacktSquot), + token(shtWord, "'", shqBackt), + token(shtWord, "`", shqPlain)) + + check("var=\"`echo \"\\`echo foo\\`\"`\"", + token(shtWord, "var=", shqPlain), + token(shtWord, "\"", shqDquot), + token(shtWord, "`", shqDquotBackt), + token(shtWord, "echo", shqDquotBackt), + token(shtSpace, " ", shqDquotBackt), + token(shtWord, "\"", shqDquotBacktDquot), + token(shtWord, "\\`echo foo\\`", shqDquotBacktDquot), // One token, since it doesn’t influence parsing. + token(shtWord, "\"", shqDquotBackt), + token(shtWord, "`", shqDquot), + token(shtWord, "\"", shqPlain)) + + check("if cond1; then action1; elif cond2; then action2; else action3; fi", + word("if"), space, word("cond1"), semicolon, space, + word("then"), space, word("action1"), semicolon, space, + word("elif"), space, word("cond2"), semicolon, space, + word("then"), space, word("action2"), semicolon, space, + word("else"), space, word("action3"), semicolon, space, + word("fi")) +} + +func (s *Suite) Test_Shtokenizer_ShAtom_Quoting(c *check.C) { + checkQuotingChange := func(input, expectedOutput string) { + p := NewShTokenizer(dummyLine, input, false) + q := shqPlain + result := "" + for { + token := p.ShAtom(q) + if token == nil { + break + } + result += token.Text + if token.Quoting != q { + q = token.Quoting + result += "[" + q.String() + "]" + } + } + c.Check(result, equals, expectedOutput) + c.Check(p.Rest(), equals, "") + } + + checkQuotingChange("hello, world", "hello, world") + checkQuotingChange("hello, \"world\"", "hello, \"[d]world\"[plain]") + checkQuotingChange("1 \"\" 2 '' 3 `` 4", "1 \"[d]\"[plain] 2 '[s]'[plain] 3 `[b]`[plain] 4") + checkQuotingChange("\"\"", "\"[d]\"[plain]") + checkQuotingChange("''", "'[s]'[plain]") + checkQuotingChange("``", "`[b]`[plain]") + checkQuotingChange("x\"x`x`x\"x'x\"x'", "x\"[d]x`[db]x`[d]x\"[plain]x'[s]x\"x'[plain]") + checkQuotingChange("x\"x`x'x'x`x\"", "x\"[d]x`[db]x'[dbs]x'[db]x`[d]x\"[plain]") + checkQuotingChange("x\\\"x\\'x\\`x\\\\", "x\\\"x\\'x\\`x\\\\") + checkQuotingChange("x\"x\\\"x\\'x\\`x\\\\", "x\"[d]x\\\"x\\'x\\`x\\\\") + checkQuotingChange("x'x\\\"x\\'x\\`x\\\\", "x'[s]x\\\"x\\'[plain]x\\`x\\\\") + checkQuotingChange("x`x\\\"x\\'x\\`x\\\\", "x`[b]x\\\"x\\'x\\`x\\\\") +} + +func (s *Suite) Test_ShTokenizer_ShToken(c *check.C) { + check := func(str string, expected ...*ShToken) { + p := NewShTokenizer(dummyLine, str, false) + for _, exp := range expected { + c.Check(p.ShToken(), deepEquals, exp) + } + c.Check(p.Rest(), equals, "") + c.Check(s.Output(), equals, "") + } + + check("", + nil) + + check("echo", + NewShToken("echo", + NewShAtom(shtWord, "echo", shqPlain))) + + check("`cat file`", + NewShToken("`cat file`", + NewShAtom(shtWord, "`", shqBackt), + NewShAtom(shtWord, "cat", shqBackt), + NewShAtom(shtSpace, " ", shqBackt), + NewShAtom(shtWord, "file", shqBackt), + NewShAtom(shtWord, "`", shqPlain))) + + check("PAGES=\"`ls -1 | ${SED} -e 's,3qt$$,3,'`\"", + NewShToken("PAGES=\"`ls -1 | ${SED} -e 's,3qt$$,3,'`\"", + NewShAtom(shtWord, "PAGES=", shqPlain), + NewShAtom(shtWord, "\"", shqDquot), + NewShAtom(shtWord, "`", shqDquotBackt), + NewShAtom(shtWord, "ls", shqDquotBackt), + NewShAtom(shtSpace, " ", shqDquotBackt), + NewShAtom(shtWord, "-1", shqDquotBackt), + NewShAtom(shtSpace, " ", shqDquotBackt), + NewShAtom(shtPipe, "|", shqDquotBackt), + NewShAtom(shtSpace, " ", shqDquotBackt), + NewShAtomVaruse("${SED}", shqDquotBackt, "SED"), + NewShAtom(shtSpace, " ", shqDquotBackt), + NewShAtom(shtWord, "-e", shqDquotBackt), + NewShAtom(shtSpace, " ", shqDquotBackt), + NewShAtom(shtWord, "'", shqDquotBacktSquot), + NewShAtom(shtWord, "s,3qt$$,3,", shqDquotBacktSquot), + NewShAtom(shtWord, "'", shqDquotBackt), + NewShAtom(shtWord, "`", shqDquot), + NewShAtom(shtWord, "\"", shqPlain))) + + check("echo hello, world", + NewShToken("echo", + NewShAtom(shtWord, "echo", shqPlain)), + NewShToken("hello,", + NewShAtom(shtWord, "hello,", shqPlain)), + NewShToken("world", + NewShAtom(shtWord, "world", shqPlain))) + + check("if cond1; then action1; elif cond2; then action2; else action3; fi", + NewShToken("if", NewShAtom(shtWord, "if", shqPlain)), + NewShToken("cond1", NewShAtom(shtWord, "cond1", shqPlain)), + NewShToken(";", NewShAtom(shtSemicolon, ";", shqPlain)), + NewShToken("then", NewShAtom(shtWord, "then", shqPlain)), + NewShToken("action1", NewShAtom(shtWord, "action1", shqPlain)), + NewShToken(";", NewShAtom(shtSemicolon, ";", shqPlain)), + NewShToken("elif", NewShAtom(shtWord, "elif", shqPlain)), + NewShToken("cond2", NewShAtom(shtWord, "cond2", shqPlain)), + NewShToken(";", NewShAtom(shtSemicolon, ";", shqPlain)), + NewShToken("then", NewShAtom(shtWord, "then", shqPlain)), + NewShToken("action2", NewShAtom(shtWord, "action2", shqPlain)), + NewShToken(";", NewShAtom(shtSemicolon, ";", shqPlain)), + NewShToken("else", NewShAtom(shtWord, "else", shqPlain)), + NewShToken("action3", NewShAtom(shtWord, "action3", shqPlain)), + NewShToken(";", NewShAtom(shtSemicolon, ";", shqPlain)), + NewShToken("fi", NewShAtom(shtWord, "fi", shqPlain))) + + check("PATH=/nonexistent env PATH=${PATH:Q} true", + NewShToken("PATH=/nonexistent", NewShAtom(shtWord, "PATH=/nonexistent", shqPlain)), + NewShToken("env", NewShAtom(shtWord, "env", shqPlain)), + NewShToken("PATH=${PATH:Q}", + NewShAtom(shtWord, "PATH=", shqPlain), + NewShAtomVaruse("${PATH:Q}", shqPlain, "PATH", "Q")), + NewShToken("true", NewShAtom(shtWord, "true", shqPlain))) + + if false { // Don’t know how to tokenize this correctly. + check("id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)", + NewShToken("id=$$(${AWK} '{print}' < ${WRKSRC}/idfile)", + NewShAtom(shtWord, "id=", shqPlain), + NewShAtom(shtWord, "$$(", shqPlain), + NewShAtomVaruse("${AWK}", shqPlain, "AWK"))) + } + check("id=`${AWK} '{print}' < ${WRKSRC}/idfile`", + NewShToken("id=`${AWK} '{print}' < ${WRKSRC}/idfile`", + NewShAtom(shtWord, "id=", shqPlain), + NewShAtom(shtWord, "`", shqBackt), + NewShAtomVaruse("${AWK}", shqBackt, "AWK"), + NewShAtom(shtSpace, " ", shqBackt), + NewShAtom(shtWord, "'", shqBacktSquot), + NewShAtom(shtWord, "{print}", shqBacktSquot), + NewShAtom(shtWord, "'", shqBackt), + NewShAtom(shtSpace, " ", shqBackt), + NewShAtom(shtRedirect, "<", shqBackt), + NewShAtom(shtSpace, " ", shqBackt), + NewShAtomVaruse("${WRKSRC}", shqBackt, "WRKSRC"), + NewShAtom(shtWord, "/idfile", shqBackt), + NewShAtom(shtWord, "`", shqPlain))) +} diff --git a/pkgtools/pkglint/files/shtypes.go b/pkgtools/pkglint/files/shtypes.go new file mode 100644 index 00000000000..08e9e63185d --- /dev/null +++ b/pkgtools/pkglint/files/shtypes.go @@ -0,0 +1,147 @@ +package main + +import ( + "fmt" +) + +//go:generate go tool yacc -o shellyacc.go -v shellyacc.log -p shyy shell.y + +type ShAtomType uint8 + +const ( + shtSpace ShAtomType = iota + shtVaruse // ${PREFIX} + shtWord // + shtSemicolon // ; + shtCaseSeparator // ;; + shtParenOpen // ( + shtParenClose // ) + shtPipe // | + shtBackground // & + shtOr // || + shtAnd // && + shtRedirect // >, <, >> + shtComment // # ... + shtSubshell // $$( +) + +func (t ShAtomType) String() string { + return [...]string{ + "space", + "varuse", + "word", + "semicolon", + "caseSeparator", + "parenOpen", "parenClose", + "pipe", "background", + "or", "and", + "redirect", + "comment", + }[t] +} + +func (t ShAtomType) IsWord() bool { + switch t { + case shtVaruse, shtWord, shtRedirect: + return true + } + return false +} + +func (t ShAtomType) IsCommandDelimiter() bool { + switch t { + case shtSemicolon, shtPipe, shtBackground, shtAnd, shtOr, shtCaseSeparator: + return true + } + return false +} + +// @Beta +type ShAtom struct { + Type ShAtomType + Text string + Quoting ShQuoting + Data interface{} +} + +func NewShAtom(typ ShAtomType, text string, quoting ShQuoting) *ShAtom { + return &ShAtom{typ, text, quoting, nil} +} + +func NewShAtomVaruse(text string, quoting ShQuoting, varname string, modifiers ...string) *ShAtom { + return &ShAtom{shtVaruse, text, quoting, NewMkVarUse(varname, modifiers...)} +} + +func (token *ShAtom) String() string { + if token.Type == shtWord && token.Quoting == shqPlain && token.Data == nil { + return fmt.Sprintf("%q", token.Text) + } + if token.Type == shtVaruse { + varuse := token.Data.(*MkVarUse) + return fmt.Sprintf("varuse(%q)", varuse.varname+varuse.Mod()) + } + return fmt.Sprintf("ShAtom(%v, %q, %s)", token.Type, token.Text, token.Quoting) +} + +// ShQuoting describes the context in which a string appears +// and how it must be unescaped to get its literal value. +type ShQuoting uint8 + +const ( + shqPlain ShQuoting = iota + shqDquot + shqSquot + shqBackt + shqDquotBackt + shqBacktDquot + shqBacktSquot + shqDquotBacktDquot + shqDquotBacktSquot + shqUnknown +) + +func (q ShQuoting) String() string { + return [...]string{ + "plain", + "d", "s", "b", + "db", "bd", "bs", + "dbd", "dbs", + "unknown", + }[q] +} + +func (q ShQuoting) ToVarUseContext() vucQuoting { + switch q { + case shqPlain: + return vucQuotPlain + case shqDquot: + return vucQuotDquot + case shqSquot: + return vucQuotSquot + case shqBackt: + return vucQuotBackt + } + return vucQuotUnknown +} + +// See http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_10_02 +type ShToken struct { + MkText string // The text as it appeared in the Makefile, after replacing `\#` with `#` + Atoms []*ShAtom +} + +func NewShToken(mkText string, atoms ...*ShAtom) *ShToken { + return &ShToken{mkText, atoms} +} + +func (token *ShToken) String() string { + return fmt.Sprintf("ShToken(%v)", token.Atoms) +} + +func (token *ShToken) IsAssignment() bool { + return matches(token.MkText, `^[A-Za-z_]\w*=`) +} + +func (token *ShToken) IsWord() bool { + return token.Atoms[0].Type.IsWord() +} diff --git a/pkgtools/pkglint/files/shtypes_test.go b/pkgtools/pkglint/files/shtypes_test.go new file mode 100644 index 00000000000..98b88a24238 --- /dev/null +++ b/pkgtools/pkglint/files/shtypes_test.go @@ -0,0 +1,13 @@ +package main + +import ( + check "gopkg.in/check.v1" +) + +func (s *Suite) Test_ShAtom_String(c *check.C) { + c.Check(shtComment.String(), equals, "comment") +} + +func (s *Suite) Test_ShQuoting_String(c *check.C) { + c.Check(shqUnknown.String(), equals, "unknown") +} diff --git a/pkgtools/pkglint/files/toplevel.go b/pkgtools/pkglint/files/toplevel.go index 3c8a0765985..2fda81f237b 100644 --- a/pkgtools/pkglint/files/toplevel.go +++ b/pkgtools/pkglint/files/toplevel.go @@ -6,7 +6,7 @@ type Toplevel struct { } func CheckdirToplevel() { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall1(G.CurrentDir)() } diff --git a/pkgtools/pkglint/files/util.go b/pkgtools/pkglint/files/util.go index 55d4dc8f62e..8614ef42e9c 100644 --- a/pkgtools/pkglint/files/util.go +++ b/pkgtools/pkglint/files/util.go @@ -56,7 +56,7 @@ func isEmptyDir(fname string) bool { func getSubdirs(fname string) []string { dirents, err := ioutil.ReadDir(fname) if err != nil { - Fatalf(fname, noLines, "Cannot be read: %s", err) + NewLineWhole(fname).Fatalf("Cannot be read: %s", err) } var subdirs []string @@ -251,7 +251,7 @@ func match5(s, re string) (matched bool, m1, m2, m3, m4, m5 string) { } func replaceFirst(s, re, replacement string) ([]string, string) { - if G.opts.DebugTrace { + if G.opts.Debug { defer tracecall(s, re, replacement)() } @@ -266,6 +266,8 @@ func replaceFirst(s, re, replacement string) ([]string, string) { return nil, s } +type PrefixReplacerMark string + type PrefixReplacer struct { rest string s string // The last match from a prefix @@ -276,43 +278,12 @@ func NewPrefixReplacer(s string) *PrefixReplacer { return &PrefixReplacer{s, "", nil} } -func (pr *PrefixReplacer) AdvanceBytes(bits0x00, bits0x20, bits0x40, bits0x60 uint32, re string) bool { - if G.TestingData != nil { - pr.verifyBits(bits0x00, bits0x20, bits0x40, bits0x60, re) - } - - rest := pr.rest - n := len(pr.rest) - i := 0 -loop: - for ; i < n; i++ { - b := rest[i] - var mask uint32 - switch b & 0xE0 { - case 0x00: - mask = bits0x00 - case 0x20: - mask = bits0x20 - case 0x40: - mask = bits0x40 - case 0x60: - mask = bits0x60 - } - if (mask>>(b&0x1F))&1 == 0 { - break loop - } - } - pr.s = rest[:i] - pr.rest = rest[i:] - return i != 0 -} - func (pr *PrefixReplacer) AdvanceStr(prefix string) bool { pr.m = nil pr.s = "" if hasPrefix(pr.rest, prefix) { - if G.opts.DebugTrace { - trace("", "PrefixReplacer.AdvanceStr", pr.rest, prefix) + if G.opts.Debug { + traceStep("PrefixReplacer.AdvanceStr(%q, %q)", pr.rest, prefix) } pr.s = prefix pr.rest = pr.rest[len(prefix):] @@ -320,21 +291,50 @@ func (pr *PrefixReplacer) AdvanceStr(prefix string) bool { } return false } + +func (pr *PrefixReplacer) AdvanceBytesFunc(fn func(c byte) bool) bool { + i := 0 + for i < len(pr.rest) && fn(pr.rest[i]) { + i++ + } + if i != 0 { + pr.s = pr.rest[:i] + pr.rest = pr.rest[i:] + return true + } + return false +} + +func (pr *PrefixReplacer) AdvanceHspace() bool { + i := 0 + rest := pr.rest + for i < len(rest) && (rest[i] == ' ' || rest[i] == '\t') { + i++ + } + if i != 0 { + pr.s = pr.rest[:i] + pr.rest = pr.rest[i:] + return true + } + return false +} + func (pr *PrefixReplacer) AdvanceRegexp(re string) bool { pr.m = nil pr.s = "" if !hasPrefix(re, "^") { panic(fmt.Sprintf("PrefixReplacer.AdvanceRegexp: regular expression %q must have prefix %q.", re, "^")) } - if matches("", re) { + if G.Testing && matches("", re) { panic(fmt.Sprintf("PrefixReplacer.AdvanceRegexp: the empty string must not match the regular expression %q.", re)) } if m := match(pr.rest, re); m != nil { - if G.opts.DebugTrace { - trace("", "PrefixReplacer.AdvanceRegexp", pr.rest, re, m[0]) + if G.opts.Debug { + traceStep("PrefixReplacer.AdvanceRegexp(%q, %q, %q)", pr.rest, re, m[0]) } pr.rest = pr.rest[len(m[0]):] pr.m = m + pr.s = m[0] return true } return false @@ -348,11 +348,11 @@ func (pr *PrefixReplacer) PeekByte() int { return int(rest[0]) } -func (pr *PrefixReplacer) Mark() string { - return pr.rest +func (pr *PrefixReplacer) Mark() PrefixReplacerMark { + return PrefixReplacerMark(pr.rest) } -func (pr *PrefixReplacer) Reset(mark string) { - pr.rest = mark +func (pr *PrefixReplacer) Reset(mark PrefixReplacerMark) { + pr.rest = string(mark) } func (pr *PrefixReplacer) Skip(n int) { pr.rest = pr.rest[n:] @@ -360,8 +360,8 @@ func (pr *PrefixReplacer) Skip(n int) { func (pr *PrefixReplacer) SkipSpace() { pr.rest = strings.TrimLeft(pr.rest, " \t") } -func (pr *PrefixReplacer) Since(mark string) string { - return mark[:len(mark)-len(pr.rest)] +func (pr *PrefixReplacer) Since(mark PrefixReplacerMark) string { + return string(mark[:len(mark)-len(pr.rest)]) } func (pr *PrefixReplacer) AdvanceRest() string { rest := pr.rest @@ -369,46 +369,6 @@ func (pr *PrefixReplacer) AdvanceRest() string { return rest } -func (pr *PrefixReplacer) verifyBits(bits0x00, bits0x20, bits0x40, bits0x60 uint32, re string) { - if G.TestingData.VerifiedBits[re] { - return - } - G.TestingData.VerifiedBits[re] = true - rec := regcomp(re) - - var actual0x00, actual0x20, actual0x40, actual0x60 uint32 - mask := uint32(0) - for i := byte(0); i < 0x80; i++ { - if rec.Match([]byte{i}) { - mask |= uint32(1) << (i & 31) - } - switch i { - case 0x1F: - actual0x00 = mask - case 0x3F: - actual0x20 = mask - case 0x5F: - actual0x40 = mask - case 0x7F: - actual0x60 = mask - } - if i&0x1F == 0x1F { - mask = 0 - } - } - if actual0x00 == bits0x00 && actual0x20 == bits0x20 && actual0x40 == bits0x40 && actual0x60 == bits0x60 { - return - } - - print(fmt.Sprintf(""+ - "Expected bits(%#08x, %#08x, %#08x, %#08x), "+ - "not bits(%#08x, %#08x, %#08x, %#08x) "+ - "for pattern %q.\n", - actual0x00, actual0x20, actual0x40, actual0x60, - bits0x00, bits0x20, bits0x40, bits0x60, - re)) -} - // Useful in combination with regex.Find*Index func negToZero(i int) int { if i >= 0 { @@ -457,29 +417,49 @@ func argsStr(args ...interface{}) string { return argsStr } -func trace(action, funcname string, args ...interface{}) { - if G.opts.DebugTrace { - io.WriteString(G.debugOut, fmt.Sprintf("TRACE: %s%s%s(%s)\n", strings.Repeat("| ", G.traceDepth), action, funcname, argsStr(args...))) +func traceIndent() string { + indent := "" + for i := 0; i < G.traceDepth; i++ { + indent += fmt.Sprintf("%d ", i+1) + } + return indent +} + +func traceStep(format string, args ...interface{}) { + if G.opts.Debug { + msg := fmt.Sprintf(format, args...) + io.WriteString(G.debugOut, fmt.Sprintf("TRACE: %s %s\n", traceIndent(), msg)) + } +} +func traceStep1(format string, arg0 string) { + switch { // Prevent inlining } + traceStep(format, arg0) +} +func traceStep2(format string, arg0, arg1 string) { + switch { //Prevent inlining + } + traceStep(format, arg0, arg1) } func tracecallInternal(args ...interface{}) func() { - if !G.opts.DebugTrace { + if !G.opts.Debug { panic("Internal pkglint error: calls to tracecall must only occur in tracing mode") } funcname := "?" if pc, _, _, ok := runtime.Caller(2); ok { if fn := runtime.FuncForPC(pc); fn != nil { - funcname = strings.TrimSuffix(fn.Name(), "main.") + funcname = strings.TrimPrefix(fn.Name(), "netbsd.org/pkglint.") } } - trace("+ ", funcname, args...) + indent := traceIndent() + io.WriteString(G.debugOut, fmt.Sprintf("TRACE: %s+ %s(%s)\n", indent, funcname, argsStr(args...))) G.traceDepth++ return func() { G.traceDepth-- - trace("- ", funcname, args...) + io.WriteString(G.debugOut, fmt.Sprintf("TRACE: %s- %s(%s)\n", indent, funcname, argsStr(args...))) } } func tracecall0() func() { @@ -503,6 +483,20 @@ func tracecall(args ...interface{}) func() { return tracecallInternal(args...) } +type Ref struct { + intf interface{} +} + +func ref(rv interface{}) Ref { + return Ref{rv} +} + +func (r Ref) String() string { + ptr := reflect.ValueOf(r.intf) + ref := reflect.Indirect(ptr) + return fmt.Sprintf("%v", ref) +} + // Emulates make(1)’s :S substitution operator. func mkopSubst(s string, left bool, from string, right bool, to string, all bool) string { re := ifelseStr(left, "^", "") + regexp.QuoteMeta(from) + ifelseStr(right, "$", "") @@ -524,34 +518,16 @@ func relpath(from, to string) string { panic("relpath" + argsStr(from, to, err1, err2, err3)) } result := filepath.ToSlash(rel) - if G.opts.DebugTrace { - trace("", "relpath", from, to, "=>", result) + if G.opts.Debug { + traceStep("relpath from %q to %q = %q", from, to, result) } return result } -func stringBoolMapKeys(m map[string]bool) []string { - var keys []string - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} - -func stringStringMapKeys(m map[string]string) []string { - var keys []string - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} - func abspath(fname string) string { abs, err := filepath.Abs(fname) if err != nil { - Fatalf(fname, noLines, "Cannot determine absolute path.") + NewLineWhole(fname).Fatalf("Cannot determine absolute path.") } return filepath.ToSlash(abs) } diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go index 04d9eef2e9b..9027a707521 100644 --- a/pkgtools/pkglint/files/vardefs.go +++ b/pkgtools/pkglint/files/vardefs.go @@ -92,8 +92,8 @@ func (gd *GlobalData) InitVartypes() { // some other variables, sorted alphabetically - sys(".CURDIR", lkNone, CheckvarPathname) - sys(".TARGET", lkNone, CheckvarPathname) + acl(".CURDIR", lkNone, CheckvarPathname, "buildlink3.mk:; *: use, use-loadtime") + acl(".TARGET", lkNone, CheckvarPathname, "buildlink3.mk:; *: use, use-loadtime") acl("ALL_ENV", lkShell, CheckvarShellWord, "") acl("ALTERNATIVES_FILE", lkNone, CheckvarFilename, "") acl("ALTERNATIVES_SRC", lkShell, CheckvarPathname, "") @@ -118,11 +118,11 @@ func (gd *GlobalData) InitVartypes() { pkg("BOOTSTRAP_PKG", lkNone, CheckvarYesNo) acl("BROKEN", lkNone, CheckvarMessage, "") pkg("BROKEN_GETTEXT_DETECTION", lkNone, CheckvarYesNo) - pkglist("BROKEN_EXCEPT_ON_PLATFORM", lkShell, CheckvarPlatformPattern) - pkglist("BROKEN_ON_PLATFORM", lkSpace, CheckvarPlatformPattern) + pkglist("BROKEN_EXCEPT_ON_PLATFORM", lkShell, CheckvarMachinePlatformPattern) + pkglist("BROKEN_ON_PLATFORM", lkSpace, CheckvarMachinePlatformPattern) sys("BSD_MAKE_ENV", lkShell, CheckvarShellWord) - acl("BUILDLINK_ABI_DEPENDS.*", lkSpace, CheckvarDependency, "*: append") - acl("BUILDLINK_API_DEPENDS.*", lkSpace, CheckvarDependency, "*: append") + acl("BUILDLINK_ABI_DEPENDS.*", lkSpace, CheckvarDependency, "builtin.mk: append, use-loadtime; *: append") + acl("BUILDLINK_API_DEPENDS.*", lkSpace, CheckvarDependency, "builtin.mk: append, use-loadtime; *: append") acl("BUILDLINK_CONTENTS_FILTER", lkNone, CheckvarShellCommand, "") sys("BUILDLINK_CFLAGS", lkShell, CheckvarCFlag) bl3list("BUILDLINK_CFLAGS.*", lkShell, CheckvarCFlag) @@ -130,21 +130,21 @@ func (gd *GlobalData) InitVartypes() { bl3list("BUILDLINK_CPPFLAGS.*", lkShell, CheckvarCFlag) acl("BUILDLINK_CONTENTS_FILTER.*", lkNone, CheckvarShellCommand, "buildlink3.mk: set") acl("BUILDLINK_DEPENDS", lkSpace, CheckvarIdentifier, "buildlink3.mk: append") - acl("BUILDLINK_DEPMETHOD.*", lkShell, CheckvarBuildlinkDepmethod, "buildlink3.mk: default, append; Makefile: set, append; Makefile.common, *.mk: append") - sys("BUILDLINK_DIR", lkNone, CheckvarPathname) + acl("BUILDLINK_DEPMETHOD.*", lkShell, CheckvarBuildlinkDepmethod, "buildlink3.mk: default, append, use; Makefile: set, append; Makefile.common, *.mk: append") + acl("BUILDLINK_DIR", lkNone, CheckvarPathname, "*: use") bl3list("BUILDLINK_FILES.*", lkShell, CheckvarPathmask) acl("BUILDLINK_FILES_CMD.*", lkNone, CheckvarShellCommand, "") - acl("BUILDLINK_INCDIRS.*", lkShell, CheckvarPathname, "buildlink3.mk: default, append") - acl("BUILDLINK_JAVA_PREFIX.*", lkNone, CheckvarPathname, "buildlink3.mk: set") - acl("BUILDLINK_LDADD.*", lkShell, CheckvarLdFlag, "builtin.mk: set, default, append, use; buildlink3.mk: append; Makefile, Makefile.common, *.mk: use") - sys("BUILDLINK_LDFLAGS", lkShell, CheckvarLdFlag) + acl("BUILDLINK_INCDIRS.*", lkShell, CheckvarPathname, "buildlink3.mk: default, append; Makefile, Makefile.common, *.mk: use") + acl("BUILDLINK_JAVA_PREFIX.*", lkNone, CheckvarPathname, "buildlink3.mk: set, use") + acl("BUILDLINK_LDADD.*", lkShell, CheckvarLdFlag, "builtin.mk: set, default, append, use; buildlink3.mk: append, use; Makefile, Makefile.common, *.mk: use") + acl("BUILDLINK_LDFLAGS", lkShell, CheckvarLdFlag, "*: use") bl3list("BUILDLINK_LDFLAGS.*", lkShell, CheckvarLdFlag) - bl3list("BUILDLINK_LIBDIRS.*", lkShell, CheckvarPathname) + acl("BUILDLINK_LIBDIRS.*", lkShell, CheckvarPathname, "buildlink3.mk, builtin.mk: append; Makefile, Makefile.common, *.mk: use") acl("BUILDLINK_LIBS.*", lkShell, CheckvarLdFlag, "buildlink3.mk: append") acl("BUILDLINK_PASSTHRU_DIRS", lkShell, CheckvarPathname, "Makefile, Makefile.common, buildlink3.mk, hacks.mk: append") acl("BUILDLINK_PASSTHRU_RPATHDIRS", lkShell, CheckvarPathname, "Makefile, Makefile.common, buildlink3.mk, hacks.mk: append") acl("BUILDLINK_PKGSRCDIR.*", lkNone, CheckvarRelativePkgDir, "buildlink3.mk: default, use-loadtime") - acl("BUILDLINK_PREFIX.*", lkNone, CheckvarPathname, "builtin.mk: set, use; buildlink3.mk:; Makefile, Makefile.common, *.mk: use") + acl("BUILDLINK_PREFIX.*", lkNone, CheckvarPathname, "builtin.mk: set, use; buildlink3.mk: use; Makefile, Makefile.common, *.mk: use") acl("BUILDLINK_RPATHDIRS.*", lkShell, CheckvarPathname, "buildlink3.mk: append") acl("BUILDLINK_TARGETS", lkShell, CheckvarIdentifier, "") acl("BUILDLINK_FNAME_TRANSFORM.*", lkNone, CheckvarSedCommands, "Makefile, buildlink3.mk, builtin.mk, hacks.mk: append") @@ -156,7 +156,7 @@ func (gd *GlobalData) InitVartypes() { pkglist("BUILD_ENV", lkShell, CheckvarShellWord) sys("BUILD_MAKE_CMD", lkNone, CheckvarShellCommand) pkglist("BUILD_MAKE_FLAGS", lkShell, CheckvarShellWord) - pkg("BUILD_TARGET", lkShell, CheckvarIdentifier) + pkglist("BUILD_TARGET", lkShell, CheckvarIdentifier) pkg("BUILD_USES_MSGFMT", lkNone, CheckvarYes) acl("BUILTIN_PKG", lkNone, CheckvarIdentifier, "builtin.mk: set, use-loadtime, use") acl("BUILTIN_PKG.*", lkNone, CheckvarPkgName, "builtin.mk: set, use-loadtime, use") @@ -191,7 +191,7 @@ func (gd *GlobalData) InitVartypes() { pkg("CMAKE_ARG_PATH", lkNone, CheckvarPathname) pkglist("CMAKE_ARGS", lkShell, CheckvarShellWord) acl("COMMENT", lkNone, CheckvarComment, "Makefile, Makefile.common: set, append") - sys("COMPILER_RPATH_FLAG", lkNone, enum("-Wl,-rpath")) + acl("COMPILER_RPATH_FLAG", lkNone, enum("-Wl,-rpath"), "*: use") pkglist("CONFIGURE_ARGS", lkShell, CheckvarShellWord) pkglist("CONFIGURE_DIRS", lkShell, CheckvarWrksrcSubdirectory) acl("CONFIGURE_ENV", lkShell, CheckvarShellWord, "Makefile, Makefile.common: append, set, use; buildlink3.mk, builtin.mk: append; *.mk: append, use") @@ -249,7 +249,7 @@ func (gd *GlobalData) InitVartypes() { sys("DO_NADA", lkNone, CheckvarShellCommand) pkg("DYNAMIC_SITES_CMD", lkNone, CheckvarShellCommand) pkg("DYNAMIC_SITES_SCRIPT", lkNone, CheckvarPathname) - sys("ECHO", lkNone, CheckvarShellCommand) + acl("ECHO", lkNone, CheckvarShellCommand, "*: use") sys("ECHO_MSG", lkNone, CheckvarShellCommand) sys("ECHO_N", lkNone, CheckvarShellCommand) pkg("EGDIR", lkNone, CheckvarPathname) // Not defined anywhere, but used in many places like this. @@ -312,7 +312,7 @@ func (gd *GlobalData) InitVartypes() { acl("FILES_SUBST_SED", lkShell, CheckvarShellWord, "") pkglist("FIX_RPATH", lkShell, CheckvarVarname) pkglist("FLEX_REQD", lkShell, CheckvarVersion) - acl("FONTS_DIRS.*", lkShell, CheckvarPathname, "Makefile: set, append; Makefile.common: append") + acl("FONTS_DIRS.*", lkShell, CheckvarPathname, "Makefile: set, append, use; Makefile.common: append, use") sys("GAMEDATAMODE", lkNone, CheckvarFileMode) sys("GAMES_GROUP", lkNone, CheckvarUserGroupName) sys("GAMEMODE", lkNone, CheckvarFileMode) @@ -332,10 +332,10 @@ func (gd *GlobalData) InitVartypes() { acl("GNU_CONFIGURE_PREFIX", lkNone, CheckvarPathname, "Makefile: set") acl("HAS_CONFIGURE", lkNone, CheckvarYes, "Makefile, Makefile.common: set") pkglist("HEADER_TEMPLATES", lkShell, CheckvarPathname) - pkg("HOMEPAGE", lkNone, CheckvarURL) + pkg("HOMEPAGE", lkNone, CheckvarHomepage) acl("IGNORE_PKG.*", lkNone, CheckvarYes, "*: set, use-loadtime") - acl("INCOMPAT_CURSES", lkSpace, CheckvarPlatformPattern, "Makefile: set, append") - acl("INCOMPAT_ICONV", lkSpace, CheckvarPlatformPattern, "") + acl("INCOMPAT_CURSES", lkSpace, CheckvarMachinePlatformPattern, "Makefile: set, append") + acl("INCOMPAT_ICONV", lkSpace, CheckvarMachinePlatformPattern, "") acl("INFO_DIR", lkNone, CheckvarPathname, "") // relative to PREFIX pkg("INFO_FILES", lkNone, CheckvarYes) sys("INSTALL", lkNone, CheckvarShellCommand) @@ -374,7 +374,7 @@ func (gd *GlobalData) InitVartypes() { sys("KRB5BASE", lkNone, CheckvarPathname) acl("KRB5_ACCEPTED", lkShell, enum("heimdal mit-krb5"), "") usr("KRB5_DEFAULT", lkNone, enum("heimdal mit-krb5")) - sys("KRB5_TYPE", lkNone, CheckvarUnchecked) + sys("KRB5_TYPE", lkNone, CheckvarIdentifier) sys("LD", lkNone, CheckvarShellCommand) pkglist("LDFLAGS*", lkShell, CheckvarLdFlag) sys("LIBGRP", lkNone, CheckvarUserGroupName) @@ -391,8 +391,10 @@ func (gd *GlobalData) InitVartypes() { sys("LINKER_RPATH_FLAG", lkNone, CheckvarShellWord) sys("LOWER_OPSYS", lkNone, CheckvarIdentifier) acl("LTCONFIG_OVERRIDE", lkShell, CheckvarPathmask, "Makefile: set, append; Makefile.common: append") - sys("MACHINE_ARCH", lkNone, CheckvarIdentifier) - sys("MACHINE_GNU_PLATFORM", lkNone, CheckvarPlatformPattern) // This one is actually not a pattern + sys("MACHINE_ARCH", lkNone, enumMachineArch) + sys("MACHINE_GNU_ARCH", lkNone, enumMachineGnuArch) + sys("MACHINE_GNU_PLATFORM", lkNone, CheckvarMachineGnuPlatform) + sys("MACHINE_PLATFORM", lkNone, CheckvarMachinePlatform) acl("MAINTAINER", lkNone, CheckvarMailAddress, "Makefile: set; Makefile.common: default") sys("MAKE", lkNone, CheckvarShellCommand) pkglist("MAKEFLAGS", lkShell, CheckvarShellWord) @@ -451,7 +453,7 @@ func (gd *GlobalData) InitVartypes() { sys("NM", lkNone, CheckvarShellCommand) sys("NONBINMODE", lkNone, CheckvarFileMode) pkg("NOT_FOR_COMPILER", lkShell, enum("ccache ccc clang distcc f2c gcc hp icc ido mipspro mipspro-ucode pcc sunpro xlc")) - pkglist("NOT_FOR_PLATFORM", lkSpace, CheckvarPlatformPattern) + pkglist("NOT_FOR_PLATFORM", lkSpace, CheckvarMachinePlatformPattern) pkg("NOT_FOR_UNPRIVILEGED", lkNone, CheckvarYesNo) acl("NO_BIN_ON_CDROM", lkNone, CheckvarRestricted, "Makefile, Makefile.common: set") acl("NO_BIN_ON_FTP", lkNone, CheckvarRestricted, "Makefile, Makefile.common: set") @@ -465,7 +467,7 @@ func (gd *GlobalData) InitVartypes() { acl("NO_SRC_ON_CDROM", lkNone, CheckvarRestricted, "Makefile, Makefile.common: set") acl("NO_SRC_ON_FTP", lkNone, CheckvarRestricted, "Makefile, Makefile.common: set") pkglist("ONLY_FOR_COMPILER", lkShell, enum("ccc clang gcc hp icc ido mipspro mipspro-ucode pcc sunpro xlc")) - pkglist("ONLY_FOR_PLATFORM", lkSpace, CheckvarPlatformPattern) + pkglist("ONLY_FOR_PLATFORM", lkSpace, CheckvarMachinePlatformPattern) pkg("ONLY_FOR_UNPRIVILEGED", lkNone, CheckvarYesNo) sys("OPSYS", lkNone, CheckvarIdentifier) acl("OPSYSVARS", lkShell, CheckvarVarname, "Makefile, Makefile.common: append") @@ -484,7 +486,7 @@ func (gd *GlobalData) InitVartypes() { acl("PATCH_DIST_ARGS", lkShell, CheckvarShellWord, "Makefile: set, append") acl("PATCH_DIST_CAT", lkNone, CheckvarShellCommand, "") acl("PATCH_DIST_STRIP*", lkNone, CheckvarShellWord, "Makefile, Makefile.common: set; buildlink3.mk:; builtin.mk:; *.mk: set") - acl("PATCH_SITES", lkShell, CheckvarURL, "Makefile: set; options.mk: set; Makefile.common: set") + acl("PATCH_SITES", lkShell, CheckvarFetchURL, "Makefile: set; options.mk: set; Makefile.common: set") acl("PATCH_STRIP", lkNone, CheckvarShellWord, "") pkg("PERL5_USE_PACKLIST", lkNone, CheckvarYesNo) acl("PERL5_PACKLIST", lkShell, CheckvarPerl5Packlist, "Makefile: set; options.mk: set, append") @@ -560,7 +562,7 @@ func (gd *GlobalData) InitVartypes() { acl("PKG_SHLIBTOOL", lkNone, CheckvarPathname, "") pkglist("PKG_SKIP_REASON", lkShell, CheckvarShellWord) acl("PKG_SUGGESTED_OPTIONS", lkShell, CheckvarOption, "options.mk: set, append; Makefile: set, append; Makefile.common: set") - acl("PKG_SUPPORTED_OPTIONS", lkShell, CheckvarOption, "options.mk: set, append; Makefile: set, append; Makefile.common: set") + acl("PKG_SUPPORTED_OPTIONS", lkShell, CheckvarOption, "options.mk: set, append, use; Makefile: set, append; Makefile.common: set") pkg("PKG_SYSCONFDIR*", lkNone, CheckvarPathname) pkglist("PKG_SYSCONFDIR_PERMS", lkShell, CheckvarShellWord) sys("PKG_SYSCONFBASEDIR", lkNone, CheckvarPathname) @@ -629,7 +631,7 @@ func (gd *GlobalData) InitVartypes() { sys("SHAREOWN", lkNone, CheckvarUserGroupName) sys("SHCOMMENT", lkNone, CheckvarShellCommand) acl("SHLIB_HANDLING", lkNone, enum("YES NO no"), "") - acl("SHLIBTOOL", lkNone, CheckvarShellCommand, "") + acl("SHLIBTOOL", lkNone, CheckvarShellCommand, "Makefile: use") acl("SHLIBTOOL_OVERRIDE", lkShell, CheckvarPathmask, "Makefile: set, append; Makefile.common: append") acl("SITES.*", lkShell, CheckvarFetchURL, "Makefile, Makefile.common, options.mk: set, append, use") pkglist("SPECIAL_PERMS", lkShell, CheckvarShellWord) @@ -677,7 +679,7 @@ func (gd *GlobalData) InitVartypes() { acl("USE_IMAKE", lkNone, CheckvarYes, "Makefile: set") pkg("USE_JAVA", lkNone, enum("run yes build")) pkg("USE_JAVA2", lkNone, enum("YES yes no 1.4 1.5 6 7 8")) - acl("USE_LANGUAGES", lkShell, enum("ada c c99 c++ fortran fortran77 java objc"), "Makefile, Makefile.common, options.mk: set") + acl("USE_LANGUAGES", lkShell, enum("ada c c99 c++ fortran fortran77 java objc"), "Makefile, Makefile.common, options.mk: set, append") pkg("USE_LIBTOOL", lkNone, CheckvarYes) pkg("USE_MAKEINFO", lkNone, CheckvarYes) pkg("USE_MSGFMT_PLURALS", lkNone, CheckvarYes) @@ -806,7 +808,7 @@ func pkg(varname string, kindOfList KindOfList, checker *VarChecker) { // A package-defined list may be appended to in all Makefiles except buildlink3.mk and builtin.mk. // Simple assignment (instead of appending) is only allowed in Makefile and Makefile.common. func pkglist(varname string, kindOfList KindOfList, checker *VarChecker) { - acl(varname, kindOfList, checker, "Makefile, Makefile.common: append, set, use; buildlink3.mk, builtin.mk:; *.mk: append, use") + acl(varname, kindOfList, checker, "Makefile, Makefile.common, options.mk: append, default, set, use; buildlink3.mk, builtin.mk:; *.mk: append, default, use") } // A user-defined or system-defined variable must not be set by any diff --git a/pkgtools/pkglint/files/vartype.go b/pkgtools/pkglint/files/vartype.go index f88e548d5c3..c385712ac67 100644 --- a/pkgtools/pkglint/files/vartype.go +++ b/pkgtools/pkglint/files/vartype.go @@ -134,6 +134,63 @@ func (vt *Vartype) String() string { } } +func (vt *Vartype) IsShell() bool { + switch vt.checker { + case CheckvarCFlag, // Subtype of ShellWord + CheckvarLdFlag, // Subtype of ShellWord + CheckvarSedCommands, + CheckvarShellCommand, + CheckvarShellCommands, + CheckvarShellWord: + return true + } + return false +} + +// The basic vartype consists only of characters that don’t +// need escaping in most contexts, like A-Za-z0-9-_. +func (vt *Vartype) IsBasicSafe() bool { + switch vt.checker { + case CheckvarBuildlinkDepmethod, + CheckvarCategory, + CheckvarDistSuffix, + CheckvarEmulPlatform, + CheckvarFileMode, + CheckvarFilename, + CheckvarIdentifier, + CheckvarInteger, + CheckvarMachineGnuPlatform, + CheckvarMachinePlatform, + CheckvarOption, + CheckvarPathname, + CheckvarPerl5Packlist, + CheckvarPkgName, + CheckvarPkgOptionsVar, + CheckvarPkgPath, + CheckvarPkgRevision, + CheckvarPrefixPathname, + CheckvarPythonDependency, + CheckvarRelativePkgDir, + CheckvarRelativePkgPath, + CheckvarStage, + CheckvarUserGroupName, + CheckvarVersion, + CheckvarWrkdirSubdirectory, + CheckvarYesNo, + CheckvarYesNoIndirectly: + return true + } + return false +} + +func (vt *Vartype) IsPlainString() bool { + switch vt.checker { + case CheckvarComment, CheckvarMessage, CheckvarString: + return true + } + return false +} + type VarChecker struct { name string checker func(*VartypeCheck) @@ -164,10 +221,14 @@ var ( CheckvarFilename = &VarChecker{"Filename", (*VartypeCheck).Filename} CheckvarFilemask = &VarChecker{"Filemask", (*VartypeCheck).Filemask} CheckvarFileMode = &VarChecker{"FileMode", (*VartypeCheck).FileMode} + CheckvarHomepage = &VarChecker{"Homepage", (*VartypeCheck).Homepage} CheckvarIdentifier = &VarChecker{"Identifier", (*VartypeCheck).Identifier} CheckvarInteger = &VarChecker{"Integer", (*VartypeCheck).Integer} CheckvarLdFlag = &VarChecker{"LdFlag", (*VartypeCheck).LdFlag} CheckvarLicense = &VarChecker{"License", (*VartypeCheck).License} + CheckvarMachineGnuPlatform = &VarChecker{"MachineGnuPlatform", (*VartypeCheck).MachineGnuPlatform} + CheckvarMachinePlatform = &VarChecker{"MachinePlatform", (*VartypeCheck).MachinePlatform} + CheckvarMachinePlatformPattern = &VarChecker{"MachinePlatformPattern", (*VartypeCheck).MachinePlatformPattern} CheckvarMailAddress = &VarChecker{"MailAddress", (*VartypeCheck).MailAddress} CheckvarMessage = &VarChecker{"Message", (*VartypeCheck).Message} CheckvarOption = &VarChecker{"Option", (*VartypeCheck).Option} @@ -179,7 +240,6 @@ var ( CheckvarPkgPath = &VarChecker{"PkgPath", (*VartypeCheck).PkgPath} CheckvarPkgOptionsVar = &VarChecker{"PkgOptionsVar", (*VartypeCheck).PkgOptionsVar} CheckvarPkgRevision = &VarChecker{"PkgRevision", (*VartypeCheck).PkgRevision} - CheckvarPlatformPattern = &VarChecker{"PlatformPattern", (*VartypeCheck).PlatformPattern} CheckvarPrefixPathname = &VarChecker{"PrefixPathname", (*VartypeCheck).PrefixPathname} CheckvarPythonDependency = &VarChecker{"PythonDependency", (*VartypeCheck).PythonDependency} CheckvarRelativePkgDir = &VarChecker{"RelativePkgDir", (*VartypeCheck).RelativePkgDir} diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go index bcf2deb5734..b60e05abf9f 100644 --- a/pkgtools/pkglint/files/vartypecheck.go +++ b/pkgtools/pkglint/files/vartypecheck.go @@ -2,6 +2,7 @@ package main import ( "path" + "sort" "strings" ) @@ -26,7 +27,6 @@ const ( opAssignAppend // += opAssignDefault // ?= opUse // - opUseLoadtime // opUseMatch // Used in the :M operator ) @@ -51,34 +51,71 @@ func (op MkOperator) String() string { } const ( - reEmulPlatform = "" + + reMachineOpsys = "" + // See mk/platform + "AIX|BSDOS|Bitrig|Cygwin|Darwin|DragonFly|FreeBSD|FreeMiNT|GNUkFreeBSD|" + + "HPUX|Haiku|IRIX|Interix|Linux|Minix|MirBSD|NetBSD|OSF1|OpenBSD|QNX|SCO_SV|SunOS|UnixWare" + + // See mk/emulator/emulator-vars.mk. + reEmulOpsys = "" + "bitrig|bsdos|cygwin|darwin|dragonfly|freebsd|" + - "haiku|hpux|interix|irix|linux|mirbsd|netbsd|openbsd|osf1|solaris" - rePlatformOs = "" + - "Bitrig|BSDOS|Cygwin|Darwin|DragonFly|FreeBSD|" + - "Haiku|HPUX|Interix|IRIX|Linux|MirBSD|NetBSD|OpenBSD|OSF1|QNX|SunOS" - rePlatformArch = "" + - "alpha|amd64|arc|arm|arm32|cobalt|convex|dreamcast|" + - "hpcmips|hpcsh|hppa|i386|ia64|" + - "m68k|m88k|mips|mips64|mips64eb|mips64el|mipseb|mipsel|mipsn32|" + - "ns32k|pc532|pmax|powerpc|rs6000|s390|sh3eb|sh3el|sparc|sparc64|vax|x86_64" + "haiku|hpux|interix|irix|linux|mirbsd|netbsd|openbsd|osf1|solaris|sunos" + + // Hardware architectures having the same name in bsd.own.mk and the GNU world. + // These are best-effort guesses, since they depend on the operating system. + reArch = "" + + "aarch64|alpha|amd64|arc|arm|cobalt|convex|dreamcast|i386|" + + "hpcmips|hpcsh|hppa|hppa64|ia64|" + + "m68k|m88k|mips|mips64|mips64el|mipseb|mipsel|mipsn32|mlrisc|" + + "ns32k|pc532|pmax|powerpc|powerpc64|rs6000|s390|sparc|sparc64|vax|x86_64" + + // See mk/bsd.prefs.mk:/^GNU_ARCH\./ + reMachineArch = "" + + reArch + "|" + + "aarch64eb|amd64|arm26|arm32|coldfire|earm|earmeb|earmhf|earmhfeb|earmv4|earmv4eb|earmv5|" + + "earmv5eb|earmv6|earmv6eb|earmv6hf|earmv6hfeb|earmv7|earmv7eb|earmv7hf|earmv7hfeb|evbarm|" + + "i386|i586|i686|m68000|mips|mips64eb|sh3eb|sh3el" + + // See mk/bsd.prefs.mk:/^GNU_ARCH\./ + reMachineGnuArch = "" + + reArch + "|" + + "aarch64_be|arm|armeb|armv4|armv4eb|armv6|armv6eb|armv7|armv7eb|" + + "i486|m5407|m68010|mips64|mipsel|sh|shle|x86_64" + + reEmulArch = reMachineArch // Just a wild guess. ) +func enumFromRe(re string) *VarChecker { + values := strings.Split(re, "|") + sort.Strings(values) + seen := make(map[string]bool) + var unique []string + for _, value := range values { + if !seen[value] { + seen[value] = true + unique = append(unique, value) + } + } + return enum(strings.Join(unique, " ")) +} + var ( - emulPlatformEnum = enum(strings.Replace(reEmulPlatform, "|", " ", -1)) - platformOsEnum = enum(strings.Replace(rePlatformOs, "|", " ", -1)) - platformArchEnum = enum(strings.Replace(rePlatformArch, "|", " ", -1)) + enumMachineOpsys = enumFromRe(reMachineOpsys) + enumMachineArch = enumFromRe(reMachineArch) + enumMachineGnuArch = enumFromRe(reMachineGnuArch) + enumEmulOpsys = enumFromRe(reEmulOpsys) + enumEmulArch = enumFromRe(reEmulArch) + enumMachineGnuPlatformOpsys = enumEmulOpsys ) func (cv *VartypeCheck) AwkCommand() { - if G.opts.DebugUnchecked { - cv.line.Debug1("Unchecked AWK command: %q", cv.value) + if G.opts.Debug { + traceStep1("Unchecked AWK command: %q", cv.value) } } func (cv *VartypeCheck) BasicRegularExpression() { - if G.opts.DebugUnchecked { - cv.line.Debug1("Unchecked basic regular expression: %q", cv.value) + if G.opts.Debug { + traceStep1("Unchecked basic regular expression: %q", cv.value) } } @@ -89,7 +126,7 @@ func (cv *VartypeCheck) BuildlinkDepmethod() { } func (cv *VartypeCheck) Category() { - if fileExists(G.CurrentDir + "/" + G.CurPkgsrcdir + "/" + cv.value + "/Makefile") { + if cv.value != "wip" && fileExists(G.CurrentDir+"/"+G.CurPkgsrcdir+"/"+cv.value+"/Makefile") { return } switch cv.value { @@ -157,7 +194,7 @@ func (cv *VartypeCheck) Comment() { func (cv *VartypeCheck) Dependency() { line, value := cv.line, cv.value - parser := NewParser(line, value) + parser := NewParser(line, value, false) deppat := parser.Dependency() if deppat != nil && deppat.wildcard == "" && (parser.Rest() == "{,nb*}" || parser.Rest() == "{,nb[0-9]*}") { line.Warn0("Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.") @@ -276,7 +313,7 @@ func (cv *VartypeCheck) EmulPlatform() { cv.comment, cv.listContext, cv.guessed} - emulPlatformEnum.checker(opsysCv) + enumEmulOpsys.checker(opsysCv) // no check for os_version @@ -290,7 +327,7 @@ func (cv *VartypeCheck) EmulPlatform() { cv.comment, cv.listContext, cv.guessed} - platformArchEnum.checker(archCv) + enumEmulArch.checker(archCv) } else { cv.line.Warn1("%q is not a valid emulation platform.", cv.value) Explain( @@ -307,7 +344,7 @@ func (cv *VartypeCheck) EmulPlatform() { func (cv *VartypeCheck) FetchURL() { cv.mkline.CheckVartypePrimitive(cv.varname, CheckvarURL, cv.op, cv.value, cv.comment, cv.listContext, cv.guessed) - for siteURL, siteName := range G.globalData.MasterSiteUrls { + for siteURL, siteName := range G.globalData.MasterSiteURLToVar { if hasPrefix(cv.value, siteURL) { subdir := cv.value[len(siteURL):] if hasPrefix(cv.value, "https://github.com/") { @@ -322,8 +359,8 @@ func (cv *VartypeCheck) FetchURL() { } if m, name, subdir := match2(cv.value, `\$\{(MASTER_SITE_[^:]*).*:=(.*)\}$`); m { - if !G.globalData.MasterSiteVars[name] { - cv.line.Error1("%s does not exist.", name) + if G.globalData.MasterSiteVarToURL[name] == "" { + cv.line.Error1("The site %s does not exist.", name) } if !hasSuffix(subdir, "/") { cv.line.Error1("The subdirectory in %s must end with a slash.", name) @@ -364,6 +401,40 @@ func (cv *VartypeCheck) FileMode() { } } +func (cv *VartypeCheck) Homepage() { + cv.mkline.CheckVartypePrimitive(cv.varname, CheckvarURL, cv.op, cv.value, cv.comment, cv.listContext, cv.guessed) + + if m, wrong, sitename, subdir := match3(cv.value, `^(\$\{(MASTER_SITE\w+)(?::=([\w\-/]+))?\})`); m { + baseURL := G.globalData.MasterSiteVarToURL[sitename] + if sitename == "MASTER_SITES" && G.Pkg != nil { + masterSites, _ := G.Pkg.varValue("MASTER_SITES") + if !containsVarRef(masterSites) { + baseURL = masterSites + } + } + fixedURL := baseURL + subdir + explain := false + if baseURL != "" { + if !cv.line.AutofixReplace(wrong, fixedURL) { + cv.line.Warn1("HOMEPAGE should not be defined in terms of MASTER_SITEs. Use %s directly.", fixedURL) + explain = true + } + } else { + cv.line.Warn0("HOMEPAGE should not be defined in terms of MASTER_SITEs.") + explain = true + } + if explain { + Explain( + "The HOMEPAGE is a single URL, while MASTER_SITES is a list of URLs.", + "As long as this list has exactly one element, this works, but as", + "soon as another site is added, the HOMEPAGE would not be a valid", + "URL anymore.", + "", + "Defining MASTER_SITES=${HOMEPAGE} is ok, though.") + } + } +} + func (cv *VartypeCheck) Identifier() { if cv.op == opUseMatch { if cv.value == cv.valueNovar && !matches(cv.value, `^[\w*?]`) { @@ -421,6 +492,60 @@ func (cv *VartypeCheck) License() { checklineLicense(cv.mkline, cv.value) } +func (cv *VartypeCheck) MachineGnuPlatform() { + if cv.value != cv.valueNovar { + return + } + + const rePart = `(?:\[[^\]]+\]|[^-\[])+` + const rePair = `^(` + rePart + `)-(` + rePart + `)$` + const reTriple = `^(` + rePart + `)-(` + rePart + `)-(` + rePart + `)$` + + pattern := cv.value + if matches(pattern, rePair) && hasSuffix(pattern, "*") { + pattern += "-*" + } + + if m, archPattern, vendorPattern, opsysPattern := match3(pattern, reTriple); m { + archCv := &VartypeCheck{ + cv.mkline, + cv.line, + "the hardware architecture part of " + cv.varname, + opUseMatch, // Always allow patterns, since this is a PlatformPattern. + archPattern, + archPattern, + cv.comment, + cv.listContext, + cv.guessed} + enumMachineGnuArch.checker(archCv) + + _ = vendorPattern + + opsysCv := &VartypeCheck{ + cv.mkline, + cv.line, + "the operating system part of " + cv.varname, + opUseMatch, // Always allow patterns, since this is a PlatformPattern. + opsysPattern, + opsysPattern, + cv.comment, + cv.listContext, + cv.guessed} + enumMachineGnuPlatformOpsys.checker(opsysCv) + + } else { + cv.line.Warn1("%q is not a valid platform pattern.", cv.value) + Explain( + "A platform pattern has the form <OPSYS>-<OS_VERSION>-<MACHINE_ARCH>.", + "Each of these components may be a shell globbing expression.", + "", + "Examples:", + "* NetBSD-[456].*-i386", + "* *-*-*", + "* Linux-*-*") + } +} + func (cv *VartypeCheck) MailAddress() { line, value := cv.line, cv.value @@ -459,8 +584,8 @@ func (cv *VartypeCheck) Option() { line, value, valueNovar := cv.line, cv.value, cv.valueNovar if value != valueNovar { - if G.opts.DebugUnchecked { - line.Debug1("Unchecked option name: %q", value) + if G.opts.Debug { + traceStep1("Unchecked option name: %q", value) } return } @@ -575,14 +700,25 @@ func (cv *VartypeCheck) PkgRevision() { } } -func (cv *VartypeCheck) PlatformPattern() { +func (cv *VartypeCheck) MachinePlatform() { + cv.MachinePlatformPattern() +} + +func (cv *VartypeCheck) MachinePlatformPattern() { if cv.value != cv.valueNovar { return } const rePart = `(?:\[[^\]]+\]|[^-\[])+` + const rePair = `^(` + rePart + `)-(` + rePart + `)$` const reTriple = `^(` + rePart + `)-(` + rePart + `)-(` + rePart + `)$` - if m, opsysPattern, _, archPattern := match3(cv.value, reTriple); m { + + pattern := cv.value + if matches(pattern, rePair) && hasSuffix(pattern, "*") { + pattern += "-*" + } + + if m, opsysPattern, _, archPattern := match3(pattern, reTriple); m { opsysCv := &VartypeCheck{ cv.mkline, cv.line, @@ -593,7 +729,7 @@ func (cv *VartypeCheck) PlatformPattern() { cv.comment, cv.listContext, cv.guessed} - platformOsEnum.checker(opsysCv) + enumMachineOpsys.checker(opsysCv) // no check for os_version @@ -607,7 +743,7 @@ func (cv *VartypeCheck) PlatformPattern() { cv.comment, cv.listContext, cv.guessed} - platformArchEnum.checker(archCv) + enumMachineArch.checker(archCv) } else { cv.line.Warn1("%q is not a valid platform pattern.", cv.value) @@ -668,7 +804,6 @@ func (cv *VartypeCheck) SedCommand() { func (cv *VartypeCheck) SedCommands() { line := cv.line mkline := cv.mkline - shline := NewShellLine(mkline) tokens, rest := splitIntoShellTokens(line, cv.value) if rest != "" { @@ -688,7 +823,6 @@ func (cv *VartypeCheck) SedCommands() { for i := 0; i < ntokens; i++ { token := tokens[i] - shline.CheckToken(token, true) switch { case token == "-e": @@ -707,8 +841,6 @@ func (cv *VartypeCheck) SedCommands() { "", "This way, short sed commands cannot be hidden at the end of a line.") } - shline.CheckToken(tokens[i-1], true) - shline.CheckToken(tokens[i], true) mkline.CheckVartypePrimitive(cv.varname, CheckvarSedCommand, cv.op, tokens[i], cv.comment, cv.listContext, cv.guessed) } else { line.Error0("The -e option to sed requires an argument.") @@ -729,7 +861,7 @@ func (cv *VartypeCheck) SedCommands() { } func (cv *VartypeCheck) ShellCommand() { - if cv.op == opUseMatch { + if cv.op == opUseMatch || cv.op == opUse { return } setE := true @@ -743,7 +875,7 @@ func (cv *VartypeCheck) ShellCommands() { func (cv *VartypeCheck) ShellWord() { if !cv.listContext { - NewShellLine(cv.mkline).CheckToken(cv.value, true) + NewShellLine(cv.mkline).CheckWord(cv.value, true) } } @@ -762,7 +894,7 @@ func (cv *VartypeCheck) Tool() { // no warning for package-defined tool definitions } else if m, toolname, tooldep := match2(cv.value, `^([-\w]+|\[)(?::(\w+))?$`); m { - if !G.globalData.Tools[toolname] { + if G.globalData.Tools.byName[toolname] == nil { cv.line.Error1("Unknown tool %q.", toolname) } switch tooldep { @@ -905,15 +1037,24 @@ func (cv *VartypeCheck) Yes() { } func (cv *VartypeCheck) YesNo() { + const ( + yes1 = "[yY][eE][sS]" + yes2 = "[Yy][Ee][Ss]" + no1 = "[nN][oO]" + no2 = "[Nn][Oo]" + ) if cv.op == opUseMatch { switch cv.value { - case "[yY][eE][sS]": - case "[Yy][Ee][Ss]": - case "[nN][oO]": - case "[Nn][Oo]": + case yes1, yes2, no1, no2: default: - cv.line.Warnf("%s should be matched against %q or %q, not %q.", cv.varname, "[yY][eE][sS]", "[nN][oO]", cv.value) + cv.line.Warnf("%s should be matched against %q or %q, not %q.", cv.varname, yes1, no1, cv.value) } + } else if cv.op == opUse { + cv.line.Warnf("%s should be matched against %q or %q, not compared with %q.", cv.varname, yes1, no1, cv.value) + Explain( + "The yes/no value can be written in either upper or lower case, and", + "both forms are actually used. As long as this is the case, when", + "checking the variable value, both must be accepted.") } else if !matches(cv.value, `^(?:YES|yes|NO|no)(?:\s+#.*)?$`) { cv.line.Warn1("%s should be set to YES, yes, NO, or no.", cv.varname) } diff --git a/pkgtools/pkglint/files/vartypecheck_test.go b/pkgtools/pkglint/files/vartypecheck_test.go index dcd0615fe32..d5a5be6cafd 100644 --- a/pkgtools/pkglint/files/vartypecheck_test.go +++ b/pkgtools/pkglint/files/vartypecheck_test.go @@ -7,15 +7,11 @@ import ( ) func (s *Suite) TestVartypeCheck_AwkCommand(c *check.C) { - s.UseCommandLine(c, "-Dunchecked") runVartypeChecks("PLIST_AWK", opAssignAppend, (*VartypeCheck).AwkCommand, "{print $0}", "{print $$0}") - c.Check(s.Output(), equals, ""+ - "ERROR: fname:1: Invalid Makefile syntax at \"$0}\".\n"+ - "DEBUG: fname:1: Unchecked AWK command: \"{print $0}\"\n"+ - "DEBUG: fname:2: Unchecked AWK command: \"{print $$0}\"\n") + c.Check(s.Output(), equals, "WARN: fname:1: $0 is ambiguous. Use ${0} if you mean a Makefile variable or $$0 if you mean a shell variable.\n") } func (s *Suite) TestVartypeCheck_BasicRegularExpression(c *check.C) { @@ -23,7 +19,7 @@ func (s *Suite) TestVartypeCheck_BasicRegularExpression(c *check.C) { ".*\\.pl$", ".*\\.pl$$") - c.Check(s.Output(), equals, "ERROR: fname:1: Invalid Makefile syntax at \"$\".\n") + c.Check(s.Output(), equals, "WARN: fname:1: Pkglint parse error in MkLine.Tokenize at \"$\".\n") } func (s *Suite) TestVartypeCheck_BuildlinkDepmethod(c *check.C) { @@ -36,15 +32,19 @@ func (s *Suite) TestVartypeCheck_BuildlinkDepmethod(c *check.C) { func (s *Suite) TestVartypeCheck_Category(c *check.C) { s.CreateTmpFile(c, "filesyscategory/Makefile", "# empty\n") + s.CreateTmpFile(c, "wip/Makefile", "# empty\n") G.CurrentDir = s.tmpdir G.CurPkgsrcdir = "." runVartypeChecks("CATEGORIES", opAssign, (*VartypeCheck).Category, "chinese", "arabic", - "filesyscategory") + "filesyscategory", + "wip") - c.Check(s.Output(), equals, "ERROR: fname:2: Invalid category \"arabic\".\n") + c.Check(s.Output(), equals, ""+ + "ERROR: fname:2: Invalid category \"arabic\".\n"+ + "ERROR: fname:4: Invalid category \"wip\".\n") } func (s *Suite) TestVartypeCheck_CFlag(c *check.C) { @@ -166,8 +166,8 @@ func (s *Suite) TestVartypeCheck_EmulPlatform(c *check.C) { "${LINUX}") c.Check(s.Output(), equals, ""+ - "WARN: fname:2: \"nextbsd\" is not valid for the operating system part of EMUL_PLATFORM. Use one of { bitrig bsdos cygwin darwin dragonfly freebsd haiku hpux interix irix linux mirbsd netbsd openbsd osf1 solaris } instead.\n"+ - "WARN: fname:2: \"8087\" is not valid for the hardware architecture part of EMUL_PLATFORM. Use one of { alpha amd64 arc arm arm32 cobalt convex dreamcast hpcmips hpcsh hppa i386 ia64 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 ns32k pc532 pmax powerpc rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } instead.\n"+ + "WARN: fname:2: \"nextbsd\" is not valid for the operating system part of EMUL_PLATFORM. Use one of { bitrig bsdos cygwin darwin dragonfly freebsd haiku hpux interix irix linux mirbsd netbsd openbsd osf1 solaris sunos } instead.\n"+ + "WARN: fname:2: \"8087\" is not valid for the hardware architecture part of EMUL_PLATFORM. Use one of { aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } instead.\n"+ "WARN: fname:3: \"${LINUX}\" is not a valid emulation platform.\n") } @@ -182,14 +182,8 @@ func (s *Suite) TestVartypeCheck_Enum(c *check.C) { } func (s *Suite) TestVartypeCheck_FetchURL(c *check.C) { - G.globalData.MasterSiteUrls = map[string]string{ - "https://github.com/": "MASTER_SITE_GITHUB", - "http://ftp.gnu.org/pub/gnu/": "MASTER_SITE_GNU", - } - G.globalData.MasterSiteVars = map[string]bool{ - "MASTER_SITE_GITHUB": true, - "MASTER_SITE_GNU": true, - } + s.RegisterMasterSite("MASTER_SITE_GNU", "http://ftp.gnu.org/pub/gnu/") + s.RegisterMasterSite("MASTER_SITE_GITHUB", "https://github.com/") runVartypeChecks("MASTER_SITES", opAssign, (*VartypeCheck).FetchURL, "https://github.com/example/project/", @@ -201,13 +195,20 @@ func (s *Suite) TestVartypeCheck_FetchURL(c *check.C) { "WARN: fname:1: Please use ${MASTER_SITE_GITHUB:=example/} instead of \"https://github.com/example/project/\" and run \""+confMake+" help topic=github\" for further tips.\n"+ "WARN: fname:2: Please use ${MASTER_SITE_GNU:=bison} instead of \"http://ftp.gnu.org/pub/gnu/bison\".\n"+ "ERROR: fname:3: The subdirectory in MASTER_SITE_GNU must end with a slash.\n"+ - "ERROR: fname:4: MASTER_SITE_INVALID does not exist.\n") + "ERROR: fname:4: The site MASTER_SITE_INVALID does not exist.\n") // PR 46570, keyword gimp-fix-ca runVartypeChecks("MASTER_SITES", opAssign, (*VartypeCheck).FetchURL, "https://example.org/download.cgi?fname=fname&sha1=12341234") c.Check(s.Output(), equals, "") + + runVartypeChecks("MASTER_SITES", opAssign, (*VartypeCheck).FetchURL, + "http://example.org/distfiles/", + "http://example.org/download?fname=distfile;version=1.0", + "http://example.org/download?fname=<distfile>;version=<version>") + + c.Check(s.Output(), equals, "WARN: fname:3: \"http://example.org/download?fname=<distfile>;version=<version>\" is not a valid URL.\n") } func (s *Suite) TestVartypeCheck_Filename(c *check.C) { @@ -230,6 +231,16 @@ func (s *Suite) TestVartypeCheck_LdFlag(c *check.C) { c.Check(s.Output(), equals, "WARN: fname:4: Unknown linker flag \"-unknown\".\n") } +func (s *Suite) TestVartypeCheck_MachineGnuPlatform(c *check.C) { + runVartypeMatchChecks("MACHINE_GNU_PLATFORM", (*VartypeCheck).MachineGnuPlatform, + "x86_64-pc-cygwin", + "Cygwin-*-amd64") + + c.Check(s.Output(), equals, ""+ + "WARN: fname:2: The pattern \"Cygwin\" cannot match any of { aarch64 aarch64_be alpha amd64 arc arm armeb armv4 armv4eb armv6 armv6eb armv7 armv7eb cobalt convex dreamcast hpcmips hpcsh hppa hppa64 i386 i486 ia64 m5407 m68010 m68k m88k mips mips64 mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh shle sparc sparc64 vax x86_64 } for the hardware architecture part of MACHINE_GNU_PLATFORM.\n"+ + "WARN: fname:2: The pattern \"amd64\" cannot match any of { bitrig bsdos cygwin darwin dragonfly freebsd haiku hpux interix irix linux mirbsd netbsd openbsd osf1 solaris sunos } for the operating system part of MACHINE_GNU_PLATFORM.\n") +} + func (s *Suite) TestVartypeCheck_MailAddress(c *check.C) { runVartypeChecks("MAINTAINER", opAssign, (*VartypeCheck).MailAddress, "pkgsrc-users@netbsd.org") @@ -287,20 +298,23 @@ func (s *Suite) TestVartypeCheck_PkgRevision(c *check.C) { c.Check(s.Output(), equals, "") } -func (s *Suite) TestVartypeCheck_PlatformPattern(c *check.C) { - runVartypeMatchChecks("ONLY_FOR_PLATFORM", (*VartypeCheck).PlatformPattern, +func (s *Suite) TestVartypeCheck_MachinePlatformPattern(c *check.C) { + runVartypeMatchChecks("ONLY_FOR_PLATFORM", (*VartypeCheck).MachinePlatformPattern, "linux-i386", "nextbsd-5.0-8087", "netbsd-7.0-l*", "NetBSD-1.6.2-i386", + "FreeBSD*", + "FreeBSD-*", "${LINUX}") c.Check(s.Output(), equals, ""+ "WARN: fname:1: \"linux-i386\" is not a valid platform pattern.\n"+ - "WARN: fname:2: The pattern \"nextbsd\" cannot match any of { Bitrig BSDOS Cygwin Darwin DragonFly FreeBSD Haiku HPUX Interix IRIX Linux MirBSD NetBSD OpenBSD OSF1 QNX SunOS } for the operating system part of ONLY_FOR_PLATFORM.\n"+ - "WARN: fname:2: The pattern \"8087\" cannot match any of { alpha amd64 arc arm arm32 cobalt convex dreamcast hpcmips hpcsh hppa i386 ia64 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 ns32k pc532 pmax powerpc rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } for the hardware architecture part of ONLY_FOR_PLATFORM.\n"+ - "WARN: fname:3: The pattern \"netbsd\" cannot match any of { Bitrig BSDOS Cygwin Darwin DragonFly FreeBSD Haiku HPUX Interix IRIX Linux MirBSD NetBSD OpenBSD OSF1 QNX SunOS } for the operating system part of ONLY_FOR_PLATFORM.\n"+ - "WARN: fname:3: The pattern \"l*\" cannot match any of { alpha amd64 arc arm arm32 cobalt convex dreamcast hpcmips hpcsh hppa i386 ia64 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 ns32k pc532 pmax powerpc rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } for the hardware architecture part of ONLY_FOR_PLATFORM.\n") + "WARN: fname:2: The pattern \"nextbsd\" cannot match any of { AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare } for the operating system part of ONLY_FOR_PLATFORM.\n"+ + "WARN: fname:2: The pattern \"8087\" cannot match any of { aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } for the hardware architecture part of ONLY_FOR_PLATFORM.\n"+ + "WARN: fname:3: The pattern \"netbsd\" cannot match any of { AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare } for the operating system part of ONLY_FOR_PLATFORM.\n"+ + "WARN: fname:3: The pattern \"l*\" cannot match any of { aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 } for the hardware architecture part of ONLY_FOR_PLATFORM.\n"+ + "WARN: fname:5: \"FreeBSD*\" is not a valid platform pattern.\n") } func (s *Suite) TestVartypeCheck_PythonDependency(c *check.C) { @@ -330,8 +344,7 @@ func (s *Suite) TestVartypeCheck_SedCommands(c *check.C) { c.Check(s.Output(), equals, ""+ "NOTE: fname:1: Please always use \"-e\" in sed commands, even if there is only one substitution.\n"+ - "NOTE: fname:2: Each sed command should appear in an assignment of its own.\n"+ - "ERROR: fname:3: Invalid shell words \"\\\"s,\" in sed commands.\n") + "NOTE: fname:2: Each sed command should appear in an assignment of its own.\n") } func (s *Suite) TestVartypeCheck_ShellCommands(c *check.C) { @@ -351,15 +364,6 @@ func (s *Suite) TestVartypeCheck_Stage(c *check.C) { c.Check(s.Output(), equals, "WARN: fname:2: Invalid stage name \"post-modern\". Use one of {pre,do,post}-{extract,patch,configure,build,test,install}.\n") } -func (s *Suite) TestVartypeCheck_URL(c *check.C) { - runVartypeChecks("MASTER_SITES", opAssign, (*VartypeCheck).URL, - "http://example.org/distfiles/", - "http://example.org/download?fname=distfile;version=1.0", - "http://example.org/download?fname=<distfile>;version=<version>") - - c.Check(s.Output(), equals, "WARN: fname:3: \"http://example.org/download?fname=<distfile>;version=<version>\" is not a valid URL.\n") -} - func (s *Suite) TestVartypeCheck_Varname(c *check.C) { runVartypeChecks("BUILD_DEFS", opAssign, (*VartypeCheck).Varname, "VARBASE", |